From dovecot at dovecot.org Thu Nov 1 12:32:42 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 01 Nov 2012 12:32:42 +0200 Subject: dovecot-2.2: lib-storage: Error logging fixes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d7239a4554e4 changeset: 15366:d7239a4554e4 user: Timo Sirainen date: Thu Nov 01 12:32:30 2012 +0200 description: lib-storage: Error logging fixes. diffstat: src/lib-storage/index/index-storage.c | 8 ++------ src/lib-storage/index/index-sync-pvt.c | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diffs (42 lines): diff -r 626a9df21e62 -r d7239a4554e4 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Wed Oct 31 14:44:24 2012 +0200 +++ b/src/lib-storage/index/index-storage.c Thu Nov 01 12:32:30 2012 +0200 @@ -202,10 +202,8 @@ if (box->index != NULL) return 0; - if (mailbox_create_missing_dir(box, MAILBOX_LIST_PATH_TYPE_INDEX) < 0) { - mail_storage_set_internal_error(box->storage); + if (mailbox_create_missing_dir(box, MAILBOX_LIST_PATH_TYPE_INDEX) < 0) return -1; - } if (index_mailbox_alloc_index(box, &box->index) < 0) return -1; @@ -502,10 +500,8 @@ path = t_strdup_until(path, p); } - if ((ret = mailbox_mkdir(box, path, type)) < 0) { - mail_storage_copy_list_error(box->storage, box->list); + if ((ret = mailbox_mkdir(box, path, type)) < 0) return -1; - } mailbox_refresh_permissions(box); if (ret == 0) { /* directory already exists */ diff -r 626a9df21e62 -r d7239a4554e4 src/lib-storage/index/index-sync-pvt.c --- a/src/lib-storage/index/index-sync-pvt.c Wed Oct 31 14:44:24 2012 +0200 +++ b/src/lib-storage/index/index-sync-pvt.c Thu Nov 01 12:32:30 2012 +0200 @@ -18,10 +18,8 @@ if (ret <= 0) return ret; /* error / no private indexes */ - if (mailbox_create_missing_dir(box, MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE) < 0) { - mail_storage_set_internal_error(box->storage); + if (mailbox_create_missing_dir(box, MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE) < 0) return -1; - } box->index_pvt = mail_index_alloc_cache_get(NULL, index_dir, t_strconcat(box->index_prefix, ".pvt", NULL)); From dovecot at dovecot.org Sat Nov 3 13:48:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 13:48:06 +0200 Subject: dovecot-2.2: lib-index: Write to transaction log using O_APPEND ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/751181168b72 changeset: 15367:751181168b72 user: Timo Sirainen date: Sat Nov 03 13:47:55 2012 +0200 description: lib-index: Write to transaction log using O_APPEND flag. Most OSes should do the writes atomically so that reads won't see partially written data. We don't currently rely on this, but it would be possible in future to remove locking entirely from writing to transaction log. diffstat: src/lib-index/mail-transaction-log-append.c | 53 +++++++++++++--------------- src/lib-index/mail-transaction-log-file.c | 24 +++++++------ src/lib-index/mail-transaction-log.h | 2 + 3 files changed, 39 insertions(+), 40 deletions(-) diffs (193 lines): diff -r d7239a4554e4 -r 751181168b72 src/lib-index/mail-transaction-log-append.c --- a/src/lib-index/mail-transaction-log-append.c Thu Nov 01 12:32:30 2012 +0200 +++ b/src/lib-index/mail-transaction-log-append.c Sat Nov 03 13:47:55 2012 +0200 @@ -32,6 +32,7 @@ buffer_append(ctx->output, data, size); mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq); + ctx->transaction_count++; } static int @@ -60,8 +61,6 @@ static int log_buffer_write(struct mail_transaction_log_append_ctx *ctx) { struct mail_transaction_log_file *file = ctx->log->head; - struct mail_transaction_header *hdr; - uint32_t first_size; if (ctx->output->used == 0) return 0; @@ -76,19 +75,11 @@ return 0; } - /* size will be written later once everything is in disk */ - hdr = buffer_get_space_unsafe(ctx->output, 0, sizeof(*hdr)); - first_size = hdr->size; - i_assert(first_size != 0); - hdr->size = 0; - - if (pwrite_full(file->fd, ctx->output->data, ctx->output->used, - file->sync_offset) < 0) { + if (write_full(file->fd, ctx->output->data, ctx->output->used) < 0) { /* write failure, fallback to in-memory indexes. */ - hdr->size = first_size; mail_index_file_set_syscall_error(ctx->log->index, file->filepath, - "pwrite_full()"); + "write_full()"); return log_buffer_move_to_memory(ctx); } @@ -96,18 +87,6 @@ file->sync_offset + ctx->output->used == file->max_tail_offset); - /* now that the whole transaction has been written, rewrite the first - record's size so the transaction becomes visible */ - hdr->size = first_size; - if (pwrite_full(file->fd, &first_size, sizeof(uint32_t), - file->sync_offset + - offsetof(struct mail_transaction_header, size)) < 0) { - mail_index_file_set_syscall_error(ctx->log->index, - file->filepath, - "pwrite_full()"); - return log_buffer_move_to_memory(ctx); - } - if ((ctx->want_fsync && file->log->index->fsync_mode != FSYNC_MODE_NEVER) || file->log->index->fsync_mode == FSYNC_MODE_ALWAYS) { @@ -119,11 +98,6 @@ } } - /* FIXME: when we're relying on O_APPEND and someone else wrote a - transaction, we'll need to wait for it to commit its transaction. - if it crashes before doing that, we'll need to overwrite it with - a dummy record */ - if (file->mmap_base == NULL && file->buffer != NULL) { /* we're reading from a file. avoid re-reading the data that we just wrote. this is also important for some NFS clients, @@ -184,6 +158,7 @@ mail_transaction_log_append_locked(struct mail_transaction_log_append_ctx *ctx) { struct mail_transaction_log_file *file = ctx->log->head; + struct mail_transaction_boundary *boundary; if (file->sync_offset < file->last_size) { /* there is some garbage at the end of the transaction log @@ -199,6 +174,21 @@ } } + /* don't include log_file_tail_offset update in the transaction */ + boundary = buffer_get_space_unsafe(ctx->output, + sizeof(struct mail_transaction_header), + sizeof(*boundary)); + boundary->size = ctx->output->used; + + if (ctx->transaction_count <= 2) { + /* 0-1 changes. don't bother with the boundary */ + unsigned int boundary_size = + sizeof(struct mail_transaction_header) + + sizeof(*boundary); + + buffer_delete(ctx->output, 0, boundary_size); + } + if (ctx->append_sync_offset) log_append_sync_offset_if_needed(ctx); @@ -213,6 +203,7 @@ struct mail_transaction_log_append_ctx **ctx_r) { struct mail_transaction_log_append_ctx *ctx; + struct mail_transaction_boundary boundary; if (!index->log_sync_locked) { if (mail_transaction_log_lock_head(index->log) < 0) @@ -223,6 +214,10 @@ ctx->output = buffer_create_dynamic(default_pool, 1024); ctx->trans_flags = flags; + memset(&boundary, 0, sizeof(boundary)); + mail_transaction_log_append_add(ctx, MAIL_TRANSACTION_BOUNDARY, + &boundary, sizeof(boundary)); + *ctx_r = ctx; return 0; } diff -r d7239a4554e4 -r 751181168b72 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Thu Nov 01 12:32:30 2012 +0200 +++ b/src/lib-index/mail-transaction-log-file.c Sat Nov 03 13:47:55 2012 +0200 @@ -653,6 +653,11 @@ int fd, ret; bool rename_existing; + if (fcntl(new_fd, F_SETFL, O_APPEND) < 0) { + log_file_set_syscall_error(file, "fcntl(O_APPEND)"); + return -1; + } + if (file->log->nfs_flush) { /* although we check also mtime and file size below, it's done only to fix broken log files. we don't bother flushing @@ -684,7 +689,7 @@ rename_existing = TRUE; } else { /* recreated. use the file if its header is ok */ - fd = nfs_safe_open(file->filepath, O_RDWR); + fd = nfs_safe_open(file->filepath, O_RDWR | O_APPEND); if (fd == -1) { if (errno != ENOENT) { log_file_set_syscall_error(file, "open()"); @@ -843,8 +848,12 @@ int ret; for (i = 0;; i++) { - file->fd = nfs_safe_open(file->filepath, - !index->readonly ? O_RDWR : O_RDONLY); + if (!index->readonly) { + file->fd = nfs_safe_open(file->filepath, + O_RDWR | O_APPEND); + } else { + file->fd = nfs_safe_open(file->filepath, O_RDONLY); + } if (file->fd == -1 && errno == EACCES) { file->fd = nfs_safe_open(file->filepath, O_RDONLY); index->readonly = TRUE; @@ -1370,18 +1379,11 @@ /* There's more data than we could sync at the moment. If the last record's size wasn't valid, we can't know if it will be updated unless we've locked the log. */ - if (trans_size != 0) { - /* pread()s or the above fstat() check for mmaps should - have guaranteed that this doesn't happen */ - mail_transaction_log_file_set_corrupted(file, - "hdr.size too large (%u)", trans_size); - return -1; - } else if (file->locked) { + if (file->locked) { mail_transaction_log_file_set_corrupted(file, "Unexpected garbage at EOF"); return -1; } - /* The size field will be updated soon */ mail_index_flush_read_cache(file->log->index, file->filepath, file->fd, file->locked); diff -r d7239a4554e4 -r 751181168b72 src/lib-index/mail-transaction-log.h --- a/src/lib-index/mail-transaction-log.h Thu Nov 01 12:32:30 2012 +0200 +++ b/src/lib-index/mail-transaction-log.h Sat Nov 03 13:47:55 2012 +0200 @@ -173,6 +173,8 @@ enum mail_transaction_type trans_flags; uint64_t new_highest_modseq; + unsigned int transaction_count; + unsigned int append_sync_offset:1; unsigned int sync_includes_this:1; unsigned int want_fsync:1; From dovecot at dovecot.org Sat Nov 3 14:11:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 14:11:39 +0200 Subject: dovecot-2.2: lib-storage: Crashfix when doing userdb lookup. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1ad12af6efe4 changeset: 15368:1ad12af6efe4 user: Timo Sirainen date: Sat Nov 03 14:11:29 2012 +0200 description: lib-storage: Crashfix when doing userdb lookup. diffstat: src/lib-storage/mail-storage-service.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 751181168b72 -r 1ad12af6efe4 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Sat Nov 03 13:47:55 2012 +0200 +++ b/src/lib-storage/mail-storage-service.c Sat Nov 03 14:11:29 2012 +0200 @@ -1010,7 +1010,8 @@ pool_unref(&user_pool); return ret; } - *ctx->userdb_next_fieldsp = userdb_fields; + if (ctx->userdb_next_fieldsp != NULL) + *ctx->userdb_next_fieldsp = userdb_fields; } else { userdb_fields = input->userdb_fields; } From dovecot at dovecot.org Sat Nov 3 18:43:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 18:43:11 +0200 Subject: dovecot-2.2: lib-index: Recent cache file changes broke writing. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/899873ebc1cf changeset: 15369:899873ebc1cf user: Timo Sirainen date: Sat Nov 03 18:43:00 2012 +0200 description: lib-index: Recent cache file changes broke writing. diffstat: src/lib-index/mail-cache.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1ad12af6efe4 -r 899873ebc1cf src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Sat Nov 03 14:11:29 2012 +0200 +++ b/src/lib-index/mail-cache.c Sat Nov 03 18:43:00 2012 +0200 @@ -671,7 +671,7 @@ file_cache_invalidate(cache->file_cache, 0, sizeof(struct mail_cache_header)); } - if (mail_cache_map(cache, 0, 0, &data) == 0) + if (mail_cache_map(cache, 0, 0, &data) > 0) cache->hdr_copy = *cache->hdr; else { (void)mail_cache_unlock(cache); From dovecot at dovecot.org Sat Nov 3 18:46:32 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 18:46:32 +0200 Subject: dovecot-2.1: lib-index: Recent cache file changes broke writing. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/2de37734d0b9 changeset: 14794:2de37734d0b9 user: Timo Sirainen date: Sat Nov 03 18:43:00 2012 +0200 description: lib-index: Recent cache file changes broke writing. diffstat: src/lib-index/mail-cache.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7d931927e4ac -r 2de37734d0b9 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Wed Oct 31 13:24:49 2012 +0200 +++ b/src/lib-index/mail-cache.c Sat Nov 03 18:43:00 2012 +0200 @@ -684,7 +684,7 @@ file_cache_invalidate(cache->file_cache, 0, sizeof(struct mail_cache_header)); } - if (mail_cache_map(cache, 0, 0, &data) == 0) + if (mail_cache_map(cache, 0, 0, &data) > 0) cache->hdr_copy = *cache->hdr; else { (void)mail_cache_unlock(cache); From dovecot at dovecot.org Sat Nov 3 19:26:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 19:26:22 +0200 Subject: dovecot-2.2: lib-index: Transaction view shouldn't return ext re... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/148dc0f6f990 changeset: 15370:148dc0f6f990 user: Timo Sirainen date: Sat Nov 03 19:13:24 2012 +0200 description: lib-index: Transaction view shouldn't return ext records if the ext was already reset. diffstat: src/lib-index/mail-index-transaction-view.c | 19 +++++++++++++++++-- 1 files changed, 17 insertions(+), 2 deletions(-) diffs (36 lines): diff -r 899873ebc1cf -r 148dc0f6f990 src/lib-index/mail-index-transaction-view.c --- a/src/lib-index/mail-index-transaction-view.c Sat Nov 03 18:43:00 2012 +0200 +++ b/src/lib-index/mail-index-transaction-view.c Sat Nov 03 19:13:24 2012 +0200 @@ -381,6 +381,19 @@ } } +static bool +tview_is_ext_reset(struct mail_index_view_transaction *tview, uint32_t ext_id) +{ + const struct mail_transaction_ext_reset *resets; + unsigned int count; + + if (!array_is_created(&tview->t->ext_resets)) + return FALSE; + + resets = array_get(&tview->t->ext_resets, &count); + return ext_id < count && resets[ext_id].new_reset_id != 0; +} + static void tview_lookup_ext_full(struct mail_index_view *view, uint32_t seq, uint32_t ext_id, struct mail_index_map **map_r, @@ -411,8 +424,10 @@ } } - /* not updated, return the existing value */ - if (seq < tview->t->first_new_seq) { + /* not updated, return the existing value, unless ext was + already reset */ + if (seq < tview->t->first_new_seq && + !tview_is_ext_reset(tview, ext_id)) { tview->super->lookup_ext_full(view, seq, ext_id, map_r, data_r, expunged_r); } else { From dovecot at dovecot.org Sat Nov 3 19:26:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 19:26:22 +0200 Subject: dovecot-2.2: lib-index: Cache record linking is now while writin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/67afcb730109 changeset: 15371:67afcb730109 user: Timo Sirainen date: Sat Nov 03 19:25:35 2012 +0200 description: lib-index: Cache record linking is now while writing new records, not later with pwrite(). This should make performance somewhat better. It also means that now the only overwrites to cache file are to the file header and the fields header. This means that we no longer have to invalidate file_cache (with mmap_disable=yes) except when reading the headers, which is always done anyway. It would be nice to move the overwritten fields in the headers to a cache header in dovecot.index file, but this is a bit tricky currently. When writing these headers the cache file needs to be locked to avoid race conditions. To avoid deadlocks the transaction log would have to be locked before the cache. The code to do all this would still be a bit complex, and to keep backwards compatibility with old Dovecot versions it would still have to do the overwriting until the backwards compatibility code is disabled. diffstat: src/lib-index/mail-cache-fields.c | 9 +- src/lib-index/mail-cache-private.h | 9 +- src/lib-index/mail-cache-sync-update.c | 91 ----------------- src/lib-index/mail-cache-transaction.c | 176 ++++++++++++++++---------------- src/lib-index/mail-cache.c | 40 ++----- 5 files changed, 103 insertions(+), 222 deletions(-) diffs (truncated from 516 to 300 lines): diff -r 148dc0f6f990 -r 67afcb730109 src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Sat Nov 03 19:13:24 2012 +0200 +++ b/src/lib-index/mail-cache-fields.c Sat Nov 03 19:25:35 2012 +0200 @@ -206,7 +206,6 @@ const void *data; uint32_t offset = 0, next_offset; unsigned int next_count = 0; - bool invalidate = FALSE; int ret; if (MAIL_CACHE_IS_UNUSABLE(cache)) { @@ -228,7 +227,6 @@ return -1; } offset = next_offset; - invalidate = TRUE; if (cache->mmap_base != NULL || cache->map_with_read) { ret = mail_cache_map(cache, offset, sizeof(*field_hdr), @@ -274,10 +272,9 @@ cache->need_compress_file_seq = cache->hdr->file_seq; if (field_hdr_r != NULL) { - if (cache->file_cache != NULL && invalidate) { - /* if this isn't the first header in file and we hadn't - read this before, we can't trust that the cached - data is valid */ + if (cache->file_cache != NULL) { + /* invalidate the cache fields area to make sure we + get the latest cache decisions/last_used fields */ file_cache_invalidate(cache->file_cache, offset, field_hdr->size); } diff -r 148dc0f6f990 -r 67afcb730109 src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Sat Nov 03 19:13:24 2012 +0200 +++ b/src/lib-index/mail-cache-private.h Sat Nov 03 19:25:35 2012 +0200 @@ -210,7 +210,7 @@ int mail_cache_write(struct mail_cache *cache, const void *data, size_t size, uoff_t offset); int mail_cache_append(struct mail_cache *cache, const void *data, size_t size, - uint32_t *offset_r); + uint32_t *offset); int mail_cache_header_fields_read(struct mail_cache *cache); int mail_cache_header_fields_update(struct mail_cache *cache); @@ -242,9 +242,6 @@ void mail_cache_file_close(struct mail_cache *cache); int mail_cache_reopen(struct mail_cache *cache); -/* Update new_offset's prev_offset field to old_offset. */ -int mail_cache_link(struct mail_cache *cache, uint32_t old_offset, - uint32_t new_offset); /* Mark record in given offset to be deleted. */ int mail_cache_delete(struct mail_cache *cache, uint32_t offset); @@ -258,10 +255,6 @@ int mail_cache_expunge_handler(struct mail_index_sync_map_ctx *sync_ctx, uint32_t seq, const void *data, void **sync_context, void *context); -int mail_cache_sync_handler(struct mail_index_sync_map_ctx *sync_ctx, - uint32_t seq, void *old_data, const void *new_data, - void **context); -void mail_cache_sync_lost_handler(struct mail_index *index); void mail_cache_set_syscall_error(struct mail_cache *cache, const char *function); diff -r 148dc0f6f990 -r 67afcb730109 src/lib-index/mail-cache-sync-update.c --- a/src/lib-index/mail-cache-sync-update.c Sat Nov 03 19:13:24 2012 +0200 +++ b/src/lib-index/mail-cache-sync-update.c Sat Nov 03 19:25:35 2012 +0200 @@ -1,17 +1,12 @@ /* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" -#include "file-cache.h" #include "mail-cache-private.h" -#include "mail-index-view-private.h" #include "mail-index-sync-private.h" struct mail_cache_sync_context { - uoff_t invalidate_highwater; - unsigned int locked:1; unsigned int lock_failed:1; - unsigned int nfs_read_cache_flushed:1; }; static void mail_cache_handler_deinit(struct mail_index_sync_map_ctx *sync_ctx, @@ -34,7 +29,6 @@ else { *context = i_new(struct mail_cache_sync_context, 1); ctx = *context; - ctx->invalidate_highwater = (uoff_t)-1; } return ctx; } @@ -104,88 +98,3 @@ (void)mail_cache_delete(cache, *cache_offset); return 0; } - -int mail_cache_sync_handler(struct mail_index_sync_map_ctx *sync_ctx, - uint32_t seq ATTR_UNUSED, - void *old_data, const void *new_data, - void **context) -{ - struct mail_index_view *view = sync_ctx->view; - struct mail_index *index = view->index; - struct mail_cache *cache = index->cache; - struct mail_cache_sync_context *ctx = *context; - const uint32_t *old_cache_offset = old_data; - const uint32_t *new_cache_offset = new_data; - uint32_t cache_file_seq, cur_seq, tail_seq; - uoff_t cur_offset, tail_offset; - int ret; - - if (new_cache_offset == NULL) { - mail_cache_handler_deinit(sync_ctx, ctx); - *context = NULL; - return 1; - } - - ctx = mail_cache_handler_init(context); - if (cache->file_cache != NULL && !MAIL_CACHE_IS_UNUSABLE(cache)) { - /* flush read cache only once per sync */ - if (!ctx->nfs_read_cache_flushed && - (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) { - ctx->nfs_read_cache_flushed = TRUE; - mail_index_flush_read_cache(index, - cache->filepath, cache->fd, - cache->locked); - } - /* don't invalidate anything that's already been invalidated - within this sync. */ - if (*new_cache_offset < ctx->invalidate_highwater) { - file_cache_invalidate(cache->file_cache, - *new_cache_offset, - ctx->invalidate_highwater - - *new_cache_offset); - ctx->invalidate_highwater = *new_cache_offset; - } - } - - if (*old_cache_offset == 0 || *old_cache_offset == *new_cache_offset || - sync_ctx->type == MAIL_INDEX_SYNC_HANDLER_VIEW) - return 1; - - mail_transaction_log_view_get_prev_pos(view->log_view, - &cur_seq, &cur_offset); - mail_transaction_log_get_mailbox_sync_pos(index->log, - &tail_seq, &tail_offset); - if (LOG_IS_BEFORE(cur_seq, cur_offset, tail_seq, tail_offset)) { - /* already been linked */ - return 1; - } - - /* we'll need to link the old and new cache records */ - ret = mail_cache_handler_lock(ctx, cache); - if (ret <= 0) - return ret < 0 ? -1 : 1; - - if (!get_cache_file_seq(view, &cache_file_seq)) - return 1; - - if (cache_file_seq != cache->hdr->file_seq) { - /* cache has been compressed, don't modify it */ - return 1; - } - - if (mail_cache_link(cache, *old_cache_offset, *new_cache_offset) < 0) - return -1; - - return 1; -} - -void mail_cache_sync_lost_handler(struct mail_index *index) -{ - struct mail_cache *cache = index->cache; - - if (!MAIL_CACHE_IS_UNUSABLE(cache)) { - mail_index_flush_read_cache(cache->index, cache->filepath, - cache->fd, cache->locked); - } - file_cache_invalidate(cache->file_cache, 0, (uoff_t)-1); -} diff -r 148dc0f6f990 -r 67afcb730109 src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Sat Nov 03 19:13:24 2012 +0200 +++ b/src/lib-index/mail-cache-transaction.c Sat Nov 03 19:25:35 2012 +0200 @@ -34,7 +34,7 @@ buffer_t *cache_data; ARRAY(uint32_t) cache_data_seq; - uint32_t prev_seq; + uint32_t prev_seq, min_seq; size_t last_rec_pos; uoff_t bytes_written; @@ -47,8 +47,6 @@ &mail_index_module_register); static int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx); -static int mail_cache_link_locked(struct mail_cache *cache, - uint32_t old_offset, uint32_t new_offset); static void mail_index_transaction_cache_reset(struct mail_index_transaction *t) { @@ -281,7 +279,7 @@ struct mail_cache *cache = ctx->cache; const struct mail_cache_record *rec = ctx->cache_data->data; const uint32_t *seqs; - uint32_t i, seq_count, old_offset; + uint32_t i, seq_count; mail_index_ext_using_reset_id(ctx->trans, ctx->cache->ext_id, ctx->cache_file_seq); @@ -292,30 +290,7 @@ seqs = array_get(&ctx->cache_data_seq, &seq_count); for (i = 0; i < seq_count; i++) { mail_index_update_ext(ctx->trans, seqs[i], cache->ext_id, - &write_offset, &old_offset); - if (old_offset != 0) { - /* we added records for this message multiple - times in this same uncommitted transaction. - only the new one will be written to - transaction log, we need to do the linking - ourself here. */ - if (old_offset > write_offset) { - if (mail_cache_link_locked(cache, old_offset, - write_offset) < 0) - return -1; - } else { - /* if we're combining multiple transactions, - make sure the one with the smallest offset - is written into index. this is required for - non-file-mmaped cache to work properly. */ - mail_index_update_ext(ctx->trans, seqs[i], - cache->ext_id, - &old_offset, NULL); - if (mail_cache_link_locked(cache, write_offset, - old_offset) < 0) - return -1; - } - } + &write_offset, NULL); write_offset += rec->size; rec = CONST_PTR_OFFSET(rec, rec->size); @@ -324,9 +299,70 @@ } static int +mail_cache_link_records(struct mail_cache_transaction_ctx *ctx, + uint32_t write_offset) +{ + struct mail_index_map *map; + struct mail_cache_record *rec; + const uint32_t *seqs, *prev_offsetp; + ARRAY_TYPE(uint32_t) seq_offsets; + uint32_t i, seq_count, reset_id, prev_offset, *offsetp; + const void *data; + bool expunged; + + i_assert(ctx->min_seq != 0); + + i_array_init(&seq_offsets, 64); + seqs = array_get(&ctx->cache_data_seq, &seq_count); + rec = buffer_get_modifiable_data(ctx->cache_data, NULL); + for (i = 0; i < seq_count; i++) { + offsetp = array_idx_modifiable(&seq_offsets, + seqs[i] - ctx->min_seq); + if (*offsetp != 0) + prev_offset = *offsetp; + else { + mail_index_lookup_ext_full(ctx->view->trans_view, seqs[i], + ctx->cache->ext_id, &map, + &data, &expunged); + prev_offsetp = data; + + if (prev_offsetp == NULL || *prev_offsetp == 0) + prev_offset = 0; + else if (mail_index_ext_get_reset_id(ctx->view->trans_view, map, + ctx->cache->ext_id, + &reset_id) && + reset_id == ctx->cache_file_seq) + prev_offset = *prev_offsetp; + else + prev_offset = 0; + if (prev_offset >= write_offset) { + mail_cache_set_corrupted(ctx->cache, + "Cache record offset points outside existing file"); + array_free(&seq_offsets); + return -1; + } + } + + if (prev_offset != 0) { + /* link this record to previous one */ + rec->prev_offset = prev_offset; + ctx->cache->hdr_copy.continued_record_count++; + ctx->cache->hdr_modified = TRUE; + } From dovecot at dovecot.org Sat Nov 3 19:35:59 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 19:35:59 +0200 Subject: dovecot-2.2: lib-index: If a cache record is larger than 64 kB, ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d2860e8fc774 changeset: 15372:d2860e8fc774 user: Timo Sirainen date: Sat Nov 03 19:35:54 2012 +0200 description: lib-index: If a cache record is larger than 64 kB, don't add it to cache file. This shouldn't affect anything except mails that probably shouldn't exist in the first place. diffstat: src/lib-index/mail-cache-compress.c | 5 +++-- src/lib-index/mail-cache-private.h | 3 +++ src/lib-index/mail-cache-transaction.c | 12 ++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diffs (60 lines): diff -r 67afcb730109 -r d2860e8fc774 src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Sat Nov 03 19:25:35 2012 +0200 +++ b/src/lib-index/mail-cache-compress.c Sat Nov 03 19:35:54 2012 +0200 @@ -253,11 +253,12 @@ while (mail_cache_lookup_iter_next(&iter, &field) > 0) mail_cache_compress_field(&ctx, &field); - cache_rec.size = buffer_get_used_size(ctx.buffer); - if (cache_rec.size == sizeof(cache_rec)) { + if (ctx.buffer->used == sizeof(cache_rec) || + ctx.buffer->used > MAIL_CACHE_RECORD_MAX_SIZE) { /* nothing cached */ ext_offset = 0; } else { + cache_rec.size = ctx.buffer->used; ext_offset = output->offset; buffer_write(ctx.buffer, 0, &cache_rec, sizeof(cache_rec)); diff -r 67afcb730109 -r d2860e8fc774 src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Sat Nov 03 19:25:35 2012 +0200 +++ b/src/lib-index/mail-cache-private.h Sat Nov 03 19:35:54 2012 +0200 @@ -24,6 +24,9 @@ the latest cache header. */ #define MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT 4 +/* If cache record becomes larger than this, don't add it. */ +#define MAIL_CACHE_RECORD_MAX_SIZE (64*1024) + #define MAIL_CACHE_LOCK_TIMEOUT 10 #define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 300 diff -r 67afcb730109 -r d2860e8fc774 src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Sat Nov 03 19:25:35 2012 +0200 +++ b/src/lib-index/mail-cache-transaction.c Sat Nov 03 19:35:54 2012 +0200 @@ -367,6 +367,13 @@ i_assert(!ctx->cache->locked); + if (array_count(&ctx->cache_data_seq) == 0) { + /* we had done some changes, but they were aborted. */ + i_assert(ctx->last_rec_pos == 0); + ctx->min_seq = 0; + return 0; + } + if (mail_cache_transaction_lock(ctx) <= 0) return -1; @@ -424,6 +431,11 @@ rec->size = size - ctx->last_rec_pos; i_assert(rec->size > sizeof(*rec)); + if (rec->size > MAIL_CACHE_RECORD_MAX_SIZE) { + buffer_set_used_size(ctx->cache_data, ctx->last_rec_pos); + return; + } + if (ctx->min_seq > ctx->prev_seq || ctx->min_seq == 0) ctx->min_seq = ctx->prev_seq; array_append(&ctx->cache_data_seq, &ctx->prev_seq, 1); From dovecot at dovecot.org Sat Nov 3 19:37:56 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 19:37:56 +0200 Subject: dovecot-2.2: lib-index: Added minor version to dovecot.index.cac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8323e81785e2 changeset: 15373:8323e81785e2 user: Timo Sirainen date: Sat Nov 03 19:37:47 2012 +0200 description: lib-index: Added minor version to dovecot.index.cache file header. diffstat: src/doveadm/doveadm-dump-index.c | 3 ++- src/lib-index/mail-cache-compress.c | 3 ++- src/lib-index/mail-cache-private.h | 8 +++++--- src/lib-index/mail-cache.c | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diffs (64 lines): diff -r d2860e8fc774 -r 8323e81785e2 src/doveadm/doveadm-dump-index.c --- a/src/doveadm/doveadm-dump-index.c Sat Nov 03 19:35:54 2012 +0200 +++ b/src/doveadm/doveadm-dump-index.c Sat Nov 03 19:37:47 2012 +0200 @@ -318,7 +318,8 @@ } hdr = cache->hdr; - printf("version .............. = %u\n", hdr->version); + printf("major version ........ = %u\n", hdr->major_version); + printf("minor version ........ = %u\n", hdr->minor_version); printf("indexid .............. = %u (%s)\n", hdr->indexid, unixdate2str(hdr->indexid)); printf("file_seq ............. = %u (%s) (%d compressions)\n", hdr->file_seq, unixdate2str(hdr->file_seq), diff -r d2860e8fc774 -r 8323e81785e2 src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Sat Nov 03 19:35:54 2012 +0200 +++ b/src/lib-index/mail-cache-compress.c Sat Nov 03 19:37:47 2012 +0200 @@ -171,7 +171,8 @@ output = o_stream_create_fd_file(fd, 0, FALSE); memset(&hdr, 0, sizeof(hdr)); - hdr.version = MAIL_CACHE_VERSION; + hdr.major_version = MAIL_CACHE_MAJOR_VERSION; + hdr.minor_version = MAIL_CACHE_MINOR_VERSION; hdr.compat_sizeof_uoff_t = sizeof(uoff_t); hdr.indexid = cache->index->indexid; hdr.file_seq = get_next_file_seq(cache, view); diff -r d2860e8fc774 -r 8323e81785e2 src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Sat Nov 03 19:35:54 2012 +0200 +++ b/src/lib-index/mail-cache-private.h Sat Nov 03 19:37:47 2012 +0200 @@ -5,7 +5,8 @@ #include "mail-index-private.h" #include "mail-cache.h" -#define MAIL_CACHE_VERSION 1 +#define MAIL_CACHE_MAJOR_VERSION 1 +#define MAIL_CACHE_MINOR_VERSION 1 /* Drop fields that haven't been accessed for n seconds */ #define MAIL_CACHE_FIELD_DROP_SECS (3600*24*30) @@ -36,9 +37,10 @@ struct mail_cache_header { /* version is increased only when you can't have backwards compatibility. */ - uint8_t version; + uint8_t major_version; uint8_t compat_sizeof_uoff_t; - uint8_t unused[2]; + uint8_t minor_version; + uint8_t unused; uint32_t indexid; uint32_t file_seq; diff -r d2860e8fc774 -r 8323e81785e2 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Sat Nov 03 19:35:54 2012 +0200 +++ b/src/lib-index/mail-cache.c Sat Nov 03 19:37:47 2012 +0200 @@ -232,7 +232,7 @@ return FALSE; } - if (hdr->version != MAIL_CACHE_VERSION) { + if (hdr->major_version != MAIL_CACHE_MAJOR_VERSION) { /* version changed - upgrade silently */ mail_cache_unlink(cache); return FALSE; From dovecot at dovecot.org Sat Nov 3 20:25:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 03 Nov 2012 20:25:55 +0200 Subject: dovecot-2.2: lib-index: Replaced cache's deleted_space with dele... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/debecd057f9c changeset: 15374:debecd057f9c user: Timo Sirainen date: Sat Nov 03 20:25:17 2012 +0200 description: lib-index: Replaced cache's deleted_space with deleted_record_count. This way when expunging messages we don't need to actually read through the cache file and calculate how many bytes will be freed, we can simply increase the deleted_record_count by 1. We'll still compress the cache file approximately after 20% of the space (records) has been deleted. This also converts the old "hole offset" header field to tracking the number of records in the cache file. Otherwise we'd have to assume that all of the messages in mailbox have a cache record, but that isn't true for users who have only temporary cache fields. This "hole offset" cannot be trusted completely, because nothing prevents old Dovecot version from simply writing a hole offset there and messing up the record counts. Because of this there are a couple of extra sanity checks for it. (Unfortunately there wasn't any easy way to simply add a new field to header without breaking backwards compatibility.) diffstat: src/doveadm/doveadm-dump-index.c | 4 +- src/lib-index/mail-cache-compress.c | 6 ++- src/lib-index/mail-cache-private.h | 15 ++++--- src/lib-index/mail-cache-sync-update.c | 2 +- src/lib-index/mail-cache-transaction.c | 59 ++++++++++---------------------- src/lib-index/mail-cache.c | 61 +++++++++++++++++++++++---------- 6 files changed, 76 insertions(+), 71 deletions(-) diffs (296 lines): diff -r 8323e81785e2 -r debecd057f9c src/doveadm/doveadm-dump-index.c --- a/src/doveadm/doveadm-dump-index.c Sat Nov 03 19:37:47 2012 +0200 +++ b/src/doveadm/doveadm-dump-index.c Sat Nov 03 20:25:17 2012 +0200 @@ -325,9 +325,9 @@ hdr->file_seq, unixdate2str(hdr->file_seq), hdr->file_seq - hdr->indexid); printf("continued_record_count = %u\n", hdr->continued_record_count); - printf("hole_offset (unused) . = %u\n", hdr->unused_old_hole_offset); + printf("record_count ......... = %u\n", hdr->record_count); printf("used_file_size (old) . = %u\n", hdr->backwards_compat_used_file_size); - printf("deleted_space ........ = %u\n", hdr->deleted_space); + printf("deleted_record_count . = %u\n", hdr->deleted_record_count); printf("field_header_offset .. = %u (0x%08x nontranslated)\n", mail_index_offset_to_uint32(hdr->field_header_offset), hdr->field_header_offset); diff -r 8323e81785e2 -r debecd057f9c src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Sat Nov 03 19:37:47 2012 +0200 +++ b/src/lib-index/mail-cache-compress.c Sat Nov 03 20:25:17 2012 +0200 @@ -163,7 +163,7 @@ struct mail_cache_record cache_rec; struct ostream *output; uint32_t message_count, seq, first_new_seq, ext_offset; - unsigned int i, used_fields_count, orig_fields_count; + unsigned int i, used_fields_count, orig_fields_count, record_count; time_t max_drop_time; view = mail_index_transaction_get_view(trans); @@ -231,7 +231,7 @@ first_new_seq = mail_cache_get_first_new_seq(view); message_count = mail_index_view_get_messages_count(view); - i_array_init(ext_offsets, message_count); + i_array_init(ext_offsets, message_count); record_count = 0; for (seq = 1; seq <= message_count; seq++) { if (mail_index_transaction_is_expunged(trans, seq)) { array_append_zero(ext_offsets); @@ -264,12 +264,14 @@ buffer_write(ctx.buffer, 0, &cache_rec, sizeof(cache_rec)); o_stream_nsend(output, ctx.buffer->data, cache_rec.size); + record_count++; } array_append(ext_offsets, &ext_offset, 1); } i_assert(orig_fields_count == cache->fields_count); + hdr.record_count = record_count; hdr.field_header_offset = mail_index_uint32_to_offset(output->offset); mail_cache_compress_get_fields(&ctx, used_fields_count); o_stream_nsend(output, ctx.buffer->data, ctx.buffer->used); diff -r 8323e81785e2 -r debecd057f9c src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Sat Nov 03 19:37:47 2012 +0200 +++ b/src/lib-index/mail-cache-private.h Sat Nov 03 20:25:17 2012 +0200 @@ -12,10 +12,10 @@ #define MAIL_CACHE_FIELD_DROP_SECS (3600*24*30) /* Never compress the file if it's smaller than this */ -#define MAIL_CACHE_COMPRESS_MIN_SIZE (1024*50) +#define MAIL_CACHE_COMPRESS_MIN_SIZE (1024*32) -/* Compress the file when deleted space reaches n% of total size */ -#define MAIL_CACHE_COMPRESS_PERCENTAGE 20 +/* Compress the file when n% of records are deleted */ +#define MAIL_CACHE_COMPRESS_DELETE_PERCENTAGE 20 /* Compress the file when n% of rows contain continued rows. 200% means that there's 2 continued rows per record. */ @@ -47,9 +47,11 @@ uint32_t continued_record_count; - uint32_t unused_old_hole_offset; + /* NOTE: old versions used this for hole offset, so we can't fully + rely on it */ + uint32_t record_count; uint32_t backwards_compat_used_file_size; - uint32_t deleted_space; + uint32_t deleted_record_count; uint32_t field_header_offset; }; @@ -247,8 +249,7 @@ void mail_cache_file_close(struct mail_cache *cache); int mail_cache_reopen(struct mail_cache *cache); -/* Mark record in given offset to be deleted. */ -int mail_cache_delete(struct mail_cache *cache, uint32_t offset); +void mail_cache_delete(struct mail_cache *cache); /* Notify the decision handling code that field was looked up for seq. This should be called even for fields that aren't currently in cache file */ diff -r 8323e81785e2 -r debecd057f9c src/lib-index/mail-cache-sync-update.c --- a/src/lib-index/mail-cache-sync-update.c Sat Nov 03 19:37:47 2012 +0200 +++ b/src/lib-index/mail-cache-sync-update.c Sat Nov 03 20:25:17 2012 +0200 @@ -95,6 +95,6 @@ if (!MAIL_CACHE_IS_UNUSABLE(cache) && cache_file_seq == cache->hdr->file_seq) - (void)mail_cache_delete(cache, *cache_offset); + mail_cache_delete(cache); return 0; } diff -r 8323e81785e2 -r debecd057f9c src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Sat Nov 03 19:37:47 2012 +0200 +++ b/src/lib-index/mail-cache-transaction.c Sat Nov 03 20:25:17 2012 +0200 @@ -37,7 +37,7 @@ uint32_t prev_seq, min_seq; size_t last_rec_pos; - uoff_t bytes_written; + unsigned int records_written; unsigned int tried_compression:1; unsigned int changes:1; @@ -133,12 +133,12 @@ *_ctx = NULL; - if (ctx->bytes_written > 0) { + if (ctx->records_written > 0) { /* we already wrote to the cache file. we can't (or don't want to) delete that data, so just mark it as deleted space */ if (mail_cache_transaction_lock(ctx) > 0) { - ctx->cache->hdr_copy.deleted_space += - ctx->bytes_written; + ctx->cache->hdr_copy.deleted_record_count += + ctx->records_written; (void)mail_cache_unlock(ctx->cache); } } @@ -347,7 +347,8 @@ /* link this record to previous one */ rec->prev_offset = prev_offset; ctx->cache->hdr_copy.continued_record_count++; - ctx->cache->hdr_modified = TRUE; + } else { + ctx->cache->hdr_copy.record_count++; } *offsetp = write_offset; @@ -355,6 +356,7 @@ rec = PTR_OFFSET(rec, rec->size); } array_free(&seq_offsets); + ctx->cache->hdr_modified = TRUE; return 0; } @@ -401,7 +403,7 @@ ret = -1; else { /* update records' cache offsets to index */ - ctx->bytes_written += ctx->last_rec_pos; + ctx->records_written++; ret = mail_cache_transaction_update_index(ctx, write_offset); } if (mail_cache_unlock(ctx->cache) < 0) @@ -476,7 +478,7 @@ ret = -1; else { /* successfully wrote everything */ - ctx->bytes_written = 0; + ctx->records_written = 0; } /* Here would be a good place to do fdatasync() to make sure everything is written before offsets are updated to index. @@ -737,42 +739,17 @@ return mail_cache_field_exists(ctx->view, seq, field_idx) == 0; } -static int mail_cache_delete_real(struct mail_cache *cache, uint32_t offset) +void mail_cache_delete(struct mail_cache *cache) { - const struct mail_cache_record *rec; - struct mail_cache_loop_track loop_track; - int ret = 0; - i_assert(cache->locked); - /* we'll only update the deleted_space in header. we can't really - do any actual deleting as other processes might still be using - the data. also it's actually useful as some index views are still - able to ask cached data from messages that have already been + /* we'll only update the deleted record count in the header. we can't + really do any actual deleting as other processes might still be + using the data. also it's actually useful as old index views are + still able to ask cached data for messages that have already been expunged. */ - memset(&loop_track, 0, sizeof(loop_track)); - while (offset != 0 && - (ret = mail_cache_get_record(cache, offset, &rec)) == 0) { - if (mail_cache_track_loops(&loop_track, offset, rec->size)) { - mail_cache_set_corrupted(cache, - "record list is circular"); - return -1; - } - - cache->hdr_copy.deleted_space += rec->size; - offset = rec->prev_offset; - } - return ret; + cache->hdr_copy.deleted_record_count++; + if (cache->hdr_copy.record_count > 0) + cache->hdr_copy.record_count--; + cache->hdr_modified = TRUE; } - -int mail_cache_delete(struct mail_cache *cache, uint32_t offset) -{ - int ret; - - i_assert(cache->locked); - T_BEGIN { - ret = mail_cache_delete_real(cache, offset); - } T_END; - cache->hdr_modified = TRUE; - return ret; -} diff -r 8323e81785e2 -r debecd057f9c src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Sat Nov 03 19:37:47 2012 +0200 +++ b/src/lib-index/mail-cache.c Sat Nov 03 20:25:17 2012 +0200 @@ -197,30 +197,55 @@ { const struct mail_cache_header *hdr = cache->hdr; struct stat st; - unsigned int cont_percentage; - uoff_t file_size, max_del_space; + unsigned int msg_count; + unsigned int records_count, cont_percentage, delete_percentage; + bool want_compress = FALSE; - if (fstat(cache->fd, &st) < 0) { - if (!ESTALE_FSTAT(errno)) - mail_cache_set_syscall_error(cache, "fstat()"); + if (hdr->minor_version == 0) { + /* compress to get ourself into the new header version */ + cache->need_compress_file_seq = hdr->file_seq; return; } - file_size = st.st_size; - cont_percentage = hdr->continued_record_count * 100 / - (cache->index->map->rec_map->records_count == 0 ? 1 : - cache->index->map->rec_map->records_count); - if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE && - file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) { - /* too many continued rows, compress */ - cache->need_compress_file_seq = hdr->file_seq; + msg_count = cache->index->map->rec_map->records_count; + if (msg_count == 0) + records_count = 1; + else if (hdr->record_count == 0 || hdr->record_count > msg_count*2) { + /* probably not the real record_count, but hole offset that + Dovecot <=v2.1 versions used to use in this position. + we already checked that minor_version>0, but this could + happen if old Dovecot was used to access mailbox after + it had been updated. */ + records_count = I_MAX(msg_count, 1); + cache->hdr_copy.record_count = msg_count; + cache->hdr_modified = TRUE; + } else { + records_count = hdr->record_count; } - /* see if we've reached the max. deleted space in file */ - max_del_space = file_size / 100 * MAIL_CACHE_COMPRESS_PERCENTAGE; - if (hdr->deleted_space >= max_del_space && - file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) - cache->need_compress_file_seq = hdr->file_seq; + cont_percentage = hdr->continued_record_count * 100 / records_count; + if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE) { + /* too many continued rows, compress */ + want_compress = TRUE; + } + + delete_percentage = hdr->deleted_record_count * 100 / + (records_count + hdr->deleted_record_count); + if (delete_percentage >= MAIL_CACHE_COMPRESS_DELETE_PERCENTAGE) { + /* too many deleted records, compress */ + want_compress = TRUE; + } + + if (want_compress) { + if (fstat(cache->fd, &st) < 0) { + if (!ESTALE_FSTAT(errno)) + mail_cache_set_syscall_error(cache, "fstat()"); + return; + } + if (st.st_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) + cache->need_compress_file_seq = hdr->file_seq; + } + } static bool mail_cache_verify_header(struct mail_cache *cache, From dovecot at dovecot.org Sun Nov 4 17:26:38 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 04 Nov 2012 17:26:38 +0200 Subject: dovecot-2.1: lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY may have c... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/8252bcd7c489 changeset: 14795:8252bcd7c489 user: Timo Sirainen date: Sun Nov 04 17:26:24 2012 +0200 description: lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY may have caused stale data to be read from cache. diffstat: src/lib-index/mail-cache.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 2de37734d0b9 -r 8252bcd7c489 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Sat Nov 03 18:43:00 2012 +0200 +++ b/src/lib-index/mail-cache.c Sun Nov 04 17:26:24 2012 +0200 @@ -751,6 +751,8 @@ if (cache->file_cache != NULL) file_cache_write(cache->file_cache, data, size, offset); + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); return 0; } From dovecot at dovecot.org Sun Nov 4 19:17:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 04 Nov 2012 19:17:12 +0200 Subject: dovecot-2.2: lib-ssl-iostream: Copy the original istream's fd to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/09da0c7d23bd changeset: 15375:09da0c7d23bd user: Timo Sirainen date: Sun Nov 04 19:16:59 2012 +0200 description: lib-ssl-iostream: Copy the original istream's fd to the ssl istream. diffstat: src/lib-ssl-iostream/istream-openssl.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (11 lines): diff -r debecd057f9c -r 09da0c7d23bd src/lib-ssl-iostream/istream-openssl.c --- a/src/lib-ssl-iostream/istream-openssl.c Sat Nov 03 20:25:17 2012 +0200 +++ b/src/lib-ssl-iostream/istream-openssl.c Sun Nov 04 19:16:59 2012 +0200 @@ -126,5 +126,6 @@ sstream->istream.read = i_stream_ssl_read; sstream->istream.istream.readable_fd = FALSE; - return i_stream_create(&sstream->istream, NULL, -1); + return i_stream_create(&sstream->istream, NULL, + i_stream_get_fd(ssl_io->plain_input)); } From dovecot at dovecot.org Sun Nov 4 20:04:32 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 04 Nov 2012 20:04:32 +0200 Subject: dovecot-2.2: lib-ssl-iostream: Fixed potential assert-crash when... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/edc604cba6e1 changeset: 15376:edc604cba6e1 user: Timo Sirainen date: Sun Nov 04 20:04:21 2012 +0200 description: lib-ssl-iostream: Fixed potential assert-crash when reading. diffstat: src/lib-ssl-iostream/istream-openssl.c | 5 +---- 1 files changed, 1 insertions(+), 4 deletions(-) diffs (15 lines): diff -r 09da0c7d23bd -r edc604cba6e1 src/lib-ssl-iostream/istream-openssl.c --- a/src/lib-ssl-iostream/istream-openssl.c Sun Nov 04 19:16:59 2012 +0200 +++ b/src/lib-ssl-iostream/istream-openssl.c Sun Nov 04 20:04:21 2012 +0200 @@ -85,10 +85,7 @@ sstream->ssl_io->input_handler = FALSE; stream->max_buffer_size = (size_t)-1; while ((ret = SSL_read(ssl_io->ssl, buffer, sizeof(buffer))) > 0) { - if (!i_stream_try_alloc(stream, ret, &size)) - i_unreached(); - i_assert(size >= (size_t)ret); - memcpy(stream->w_buffer + stream->pos, buffer, ret); + memcpy(i_stream_alloc(stream, ret), buffer, ret); stream->pos += ret; total_ret += ret; } From dovecot at dovecot.org Mon Nov 5 14:28:03 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 05 Nov 2012 14:28:03 +0200 Subject: dovecot-2.2: lib-ssl-iostream: [io]streams were added an unneces... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/84ea8ea6b9dc changeset: 15377:84ea8ea6b9dc user: Timo Sirainen date: Mon Nov 05 14:27:52 2012 +0200 description: lib-ssl-iostream: [io]streams were added an unnecessary reference, causing memory leaks. diffstat: src/lib-ssl-iostream/iostream-openssl.c | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diffs (13 lines): diff -r edc604cba6e1 -r 84ea8ea6b9dc src/lib-ssl-iostream/iostream-openssl.c --- a/src/lib-ssl-iostream/iostream-openssl.c Sun Nov 04 20:04:21 2012 +0200 +++ b/src/lib-ssl-iostream/iostream-openssl.c Mon Nov 05 14:27:52 2012 +0200 @@ -212,9 +212,6 @@ SSL_set_bio(ssl_io->ssl, bio_int, bio_int); SSL_set_ex_data(ssl_io->ssl, dovecot_ssl_extdata_index, ssl_io); - i_stream_ref(ssl_io->plain_input); - o_stream_ref(ssl_io->plain_output); - T_BEGIN { ret = ssl_iostream_set(ssl_io, set); } T_END; From dovecot at dovecot.org Mon Nov 5 14:28:33 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 05 Nov 2012 14:28:33 +0200 Subject: dovecot-2.1: lib-ssl-iostream: [io]streams were added an unneces... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/ba1287f46c98 changeset: 14796:ba1287f46c98 user: Timo Sirainen date: Mon Nov 05 14:28:24 2012 +0200 description: lib-ssl-iostream: [io]streams were added an unnecessary reference, causing memory leaks. diffstat: src/lib-ssl-iostream/iostream-openssl.c | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diffs (13 lines): diff -r 8252bcd7c489 -r ba1287f46c98 src/lib-ssl-iostream/iostream-openssl.c --- a/src/lib-ssl-iostream/iostream-openssl.c Sun Nov 04 17:26:24 2012 +0200 +++ b/src/lib-ssl-iostream/iostream-openssl.c Mon Nov 05 14:28:24 2012 +0200 @@ -207,9 +207,6 @@ SSL_set_bio(ssl_io->ssl, bio_int, bio_int); SSL_set_ex_data(ssl_io->ssl, dovecot_ssl_extdata_index, ssl_io); - i_stream_ref(ssl_io->plain_input); - o_stream_ref(ssl_io->plain_output); - T_BEGIN { ret = ssl_iostream_set(ssl_io, set); } T_END; From dovecot at dovecot.org Mon Nov 5 18:17:03 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 05 Nov 2012 18:17:03 +0200 Subject: dovecot-2.2: lib-ssl-iostream: Call all of the OpenSSL deinit fu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/94778985bb6a changeset: 15378:94778985bb6a user: Timo Sirainen date: Mon Nov 05 18:16:56 2012 +0200 description: lib-ssl-iostream: Call all of the OpenSSL deinit functions at exit diffstat: src/lib-ssl-iostream/iostream-openssl-context.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 84ea8ea6b9dc -r 94778985bb6a src/lib-ssl-iostream/iostream-openssl-context.c --- a/src/lib-ssl-iostream/iostream-openssl-context.c Mon Nov 05 14:27:52 2012 +0200 +++ b/src/lib-ssl-iostream/iostream-openssl-context.c Mon Nov 05 18:16:56 2012 +0200 @@ -485,6 +485,8 @@ ENGINE_finish(ssl_iostream_engine); ENGINE_cleanup(); EVP_cleanup(); + CRYPTO_cleanup_all_ex_data(); + ERR_remove_state(0); ERR_free_strings(); } From dovecot at dovecot.org Tue Nov 6 01:04:47 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 01:04:47 +0200 Subject: dovecot-2.2: lib-ssl-iostream now dynamically loads openssl libr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/68d21f872fd7 changeset: 15379:68d21f872fd7 user: Timo Sirainen date: Tue Nov 06 01:04:24 2012 +0200 description: lib-ssl-iostream now dynamically loads openssl library instead of linking to it. This allowed removing the separate libdovecot-ssl library. In future if GnuTLS/NSS support is added it would also allow switching between them dynamically. diffstat: configure.ac | 6 +- src/auth/Makefile.am | 3 +- src/lib-dovecot/Makefile.am | 18 +- src/lib-master/Makefile.am | 6 +- src/lib-ssl-iostream/Makefile.am | 21 +- src/lib-ssl-iostream/iostream-openssl-common.c | 135 ++++++++++++ src/lib-ssl-iostream/iostream-openssl-context.c | 98 ++------ src/lib-ssl-iostream/iostream-openssl-params.c | 14 +- src/lib-ssl-iostream/iostream-openssl.c | 268 +++++++++-------------- src/lib-ssl-iostream/iostream-openssl.h | 40 ++- src/lib-ssl-iostream/iostream-ssl-private.h | 40 +++ src/lib-ssl-iostream/iostream-ssl.c | 147 +++++++++++++ src/lib-ssl-iostream/istream-openssl.c | 6 +- src/lib-ssl-iostream/ostream-openssl.c | 16 +- src/login-common/Makefile.am | 5 + 15 files changed, 520 insertions(+), 303 deletions(-) diffs (truncated from 1509 to 300 lines): diff -r 94778985bb6a -r 68d21f872fd7 configure.ac --- a/configure.ac Mon Nov 05 18:16:56 2012 +0200 +++ b/configure.ac Tue Nov 06 01:04:24 2012 +0200 @@ -2499,17 +2499,15 @@ LIBDOVECOT="$LIBDOVECOT_DEPS" LIBDOVECOT_STORAGE_DEPS='$(top_builddir)/src/lib-storage/libdovecot-storage.la $(top_builddir)/src/lib-imap-storage/libimap-storage.la' LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/libdovecot-login.la' - LIBDOVECOT_SSL='$(top_builddir)/src/lib-master/libmaster_ssl.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la' LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la' else - LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' + LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV)" LIBDOVECOT_STORAGE_LAST='$(top_builddir)/src/lib-storage/list/libstorage_list.la $(top_builddir)/src/lib-storage/index/libstorage_index.la $(top_builddir)/src/lib-storage/libstorage.la $(top_builddir)/src/lib-index/libindex.la $(top_builddir)/src/lib-imap-storage/libimap-storage.la' LIBDOVECOT_STORAGE_FIRST='$(top_builddir)/src/lib-storage/libstorage_service.la $(top_builddir)/src/lib-storage/register/libstorage_register.la' LIBDOVECOT_STORAGE_DEPS="$LIBDOVECOT_STORAGE_FIRST $LINKED_STORAGE_LIBS $LIBDOVECOT_STORAGE_LAST" - LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la' - LIBDOVECOT_SSL='$(top_builddir)/src/lib-master/libmaster_ssl.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la' + LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la' LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/liblda.la' fi diff -r 94778985bb6a -r 68d21f872fd7 src/auth/Makefile.am --- a/src/auth/Makefile.am Mon Nov 05 18:16:56 2012 +0200 +++ b/src/auth/Makefile.am Tue Nov 06 01:04:24 2012 +0200 @@ -176,8 +176,7 @@ libauthdb_imap_la_LDFLAGS = -module -avoid-version libauthdb_imap_la_LIBADD = \ ../lib-imap-client/libimap_client.la \ - ../lib-ssl-iostream/libssl_iostream.la \ - $(LIBDOVECOT) $(SSL_LIBS) + $(LIBDOVECOT) libauthdb_imap_la_CPPFLAGS = \ $(AM_CPPFLAGS) \ -I$(top_srcdir)/src/lib-imap \ diff -r 94778985bb6a -r 68d21f872fd7 src/lib-dovecot/Makefile.am --- a/src/lib-dovecot/Makefile.am Mon Nov 05 18:16:56 2012 +0200 +++ b/src/lib-dovecot/Makefile.am Tue Nov 06 01:04:24 2012 +0200 @@ -8,32 +8,18 @@ ../lib-fs/libfs.la \ ../lib-charset/libcharset.la \ ../lib-master/libmaster.la \ + ../lib-ssl-iostream/libssl_iostream.la ../lib-test/libtest.la \ ../lib/liblib.la -ssl_libs = \ - ../lib-master/libmaster_ssl.la \ - ../lib-ssl-iostream/libssl_iostream.la - -pkglib_LTLIBRARIES = libdovecot.la libdovecot-ssl.la +pkglib_LTLIBRARIES = libdovecot.la libdovecot_la_SOURCES = -libdovecot_ssl_la_SOURCES = libdovecot_la_LIBADD = \ $(libs) \ $(MODULE_LIBS) \ $(LTLIBICONV) -libdovecot_ssl_la_LIBADD = \ - libdovecot.la \ - ../lib/liblib.la \ - $(MODULE_LIBS) \ - $(ssl_libs) \ - $(SSL_LIBS) - libdovecot_la_DEPENDENCIES = $(libs) -libdovecot_ssl_la_DEPENDENCIES = $(ssl_libs) libdovecot.la - libdovecot_la_LDFLAGS = -export-dynamic -libdovecot_ssl_la_LDFLAGS = -export-dynamic diff -r 94778985bb6a -r 68d21f872fd7 src/lib-master/Makefile.am --- a/src/lib-master/Makefile.am Mon Nov 05 18:16:56 2012 +0200 +++ b/src/lib-master/Makefile.am Tue Nov 06 01:04:24 2012 +0200 @@ -1,6 +1,6 @@ pkgsysconfdir = $(sysconfdir)/dovecot -noinst_LTLIBRARIES = libmaster.la libmaster_ssl.la +noinst_LTLIBRARIES = libmaster.la AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ @@ -22,13 +22,11 @@ master-service.c \ master-service-settings.c \ master-service-settings-cache.c \ + master-service-ssl.c \ master-service-ssl-settings.c \ mountpoint-list.c \ syslog-util.c -libmaster_ssl_la_SOURCES = \ - master-service-ssl.c - headers = \ anvil-client.h \ ipc-client.h \ diff -r 94778985bb6a -r 68d21f872fd7 src/lib-ssl-iostream/Makefile.am --- a/src/lib-ssl-iostream/Makefile.am Mon Nov 05 18:16:56 2012 +0200 +++ b/src/lib-ssl-iostream/Makefile.am Tue Nov 06 01:04:24 2012 +0200 @@ -1,29 +1,34 @@ noinst_LTLIBRARIES = libssl_iostream.la +NOPLUGIN_LDFLAGS = + AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/lib-test + -I$(top_srcdir)/src/lib-test \ + -DMODULE_DIR=\""$(moduledir)"\" if BUILD_OPENSSL -ssl_sources = \ +module_LTLIBRARIES = libssl_iostream_openssl.la + +libssl_iostream_openssl_la_LDFLAGS = -module -avoid-version +libssl_iostream_openssl_la_LIBADD = $(SSL_LIBS) +libssl_iostream_openssl_la_SOURCES = \ iostream-openssl.c \ + iostream-openssl-common.c \ iostream-openssl-context.c \ iostream-openssl-params.c \ istream-openssl.c \ ostream-openssl.c -else -ssl_sources = \ - iostream-ssl-none.c endif libssl_iostream_la_SOURCES = \ + iostream-ssl.c \ $(ssl_sources) -libssl_iostream_la_LIBADD = \ - $(SSL_LIBS) headers = \ iostream-openssl.h \ - iostream-ssl.h + iostream-ssl.h \ + iostream-ssl-private.h pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) diff -r 94778985bb6a -r 68d21f872fd7 src/lib-ssl-iostream/iostream-openssl-common.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-ssl-iostream/iostream-openssl-common.c Tue Nov 06 01:04:24 2012 +0200 @@ -0,0 +1,135 @@ +/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "iostream-openssl.h" + +#include + +enum { + DOVECOT_SSL_PROTO_SSLv2 = 0x01, + DOVECOT_SSL_PROTO_SSLv3 = 0x02, + DOVECOT_SSL_PROTO_TLSv1 = 0x04, + DOVECOT_SSL_PROTO_ALL = 0x07 +}; + +int openssl_get_protocol_options(const char *protocols) +{ + const char *const *tmp; + int proto, op = 0, include = 0, exclude = 0; + bool neg; + + tmp = t_strsplit_spaces(protocols, " "); + for (; *tmp != NULL; tmp++) { + const char *name = *tmp; + + if (*name != '!') + neg = FALSE; + else { + name++; + neg = TRUE; + } + if (strcasecmp(name, SSL_TXT_SSLV2) == 0) + proto = DOVECOT_SSL_PROTO_SSLv2; + else if (strcasecmp(name, SSL_TXT_SSLV3) == 0) + proto = DOVECOT_SSL_PROTO_SSLv3; + else if (strcasecmp(name, SSL_TXT_TLSV1) == 0) + proto = DOVECOT_SSL_PROTO_TLSv1; + else { + i_fatal("Invalid ssl_protocols setting: " + "Unknown protocol '%s'", name); + } + if (neg) + exclude |= proto; + else + include |= proto; + } + if (include != 0) { + /* exclude everything, except those that are included + (and let excludes still override those) */ + exclude |= DOVECOT_SSL_PROTO_ALL & ~include; + } + if ((exclude & DOVECOT_SSL_PROTO_SSLv2) != 0) op |= SSL_OP_NO_SSLv2; + if ((exclude & DOVECOT_SSL_PROTO_SSLv3) != 0) op |= SSL_OP_NO_SSLv3; + if ((exclude & DOVECOT_SSL_PROTO_TLSv1) != 0) op |= SSL_OP_NO_TLSv1; + return op; +} + +static const char *asn1_string_to_c(ASN1_STRING *asn_str) +{ + const char *cstr; + unsigned int len; + + len = ASN1_STRING_length(asn_str); + cstr = t_strndup(ASN1_STRING_data(asn_str), len); + if (strlen(cstr) != len) { + /* NULs in the name - could be some MITM attack. + never allow. */ + return ""; + } + return cstr; +} + +static const char *get_general_dns_name(const GENERAL_NAME *name) +{ + if (ASN1_STRING_type(name->d.ia5) != V_ASN1_IA5STRING) + return ""; + + return asn1_string_to_c(name->d.ia5); +} + +static const char *get_cname(X509 *cert) +{ + X509_NAME *name; + X509_NAME_ENTRY *entry; + ASN1_STRING *str; + int cn_idx; + + name = X509_get_subject_name(cert); + if (name == NULL) + return ""; + cn_idx = X509_NAME_get_index_by_NID(name, NID_commonName, -1); + if (cn_idx == -1) + return ""; + entry = X509_NAME_get_entry(name, cn_idx); + i_assert(entry != NULL); + str = X509_NAME_ENTRY_get_data(entry); + i_assert(str != NULL); + return asn1_string_to_c(str); +} + +int openssl_cert_match_name(SSL *ssl, const char *verify_name) +{ + X509 *cert; + STACK_OF(GENERAL_NAME) *gnames; + const GENERAL_NAME *gn; + const char *dnsname; + bool dns_names = FALSE; + unsigned int i, count; + int ret; + + cert = SSL_get_peer_certificate(ssl); + i_assert(cert != NULL); + + /* verify against SubjectAltNames */ + gnames = X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL); + count = gnames == NULL ? 0 : sk_GENERAL_NAME_num(gnames); + for (i = 0; i < count; i++) { + gn = sk_GENERAL_NAME_value(gnames, i); + if (gn->type == GEN_DNS) { + dns_names = TRUE; + dnsname = get_general_dns_name(gn); + if (strcmp(dnsname, verify_name) == 0) + break; + } + } + sk_GENERAL_NAME_pop_free(gnames, GENERAL_NAME_free); + + /* verify against CommonName only when there wasn't any DNS + SubjectAltNames */ + if (dns_names) + ret = i < count ? 0 : -1; + else + ret = strcmp(get_cname(cert), verify_name) == 0 ? 0 : -1; + X509_free(cert); + return ret; +} diff -r 94778985bb6a -r 68d21f872fd7 src/lib-ssl-iostream/iostream-openssl-context.c --- a/src/lib-ssl-iostream/iostream-openssl-context.c Mon Nov 05 18:16:56 2012 +0200 +++ b/src/lib-ssl-iostream/iostream-openssl-context.c Tue Nov 06 01:04:24 2012 +0200 @@ -23,7 +23,7 @@ static void ssl_iostream_init_global(const struct ssl_iostream_settings *set); -const char *ssl_iostream_error(void) +const char *openssl_iostream_error(void) { unsigned long err; char *buf; @@ -44,7 +44,7 @@ return buf; } From dovecot at dovecot.org Tue Nov 6 01:11:42 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 01:11:42 +0200 Subject: dovecot-2.2: Reverted 269104a0821b (Build imapc and pop3c always... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7c75559cd8f6 changeset: 15380:7c75559cd8f6 user: Timo Sirainen date: Tue Nov 06 01:11:03 2012 +0200 description: Reverted 269104a0821b (Build imapc and pop3c always as plugins.) This is no longer needed now that lib-ssl-iostream can be linked without linking openssl. diffstat: configure.ac | 25 +++++++++++++++----- src/lib-storage/index/imapc/Makefile.am | 17 +------------- src/lib-storage/index/imapc/imapc-list.c | 2 + src/lib-storage/index/imapc/imapc-list.h | 2 - src/lib-storage/index/imapc/imapc-plugin.c | 21 ----------------- src/lib-storage/index/imapc/imapc-plugin.h | 7 ----- src/lib-storage/index/imapc/imapc-setting-storage.c | 22 ------------------ src/lib-storage/index/imapc/imapc-settings.h | 2 - src/lib-storage/index/imapc/imapc-storage.c | 3 ++ src/lib-storage/index/imapc/imapc-storage.h | 3 -- src/lib-storage/index/pop3c/Makefile.am | 16 +------------ src/lib-storage/index/pop3c/pop3c-plugin.c | 19 --------------- src/lib-storage/index/pop3c/pop3c-plugin.h | 7 ----- src/lib-storage/index/pop3c/pop3c-setting-storage.c | 22 ------------------ src/lib-storage/index/pop3c/pop3c-settings.h | 2 - src/lib-storage/index/pop3c/pop3c-storage.c | 3 ++ src/lib-storage/index/pop3c/pop3c-storage.h | 2 - 17 files changed, 29 insertions(+), 146 deletions(-) diffs (truncated from 370 to 300 lines): diff -r 68d21f872fd7 -r 7c75559cd8f6 configure.ac --- a/configure.ac Tue Nov 06 01:04:24 2012 +0200 +++ b/configure.ac Tue Nov 06 01:11:03 2012 +0200 @@ -247,14 +247,14 @@ want_gc=no) AC_ARG_WITH(storages, -AS_HELP_STRING([--with-storages], [Build with specified mail storage formats (mdbox sdbox maildir mbox cydir)]), [ +AS_HELP_STRING([--with-storages], [Build with specified mail storage formats (mdbox sdbox maildir mbox cydir imapc pop3c)]), [ if test "$withval" = "yes" || test "$withval" = "no"; then AC_MSG_ERROR([--with-storages needs storage list as parameter]) fi mail_storages="shared `echo "$withval"|sed 's/,/ /g'`" ], - mail_storages="shared mdbox sdbox maildir mbox cydir") + mail_storages="shared mdbox sdbox maildir mbox cydir imapc pop3c") AC_SUBST(mail_storages) -mail_storages="$mail_storages imapc_stub pop3c_stub raw fail" +mail_storages="$mail_storages raw fail" # drop duplicates duplicates=`(for i in $mail_storages; do echo $i; done)|sort|uniq -d|xargs echo` if test "$duplicates" != ""; then @@ -2459,8 +2459,8 @@ sdbox_libs='$(top_builddir)/src/lib-storage/index/dbox-single/libstorage_dbox_single.la' mdbox_libs='$(top_builddir)/src/lib-storage/index/dbox-multi/libstorage_dbox_multi.la' cydir_libs='$(top_builddir)/src/lib-storage/index/cydir/libstorage_cydir.la' -imapc_stub_libs='$(top_builddir)/src/lib-storage/index/imapc/libstorage_imapc.la' -pop3c_stub_libs='$(top_builddir)/src/lib-storage/index/pop3c/libstorage_pop3c.la' +imapc_libs='$(top_builddir)/src/lib-storage/index/imapc/libstorage_imapc.la $(top_builddir)/src/lib-imap-client/libimap_client.la' +pop3c_libs='$(top_builddir)/src/lib-storage/index/pop3c/libstorage_pop3c.la' raw_libs='$(top_builddir)/src/lib-storage/index/raw/libstorage_raw.la' shared_libs='$(top_builddir)/src/lib-storage/index/shared/libstorage_shared.la' @@ -2480,8 +2480,21 @@ LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS $dbox_common_libs" dbox_common_libs="" fi + if test $storage = imapc; then + mailbox_list_drivers="$mailbox_list_drivers imapc" + want_ssl_libs=yes + fi + if test $storage = pop3c; then + want_ssl_libs=yes + fi done +LINKED_STORAGE_LDADD= +if test "$want_ssl_libs" = yes; then + LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS \$(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la" + LINKED_STORAGE_LDADD="$SSL_LIBS" +fi AC_SUBST(LINKED_STORAGE_LIBS) +AC_SUBST(LINKED_STORAGE_LDADD) AC_SUBST(mailbox_list_drivers) AC_DEFINE_UNQUOTED(MAIL_STORAGES, "$mail_storages", List of compiled in mail storages) @@ -2511,7 +2524,7 @@ LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/liblda.la' fi -LIBDOVECOT_STORAGE="$LIBDOVECOT_STORAGE_DEPS" +LIBDOVECOT_STORAGE="$LIBDOVECOT_STORAGE_DEPS $LINKED_STORAGE_LDADD" LIBDOVECOT_SQL='$(top_builddir)/src/lib-sql/libsql.la' AC_SUBST(LIBDOVECOT) AC_SUBST(LIBDOVECOT_DEPS) diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/Makefile.am --- a/src/lib-storage/index/imapc/Makefile.am Tue Nov 06 01:04:24 2012 +0200 +++ b/src/lib-storage/index/imapc/Makefile.am Tue Nov 06 01:11:03 2012 +0200 @@ -1,5 +1,4 @@ noinst_LTLIBRARIES = libstorage_imapc.la -module_LTLIBRARIES = lib20_imapc_plugin.la AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ @@ -13,27 +12,16 @@ -I$(top_srcdir)/src/lib-storage/list \ -I$(top_srcdir)/src/lib-storage/index -NOPLUGIN_LDFLAGS = -lib20_imapc_plugin_la_LDFLAGS = -module -avoid-version - libstorage_imapc_la_SOURCES = \ - imapc-settings.c \ - imapc-setting-storage.c - -lib20_imapc_plugin_la_SOURCES = \ imapc-list.c \ imapc-mail.c \ imapc-mail-fetch.c \ imapc-mailbox.c \ - imapc-plugin.c \ imapc-save.c \ + imapc-settings.c \ imapc-sync.c \ imapc-storage.c -lib20_imapc_plugin_la_LIBADD = \ - ../../../lib-imap-client/libimap_client.la \ - ../../../lib-ssl-iostream/libssl_iostream.la - headers = \ imapc-list.h \ imapc-mail.h \ @@ -43,6 +31,3 @@ pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) - -noinst_HEADERS = \ - imapc-plugin.h diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Tue Nov 06 01:04:24 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-list.c Tue Nov 06 01:11:03 2012 +0200 @@ -20,6 +20,8 @@ struct mailbox_info info; }; +extern struct mailbox_list imapc_mailbox_list; + static struct mailbox_list *imapc_list_alloc(void) { struct imapc_mailbox_list *list; diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/imapc-list.h --- a/src/lib-storage/index/imapc/imapc-list.h Tue Nov 06 01:04:24 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-list.h Tue Nov 06 01:11:03 2012 +0200 @@ -22,8 +22,6 @@ unsigned int index_list_failed:1; }; -extern struct mailbox_list imapc_mailbox_list; - int imapc_list_get_mailbox_flags(struct mailbox_list *list, const char *name, enum mailbox_info_flags *flags_r); diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/imapc-plugin.c --- a/src/lib-storage/index/imapc/imapc-plugin.c Tue Nov 06 01:04:24 2012 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,21 +0,0 @@ -/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "imapc-list.h" -#include "imapc-storage.h" -#include "imapc-plugin.h" - -const char *imapc_plugin_version = DOVECOT_ABI_VERSION; - -void imapc_plugin_init(struct module *module ATTR_UNUSED) -{ - mail_storage_class_unregister(&imapc_stub_storage); - mail_storage_class_register(&imapc_storage); - mailbox_list_register(&imapc_mailbox_list); -} - -void imapc_plugin_deinit(void) -{ - mail_storage_class_unregister(&imapc_storage); - mailbox_list_unregister(&imapc_mailbox_list); -} diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/imapc-plugin.h --- a/src/lib-storage/index/imapc/imapc-plugin.h Tue Nov 06 01:04:24 2012 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -#ifndef IMAPC_PLUGIN_H -#define IMAPC_PLUGIN_H - -void imapc_plugin_init(struct module *module); -void imapc_plugin_deinit(void); - -#endif diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/imapc-setting-storage.c --- a/src/lib-storage/index/imapc/imapc-setting-storage.c Tue Nov 06 01:04:24 2012 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "imapc-storage.h" -#include "imapc-settings.h" - -struct mail_storage imapc_stub_storage = { - .name = IMAPC_STORAGE_NAME, - .class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT, - - .v = { - imapc_get_setting_parser_info, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL - } -}; diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/imapc-settings.h --- a/src/lib-storage/index/imapc/imapc-settings.h Tue Nov 06 01:04:24 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-settings.h Tue Nov 06 01:11:03 2012 +0200 @@ -30,8 +30,6 @@ enum imapc_features parsed_features; }; -extern struct mail_storage imapc_stub_storage; - const struct setting_parser_info *imapc_get_setting_parser_info(void); #endif diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Tue Nov 06 01:04:24 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Tue Nov 06 01:11:03 2012 +0200 @@ -27,6 +27,9 @@ enum mail_error error; }; +extern struct mail_storage imapc_storage; +extern struct mailbox imapc_mailbox; + static struct imapc_resp_code_map imapc_resp_code_map[] = { { IMAP_RESP_CODE_UNAVAILABLE, MAIL_ERROR_TEMP }, { IMAP_RESP_CODE_AUTHFAILED, MAIL_ERROR_PERM }, diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/imapc/imapc-storage.h --- a/src/lib-storage/index/imapc/imapc-storage.h Tue Nov 06 01:04:24 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.h Tue Nov 06 01:11:03 2012 +0200 @@ -110,9 +110,6 @@ int ret; }; -extern struct mail_storage imapc_storage; -extern struct mailbox imapc_mailbox; - struct mail_save_context * imapc_save_alloc(struct mailbox_transaction_context *_t); int imapc_save_begin(struct mail_save_context *ctx, struct istream *input); diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/pop3c/Makefile.am --- a/src/lib-storage/index/pop3c/Makefile.am Tue Nov 06 01:04:24 2012 +0200 +++ b/src/lib-storage/index/pop3c/Makefile.am Tue Nov 06 01:11:03 2012 +0200 @@ -1,5 +1,4 @@ noinst_LTLIBRARIES = libstorage_pop3c.la -module_LTLIBRARIES = lib20_pop3c_plugin.la AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ @@ -12,23 +11,13 @@ -I$(top_srcdir)/src/lib-storage \ -I$(top_srcdir)/src/lib-storage/index -NOPLUGIN_LDFLAGS = -lib20_pop3c_plugin_la_LDFLAGS = -module -avoid-version - libstorage_pop3c_la_SOURCES = \ - pop3c-settings.c \ - pop3c-setting-storage.c - -lib20_pop3c_plugin_la_SOURCES = \ pop3c-client.c \ pop3c-mail.c \ - pop3c-plugin.c \ + pop3c-settings.c \ pop3c-storage.c \ pop3c-sync.c -lib20_pop3c_plugin_la_LIBADD = \ - ../../../lib-ssl-iostream/libssl_iostream.la - headers = \ pop3c-client.h \ pop3c-settings.h \ @@ -37,6 +26,3 @@ pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) - -noinst_HEADERS = \ - pop3c-plugin.h diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/pop3c/pop3c-plugin.c --- a/src/lib-storage/index/pop3c/pop3c-plugin.c Tue Nov 06 01:04:24 2012 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,19 +0,0 @@ -/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "pop3c-storage.h" -#include "pop3c-settings.h" -#include "pop3c-plugin.h" - -const char *pop3c_plugin_version = DOVECOT_ABI_VERSION; - -void pop3c_plugin_init(struct module *module ATTR_UNUSED) -{ - mail_storage_class_unregister(&pop3c_stub_storage); - mail_storage_class_register(&pop3c_storage); -} - -void pop3c_plugin_deinit(void) -{ - mail_storage_class_unregister(&pop3c_storage); -} diff -r 68d21f872fd7 -r 7c75559cd8f6 src/lib-storage/index/pop3c/pop3c-plugin.h --- a/src/lib-storage/index/pop3c/pop3c-plugin.h Tue Nov 06 01:04:24 2012 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ From dovecot at dovecot.org Tue Nov 6 01:16:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 01:16:25 +0200 Subject: dovecot-2.2: dovecot-config: Removed LIBDOVECOT_SSL, which is no... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1ab783418ff6 changeset: 15381:1ab783418ff6 user: Timo Sirainen date: Tue Nov 06 01:16:20 2012 +0200 description: dovecot-config: Removed LIBDOVECOT_SSL, which is no longer needed. diffstat: dovecot-config.in.in | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (19 lines): diff -r 7c75559cd8f6 -r 1ab783418ff6 dovecot-config.in.in --- a/dovecot-config.in.in Tue Nov 06 01:11:03 2012 +0200 +++ b/dovecot-config.in.in Tue Nov 06 01:16:20 2012 +0200 @@ -7,7 +7,6 @@ LIBDOVECOT="@LIBDOVECOT@ @MODULE_LIBS@" LIBDOVECOT_LOGIN="@LIBDOVECOT_LOGIN@ @SSL_LIBS@" LIBDOVECOT_SQL="@LIBDOVECOT_SQL@" -LIBDOVECOT_SSL="@LIBDOVECOT_SSL@" LIBDOVECOT_COMPRESS="@LIBDOVECOT_COMPRESS@" LIBDOVECOT_LDA="@LIBDOVECOT_LDA@" LIBDOVECOT_STORAGE="@LIBDOVECOT_STORAGE@" @@ -15,7 +14,6 @@ LIBDOVECOT_DEPS="@LIBDOVECOT_DEPS@" LIBDOVECOT_LOGIN_DEPS="@LIBDOVECOT_LOGIN@" LIBDOVECOT_SQL_DEPS="@LIBDOVECOT_SQL@" -LIBDOVECOT_SSL_DEPS="@LIBDOVECOT_SSL@" LIBDOVECOT_COMPRESS_DEPS="@LIBDOVECOT_COMPRESS@" LIBDOVECOT_LDA_DEPS="@LIBDOVECOT_LDA@" LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE_DEPS@" From dovecot at dovecot.org Tue Nov 6 01:40:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 01:40:29 +0200 Subject: dovecot-2.2: Makefile: Fixed building libdovecot.so Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7704a4f935ea changeset: 15382:7704a4f935ea user: Timo Sirainen date: Tue Nov 06 01:40:17 2012 +0200 description: Makefile: Fixed building libdovecot.so diffstat: src/lib-dovecot/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1ab783418ff6 -r 7704a4f935ea src/lib-dovecot/Makefile.am --- a/src/lib-dovecot/Makefile.am Tue Nov 06 01:16:20 2012 +0200 +++ b/src/lib-dovecot/Makefile.am Tue Nov 06 01:40:17 2012 +0200 @@ -8,7 +8,7 @@ ../lib-fs/libfs.la \ ../lib-charset/libcharset.la \ ../lib-master/libmaster.la \ - ../lib-ssl-iostream/libssl_iostream.la + ../lib-ssl-iostream/libssl_iostream.la \ ../lib-test/libtest.la \ ../lib/liblib.la From dovecot at dovecot.org Tue Nov 6 02:33:50 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 02:33:50 +0200 Subject: dovecot-2.2: Makefile: Build lib-ssl-iostream before lib-http Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d9d5b09a0b5b changeset: 15383:d9d5b09a0b5b user: Timo Sirainen date: Tue Nov 06 02:33:38 2012 +0200 description: Makefile: Build lib-ssl-iostream before lib-http diffstat: src/Makefile.am | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 7704a4f935ea -r d9d5b09a0b5b src/Makefile.am --- a/src/Makefile.am Tue Nov 06 01:40:17 2012 +0200 +++ b/src/Makefile.am Tue Nov 06 02:33:38 2012 +0200 @@ -9,10 +9,10 @@ lib-imap \ lib-imap-storage \ lib-master \ - lib-http \ lib-dict \ lib-settings \ - lib-ssl-iostream + lib-ssl-iostream \ + lib-http SUBDIRS = \ $(LIBDOVECOT_SUBDIRS) \ From dovecot at dovecot.org Tue Nov 6 14:20:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 14:20:11 +0200 Subject: dovecot-2.2: lib-ssl-iostream: Compiler warning fix when compili... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cca85adcb583 changeset: 15384:cca85adcb583 user: Timo Sirainen date: Tue Nov 06 14:20:00 2012 +0200 description: lib-ssl-iostream: Compiler warning fix when compiling without ssl diffstat: src/lib-ssl-iostream/iostream-ssl.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (13 lines): diff -r d9d5b09a0b5b -r cca85adcb583 src/lib-ssl-iostream/iostream-ssl.c --- a/src/lib-ssl-iostream/iostream-ssl.c Tue Nov 06 02:33:38 2012 +0200 +++ b/src/lib-ssl-iostream/iostream-ssl.c Tue Nov 06 14:20:00 2012 +0200 @@ -5,7 +5,9 @@ #include "iostream-ssl-private.h" static bool ssl_module_loaded = FALSE; +#ifdef HAVE_SSL static struct module *ssl_module = NULL; +#endif static const struct iostream_ssl_vfuncs *ssl_vfuncs = NULL; static int ssl_module_load(void) From dovecot at dovecot.org Tue Nov 6 17:40:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 17:40:52 +0200 Subject: dovecot-2.2: istream-seekable: Underlying stream errors weren't ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/47085487f3d8 changeset: 15385:47085487f3d8 user: Timo Sirainen date: Tue Nov 06 17:40:34 2012 +0200 description: istream-seekable: Underlying stream errors weren't handled correctly. diffstat: src/lib/istream-seekable.c | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diffs (24 lines): diff -r cca85adcb583 -r 47085487f3d8 src/lib/istream-seekable.c --- a/src/lib/istream-seekable.c Tue Nov 06 14:20:00 2012 +0200 +++ b/src/lib/istream-seekable.c Tue Nov 06 17:40:34 2012 +0200 @@ -139,9 +139,8 @@ return -1; } - while ((ret = i_stream_read(sstream->cur_input)) < 0) { - if (!sstream->cur_input->eof) { - /* full / error */ + while ((ret = i_stream_read(sstream->cur_input)) == -1) { + if (sstream->cur_input->stream_errno != 0) { sstream->istream.istream.stream_errno = sstream->cur_input->stream_errno; return -1; @@ -183,7 +182,7 @@ if (size == 0) { /* read more to buffer */ *ret_r = read_more(sstream); - if (*ret_r <= 0) + if (*ret_r == 0 || *ret_r == -1) return TRUE; } From dovecot at dovecot.org Tue Nov 6 22:13:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 22:13:25 +0200 Subject: dovecot-2.1: imap: Avoid sending multiple unnecessary VANISHED l... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/bcd2d140ea10 changeset: 14797:bcd2d140ea10 user: Timo Sirainen date: Tue Nov 06 22:13:13 2012 +0200 description: imap: Avoid sending multiple unnecessary VANISHED lines when sync had lots of changes. diffstat: src/imap/imap-sync.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r ba1287f46c98 -r bcd2d140ea10 src/imap/imap-sync.c --- a/src/imap/imap-sync.c Mon Nov 05 14:28:24 2012 +0200 +++ b/src/imap/imap-sync.c Tue Nov 06 22:13:13 2012 +0200 @@ -472,8 +472,10 @@ ctx->seq = 0; } - if (array_is_created(&ctx->expunges)) - imap_sync_vanished(ctx); + if (ret > 0) { + if (array_is_created(&ctx->expunges)) + imap_sync_vanished(ctx); + } return ret; } From dovecot at dovecot.org Tue Nov 6 22:15:58 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 22:15:58 +0200 Subject: dovecot-2.2: imap: Various fixes for handling expunges in mailbo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5089affc8ae5 changeset: 15386:5089affc8ae5 user: Timo Sirainen date: Tue Nov 06 22:15:51 2012 +0200 description: imap: Various fixes for handling expunges in mailbox sync. diffstat: src/imap/imap-sync.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diffs (61 lines): diff -r 47085487f3d8 -r 5089affc8ae5 src/imap/imap-sync.c --- a/src/imap/imap-sync.c Tue Nov 06 17:40:34 2012 +0200 +++ b/src/imap/imap-sync.c Tue Nov 06 22:15:51 2012 +0200 @@ -296,7 +296,7 @@ client->highest_fetch_modseq = 0; } -static int imap_sync_finish(struct imap_sync_context *ctx) +static int imap_sync_finish(struct imap_sync_context *ctx, bool aborting) { struct client *client = ctx->client; int ret = ctx->failed ? -1 : 0; @@ -327,7 +327,7 @@ client_disconnect_with_error(client, "Mailbox UIDVALIDITY changed"); } - if (!ctx->no_newmail) { + if (!ctx->no_newmail && !aborting) { if (ctx->status.messages < ctx->messages_count) i_panic("Message count decreased"); if (ctx->status.messages != ctx->messages_count && @@ -371,7 +371,7 @@ { int ret; - ret = imap_sync_finish(ctx); + ret = imap_sync_finish(ctx, TRUE); imap_client_notify_finished(ctx->client); if ((ctx->client->enabled_features & MAILBOX_FEATURE_QRESYNC) != 0) @@ -506,14 +506,14 @@ for (; ctx->seq >= ctx->sync_rec.seq1; ctx->seq--) { if (ret == 0) { /* buffer full, continue later */ - break; + return 0; } str_truncate(str, 0); str_printfa(str, "* %u EXPUNGE", ctx->seq); ret = client_send_line_next(ctx->client, str_c(str)); } - return ret; + return 1; } int imap_sync_more(struct imap_sync_context *ctx) @@ -613,10 +613,10 @@ ctx->seq = 0; } - if (array_is_created(&ctx->expunges)) - imap_sync_vanished(ctx); if (ret > 0) { - if (imap_sync_finish(ctx) < 0) + if (array_is_created(&ctx->expunges)) + imap_sync_vanished(ctx); + if (imap_sync_finish(ctx, FALSE) < 0) return -1; return imap_sync_more(ctx); } From dovecot at dovecot.org Tue Nov 6 23:23:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 23:23:31 +0200 Subject: dovecot-2.2: auth: Require auth-token-secret.dat owner to be aut... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2d0d0318b341 changeset: 15387:2d0d0318b341 user: Timo Sirainen date: Tue Nov 06 23:23:19 2012 +0200 description: auth: Require auth-token-secret.dat owner to be auth process's uid, not necessarily root. diffstat: src/auth/auth-token.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 5089affc8ae5 -r 2d0d0318b341 src/auth/auth-token.c --- a/src/auth/auth-token.c Tue Nov 06 22:15:51 2012 +0200 +++ b/src/auth/auth-token.c Tue Nov 06 23:23:19 2012 +0200 @@ -70,7 +70,8 @@ } /* check security parameters for compromise */ - if ((st.st_mode & 07777) != 0600 || st.st_uid != 0 || st.st_nlink > 1 || + if ((st.st_mode & 07777) != 0600 || + st.st_uid != geteuid() || st.st_nlink > 1 || !S_ISREG(lst.st_mode) || st.st_ino != lst.st_ino || !CMP_DEV_T(st.st_dev, lst.st_dev)) { i_error("Compromised token secret file: %s", path); From dovecot at dovecot.org Tue Nov 6 23:50:10 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 23:50:10 +0200 Subject: dovecot-2.2: module_dir_deinit(): If no modules were actually in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6fc2502d6021 changeset: 15388:6fc2502d6021 user: Timo Sirainen date: Tue Nov 06 23:49:24 2012 +0200 description: module_dir_deinit(): If no modules were actually initialized, avoid doing memory allocation. This allows calling module_dir_unload() with atexit(), as long as the module doesn't need to call deinit(). diffstat: src/lib/module-dir.c | 19 +++++++++++-------- 1 files changed, 11 insertions(+), 8 deletions(-) diffs (42 lines): diff -r 2d0d0318b341 -r 6fc2502d6021 src/lib/module-dir.c --- a/src/lib/module-dir.c Tue Nov 06 23:23:19 2012 +0200 +++ b/src/lib/module-dir.c Tue Nov 06 23:49:24 2012 +0200 @@ -494,8 +494,10 @@ struct module *module, **rev; unsigned int i, count = 0; - for (module = modules; module != NULL; module = module->next) - count++; + for (module = modules; module != NULL; module = module->next) { + if (module->deinit != NULL && module->initialized) + count++; + } if (count == 0) return; @@ -503,18 +505,19 @@ /* @UNSAFE: deinitialize in reverse order */ T_BEGIN { rev = t_new(struct module *, count); - for (i = 0, module = modules; i < count; i++) { - rev[count-i-1] = module; + for (i = 0, module = modules; i < count; ) { + if (module->deinit != NULL && module->initialized) { + rev[count-i-1] = module; + i++; + } module = module->next; } for (i = 0; i < count; i++) { module = rev[i]; - if (module->deinit != NULL && module->initialized) { - module->deinit(); - module->initialized = FALSE; - } + module->deinit(); + module->initialized = FALSE; } } T_END; } From dovecot at dovecot.org Tue Nov 6 23:50:10 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 23:50:10 +0200 Subject: dovecot-2.2: lib-ssl-iostream: Unload ssl_iostream_openssl plugi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7d28b19fe28d changeset: 15389:7d28b19fe28d user: Timo Sirainen date: Tue Nov 06 23:49:57 2012 +0200 description: lib-ssl-iostream: Unload ssl_iostream_openssl plugin at exit. diffstat: src/lib-ssl-iostream/iostream-ssl.c | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diffs (34 lines): diff -r 6fc2502d6021 -r 7d28b19fe28d src/lib-ssl-iostream/iostream-ssl.c --- a/src/lib-ssl-iostream/iostream-ssl.c Tue Nov 06 23:49:24 2012 +0200 +++ b/src/lib-ssl-iostream/iostream-ssl.c Tue Nov 06 23:49:57 2012 +0200 @@ -4,12 +4,21 @@ #include "module-dir.h" #include "iostream-ssl-private.h" +#include + static bool ssl_module_loaded = FALSE; #ifdef HAVE_SSL static struct module *ssl_module = NULL; #endif static const struct iostream_ssl_vfuncs *ssl_vfuncs = NULL; +#ifdef HAVE_SSL +static void ssl_module_unload(void) +{ + module_dir_unload(&ssl_module); +} +#endif + static int ssl_module_load(void) { #ifdef HAVE_SSL @@ -24,6 +33,8 @@ ssl_vfuncs = module_get_symbol(ssl_module, "ssl_vfuncs"); if (ssl_vfuncs == NULL) i_fatal("%s: ssl_vfuncs symbol not found", plugin_name); + + atexit(ssl_module_unload); ssl_module_loaded = TRUE; return 0; #else From dovecot at dovecot.org Tue Nov 6 23:53:17 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 06 Nov 2012 23:53:17 +0200 Subject: dovecot-2.2: module_dir_unload(): Don't dlclose() modules if GDB... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/af331bc452d0 changeset: 15390:af331bc452d0 user: Timo Sirainen date: Tue Nov 06 23:53:11 2012 +0200 description: module_dir_unload(): Don't dlclose() modules if GDB environment is set. diffstat: src/lib/module-dir.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (19 lines): diff -r 7d28b19fe28d -r af331bc452d0 src/lib/module-dir.c --- a/src/lib/module-dir.c Tue Nov 06 23:49:57 2012 +0200 +++ b/src/lib/module-dir.c Tue Nov 06 23:53:11 2012 +0200 @@ -60,8 +60,13 @@ { if (module->deinit != NULL && module->initialized) module->deinit(); - if (dlclose(module->handle) != 0) - i_error("dlclose(%s) failed: %m", module->path); + /* dlclose()ing removes all symbols from valgrind's visibility. + if GDB environment is set, don't actually unload the module + (the GDB environment is used elsewhere too) */ + if (getenv("GDB") == NULL) { + if (dlclose(module->handle) != 0) + i_error("dlclose(%s) failed: %m", module->path); + } i_free(module->path); i_free(module->name); i_free(module); From dovecot at dovecot.org Wed Nov 7 17:05:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 07 Nov 2012 17:05:53 +0200 Subject: dovecot-2.1: mdbox: Don't leave extra records to dovecot.map.ind... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/6c1b4b9f527c changeset: 14798:6c1b4b9f527c user: Timo Sirainen date: Wed Nov 07 17:05:47 2012 +0200 description: mdbox: Don't leave extra records to dovecot.map.index if mailbox index locking fails. Fixes errors such as: Corrupted dbox file /mdbox/storage/m.1 (around offset=2652): EOF reading msg header (got 0/30 bytes) diffstat: src/lib-storage/index/dbox-multi/mdbox-save.c | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) diffs (37 lines): diff -r bcd2d140ea10 -r 6c1b4b9f527c src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Tue Nov 06 22:13:13 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Nov 07 17:05:47 2012 +0200 @@ -296,16 +296,6 @@ mdbox_transaction_save_rollback(_ctx); return -1; } - - /* assign map UIDs for newly saved messages. they're written to - transaction log immediately within this function, but the map - is left locked. */ - if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, - &last_map_uid) < 0) { - mdbox_transaction_save_rollback(_ctx); - return -1; - } - /* lock the mailbox after map to avoid deadlocks. if we've noticed any corruption, deal with it later, otherwise we won't have up-to-date atomic->sync_view */ @@ -318,6 +308,16 @@ return -1; } + /* assign map UIDs for newly saved messages after we've successfully + acquired all the locks. the transaction is now very unlikely to + fail. the UIDs are written to the transaction log immediately within + this function, but the map is left locked. */ + if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, + &last_map_uid) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + /* assign UIDs for new messages */ hdr = mail_index_get_header(ctx->sync_ctx->sync_view); mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, From dovecot at dovecot.org Wed Nov 7 17:37:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 07 Nov 2012 17:37:28 +0200 Subject: dovecot-2.1: var_expand*(): Don't use a data stack frame when ex... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/3a33e686fc38 changeset: 14799:3a33e686fc38 user: Timo Sirainen date: Wed Nov 07 17:37:16 2012 +0200 description: var_expand*(): Don't use a data stack frame when expanding long %{variables} This avoids potential crashes if the destination string is also allocated from data stack and requires growing. diffstat: src/lib/var-expand.c | 50 ++++++++++++++++++++++++-------------------------- 1 files changed, 24 insertions(+), 26 deletions(-) diffs (71 lines): diff -r 6c1b4b9f527c -r 3a33e686fc38 src/lib/var-expand.c --- a/src/lib/var-expand.c Wed Nov 07 17:05:47 2012 +0200 +++ b/src/lib/var-expand.c Wed Nov 07 17:37:16 2012 +0200 @@ -160,7 +160,7 @@ const void *key_start, unsigned int key_len, void *context) { const struct var_expand_table *t; - const char *value = NULL; + const char *key, *value = NULL; if (table != NULL) { for (t = table; !TABLE_LAST(t); t++) { @@ -171,35 +171,33 @@ } } } + key = t_strndup(key_start, key_len); /* built-in variables: */ - T_BEGIN { - const char *key = t_strndup(key_start, key_len); + switch (key_len) { + case 3: + if (strcmp(key, "pid") == 0) + value = my_pid; + else if (strcmp(key, "uid") == 0) + value = dec2str(geteuid()); + else if (strcmp(key, "gid") == 0) + value = dec2str(getegid()); + break; + case 8: + if (strcmp(key, "hostname") == 0) + value = my_hostname; + break; + } - switch (key_len) { - case 3: - if (strcmp(key, "pid") == 0) - value = my_pid; - else if (strcmp(key, "uid") == 0) - value = dec2str(geteuid()); - else if (strcmp(key, "gid") == 0) - value = dec2str(getegid()); - break; - case 8: - if (strcmp(key, "hostname") == 0) - value = my_hostname; - break; - } - if (value == NULL) { - const char *data = strchr(key, ':'); + if (value == NULL) { + const char *data = strchr(key, ':'); - if (data != NULL) - key = t_strdup_until(key, data++); - else - data = ""; - value = var_expand_func(func_table, key, data, context); - } - } T_END; + if (data != NULL) + key = t_strdup_until(key, data++); + else + data = ""; + value = var_expand_func(func_table, key, data, context); + } return value; } From dovecot at dovecot.org Wed Nov 7 17:44:44 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 07 Nov 2012 17:44:44 +0200 Subject: dovecot-2.1: Makefile: Add -lssl to installed dovecot-config's L... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/22caa1317eb1 changeset: 14800:22caa1317eb1 user: Timo Sirainen date: Wed Nov 07 17:44:32 2012 +0200 description: Makefile: Add -lssl to installed dovecot-config's LIBDOVECOT_LOGIN if needed. diffstat: Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3a33e686fc38 -r 22caa1317eb1 Makefile.am --- a/Makefile.am Wed Nov 07 17:37:16 2012 +0200 +++ b/Makefile.am Wed Nov 07 17:44:32 2012 +0200 @@ -63,7 +63,7 @@ grep -v '^LIBDOVECOT_.*_INCLUDE' dovecot-config | \ grep -v '^LIBDOVECOT.*_DEPS' | sed \ -e "s|^\(LIBDOVECOT\)=.*$$|\1='-L$(pkglibdir) -ldovecot'|" \ - -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login|" \ + -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login $(SSL_LIBS)|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ From dovecot at dovecot.org Wed Nov 7 18:14:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 07 Nov 2012 18:14:31 +0200 Subject: dovecot-2.1: auth: Give a better error message for "client doesn... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c811aab61355 changeset: 14801:c811aab61355 user: Timo Sirainen date: Wed Nov 07 18:14:20 2012 +0200 description: auth: Give a better error message for "client doesn't have lookup permissions". diffstat: src/auth/auth-master-connection.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 22caa1317eb1 -r c811aab61355 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Nov 07 17:44:32 2012 +0200 +++ b/src/auth/auth-master-connection.c Wed Nov 07 18:14:20 2012 +0200 @@ -239,7 +239,8 @@ auth_request_log_error(auth_request, "userdb", "client doesn't have lookup permissions for this user: %s " - "(change userdb socket permissions)", reason); + "(to bypass this check, set: service auth { unix_listener %s { mode=0777 } })", + reason, conn->path); return -1; } From dovecot at dovecot.org Fri Nov 9 00:31:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 09 Nov 2012 00:31:23 +0200 Subject: dovecot-2.2: iostream-rawlog: Don't assert-crash if trying to wr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/482da7a91b1c changeset: 15391:482da7a91b1c user: Timo Sirainen date: Fri Nov 09 00:31:07 2012 +0200 description: iostream-rawlog: Don't assert-crash if trying to write 0 bytes. diffstat: src/lib/iostream-rawlog.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r af331bc452d0 -r 482da7a91b1c src/lib/iostream-rawlog.c --- a/src/lib/iostream-rawlog.c Tue Nov 06 23:53:11 2012 +0200 +++ b/src/lib/iostream-rawlog.c Fri Nov 09 00:31:07 2012 +0200 @@ -95,7 +95,8 @@ size_t pos; bool line_ends; - i_assert(size > 0); + if (size == 0) + return; io_loop_time_refresh(); if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_BUFFERED) == 0) { From dovecot at dovecot.org Fri Nov 16 17:40:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 16 Nov 2012 17:40:46 +0200 Subject: dovecot-2.2: lib-imap: imap_append_string_for_humans() was broke... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ea084c04e694 changeset: 15392:ea084c04e694 user: Timo Sirainen date: Fri Nov 16 17:39:25 2012 +0200 description: lib-imap: imap_append_string_for_humans() was broken for empty input string. diffstat: src/lib-imap/imap-quote.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 482da7a91b1c -r ea084c04e694 src/lib-imap/imap-quote.c --- a/src/lib-imap/imap-quote.c Fri Nov 09 00:31:07 2012 +0200 +++ b/src/lib-imap/imap-quote.c Fri Nov 16 17:39:25 2012 +0200 @@ -156,7 +156,7 @@ break; } } - if (last_lwsp) { + if (last_lwsp && i > 0) { modify = TRUE; remove_count++; } From pigeonhole at rename-it.nl Tue Nov 20 01:17:51 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 20 Nov 2012 00:17:51 +0100 Subject: dovecot-2.0-pigeonhole: Fixed -m argument for sieve-test tool. Message-ID: details: http://hg.rename-it.nl/dovecot-2.0-pigeonhole/rev/e42a38f02d28 changeset: 1559:e42a38f02d28 user: Stephan Bosch date: Tue Nov 20 00:17:45 2012 +0100 description: Fixed -m argument for sieve-test tool. diffstat: src/sieve-tools/sieve-test.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 08a2d2718a65 -r e42a38f02d28 src/sieve-tools/sieve-test.c --- a/src/sieve-tools/sieve-test.c Wed Mar 07 22:03:01 2012 +0100 +++ b/src/sieve-tools/sieve-test.c Tue Nov 20 00:17:45 2012 +0100 @@ -266,7 +266,7 @@ /* Compose script environment */ memset(&scriptenv, 0, sizeof(scriptenv)); - scriptenv.default_mailbox = "INBOX"; + scriptenv.default_mailbox = mailbox; scriptenv.user = sieve_tool_get_mail_user(sieve_tool); scriptenv.username = sieve_tool_get_username(sieve_tool); scriptenv.hostname = "host.example.com"; From dovecot at dovecot.org Fri Nov 23 08:32:21 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 23 Nov 2012 08:32:21 +0200 Subject: dovecot-2.1: auth: Added a way to set default values for nonexis... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c722bd39098b changeset: 14802:c722bd39098b user: Timo Sirainen date: Fri Nov 23 08:32:13 2012 +0200 description: auth: Added a way to set default values for nonexistent LDAP attributes. %{ldap:attrName:default_value} expands to default_value if attrName doesn't exist. diffstat: src/auth/db-ldap.c | 28 +++++++++++++++++++++------- 1 files changed, 21 insertions(+), 7 deletions(-) diffs (61 lines): diff -r c811aab61355 -r c722bd39098b src/auth/db-ldap.c --- a/src/auth/db-ldap.c Wed Nov 07 18:14:20 2012 +0200 +++ b/src/auth/db-ldap.c Fri Nov 23 08:32:13 2012 +0200 @@ -990,7 +990,7 @@ char *ldap_attr; if (*data != '\0') { - ldap_attr = p_strdup(ctx->pool, data); + ldap_attr = p_strdup(ctx->pool, t_strcut(data, ':')); array_append(&ctx->attr_names, &ldap_attr, 1); } return NULL; @@ -1200,28 +1200,42 @@ return ctx; } +static const char *db_ldap_field_get_default(const char *data) +{ + const char *p; + + p = strchr(data, ':'); + if (p == NULL) + return ""; + else { + /* default value given */ + return p+1; + } +} + static const char *db_ldap_field_expand(const char *data, void *context) { struct db_ldap_result_iterate_context *ctx = context; struct db_ldap_value *ldap_value; + const char *field_name = t_strcut(data, ':'); - ldap_value = hash_table_lookup(ctx->ldap_attrs, data); + ldap_value = hash_table_lookup(ctx->ldap_attrs, field_name); if (ldap_value == NULL) { - /* ldap attribute wasn't requested */ + /* requested ldap attribute wasn't returned at all */ if (ctx->debug) - str_printfa(ctx->debug, "; %s missing", data); - return ""; + str_printfa(ctx->debug, "; %s missing", field_name); + return db_ldap_field_get_default(data); } ldap_value->used = TRUE; if (ldap_value->values[0] == NULL) { /* no value for ldap attribute */ - return ""; + return db_ldap_field_get_default(data); } if (ldap_value->values[1] != NULL) { auth_request_log_warning(ctx->auth_request, "ldap", "Multiple values found for '%s', using value '%s'", - data, ldap_value->values[0]); + field_name, ldap_value->values[0]); } return ldap_value->values[0]; } From dovecot at dovecot.org Fri Nov 23 08:52:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 23 Nov 2012 08:52:12 +0200 Subject: dovecot-2.1: login: Try to avoid busy-looping on SSL_accept() wh... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/e95479f439aa changeset: 14803:e95479f439aa user: Timo Sirainen date: Fri Nov 23 08:52:06 2012 +0200 description: login: Try to avoid busy-looping on SSL_accept() when client doesn't behave nicely. diffstat: src/login-common/ssl-proxy-openssl.c | 17 +++++++++++------ 1 files changed, 11 insertions(+), 6 deletions(-) diffs (63 lines): diff -r c722bd39098b -r e95479f439aa src/login-common/ssl-proxy-openssl.c --- a/src/login-common/ssl-proxy-openssl.c Fri Nov 23 08:32:13 2012 +0200 +++ b/src/login-common/ssl-proxy-openssl.c Fri Nov 23 08:52:06 2012 +0200 @@ -394,8 +394,9 @@ return ssl_err2str(err, data, flags); } -static void ssl_handle_error(struct ssl_proxy *proxy, int ret, - const char *func_name) +static void +ssl_handle_error(struct ssl_proxy *proxy, int ret, bool remove_wrong_direction, + const char *func_name) { const char *errstr = NULL; int err; @@ -408,9 +409,13 @@ switch (err) { case SSL_ERROR_WANT_READ: ssl_set_io(proxy, SSL_ADD_INPUT); + if (remove_wrong_direction) + ssl_set_io(proxy, SSL_REMOVE_OUTPUT); break; case SSL_ERROR_WANT_WRITE: ssl_set_io(proxy, SSL_ADD_OUTPUT); + if (remove_wrong_direction) + ssl_set_io(proxy, SSL_REMOVE_INPUT); break; case SSL_ERROR_SYSCALL: /* eat up the error queue */ @@ -458,13 +463,13 @@ if (proxy->client_proxy) { ret = SSL_connect(proxy->ssl); if (ret != 1) { - ssl_handle_error(proxy, ret, "SSL_connect()"); + ssl_handle_error(proxy, ret, TRUE, "SSL_connect()"); return; } } else { ret = SSL_accept(proxy->ssl); if (ret != 1) { - ssl_handle_error(proxy, ret, "SSL_accept()"); + ssl_handle_error(proxy, ret, TRUE, "SSL_accept()"); return; } } @@ -491,7 +496,7 @@ sizeof(proxy->plainout_buf) - proxy->plainout_size); if (ret <= 0) { - ssl_handle_error(proxy, ret, "SSL_read()"); + ssl_handle_error(proxy, ret, FALSE, "SSL_read()"); break; } else { i_free_and_null(proxy->last_error); @@ -507,7 +512,7 @@ ret = SSL_write(proxy->ssl, proxy->sslout_buf, proxy->sslout_size); if (ret <= 0) - ssl_handle_error(proxy, ret, "SSL_write()"); + ssl_handle_error(proxy, ret, FALSE, "SSL_write()"); else { i_free_and_null(proxy->last_error); proxy->sslout_size -= ret; From dovecot at dovecot.org Fri Nov 23 23:49:50 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 23 Nov 2012 23:49:50 +0200 Subject: dovecot-2.2: bsdauth: Use "auth-" as the auth type. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b40bda50541c changeset: 15393:b40bda50541c user: Timo Sirainen date: Fri Nov 23 23:49:39 2012 +0200 description: bsdauth: Use "auth-" as the auth type. diffstat: src/auth/passdb-bsdauth.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (21 lines): diff -r ea084c04e694 -r b40bda50541c src/auth/passdb-bsdauth.c --- a/src/auth/passdb-bsdauth.c Fri Nov 16 17:39:25 2012 +0200 +++ b/src/auth/passdb-bsdauth.c Fri Nov 23 23:49:39 2012 +0200 @@ -18,6 +18,7 @@ verify_plain_callback_t *callback) { struct passwd pw; + const char *type; int result; auth_request_log_debug(request, "bsdauth", "lookup"); @@ -35,7 +36,8 @@ } /* check if the password is valid */ - result = auth_userokay(request->user, NULL, NULL, + type = t_strdup_printf("auth-%s", request->service); + result = auth_userokay(request->user, NULL, t_strdup_noconst(type), t_strdup_noconst(password)); /* clear the passwords from memory */ From dovecot at dovecot.org Sat Nov 24 00:33:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 24 Nov 2012 00:33:06 +0200 Subject: dovecot-2.2: lib-http: Added initial HTTP client implementation. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/107c8b2c9594 changeset: 15394:107c8b2c9594 user: Stephan Bosch date: Sat Nov 24 00:30:14 2012 +0200 description: lib-http: Added initial HTTP client implementation. diffstat: src/lib-http/Makefile.am | 92 +++- src/lib-http/http-client-connection.c | 788 +++++++++++++++++++++++++++++++ src/lib-http/http-client-host.c | 388 +++++++++++++++ src/lib-http/http-client-peer.c | 358 ++++++++++++++ src/lib-http/http-client-private.h | 250 +++++++++ src/lib-http/http-client-request.c | 386 +++++++++++++++ src/lib-http/http-client.c | 182 +++++++ src/lib-http/http-client.h | 83 +++ src/lib-http/http-header-parser.c | 282 +++++++++++ src/lib-http/http-header-parser.h | 15 + src/lib-http/http-parser.c | 132 +++++ src/lib-http/http-parser.h | 53 ++ src/lib-http/http-response-parser.c | 503 +++++++++++++++++++ src/lib-http/http-response-parser.h | 37 + src/lib-http/http-transfer-chunked.c | 479 ++++++++++++++++++ src/lib-http/http-transfer.h | 8 + src/lib-http/test-http-client.c | 226 ++++++++ src/lib-http/test-http-header-parser.c | 18 +- src/lib-http/test-http-response-parser.c | 206 ++++++++ src/lib-http/test-http-responses.c | 39 + src/lib-http/test-http-transfer.c | 126 ++++ 21 files changed, 4640 insertions(+), 11 deletions(-) diffs (truncated from 4806 to 300 lines): diff -r b40bda50541c -r 107c8b2c9594 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Fri Nov 23 23:49:39 2012 +0200 +++ b/src/lib-http/Makefile.am Sat Nov 24 00:30:14 2012 +0200 @@ -8,26 +8,51 @@ libhttp_la_SOURCES = \ http-date.c \ - http-url.c + http-url.c \ + http-parser.c \ + http-header-parser.c \ + http-transfer-chunked.c \ + http-response-parser.c \ + http-client-request.c \ + http-client-connection.c \ + http-client-peer.c \ + http-client-host.c \ + http-client.c headers = \ http-date.h \ - http-url.h + http-url.h \ + http-parser.h \ + http-header-parser.h \ + http-transfer.h \ + http-response-parser.h \ + http-client-private.h \ + http-client.h pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) test_programs = \ test-http-date \ - test-http-url + test-http-url \ + test-http-header-parser \ + test-http-transfer \ + test-http-response-parser \ + test-http-client -noinst_PROGRAMS = $(test_programs) +noinst_PROGRAMS = $(test_programs) test-http-responses test_libs = \ ../lib-test/libtest.la \ - ../lib/liblib.la + ../lib/liblib.la \ + ../lib/safe-memset.lo \ + $(MODULE_LIBS) -test_deps = $(noinst_LTLIBRARIES) $(test_libs) +test_deps = \ + $(noinst_LTLIBRARIES) + ../lib-test/libtest.la \ + ../lib/liblib.la \ + ../lib/safe-memset.lo test_http_url_SOURCES = test-http-url.c test_http_url_LDADD = http-url.lo $(test_libs) @@ -37,6 +62,61 @@ test_http_date_LDADD = http-date.lo $(test_libs) test_http_date_DEPENDENCIES = $(test_deps) +test_http_header_parser_SOURCES = test-http-header-parser.c +test_http_header_parser_LDADD = http-parser.lo http-header-parser.lo $(test_libs) +test_http_header_parser_DEPENDENCIES = $(test_deps) + +test_http_transfer_SOURCES = test-http-transfer.c +test_http_transfer_LDADD = \ + http-parser.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + $(test_libs) +test_http_transfer_DEPENDENCIES = $(test_deps) + +test_http_response_parser_SOURCES = test-http-response-parser.c +test_http_response_parser_LDADD = \ + http-date.lo \ + http-parser.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-response-parser.lo \ + $(test_libs) +test_http_response_parser_DEPENDENCIES = $(test_deps) + +test_http_client_SOURCES = test-http-client.c +test_http_client_LDFLAGS = -export-dynamic +test_http_client_LDADD = \ + libhttp.la \ + ../lib-dns/libdns.la \ + ../lib-ssl-iostream/libssl_iostream.la \ + $(test_libs) +test_http_client_DEPENDENCIES = \ + libhttp.la \ + ../lib-dns/libdns.la \ + ../lib-ssl-iostream/libssl_iostream.la \ + $(test_deps) + +test_http_responses_SOURCES = test-http-responses.c +test_http_responses_LDADD = \ + http-date.lo \ + http-parser.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-response-parser.lo \ + ../lib/liblib.la \ + ../lib/safe-memset.lo \ + $(MODULE_LIBS) +test_http_responses_DEPENDENCIES = \ + http-date.lo \ + http-parser.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-response-parser.lo \ + $(noinst_LTLIBRARIES) \ + ../lib/liblib.la \ + ../lib/safe-memset.lo + check: check-am check-test check-test: all-am for bin in $(test_programs); do \ diff -r b40bda50541c -r 107c8b2c9594 src/lib-http/http-client-connection.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-http/http-client-connection.c Sat Nov 24 00:30:14 2012 +0200 @@ -0,0 +1,788 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "hash.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "iostream-rawlog.h" +#include "iostream-ssl.h" +#include "http-response-parser.h" + +#include "http-client-private.h" + +/* + * Logging + */ + +static inline void +http_client_connection_debug(struct http_client_connection *conn, + const char *format, ...) ATTR_FORMAT(2, 3); +static inline void +http_client_connection_error(struct http_client_connection *conn, + const char *format, ...) ATTR_FORMAT(2, 3); + +static inline void +http_client_connection_debug(struct http_client_connection *conn, + const char *format, ...) +{ + va_list args; + + if (conn->client->set.debug) { + + va_start(args, format); + i_debug("http-client: conn %s: %s", + http_client_connection_label(conn), t_strdup_vprintf(format, args)); + va_end(args); + } +} + +static inline void +http_client_connection_error(struct http_client_connection *conn, + const char *format, ...) +{ + va_list args; + + va_start(args, format); + i_error("http-client: conn %s: %s", + http_client_connection_label(conn), t_strdup_vprintf(format, args)); + va_end(args); +} + + +/* + * Connection + */ + +static void http_client_connection_input(struct connection *_conn); + +bool http_client_connection_is_ready(struct http_client_connection *conn) +{ + return (conn->connected && !conn->output_locked && + array_count(&conn->request_wait_list) < + conn->client->set.max_pipelined_requests); +} + +static void +http_client_connection_retry_requests(struct http_client_connection *conn, + unsigned int status, const char *error) +{ + struct http_client_request **req; + + array_foreach_modifiable(&conn->request_wait_list, req) { + http_client_request_retry(*req, status, error); + } + array_clear(&conn->request_wait_list); +} + +static void +http_client_connection_server_close(struct http_client_connection **_conn) +{ + struct http_client_connection *conn = *_conn; + struct http_client_request **req; + + conn->connected = FALSE; + conn->closing = TRUE; + + http_client_connection_debug(conn, + "Server explicitly closed connection"); + + array_foreach_modifiable(&conn->request_wait_list, req) { + http_client_request_resubmit(*req); + } + array_clear(&conn->request_wait_list); + + if (conn->client->ioloop != NULL) + io_loop_stop(conn->client->ioloop); + + http_client_connection_free(_conn); +} + +static void +http_client_connection_abort_temp_error(struct http_client_connection **_conn, + unsigned int status, const char *error) +{ + struct http_client_connection *conn = *_conn; + + conn->connected = FALSE; + conn->closing = TRUE; + + http_client_connection_retry_requests(conn, status, error); + http_client_connection_free(_conn); +} + +static void +http_client_connection_abort_error(struct http_client_connection **_conn, + unsigned int status, const char *error) +{ + struct http_client_connection *conn = *_conn; + struct http_client_request **req; + + conn->connected = FALSE; + conn->closing = TRUE; + + array_foreach_modifiable(&conn->request_wait_list, req) { + http_client_request_error(*req, status, error); + } + array_clear(&conn->request_wait_list); + http_client_connection_free(_conn); +} + +static void +http_client_connection_idle_timeout(struct http_client_connection *conn) +{ + http_client_connection_debug(conn, "Idle connection timed out"); + + http_client_connection_free(&conn); +} + +static void +http_client_connection_check_idle(struct http_client_connection *conn) +{ + unsigned int timeout, count; + + if (array_count(&conn->request_wait_list) == 0 && + conn->incoming_payload == NULL && + conn->client->set.max_idle_time_msecs > 0) { + + if (conn->to_idle != NULL) { + /* timeout already set */ + return; + } + + http_client_connection_debug(conn, + "No more requests queued; going idle"); + + if (conn->client->ioloop != NULL) + io_loop_stop(conn->client->ioloop); + + count = array_count(&conn->peer->conns); + i_assert(count > 0); + + /* set timeout for this connection */ + if (count > conn->client->set.max_parallel_connections) { + /* instant death for (urgent) connections above limit */ + timeout = 0; + } else { + /* kill duplicate connections quicker; + linearly based on the number of connections */ + timeout = (conn->client->set.max_parallel_connections - count) * + (conn->client->set.max_idle_time_msecs / From dovecot at dovecot.org Sat Nov 24 00:33:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 24 Nov 2012 00:33:06 +0200 Subject: dovecot-2.2: lib-http: Compiler warning fixes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa32c5f9cf71 changeset: 15395:fa32c5f9cf71 user: Timo Sirainen date: Sat Nov 24 00:32:56 2012 +0200 description: lib-http: Compiler warning fixes diffstat: src/lib-http/Makefile.am | 22 +-------------------- src/lib-http/http-client-host.c | 3 +- src/lib-http/test-http-responses.c | 39 -------------------------------------- 3 files changed, 2 insertions(+), 62 deletions(-) diffs (98 lines): diff -r 107c8b2c9594 -r fa32c5f9cf71 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Sat Nov 24 00:30:14 2012 +0200 +++ b/src/lib-http/Makefile.am Sat Nov 24 00:32:56 2012 +0200 @@ -40,7 +40,7 @@ test-http-response-parser \ test-http-client -noinst_PROGRAMS = $(test_programs) test-http-responses +noinst_PROGRAMS = $(test_programs) test_libs = \ ../lib-test/libtest.la \ @@ -97,26 +97,6 @@ ../lib-ssl-iostream/libssl_iostream.la \ $(test_deps) -test_http_responses_SOURCES = test-http-responses.c -test_http_responses_LDADD = \ - http-date.lo \ - http-parser.lo \ - http-header-parser.lo \ - http-transfer-chunked.lo \ - http-response-parser.lo \ - ../lib/liblib.la \ - ../lib/safe-memset.lo \ - $(MODULE_LIBS) -test_http_responses_DEPENDENCIES = \ - http-date.lo \ - http-parser.lo \ - http-header-parser.lo \ - http-transfer-chunked.lo \ - http-response-parser.lo \ - $(noinst_LTLIBRARIES) \ - ../lib/liblib.la \ - ../lib/safe-memset.lo - check: check-am check-test check-test: all-am for bin in $(test_programs); do \ diff -r 107c8b2c9594 -r fa32c5f9cf71 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Sat Nov 24 00:30:14 2012 +0200 +++ b/src/lib-http/http-client-host.c Sat Nov 24 00:32:56 2012 +0200 @@ -109,11 +109,10 @@ array_foreach_modifiable(queue, req_idx) { if (*req_idx == req) { idx = array_foreach_idx(queue, req_idx); + array_delete(queue, idx, 1); break; } } - - array_delete(queue, idx, 1); } /* diff -r 107c8b2c9594 -r fa32c5f9cf71 src/lib-http/test-http-responses.c --- a/src/lib-http/test-http-responses.c Sat Nov 24 00:30:14 2012 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,39 +0,0 @@ -/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ - -#include "test-lib.h" -#include "buffer.h" -#include "str.h" -#include "str-sanitize.h" -#include "istream.h" -#include "ostream.h" -#include "test-common.h" -#include "http-response-parser.h" - -#include - -// FIXME: debug tool; can be removed - -int main(int argc, char **argv) -{ - struct istream *input; - struct http_response_parser *parser; - struct http_response *response; - const char *payload, *error = NULL; - int ret; - - if (argc < 2) - return 1; - - input = i_stream_create_file(argv[1], 32); - parser = http_response_parser_init(input); - - payload = NULL; - while ((ret=http_response_parse_next(parser, FALSE, &response, &error)) > 0) { - printf("RESPONSE: %u %s\n", response->status, response->reason); - } - - printf("RET: %d %s\n", ret, error); - - http_response_parser_deinit(&parser); -} - From pigeonhole at rename-it.nl Mon Nov 26 22:17:11 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 26 Nov 2012 21:17:11 +0100 Subject: dovecot-2.1-pigeonhole: lib-sieve: vacation extension: Fixed det... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/b56711807edc changeset: 1667:b56711807edc user: Stephan Bosch date: Mon Nov 26 21:16:54 2012 +0100 description: lib-sieve: vacation extension: Fixed determination of From: address for when sieve_vacation_dont_check_recipient is active. diffstat: src/lib-sieve/plugins/vacation/cmd-vacation.c | 76 ++++++++++++++------------ 1 files changed, 41 insertions(+), 35 deletions(-) diffs (105 lines): diff -r 0dc8453e07c4 -r b56711807edc src/lib-sieve/plugins/vacation/cmd-vacation.c --- a/src/lib-sieve/plugins/vacation/cmd-vacation.c Wed Oct 17 22:11:16 2012 +0200 +++ b/src/lib-sieve/plugins/vacation/cmd-vacation.c Mon Nov 26 21:16:54 2012 +0100 @@ -1031,7 +1031,7 @@ const char *recipient = sieve_message_get_final_recipient(aenv->msgctx); const char *const *hdsp; const char *const *headers; - const char *reply_from = NULL; + const char *reply_from = NULL, *orig_recipient = NULL; /* Is the recipient unset? */ @@ -1142,52 +1142,58 @@ return TRUE; } + /* Fetch original recipient if necessary */ + if ( config->use_original_recipient ) + orig_recipient = sieve_message_get_orig_recipient(aenv->msgctx); + /* Is the original message directly addressed to the user or the addresses * specified using the :addresses tag? */ - if ( !config->dont_check_recipient ) { - const char *orig_recipient = NULL; + hdsp = _my_address_headers; + while ( *hdsp != NULL ) { + if ( mail_get_headers + (mail, *hdsp, &headers) >= 0 && headers[0] != NULL ) { - if ( config->use_original_recipient ) - orig_recipient = sieve_message_get_orig_recipient(aenv->msgctx); + /* Final recipient directly listed in headers? */ + if ( _contains_my_address(headers, recipient) ) { + reply_from = recipient; + break; + } - hdsp = _my_address_headers; - while ( *hdsp != NULL ) { - if ( mail_get_headers - (mail, *hdsp, &headers) >= 0 && headers[0] != NULL ) { + /* Original recipient directly listed in headers? */ + if ( orig_recipient != NULL && + _contains_my_address(headers, orig_recipient) ) { + reply_from = orig_recipient; + break; + } - if ( _contains_my_address(headers, recipient) ) { - reply_from = recipient; - break; + /* User-provided :addresses listed in headers? */ + if ( ctx->addresses != NULL ) { + bool found = FALSE; + const char * const *my_address = ctx->addresses; + + while ( !found && *my_address != NULL ) { + if ( (found=_contains_my_address(headers, *my_address)) ) + reply_from = *my_address; + my_address++; } - if ( orig_recipient != NULL && _contains_my_address(headers, orig_recipient) ) { - reply_from = orig_recipient; - break; - } + if ( found ) break; + } + } + hdsp++; + } - if ( ctx->addresses != NULL ) { - bool found = FALSE; - const char * const *my_address = ctx->addresses; + /* My address not found in the headers; we got an implicit delivery */ + if ( *hdsp == NULL ) { + if ( config->dont_check_recipient ) { + /* Send reply from envelope recipient address */ + reply_from = recipient; - while ( !found && *my_address != NULL ) { - if ( (found=_contains_my_address(headers, *my_address)) ) - reply_from = *my_address; - my_address++; - } - - if ( found ) break; - } - } - hdsp++; - } - - - /* My address not found in the headers; we got an implicit delivery */ - if ( *hdsp == NULL ) { + } else { const char *original_recipient = ""; - /* No, bail out */ + /* Bail out */ if ( config->use_original_recipient ) { original_recipient = t_strdup_printf("original-recipient=<%s>, ", From dovecot at dovecot.org Mon Nov 26 23:03:18 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:03:18 +0200 Subject: dovecot-2.2: lib-fs: Iterating a nonexistent directory should be... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bd01ca99bee5 changeset: 15397:bd01ca99bee5 user: Timo Sirainen date: Mon Nov 26 23:01:28 2012 +0200 description: lib-fs: Iterating a nonexistent directory should be the same as iterating an empty directory. diffstat: src/lib-fs/fs-api.h | 3 ++- src/lib-fs/fs-posix.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diffs (25 lines): diff -r 38f3bd4bf6e3 -r bd01ca99bee5 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Mon Nov 26 22:59:37 2012 +0200 +++ b/src/lib-fs/fs-api.h Mon Nov 26 23:01:28 2012 +0200 @@ -157,7 +157,8 @@ void fs_unlock(struct fs_lock **lock); /* Iterate through all files (but not directories) in the given directory. - Doesn't recurse to child directories. */ + Doesn't recurse to child directories. It's not an error to iterate a + nonexistent directory. */ struct fs_iter *fs_iter_init(struct fs *fs, const char *path); /* Returns 0 if ok, -1 if iteration failed. */ int fs_iter_deinit(struct fs_iter **iter); diff -r 38f3bd4bf6e3 -r bd01ca99bee5 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Mon Nov 26 22:59:37 2012 +0200 +++ b/src/lib-fs/fs-posix.c Mon Nov 26 23:01:28 2012 +0200 @@ -652,7 +652,7 @@ iter->iter.fs = fs; iter->path = i_strdup(path); iter->dir = opendir(path); - if (iter->dir == NULL) { + if (iter->dir == NULL && errno != ENOENT) { iter->err = errno; fs_set_error(fs, "opendir(%s) failed: %m", path); } From dovecot at dovecot.org Mon Nov 26 23:03:18 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:03:18 +0200 Subject: dovecot-2.2: lib-fs: fs_wait_async() can now return an error. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/38f3bd4bf6e3 changeset: 15396:38f3bd4bf6e3 user: Timo Sirainen date: Mon Nov 26 22:59:37 2012 +0200 description: lib-fs: fs_wait_async() can now return an error. diffstat: src/lib-fs/fs-api-private.h | 2 +- src/lib-fs/fs-api.c | 8 +++++--- src/lib-fs/fs-api.h | 5 +++-- src/lib-fs/fs-sis-queue.c | 4 ++-- src/lib-fs/fs-sis.c | 4 ++-- 5 files changed, 13 insertions(+), 10 deletions(-) diffs (80 lines): diff -r fa32c5f9cf71 -r 38f3bd4bf6e3 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Sat Nov 24 00:32:56 2012 +0200 +++ b/src/lib-fs/fs-api-private.h Mon Nov 26 22:59:37 2012 +0200 @@ -19,7 +19,7 @@ void (*set_async_callback)(struct fs_file *file, fs_file_async_callback_t *callback, void *context); - void (*wait_async)(struct fs *fs); + int (*wait_async)(struct fs *fs); void (*set_metadata)(struct fs_file *file, const char *key, const char *value); diff -r fa32c5f9cf71 -r 38f3bd4bf6e3 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Sat Nov 24 00:32:56 2012 +0200 +++ b/src/lib-fs/fs-api.c Mon Nov 26 22:59:37 2012 +0200 @@ -190,10 +190,12 @@ callback(context); } -void fs_wait_async(struct fs *fs) +int fs_wait_async(struct fs *fs) { - if (fs->v.wait_async != NULL) - fs->v.wait_async(fs); + if (fs->v.wait_async == NULL) + return 0; + else + return fs->v.wait_async(fs); } int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r) diff -r fa32c5f9cf71 -r 38f3bd4bf6e3 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Sat Nov 24 00:32:56 2012 +0200 +++ b/src/lib-fs/fs-api.h Mon Nov 26 22:59:37 2012 +0200 @@ -127,8 +127,9 @@ fs_file_async_callback_t *callback, void *context); /* Wait until some file can be read/written to more before returning. - It's an error to call this when there are no pending async operations. */ -void fs_wait_async(struct fs *fs); + It's an error to call this when there are no pending async operations. + Returns 0 if ok, -1 if timed out. */ +int fs_wait_async(struct fs *fs); /* Returns 1 if file exists, 0 if not, -1 if error occurred. */ int fs_exists(struct fs_file *file); diff -r fa32c5f9cf71 -r 38f3bd4bf6e3 src/lib-fs/fs-sis-queue.c --- a/src/lib-fs/fs-sis-queue.c Sat Nov 24 00:32:56 2012 +0200 +++ b/src/lib-fs/fs-sis-queue.c Mon Nov 26 22:59:37 2012 +0200 @@ -127,11 +127,11 @@ fs_file_set_async_callback(file->super, callback, context); } -static void fs_sis_queue_wait_async(struct fs *_fs) +static int fs_sis_queue_wait_async(struct fs *_fs) { struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs; - fs_wait_async(fs->super); + return fs_wait_async(fs->super); } static void diff -r fa32c5f9cf71 -r 38f3bd4bf6e3 src/lib-fs/fs-sis.c --- a/src/lib-fs/fs-sis.c Sat Nov 24 00:32:56 2012 +0200 +++ b/src/lib-fs/fs-sis.c Mon Nov 26 22:59:37 2012 +0200 @@ -164,11 +164,11 @@ fs_file_set_async_callback(file->super, callback, context); } -static void fs_sis_wait_async(struct fs *_fs) +static int fs_sis_wait_async(struct fs *_fs) { struct sis_fs *fs = (struct sis_fs *)_fs; - fs_wait_async(fs->super); + return fs_wait_async(fs->super); } static void From dovecot at dovecot.org Mon Nov 26 23:03:18 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:03:18 +0200 Subject: dovecot-2.2: lib-fs: Fixed iteration with POSIX backend. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bc497e533da4 changeset: 15398:bc497e533da4 user: Timo Sirainen date: Mon Nov 26 23:01:53 2012 +0200 description: lib-fs: Fixed iteration with POSIX backend. diffstat: src/lib-fs/fs-posix.c | 12 +++++++++--- 1 files changed, 9 insertions(+), 3 deletions(-) diffs (23 lines): diff -r bd01ca99bee5 -r bc497e533da4 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Mon Nov 26 23:01:28 2012 +0200 +++ b/src/lib-fs/fs-posix.c Mon Nov 26 23:01:53 2012 +0200 @@ -686,10 +686,16 @@ errno = 0; while ((d = readdir(iter->dir)) != NULL) { #ifdef HAVE_DIRENT_D_TYPE - if (d->d_type != DT_DIR) { - if (d->d_type == DT_UNKNOWN && - fs_posix_iter_want(iter->path, d->d_name)) + switch (d->d_type) { + case DT_UNKNOWN: + if (!fs_posix_iter_want(iter->path, d->d_name)) + break; + /* fall through */ + case DT_REG: + case DT_LNK: return d->d_name; + default: + break; } #else if (fs_posix_iter_want(iter->path, d->d_name)) From dovecot at dovecot.org Mon Nov 26 23:03:18 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:03:18 +0200 Subject: dovecot-2.2: lib-fs: Added ITER and RELIABLEITER properties for ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/39578785ee92 changeset: 15399:39578785ee92 user: Timo Sirainen date: Mon Nov 26 23:02:51 2012 +0200 description: lib-fs: Added ITER and RELIABLEITER properties for backends. diffstat: src/lib-fs/fs-api.h | 7 ++++++- src/lib-fs/fs-posix.c | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diffs (29 lines): diff -r bc497e533da4 -r 39578785ee92 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Mon Nov 26 23:01:53 2012 +0200 +++ b/src/lib-fs/fs-api.h Mon Nov 26 23:02:51 2012 +0200 @@ -11,7 +11,12 @@ FS_PROPERTY_LOCKS = 0x02, FS_PROPERTY_FASTCOPY = 0x04, FS_PROPERTY_RENAME = 0x08, - FS_PROPERTY_STAT = 0x10 + FS_PROPERTY_STAT = 0x10, + /* Iteration is possible */ + FS_PROPERTY_ITER = 0x20, + /* Iteration always returns all of the files (instead of possibly + slightly out of date view) */ + FS_PROPERTY_RELIABLEITER= 0x40 }; enum fs_open_mode { diff -r bc497e533da4 -r 39578785ee92 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Mon Nov 26 23:01:53 2012 +0200 +++ b/src/lib-fs/fs-posix.c Mon Nov 26 23:02:51 2012 +0200 @@ -124,7 +124,7 @@ static enum fs_properties fs_posix_get_properties(struct fs *fs ATTR_UNUSED) { return FS_PROPERTY_LOCKS | FS_PROPERTY_FASTCOPY | FS_PROPERTY_RENAME | - FS_PROPERTY_STAT; + FS_PROPERTY_STAT | FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER; } static int fs_posix_mkdir_parents(struct posix_fs *fs, const char *path) From dovecot at dovecot.org Mon Nov 26 23:03:18 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:03:18 +0200 Subject: dovecot-2.2: lib-fs: Added a temp_dir setting that the caller ca... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5552ac605201 changeset: 15400:5552ac605201 user: Timo Sirainen date: Mon Nov 26 23:03:09 2012 +0200 description: lib-fs: Added a temp_dir setting that the caller can specify. diffstat: src/lib-fs/fs-api.h | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 39578785ee92 -r 5552ac605201 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Mon Nov 26 23:02:51 2012 +0200 +++ b/src/lib-fs/fs-api.h Mon Nov 26 23:03:09 2012 +0200 @@ -52,6 +52,10 @@ struct fs_settings { /* Dovecot instance's base_dir */ const char *base_dir; + /* Directory where temporary files can be created at any time + (e.g. /tmp or mail_temp_dir) */ + const char *temp_dir; + /* Automatically try to rmdir() directories up to this path when deleting files. */ const char *root_path; From dovecot at dovecot.org Mon Nov 26 23:08:17 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:08:17 +0200 Subject: dovecot-2.2: Added iostream-temp for easily creating a temporary... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/faa3a83282fb changeset: 15402:faa3a83282fb user: Timo Sirainen date: Mon Nov 26 23:08:10 2012 +0200 description: Added iostream-temp for easily creating a temporary istream (to memory/file). diffstat: src/lib/Makefile.am | 2 + src/lib/iostream-temp.c | 149 ++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/iostream-temp.h | 12 +++ 3 files changed, 163 insertions(+), 0 deletions(-) diffs (188 lines): diff -r 22cfb7b347a8 -r faa3a83282fb src/lib/Makefile.am --- a/src/lib/Makefile.am Mon Nov 26 23:06:24 2012 +0200 +++ b/src/lib/Makefile.am Mon Nov 26 23:08:10 2012 +0200 @@ -52,6 +52,7 @@ ipwd.c \ iostream.c \ iostream-rawlog.c \ + iostream-temp.c \ iso8601-date.c \ istream.c \ istream-base64-decoder.c \ @@ -179,6 +180,7 @@ iostream-private.h \ iostream-rawlog.h \ iostream-rawlog-private.h \ + iostream-temp.h \ iso8601-date.h \ istream.h \ istream-base64.h \ diff -r 22cfb7b347a8 -r faa3a83282fb src/lib/iostream-temp.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/iostream-temp.c Mon Nov 26 23:08:10 2012 +0200 @@ -0,0 +1,149 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "safe-mkstemp.h" +#include "write-full.h" +#include "istream.h" +#include "ostream-private.h" +#include "iostream-temp.h" + +#include + +#define IOSTREAM_TEMP_MAX_BUF_SIZE (1024*128) + +struct temp_ostream { + struct ostream_private ostream; + char *temp_path_prefix; + buffer_t *buf; + int fd; + bool fd_tried; +}; + +static void o_stream_temp_close(struct iostream_private *stream) +{ + struct temp_ostream *tstream = (struct temp_ostream *)stream; + + if (tstream->fd != -1) + i_close_fd(&tstream->fd); + if (tstream->buf != NULL) + buffer_free(&tstream->buf); + i_free(tstream->temp_path_prefix); +} + +static int o_stream_temp_move_to_fd(struct temp_ostream *tstream) +{ + string_t *path; + + if (tstream->fd_tried) + return -1; + tstream->fd_tried = TRUE; + + path = t_str_new(128); + str_append(path, tstream->temp_path_prefix); + tstream->fd = safe_mkstemp_hostpid(path, 0600, (uid_t)-1, (gid_t)-1); + if (tstream->fd == -1) { + i_error("safe_mkstemp(%s) failed: %m", str_c(path)); + return -1; + } + if (unlink(str_c(path)) < 0) { + i_error("unlink(%s) failed: %m", str_c(path)); + i_close_fd(&tstream->fd); + return -1; + } + if (write_full(tstream->fd, tstream->buf->data, tstream->buf->used) < 0) { + i_error("write(%s) failed: %m", str_c(path)); + i_close_fd(&tstream->fd); + return -1; + } + buffer_free(&tstream->buf); + return 0; +} + +static ssize_t +o_stream_temp_fd_sendv(struct temp_ostream *tstream, + const struct const_iovec *iov, unsigned int iov_count) +{ + size_t bytes = 0; + unsigned int i; + + for (i = 0; i < iov_count; i++) { + if (write_full(tstream->fd, iov[i].iov_base, iov[i].iov_len) < 0) { + tstream->ostream.ostream.stream_errno = errno; + return -1; + } + bytes += iov[i].iov_len; + tstream->ostream.ostream.offset += iov[i].iov_len; + } + return bytes; +} + +static ssize_t +o_stream_temp_sendv(struct ostream_private *stream, + const struct const_iovec *iov, unsigned int iov_count) +{ + struct temp_ostream *tstream = (struct temp_ostream *)stream; + ssize_t ret = 0; + unsigned int i; + + if (tstream->fd != -1) + return o_stream_temp_fd_sendv(tstream, iov, iov_count); + + for (i = 0; i < iov_count; i++) { + if (tstream->buf->used + iov[i].iov_len > IOSTREAM_TEMP_MAX_BUF_SIZE) { + if (o_stream_temp_move_to_fd(tstream) == 0) { + return o_stream_temp_fd_sendv(tstream, iov+i, + iov_count-i); + } + /* failed to move to temp fd, just keep it in memory */ + } + buffer_append(tstream->buf, iov[i].iov_base, iov[i].iov_len); + ret += iov[i].iov_len; + stream->ostream.offset += iov[i].iov_len; + } + return ret; +} + +struct ostream *iostream_temp_create(const char *temp_path_prefix) +{ + struct temp_ostream *tstream; + struct ostream *output; + + tstream = i_new(struct temp_ostream, 1); + tstream->ostream.sendv = o_stream_temp_sendv; + tstream->ostream.iostream.close = o_stream_temp_close; + tstream->temp_path_prefix = i_strdup(temp_path_prefix); + tstream->buf = buffer_create_dynamic(default_pool, 8192); + tstream->fd = -1; + + output = o_stream_create(&tstream->ostream, NULL, -1); + o_stream_set_name(output, "(temp iostream)"); + return output; +} + +static void iostream_temp_buf_destroyed(buffer_t *buf) +{ + buffer_free(&buf); +} + +struct istream *iostream_temp_finish(struct ostream **output, + size_t max_buffer_size) +{ + struct temp_ostream *tstream = + (struct temp_ostream *)(*output)->real_stream; + struct istream *input; + + if (tstream->fd != -1) { + input = i_stream_create_fd(tstream->fd, max_buffer_size, TRUE); + tstream->fd = -1; + } else { + input = i_stream_create_from_data(tstream->buf->data, + tstream->buf->used); + i_stream_set_destroy_callback(input, iostream_temp_buf_destroyed, + tstream->buf); + tstream->buf = NULL; + } + o_stream_destroy(output); + return input; +} diff -r 22cfb7b347a8 -r faa3a83282fb src/lib/iostream-temp.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/iostream-temp.h Mon Nov 26 23:08:10 2012 +0200 @@ -0,0 +1,12 @@ +#ifndef IOSTREAM_TEMP_H +#define IOSTREAM_TEMP_H + +/* Start writing to given output stream. The data is initially written to + memory, and later to a temporary file that is immediately unlinked. */ +struct ostream *iostream_temp_create(const char *temp_path_prefix); +/* Finished writing to stream. Return input stream for it and free the + output stream. */ +struct istream *iostream_temp_finish(struct ostream **output, + size_t max_buffer_size); + +#endif From dovecot at dovecot.org Mon Nov 26 23:08:16 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:08:16 +0200 Subject: dovecot-2.2: liblib: Added assert-check to epoll code to make su... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22cfb7b347a8 changeset: 15401:22cfb7b347a8 user: Timo Sirainen date: Mon Nov 26 23:06:24 2012 +0200 description: liblib: Added assert-check to epoll code to make sure we don't wait infinitely with no events. diffstat: src/lib/ioloop-epoll.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 5552ac605201 -r 22cfb7b347a8 src/lib/ioloop-epoll.c --- a/src/lib/ioloop-epoll.c Mon Nov 26 23:03:09 2012 +0200 +++ b/src/lib/ioloop-epoll.c Mon Nov 26 23:06:24 2012 +0200 @@ -179,8 +179,8 @@ } else { /* no I/Os, but we should have some timeouts. just wait for them. */ - if (msecs > 0) - usleep(msecs*1000); + i_assert(msecs >= 0); + usleep(msecs*1000); ret = 0; } From dovecot at dovecot.org Mon Nov 26 23:08:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:08:55 +0200 Subject: dovecot-2.2: unlink_directory(): Added UNLINK_DIRECTORY_FLAG_FIL... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a39024316d8e changeset: 15403:a39024316d8e user: Timo Sirainen date: Mon Nov 26 23:08:50 2012 +0200 description: unlink_directory(): Added UNLINK_DIRECTORY_FLAG_FILES_ONLY diffstat: src/lib/unlink-directory.c | 6 +++++- src/lib/unlink-directory.h | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diffs (37 lines): diff -r faa3a83282fb -r a39024316d8e src/lib/unlink-directory.c --- a/src/lib/unlink-directory.c Mon Nov 26 23:08:10 2012 +0200 +++ b/src/lib/unlink-directory.c Mon Nov 26 23:08:50 2012 +0200 @@ -115,7 +115,8 @@ if (errno != ENOENT) break; errno = 0; - } else if (S_ISDIR(st.st_mode)) { + } else if (S_ISDIR(st.st_mode) && + (flags & UNLINK_DIRECTORY_FLAG_FILES_ONLY) == 0) { if (unlink_directory_r(d->d_name, flags) < 0) { if (errno != ENOENT) break; @@ -134,6 +135,9 @@ } errno = 0; } + } else if (S_ISDIR(st.st_mode) && + (flags & UNLINK_DIRECTORY_FLAG_FILES_ONLY) != 0) { + /* skip directory */ } else if (old_errno == EBUSY && strncmp(d->d_name, ".nfs", 4) == 0) { /* can't delete NFS files that are still diff -r faa3a83282fb -r a39024316d8e src/lib/unlink-directory.h --- a/src/lib/unlink-directory.h Mon Nov 26 23:08:10 2012 +0200 +++ b/src/lib/unlink-directory.h Mon Nov 26 23:08:50 2012 +0200 @@ -5,7 +5,9 @@ /* After unlinking all files, rmdir() the directory itself */ UNLINK_DIRECTORY_FLAG_RMDIR = 0x01, /* Don't unlink any files beginning with "." */ - UNLINK_DIRECTORY_FLAG_SKIP_DOTFILES = 0x02 + UNLINK_DIRECTORY_FLAG_SKIP_DOTFILES = 0x02, + /* Don't recurse into subdirectories */ + UNLINK_DIRECTORY_FLAG_FILES_ONLY = 0x04 }; /* Unlink directory and/or everything under it. From dovecot at dovecot.org Mon Nov 26 23:16:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:16:11 +0200 Subject: dovecot-2.2: lib-dns: Added DNS_CLIENT_SOCKET_NAME macro for the... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6d2cfcaad33c changeset: 15404:6d2cfcaad33c user: Timo Sirainen date: Mon Nov 26 23:09:25 2012 +0200 description: lib-dns: Added DNS_CLIENT_SOCKET_NAME macro for the default socket name. diffstat: src/lib-dns/dns-lookup.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r a39024316d8e -r 6d2cfcaad33c src/lib-dns/dns-lookup.h --- a/src/lib-dns/dns-lookup.h Mon Nov 26 23:08:50 2012 +0200 +++ b/src/lib-dns/dns-lookup.h Mon Nov 26 23:09:25 2012 +0200 @@ -1,6 +1,8 @@ #ifndef DNS_LOOKUP_H #define DNS_LOOKUP_H +#define DNS_CLIENT_SOCKET_NAME "dns-client" + struct dns_lookup; struct dns_lookup_settings { From dovecot at dovecot.org Mon Nov 26 23:16:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:16:11 +0200 Subject: dovecot-2.2: lib-http: Removed explicit safe-memset.lo links Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a3653f58d4e6 changeset: 15405:a3653f58d4e6 user: Timo Sirainen date: Mon Nov 26 23:11:57 2012 +0200 description: lib-http: Removed explicit safe-memset.lo links At least in all my systems they just complain about duplicates.. diffstat: src/lib-http/Makefile.am | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) diffs (19 lines): diff -r 6d2cfcaad33c -r a3653f58d4e6 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Mon Nov 26 23:09:25 2012 +0200 +++ b/src/lib-http/Makefile.am Mon Nov 26 23:11:57 2012 +0200 @@ -45,14 +45,12 @@ test_libs = \ ../lib-test/libtest.la \ ../lib/liblib.la \ - ../lib/safe-memset.lo \ $(MODULE_LIBS) test_deps = \ $(noinst_LTLIBRARIES) ../lib-test/libtest.la \ - ../lib/liblib.la \ - ../lib/safe-memset.lo + ../lib/liblib.la test_http_url_SOURCES = test-http-url.c test_http_url_LDADD = http-url.lo $(test_libs) From dovecot at dovecot.org Mon Nov 26 23:16:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:16:11 +0200 Subject: dovecot-2.2: Added libhttp to libdovecot.so Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d1bff2c44fe4 changeset: 15406:d1bff2c44fe4 user: Timo Sirainen date: Mon Nov 26 23:12:29 2012 +0200 description: Added libhttp to libdovecot.so diffstat: configure.ac | 2 +- src/lib-dovecot/Makefile.am | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diffs (33 lines): diff -r a3653f58d4e6 -r d1bff2c44fe4 configure.ac --- a/configure.ac Mon Nov 26 23:11:57 2012 +0200 +++ b/configure.ac Mon Nov 26 23:12:29 2012 +0200 @@ -2515,7 +2515,7 @@ LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la' else - LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' + LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV)" LIBDOVECOT_STORAGE_LAST='$(top_builddir)/src/lib-storage/list/libstorage_list.la $(top_builddir)/src/lib-storage/index/libstorage_index.la $(top_builddir)/src/lib-storage/libstorage.la $(top_builddir)/src/lib-index/libindex.la $(top_builddir)/src/lib-imap-storage/libimap-storage.la' LIBDOVECOT_STORAGE_FIRST='$(top_builddir)/src/lib-storage/libstorage_service.la $(top_builddir)/src/lib-storage/register/libstorage_register.la' diff -r a3653f58d4e6 -r d1bff2c44fe4 src/lib-dovecot/Makefile.am --- a/src/lib-dovecot/Makefile.am Mon Nov 26 23:11:57 2012 +0200 +++ b/src/lib-dovecot/Makefile.am Mon Nov 26 23:12:29 2012 +0200 @@ -1,13 +1,15 @@ +# when adding libraries, update LIBDOVECOT also in configure.in libs = \ + ../lib-master/libmaster.la \ + ../lib-fs/libfs.la \ ../lib-settings/libsettings.la \ + ../lib-http/libhttp.la \ ../lib-dict/libdict.la \ ../lib-imap/libimap.la \ ../lib-mail/libmail.la \ ../lib-auth/libauth.la \ ../lib-dns/libdns.la \ - ../lib-fs/libfs.la \ ../lib-charset/libcharset.la \ - ../lib-master/libmaster.la \ ../lib-ssl-iostream/libssl_iostream.la \ ../lib-test/libtest.la \ ../lib/liblib.la From dovecot at dovecot.org Mon Nov 26 23:16:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:16:12 +0200 Subject: dovecot-2.2: lib-storage: Code cleanup: dbox_save_context.cur_fi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7fb33baa8201 changeset: 15407:7fb33baa8201 user: Timo Sirainen date: Mon Nov 26 23:15:01 2012 +0200 description: lib-storage: Code cleanup: dbox_save_context.cur_file doesn't need to be in dbox-common. diffstat: src/lib-storage/index/dbox-common/dbox-save.h | 1 - src/lib-storage/index/dbox-multi/mdbox-save.c | 5 +++-- src/lib-storage/index/dbox-single/sdbox-save.c | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diffs (74 lines): diff -r d1bff2c44fe4 -r 7fb33baa8201 src/lib-storage/index/dbox-common/dbox-save.h --- a/src/lib-storage/index/dbox-common/dbox-save.h Mon Nov 26 23:12:29 2012 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-save.h Mon Nov 26 23:15:01 2012 +0200 @@ -12,7 +12,6 @@ struct istream *input; struct mail *mail; - struct dbox_file *cur_file; struct ostream *dbox_output; unsigned int failed:1; diff -r d1bff2c44fe4 -r 7fb33baa8201 src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Mon Nov 26 23:12:29 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Mon Nov 26 23:15:01 2012 +0200 @@ -32,6 +32,7 @@ struct mdbox_mailbox *mbox; struct mdbox_sync_context *sync_ctx; + struct dbox_file *cur_file; struct dbox_file_append_context *cur_file_append; struct mdbox_map_append_context *append_ctx; @@ -105,9 +106,9 @@ if (ctx != NULL) { /* use the existing allocated structure */ + ctx->cur_file = NULL; ctx->ctx.failed = FALSE; ctx->ctx.finished = FALSE; - ctx->ctx.cur_file = NULL; ctx->ctx.dbox_output = NULL; ctx->cur_file_append = NULL; return &ctx->ctx.ctx; @@ -148,7 +149,7 @@ i_assert(ctx->ctx.dbox_output->offset <= (uint32_t)-1); append_offset = ctx->ctx.dbox_output->offset; - ctx->ctx.cur_file = ctx->cur_file_append->file; + ctx->cur_file = ctx->cur_file_append->file; dbox_save_begin(&ctx->ctx, input); save_mail = array_append_space(&ctx->mails); diff -r d1bff2c44fe4 -r 7fb33baa8201 src/lib-storage/index/dbox-single/sdbox-save.c --- a/src/lib-storage/index/dbox-single/sdbox-save.c Mon Nov 26 23:12:29 2012 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Mon Nov 26 23:15:01 2012 +0200 @@ -25,6 +25,8 @@ struct sdbox_mailbox *mbox; struct sdbox_sync_context *sync_ctx; + + struct dbox_file *cur_file; struct dbox_file_append_context *append_ctx; uint32_t first_saved_seq; @@ -59,9 +61,9 @@ if (ctx != NULL) { /* use the existing allocated structure */ + ctx->cur_file = NULL; ctx->ctx.failed = FALSE; ctx->ctx.finished = FALSE; - ctx->ctx.cur_file = NULL; ctx->ctx.dbox_output = NULL; return &ctx->ctx.ctx; } @@ -110,7 +112,7 @@ ctx->ctx.failed = TRUE; return -1; } - ctx->ctx.cur_file = file; + ctx->cur_file = file; dbox_save_begin(&ctx->ctx, input); sdbox_save_add_file(_ctx, file); From dovecot at dovecot.org Mon Nov 26 23:16:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:16:12 +0200 Subject: dovecot-2.2: sdbox: Verify that mail_attachment_fs backend suppo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/80a8bed29ad9 changeset: 15408:80a8bed29ad9 user: Timo Sirainen date: Mon Nov 26 23:16:00 2012 +0200 description: sdbox: Verify that mail_attachment_fs backend supports RENAME property. This isn't needed by mdbox. diffstat: src/lib-storage/index/dbox-single/sdbox-storage.c | 26 +++++++++++++++++++++- 1 files changed, 24 insertions(+), 2 deletions(-) diffs (57 lines): diff -r 7fb33baa8201 -r 80a8bed29ad9 src/lib-storage/index/dbox-single/sdbox-storage.c --- a/src/lib-storage/index/dbox-single/sdbox-storage.c Mon Nov 26 23:15:01 2012 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Mon Nov 26 23:16:00 2012 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "fs-api.h" #include "master-service.h" #include "mail-index-modseq.h" #include "mail-search-build.h" @@ -28,6 +29,27 @@ return &storage->storage.storage; } +static int sdbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, + const char **error_r) +{ + struct dbox_storage *storage = (struct dbox_storage *)_storage; + enum fs_properties props; + + if (dbox_storage_create(_storage, ns, error_r) < 0) + return -1; + + if (storage->attachment_fs != NULL) { + props = fs_get_properties(storage->attachment_fs); + if ((props & FS_PROPERTY_RENAME) == 0) { + *error_r = "mail_attachment_fs: " + "Backend doesn't support renaming"; + return -1; + } + } + return 0; +} + static const char * sdbox_storage_find_root_dir(const struct mail_namespace *ns) { @@ -372,7 +394,7 @@ .v = { NULL, sdbox_storage_alloc, - dbox_storage_create, + sdbox_storage_create, dbox_storage_destroy, NULL, dbox_storage_get_list_settings, @@ -389,7 +411,7 @@ .v = { NULL, sdbox_storage_alloc, - dbox_storage_create, + sdbox_storage_create, dbox_storage_destroy, NULL, dbox_storage_get_list_settings, From dovecot at dovecot.org Mon Nov 26 23:21:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:21:05 +0200 Subject: dovecot-2.2: lib-storage: If MAIL_STORAGE_CLASS_FLAG_NO_ROOT is ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0bda8a2d178 changeset: 15410:b0bda8a2d178 user: Timo Sirainen date: Mon Nov 26 23:18:30 2012 +0200 description: lib-storage: If MAIL_STORAGE_CLASS_FLAG_NO_ROOT is set, don't even try to mkdir it. diffstat: src/lib-storage/mail-storage.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r c86b8807fa2f -r b0bda8a2d178 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Mon Nov 26 23:16:41 2012 +0200 +++ b/src/lib-storage/mail-storage.c Mon Nov 26 23:18:30 2012 +0200 @@ -355,8 +355,10 @@ list_set.layout, *error_r); return -1; } - if (mail_storage_create_root(ns->list, flags, error_r) < 0) - return -1; + if ((storage_class->class_flags & MAIL_STORAGE_CLASS_FLAG_NO_ROOT) == 0) { + if (mail_storage_create_root(ns->list, flags, error_r) < 0) + return -1; + } } storage = mail_storage_find(ns->user, storage_class, &list_set); From dovecot at dovecot.org Mon Nov 26 23:21:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:21:05 +0200 Subject: dovecot-2.2: lib-storage: Don't crash when trying to sync privat... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c86b8807fa2f changeset: 15409:c86b8807fa2f user: Timo Sirainen date: Mon Nov 26 23:16:41 2012 +0200 description: lib-storage: Don't crash when trying to sync private index, if syncing had already failed. diffstat: src/lib-storage/index/index-sync.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 80a8bed29ad9 -r c86b8807fa2f src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Mon Nov 26 23:16:00 2012 +0200 +++ b/src/lib-storage/index/index-sync.c Mon Nov 26 23:16:41 2012 +0200 @@ -387,7 +387,8 @@ /* sync private index if needed. do this after real sync to make sure that all the new messages are added to the private index, so their flags can be updated. */ - (void)index_storage_mailbox_sync_pvt(_ctx->box); + if (ret == 0) + (void)index_storage_mailbox_sync_pvt(_ctx->box); /* update search results after private index is updated */ index_sync_search_results_update(ctx); From dovecot at dovecot.org Mon Nov 26 23:21:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:21:06 +0200 Subject: dovecot-2.2: lib-storage: Comment update for mail_prefetch() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f64a91637f9b changeset: 15411:f64a91637f9b user: Timo Sirainen date: Mon Nov 26 23:19:04 2012 +0200 description: lib-storage: Comment update for mail_prefetch() diffstat: src/lib-storage/mail-storage-private.h | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r b0bda8a2d178 -r f64a91637f9b src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Mon Nov 26 23:18:30 2012 +0200 +++ b/src/lib-storage/mail-storage-private.h Mon Nov 26 23:19:04 2012 +0200 @@ -575,7 +575,9 @@ void mail_storage_copy_error(struct mail_storage *dest, struct mail_storage *src); -/* Returns TRUE if everything should already be in memory after this call. */ +/* Returns TRUE if everything should already be in memory after this call + or if prefetching is not supported, i.e. the caller shouldn't do more + prefetching before this message is handled. */ bool mail_prefetch(struct mail *mail); void mail_set_aborted(struct mail *mail); void mail_set_expunged(struct mail *mail); From dovecot at dovecot.org Mon Nov 26 23:21:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:21:06 +0200 Subject: dovecot-2.2: lib-storage: Fixes to LAYOUT=index backend. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a725b63608fb changeset: 15412:a725b63608fb user: Timo Sirainen date: Mon Nov 26 23:20:59 2012 +0200 description: lib-storage: Fixes to LAYOUT=index backend. diffstat: src/lib-storage/list/mailbox-list-index-backend.c | 12 +++++++- src/lib-storage/list/mailbox-list-index-iter.c | 2 +- src/lib-storage/list/mailbox-list-index.c | 32 +++++++++++++++------- src/lib-storage/list/mailbox-list-index.h | 1 - src/lib-storage/list/mailbox-list-none.c | 1 - src/lib-storage/mailbox-list-private.h | 2 + 6 files changed, 35 insertions(+), 15 deletions(-) diffs (175 lines): diff -r f64a91637f9b -r a725b63608fb src/lib-storage/list/mailbox-list-index-backend.c --- a/src/lib-storage/list/mailbox-list-index-backend.c Mon Nov 26 23:19:04 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index-backend.c Mon Nov 26 23:20:59 2012 +0200 @@ -36,10 +36,18 @@ static int index_list_init(struct mailbox_list *_list, const char **error_r) { + const char *dir; + if (!_list->mail_set->mailbox_list_index) { *error_r = "LAYOUT=index requires mailbox_list_index=yes"; return -1; } + if (mailbox_list_get_root_path(_list, MAILBOX_LIST_PATH_TYPE_INDEX, &dir) && + mailbox_list_mkdir_root(_list, dir, MAILBOX_LIST_PATH_TYPE_INDEX) < 0) { + *error_r = t_strdup_printf("Failed to create the index root directory: %s", + mailbox_list_get_last_error(_list, NULL)); + return -1; + } return 0; } @@ -542,7 +550,7 @@ struct mailbox_list_iterate_context *ctx; pool_t pool; - pool = pool_alloconly_create("mailbox list index iter", 1024); + pool = pool_alloconly_create("mailbox list index backend iter", 1024); ctx = p_new(pool, struct mailbox_list_iterate_context, 1); ctx->pool = pool; ctx->list = list; @@ -565,7 +573,7 @@ struct mailbox_list index_mailbox_list = { .name = MAILBOX_LIST_NAME_INDEX, - .props = 0, + .props = MAILBOX_LIST_PROP_NO_ROOT, .mailbox_name_max_length = MAILBOX_LIST_NAME_MAX_LENGTH, { diff -r f64a91637f9b -r a725b63608fb src/lib-storage/list/mailbox-list-index-iter.c --- a/src/lib-storage/list/mailbox-list-index-iter.c Mon Nov 26 23:19:04 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index-iter.c Mon Nov 26 23:20:59 2012 +0200 @@ -40,7 +40,7 @@ pool_t pool; char ns_sep = mail_namespace_get_sep(list->ns); - pool = pool_alloconly_create("mailbox list index iter", 1024); + pool = pool_alloconly_create("mailbox list index iter", 2048); ctx = p_new(pool, struct mailbox_list_index_iterate_context, 1); ctx->ctx.pool = pool; ctx->ctx.list = list; diff -r f64a91637f9b -r a725b63608fb src/lib-storage/list/mailbox-list-index.c --- a/src/lib-storage/list/mailbox-list-index.c Mon Nov 26 23:19:04 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index.c Mon Nov 26 23:20:59 2012 +0200 @@ -36,7 +36,7 @@ ilist->sync_log_file_offset = 0; } -static void mailbox_list_index_index_open(struct mailbox_list *list) +static int mailbox_list_index_index_open(struct mailbox_list *list) { struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(list); const struct mail_storage_settings *set = list->mail_set; @@ -44,10 +44,14 @@ unsigned int lock_timeout; if (ilist->opened) - return; - ilist->opened = TRUE; + return 0; index_flags = mail_storage_settings_to_index_flags(set); + if (strcmp(list->name, MAILBOX_LIST_NAME_INDEX) == 0) { + /* LAYOUT=index. this is the only location for the mailbox + data, so we must never move it into memory. */ + index_flags |= MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY; + } lock_timeout = set->mail_max_lock_timeout == 0 ? -1U : set->mail_max_lock_timeout; @@ -56,12 +60,17 @@ if (mail_index_open_or_create(ilist->index, index_flags) < 0) { if (mail_index_move_to_memory(ilist->index) < 0) { /* try opening once more. it should be created - directly into memory now. */ + directly into memory now, except if it fails with + LAYOUT=index backend. */ if (mail_index_open_or_create(ilist->index, - index_flags) < 0) - i_panic("in-memory index creation failed"); + index_flags) < 0) { + mailbox_list_set_internal_error(list); + return -1; + } } } + ilist->opened = TRUE; + return 0; } struct mailbox_list_index_node * @@ -344,7 +353,8 @@ return 0; } - mailbox_list_index_index_open(list); + if (mailbox_list_index_index_open(list) < 0) + return -1; if (mail_index_refresh(ilist->index) < 0) { mailbox_list_index_set_index_error(list); return -1; @@ -380,7 +390,7 @@ if (!ilist->has_backing_store) return; - mailbox_list_index_index_open(list); + (void)mailbox_list_index_index_open(list); view = mail_index_view_open(ilist->index); if (!mailbox_list_index_need_refresh(ilist, view)) { @@ -481,7 +491,8 @@ /* update the "subscriptions changed" counter/timestamp. its purpose is to trigger NOTIFY watcher to handle SubscriptionChange events */ - mailbox_list_index_index_open(_list); + if (mailbox_list_index_index_open(_list) < 0) + return -1; view = mail_index_view_open(ilist->index); mail_index_get_header_ext(view, ilist->subs_hdr_ext_id, &data, &size); if (size != sizeof(counter)) @@ -510,7 +521,8 @@ /* layout=index doesn't have any backing store */ has_backing_store = strcmp(list->name, MAILBOX_LIST_NAME_INDEX) != 0; - if (!list->mail_set->mailbox_list_index) { + if (!list->mail_set->mailbox_list_index || + strcmp(list->name, MAILBOX_LIST_NAME_NONE) == 0) { /* reserve the module context anyway, so syncing code knows that the index is disabled */ i_assert(has_backing_store); diff -r f64a91637f9b -r a725b63608fb src/lib-storage/list/mailbox-list-index.h --- a/src/lib-storage/list/mailbox-list-index.h Mon Nov 26 23:19:04 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index.h Mon Nov 26 23:20:59 2012 +0200 @@ -27,7 +27,6 @@ #include "mail-storage.h" #include "mailbox-list-private.h" -#define MAILBOX_LIST_NAME_INDEX "index" #define MAILBOX_LIST_INDEX_HIERARHCY_SEP '~' #define MAILBOX_LIST_INDEX_PREFIX "dovecot.list.index" diff -r f64a91637f9b -r a725b63608fb src/lib-storage/list/mailbox-list-none.c --- a/src/lib-storage/list/mailbox-list-none.c Mon Nov 26 23:19:04 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-none.c Mon Nov 26 23:20:59 2012 +0200 @@ -5,7 +5,6 @@ #include "imap-match.h" #include "mailbox-list-private.h" -#define MAILBOX_LIST_NAME_NONE "none" #define GLOBAL_TEMP_PREFIX ".temp." struct noop_list_iterate_context { diff -r f64a91637f9b -r a725b63608fb src/lib-storage/mailbox-list-private.h --- a/src/lib-storage/mailbox-list-private.h Mon Nov 26 23:19:04 2012 +0200 +++ b/src/lib-storage/mailbox-list-private.h Mon Nov 26 23:20:59 2012 +0200 @@ -11,6 +11,8 @@ #define MAILBOX_LIST_NAME_MAILDIRPLUSPLUS "maildir++" #define MAILBOX_LIST_NAME_IMAPDIR "imapdir" #define MAILBOX_LIST_NAME_FS "fs" +#define MAILBOX_LIST_NAME_INDEX "index" +#define MAILBOX_LIST_NAME_NONE "none" #define MAILBOX_LOG_FILE_NAME "dovecot.mailbox.log" From dovecot at dovecot.org Mon Nov 26 23:25:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:25:55 +0200 Subject: dovecot-2.2: lib-imap-storage: When returning an error istream, ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ecf2f2044455 changeset: 15413:ecf2f2044455 user: Timo Sirainen date: Mon Nov 26 23:23:52 2012 +0200 description: lib-imap-storage: When returning an error istream, give it a useful name. diffstat: src/lib-imap-storage/imap-msgpart.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (26 lines): diff -r a725b63608fb -r ecf2f2044455 src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Mon Nov 26 23:20:59 2012 +0200 +++ b/src/lib-imap-storage/imap-msgpart.c Mon Nov 26 23:23:52 2012 +0200 @@ -412,7 +412,7 @@ const struct imap_msgpart *msgpart) { struct mail_msgpart_partial_cache *cache = &mail->box->partial_cache; - struct istream *crlf_input; + struct istream *crlf_input, *errinput; uoff_t physical_start = input->v_offset; uoff_t virtual_skip = msgpart->partial_offset; bool cr_skipped; @@ -432,8 +432,11 @@ message parts. */ skip_using_parts(mail, input, physical_start, &virtual_skip); } - if (message_skip_virtual(input, virtual_skip, &cr_skipped) < 0) - return i_stream_create_error(errno); + if (message_skip_virtual(input, virtual_skip, &cr_skipped) < 0) { + errinput = i_stream_create_error(errno); + i_stream_set_name(errinput, i_stream_get_name(input)); + return errinput; + } if ((msgpart->partial_offset != 0 || msgpart->partial_size != (uoff_t)-1) && !input->eof) { From dovecot at dovecot.org Mon Nov 26 23:25:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 26 Nov 2012 23:25:55 +0200 Subject: dovecot-2.2: doveadm dump: Added support for "obox" index records. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9c22246ed431 changeset: 15414:9c22246ed431 user: Timo Sirainen date: Mon Nov 26 23:24:48 2012 +0200 description: doveadm dump: Added support for "obox" index records. diffstat: src/doveadm/doveadm-dump-index.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (25 lines): diff -r ecf2f2044455 -r 9c22246ed431 src/doveadm/doveadm-dump-index.c --- a/src/doveadm/doveadm-dump-index.c Mon Nov 26 23:23:52 2012 +0200 +++ b/src/doveadm/doveadm-dump-index.c Mon Nov 26 23:24:48 2012 +0200 @@ -45,6 +45,10 @@ uint32_t map_uid; uint32_t save_date; }; +struct obox_mail_index_record { + unsigned char guid[GUID_128_SIZE]; + unsigned char oid[GUID_128_SIZE]; +}; struct mailbox_list_index_record { uint32_t name_id; uint32_t parent_uid; @@ -557,6 +561,10 @@ const struct mdbox_mail_index_record *drec = data; printf(" : map_uid = %u\n", drec->map_uid); printf(" : save_date = %u (%s)\n", drec->save_date, unixdate2str(drec->save_date)); + } else if (strcmp(ext[i].name, "obox") == 0) { + const struct obox_mail_index_record *orec = data; + printf(" : guid = %s\n", guid_128_to_string(orec->guid)); + printf(" : oid = %s\n", guid_128_to_string(orec->oid)); } else if (strcmp(ext[i].name, "list") == 0) { const struct mailbox_list_index_record *lrec = data; printf(" : name_id = %u\n", lrec->name_id); From dovecot at dovecot.org Tue Nov 27 00:44:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 00:44:46 +0200 Subject: dovecot-2.2: liblib: Added module_dir_find() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8f0994194f45 changeset: 15415:8f0994194f45 user: Timo Sirainen date: Tue Nov 27 00:31:40 2012 +0200 description: liblib: Added module_dir_find() diffstat: src/lib/module-dir.c | 11 ++++++++--- src/lib/module-dir.h | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diffs (42 lines): diff -r 9c22246ed431 -r 8f0994194f45 src/lib/module-dir.c --- a/src/lib/module-dir.c Mon Nov 26 23:24:48 2012 +0200 +++ b/src/lib/module-dir.c Tue Nov 27 00:31:40 2012 +0200 @@ -315,7 +315,7 @@ } } -static bool module_is_loaded(struct module *modules, const char *name) +struct module *module_dir_find(struct module *modules, const char *name) { struct module *module; unsigned int len = strlen(name); @@ -324,10 +324,15 @@ if (strncmp(module->name, name, len) == 0) { if (module->name[len] == '\0' || strcmp(module->name + len, "_plugin") == 0) - return TRUE; + return module; } } - return FALSE; + return NULL; +} + +static bool module_is_loaded(struct module *modules, const char *name) +{ + return module_dir_find(modules, name) != NULL; } static void module_names_fix(const char **module_names) diff -r 9c22246ed431 -r 8f0994194f45 src/lib/module-dir.h --- a/src/lib/module-dir.h Mon Nov 26 23:24:48 2012 +0200 +++ b/src/lib/module-dir.h Tue Nov 27 00:31:40 2012 +0200 @@ -57,6 +57,8 @@ void module_dir_deinit(struct module *modules); /* Unload all modules */ void module_dir_unload(struct module **modules); +/* Find a module by name. */ +struct module *module_dir_find(struct module *modules, const char *name); void *module_get_symbol(struct module *module, const char *symbol); void *module_get_symbol_quiet(struct module *module, const char *symbol); From dovecot at dovecot.org Tue Nov 27 00:44:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 00:44:46 +0200 Subject: dovecot-2.2: liblib: Added lib_atexit() to register process dein... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2aef2557bdf1 changeset: 15416:2aef2557bdf1 user: Timo Sirainen date: Tue Nov 27 00:43:23 2012 +0200 description: liblib: Added lib_atexit() to register process deinit callbacks. diffstat: src/lib/lib.c | 53 +++++++++++++++++++++++++++++++++++++++++------------ src/lib/lib.h | 8 ++++++++ 2 files changed, 49 insertions(+), 12 deletions(-) diffs (111 lines): diff -r 8f0994194f45 -r 2aef2557bdf1 src/lib/lib.c --- a/src/lib/lib.c Tue Nov 27 00:31:40 2012 +0200 +++ b/src/lib/lib.c Tue Nov 27 00:43:23 2012 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2001-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "env-util.h" #include "hostpid.h" #include "ipwd.h" @@ -10,6 +11,8 @@ #include #include +static ARRAY(lib_atexit_callback_t *) atexit_callbacks = ARRAY_INIT; + size_t nearest_power(size_t num) { size_t n = 1; @@ -20,6 +23,36 @@ return n; } +int close_keep_errno(int *fd) +{ + int ret, old_errno = errno; + + i_assert(*fd != -1); + + ret = close(*fd); + *fd = -1; + errno = old_errno; + return ret; +} + +void lib_atexit(lib_atexit_callback_t *callback) +{ + lib_atexit_callback_t *const *callbacks; + unsigned int i, count; + + if (!array_is_created(&atexit_callbacks)) + i_array_init(&atexit_callbacks, 8); + else { + /* skip if it's already added */ + callbacks = array_get(&atexit_callbacks, &count); + for (i = count; i > 0; i--) { + if (callbacks[i-1] == callback) + return; + } + } + array_append(&atexit_callbacks, &callback, 1); +} + void lib_init(void) { struct timeval tv; @@ -33,20 +66,16 @@ hostpid_init(); } -int close_keep_errno(int *fd) -{ - int ret, old_errno = errno; - - i_assert(*fd != -1); - - ret = close(*fd); - *fd = -1; - errno = old_errno; - return ret; -} - void lib_deinit(void) { + lib_atexit_callback_t *const *cbp; + + if (array_is_created(&atexit_callbacks)) { + array_foreach(&atexit_callbacks, cbp) + (**cbp)(); + array_free(&atexit_callbacks); + } + ipwd_deinit(); data_stack_deinit(); env_deinit(); diff -r 8f0994194f45 -r 2aef2557bdf1 src/lib/lib.h --- a/src/lib/lib.h Tue Nov 27 00:31:40 2012 +0200 +++ b/src/lib/lib.h Tue Nov 27 00:43:23 2012 +0200 @@ -35,6 +35,8 @@ struct istream; struct ostream; +typedef void lib_atexit_callback_t(void); + #include "array-decl.h" /* ARRAY*()s may exist in any header */ #include "hash-decl.h" /* HASH_TABLE*()s may exist in any header */ #include "strfuncs.h" @@ -43,6 +45,12 @@ size_t nearest_power(size_t num) ATTR_CONST; int close_keep_errno(int *fd); +/* Call the given callback at the beginning of lib_deinit(). The main + difference to atexit() is that liblib's memory allocation and logging + functions are still available. Also if lib_atexit() is called multiple times + to the same callback, it's added only once. */ +void lib_atexit(lib_atexit_callback_t *callback); + void lib_init(void); void lib_deinit(void); From dovecot at dovecot.org Tue Nov 27 00:44:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 00:44:46 +0200 Subject: dovecot-2.2: lib-fs: Automatically try to load missing driver as... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/47c0a5a19211 changeset: 15417:47c0a5a19211 user: Timo Sirainen date: Tue Nov 27 00:43:53 2012 +0200 description: lib-fs: Automatically try to load missing driver as plugin. diffstat: src/lib-fs/Makefile.am | 3 +- src/lib-fs/fs-api-private.h | 6 +- src/lib-fs/fs-api.c | 85 ++++++++++++++++++++++++++++++++++++++------ src/lib-fs/fs-posix.c | 2 +- src/lib-fs/fs-sis-queue.c | 2 +- src/lib-fs/fs-sis.c | 2 +- 6 files changed, 80 insertions(+), 20 deletions(-) diffs (176 lines): diff -r 2aef2557bdf1 -r 47c0a5a19211 src/lib-fs/Makefile.am --- a/src/lib-fs/Makefile.am Tue Nov 27 00:43:23 2012 +0200 +++ b/src/lib-fs/Makefile.am Tue Nov 27 00:43:53 2012 +0200 @@ -1,7 +1,8 @@ noinst_LTLIBRARIES = libfs.la AM_CPPFLAGS = \ - -I$(top_srcdir)/src/lib + -I$(top_srcdir)/src/lib \ + -DMODULE_DIR=\""$(moduledir)"\" libfs_la_SOURCES = \ fs-api.c \ diff -r 2aef2557bdf1 -r 47c0a5a19211 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Tue Nov 27 00:43:23 2012 +0200 +++ b/src/lib-fs/fs-api-private.h Tue Nov 27 00:43:53 2012 +0200 @@ -74,9 +74,9 @@ struct fs *fs; }; -extern struct fs fs_class_posix; -extern struct fs fs_class_sis; -extern struct fs fs_class_sis_queue; +extern const struct fs fs_class_posix; +extern const struct fs fs_class_sis; +extern const struct fs fs_class_sis_queue; void fs_set_error(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3); void fs_set_critical(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3); diff -r 2aef2557bdf1 -r 47c0a5a19211 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Tue Nov 27 00:43:23 2012 +0200 +++ b/src/lib-fs/fs-api.c Tue Nov 27 00:43:53 2012 +0200 @@ -1,14 +1,13 @@ /* Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" +#include "module-dir.h" #include "str.h" #include "fs-api-private.h" -static struct fs *fs_classes[] = { - &fs_class_posix, - &fs_class_sis, - &fs_class_sis_queue -}; +static struct module *fs_modules = NULL; +static ARRAY(const struct fs *) fs_classes; static int fs_alloc(const struct fs *fs_class, const char *args, @@ -36,20 +35,80 @@ return 0; } +static void fs_class_register(const struct fs *fs_class) +{ + array_append(&fs_classes, &fs_class, 1); +} + +static void fs_classes_init(void) +{ + i_array_init(&fs_classes, 8); + fs_class_register(&fs_class_posix); + fs_class_register(&fs_class_sis); + fs_class_register(&fs_class_sis_queue); +} + +static const struct fs *fs_class_find(const char *driver) +{ + const struct fs *const *classp; + + if (!array_is_created(&fs_classes)) + fs_classes_init(); + + array_foreach(&fs_classes, classp) { + if (strcmp((*classp)->name, driver) == 0) + return *classp; + } + return NULL; +} + +static void fs_class_deinit_modules(void) +{ + module_dir_unload(&fs_modules); +} + +static void fs_class_try_load_plugin(const char *driver) +{ + const char *module_name = t_strdup_printf("fs_%s", driver); + struct module *module; + struct module_dir_load_settings mod_set; + const struct fs *fs_class; + + memset(&mod_set, 0, sizeof(mod_set)); + mod_set.abi_version = DOVECOT_ABI_VERSION; + mod_set.ignore_missing = TRUE; + + fs_modules = module_dir_load_missing(fs_modules, MODULE_DIR, + module_name, &mod_set); + module_dir_init(fs_modules); + + module = module_dir_find(fs_modules, module_name); + fs_class = module_get_symbol(module, + t_strdup_printf("fs_class_%s", driver)); + if (fs_class != NULL) + fs_class_register(fs_class); + + lib_atexit(fs_class_deinit_modules); +} + int fs_init(const char *driver, const char *args, const struct fs_settings *set, struct fs **fs_r, const char **error_r) { - unsigned int i; + const struct fs *fs_class; - for (i = 0; i < N_ELEMENTS(fs_classes); i++) { - if (strcmp(fs_classes[i]->name, driver) == 0) { - return fs_alloc(fs_classes[i], args, - set, fs_r, error_r); - } + fs_class = fs_class_find(driver); + if (fs_class == NULL) { + T_BEGIN { + fs_class_try_load_plugin(driver); + } T_END; + fs_class = fs_class_find(driver); } - *error_r = t_strdup_printf("Unknown fs driver: %s", driver); - return -1; + if (fs_class == NULL) { + *error_r = t_strdup_printf("Unknown fs driver: %s", driver); + return -1; + } + return fs_alloc(fs_class, args, set, fs_r, error_r); } void fs_deinit(struct fs **_fs) diff -r 2aef2557bdf1 -r 47c0a5a19211 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Tue Nov 27 00:43:23 2012 +0200 +++ b/src/lib-fs/fs-posix.c Tue Nov 27 00:43:53 2012 +0200 @@ -724,7 +724,7 @@ return ret; } -struct fs fs_class_posix = { +const struct fs fs_class_posix = { .name = "posix", .v = { fs_posix_alloc, diff -r 2aef2557bdf1 -r 47c0a5a19211 src/lib-fs/fs-sis-queue.c --- a/src/lib-fs/fs-sis-queue.c Tue Nov 27 00:43:23 2012 +0200 +++ b/src/lib-fs/fs-sis-queue.c Tue Nov 27 00:43:53 2012 +0200 @@ -335,7 +335,7 @@ return fs_iter_init(fs->super, path); } -struct fs fs_class_sis_queue = { +const struct fs fs_class_sis_queue = { .name = "sis-queue", .v = { fs_sis_queue_alloc, diff -r 2aef2557bdf1 -r 47c0a5a19211 src/lib-fs/fs-sis.c --- a/src/lib-fs/fs-sis.c Tue Nov 27 00:43:23 2012 +0200 +++ b/src/lib-fs/fs-sis.c Tue Nov 27 00:43:53 2012 +0200 @@ -470,7 +470,7 @@ return fs_iter_init(fs->super, path); } -struct fs fs_class_sis = { +const struct fs fs_class_sis = { .name = "sis", .v = { fs_sis_alloc, From dovecot at dovecot.org Tue Nov 27 00:44:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 00:44:46 +0200 Subject: dovecot-2.2: dovecot-config: LIBDOVECOT_INCLUDE was missing path... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/25e8459431fc changeset: 15418:25e8459431fc user: Timo Sirainen date: Tue Nov 27 00:44:34 2012 +0200 description: dovecot-config: LIBDOVECOT_INCLUDE was missing path to lib-http diffstat: dovecot-config.in.in | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 47c0a5a19211 -r 25e8459431fc dovecot-config.in.in --- a/dovecot-config.in.in Tue Nov 27 00:43:53 2012 +0200 +++ b/dovecot-config.in.in Tue Nov 27 00:44:34 2012 +0200 @@ -18,7 +18,7 @@ LIBDOVECOT_LDA_DEPS="@LIBDOVECOT_LDA@" LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE_DEPS@" -LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test" +LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test" LIBDOVECOT_LDA_INCLUDE="-I$(incdir)/src/lib-lda -I$(incdir)/src/lda" LIBDOVECOT_DOVEADM_INCLUDE="-I$(incdir)/src/doveadm" LIBDOVECOT_STORAGE_INCLUDE="-I$(incdir)/src/lib-index -I$(incdir)/src/lib-storage -I$(incdir)/src/lib-storage/list -I$(incdir)/src/lib-storage/index -I$(incdir)/src/lib-storage/index/raw" From dovecot at dovecot.org Tue Nov 27 02:20:59 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 02:20:59 +0200 Subject: dovecot-2.1: man: Added -d parameter to doveadm expunge. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/1ac96aad5027 changeset: 14804:1ac96aad5027 user: Timo Sirainen date: Tue Nov 27 02:20:37 2012 +0200 description: man: Added -d parameter to doveadm expunge. Patch by Pascal Volk. diffstat: doc/man/doveadm-expunge.1.in | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (22 lines): diff -r e95479f439aa -r 1ac96aad5027 doc/man/doveadm-expunge.1.in --- a/doc/man/doveadm-expunge.1.in Fri Nov 23 08:52:06 2012 +0200 +++ b/doc/man/doveadm-expunge.1.in Tue Nov 27 02:20:37 2012 +0200 @@ -1,5 +1,5 @@ -.\" Copyright (c) 2010 Dovecot authors, see the included COPYING file -.TH DOVEADM\-EXPUNGE 1 "2010-11-25" "Dovecot v2.1" "Dovecot" +.\" Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file +.TH DOVEADM\-EXPUNGE 1 "2012-11-27" "Dovecot v2.1" "Dovecot" .SH NAME doveadm\-expunge \- Expunge messages matching given search query .\"------------------------------------------------------------------------ @@ -47,6 +47,10 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- +.TP +.B \-d +Delete the mailbox if it is empty after expunging. +.\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- @INCLUDE:option-u-user@ From dovecot at dovecot.org Tue Nov 27 02:42:02 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 02:42:02 +0200 Subject: dovecot-2.1: quota: mailbox_get_status(STATUS_CHECK_OVER_QUOTA) ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/bebe54e1d640 changeset: 14805:bebe54e1d640 user: Timo Sirainen date: Tue Nov 27 02:41:53 2012 +0200 description: quota: mailbox_get_status(STATUS_CHECK_OVER_QUOTA) returned wrong value on success. This fixes lmtp_rcpt_check_quota=yes diffstat: src/plugins/quota/quota-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1ac96aad5027 -r bebe54e1d640 src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Tue Nov 27 02:20:37 2012 +0200 +++ b/src/plugins/quota/quota-storage.c Tue Nov 27 02:41:53 2012 +0200 @@ -87,7 +87,7 @@ if ((items & ~STATUS_CHECK_OVER_QUOTA) == 0) { /* don't bother calling parent, it may unnecessarily try to open the mailbox */ - return ret; + return ret < 0 ? -1 : 0; } } From dovecot at dovecot.org Tue Nov 27 03:49:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 03:49:46 +0200 Subject: dovecot-2.1: liblib: Added uni_utf8_short_*() for handling UTF8 ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/172295f5a78b changeset: 14806:172295f5a78b user: Timo Sirainen date: Tue Nov 27 03:48:15 2012 +0200 description: liblib: Added uni_utf8_short_*() for handling UTF8 data where [56]-byte sequences are invalid. diffstat: src/lib/unichar.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/unichar.h | 7 ++++++ 2 files changed, 71 insertions(+), 0 deletions(-) diffs (86 lines): diff -r bebe54e1d640 -r 172295f5a78b src/lib/unichar.c --- a/src/lib/unichar.c Tue Nov 27 02:41:53 2012 +0200 +++ b/src/lib/unichar.c Tue Nov 27 03:48:15 2012 +0200 @@ -420,3 +420,67 @@ return uni_utf8_find_invalid_pos(data, size, &i) == 0; } + +static int +uni_utf8_short_find_invalid_pos(const unsigned char *input, size_t size, + size_t *pos_r) +{ + size_t i, len; + + /* find the first invalid utf8 sequence */ + for (i = 0; i < size;) { + if (input[i] < 0x80) + i++; + else { + len = is_valid_utf8_seq(input + i, size-i); + if (unlikely(len == 0 || len > 4)) { + *pos_r = i; + return -1; + } + i += len; + } + } + return 0; +} + +bool uni_utf8_short_get_valid_data(const unsigned char *input, size_t size, + buffer_t *buf) +{ + size_t i, len; + + if (uni_utf8_short_find_invalid_pos(input, size, &i) == 0) + return TRUE; + + /* broken utf-8 input - skip the broken characters */ + while (i < size) { + if (input[i] < 0x80) { + buffer_append_c(buf, input[i++]); + continue; + } + + len = is_valid_utf8_seq(input + i, size-i); + if (len == 0 || len > 4) { + i += I_MAX(len, 1); + output_add_replacement_char(buf); + continue; + } + buffer_append(buf, input + i, len); + i += len; + } + return FALSE; +} + +bool uni_utf8_short_str_is_valid(const char *str) +{ + size_t i; + + return uni_utf8_short_find_invalid_pos((const unsigned char *)str, + strlen(str), &i) == 0; +} + +bool uni_utf8_short_data_is_valid(const unsigned char *data, size_t size) +{ + size_t i; + + return uni_utf8_find_invalid_pos(data, size, &i) == 0; +} diff -r bebe54e1d640 -r 172295f5a78b src/lib/unichar.h --- a/src/lib/unichar.h Tue Nov 27 02:41:53 2012 +0200 +++ b/src/lib/unichar.h Tue Nov 27 03:48:15 2012 +0200 @@ -88,4 +88,11 @@ /* Returns TRUE if data contains only valid UTF-8 input. */ bool uni_utf8_data_is_valid(const unsigned char *data, size_t size); +/* Same as the non-short variants, but assume 5-byte and 6-byte UTF8 + sequences are illegal. */ +bool uni_utf8_short_get_valid_data(const unsigned char *input, size_t size, + buffer_t *buf); +bool uni_utf8_short_str_is_valid(const char *str); +bool uni_utf8_short_data_is_valid(const unsigned char *data, size_t size); + #endif From dovecot at dovecot.org Tue Nov 27 03:49:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 03:49:46 +0200 Subject: dovecot-2.1: fts: Added FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 to req... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/01550514f189 changeset: 14807:01550514f189 user: Timo Sirainen date: Tue Nov 27 03:49:25 2012 +0200 description: fts: Added FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 to require sending only short UTF8 data to backend. diffstat: src/plugins/fts/fts-api-private.h | 4 +++- src/plugins/fts/fts-build-mail.c | 7 +++++-- src/plugins/fts/fts-parser.c | 24 ++++++++++++++++++------ src/plugins/fts/fts-parser.h | 3 ++- 4 files changed, 28 insertions(+), 10 deletions(-) diffs (116 lines): diff -r 172295f5a78b -r 01550514f189 src/plugins/fts/fts-api-private.h --- a/src/plugins/fts/fts-api-private.h Tue Nov 27 03:48:15 2012 +0200 +++ b/src/plugins/fts/fts-api-private.h Tue Nov 27 03:49:25 2012 +0200 @@ -59,7 +59,9 @@ /* Send only fully indexable words rather than randomly sized blocks */ FTS_BACKEND_FLAG_BUILD_FULL_WORDS = 0x04, /* Fuzzy search works */ - FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08 + FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08, + /* Don't allow 5-byte or 6-byte UTF8 sequences */ + FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 = 0x10 }; struct fts_backend { diff -r 172295f5a78b -r 01550514f189 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Tue Nov 27 03:48:15 2012 +0200 +++ b/src/plugins/fts/fts-build-mail.c Tue Nov 27 03:49:25 2012 +0200 @@ -144,6 +144,7 @@ struct mail_storage *storage; const char *content_type; struct fts_backend_build_key key; + bool require_short_utf8; i_assert(ctx->body_parser == NULL); @@ -158,9 +159,11 @@ return FALSE; } - + require_short_utf8 = (ctx->update_ctx->backend->flags & + FTS_BACKEND_FLAG_BUILD_SHORT_UTF8) != 0; + storage = mailbox_get_storage(ctx->mail->box); - if (fts_parser_init(mail_storage_get_user(storage), + if (fts_parser_init(mail_storage_get_user(storage), require_short_utf8, content_type, ctx->content_disposition, &ctx->body_parser)) { /* extract text using the the returned parser */ diff -r 172295f5a78b -r 01550514f189 src/plugins/fts/fts-parser.c --- a/src/plugins/fts/fts-parser.c Tue Nov 27 03:48:15 2012 +0200 +++ b/src/plugins/fts/fts-parser.c Tue Nov 27 03:49:25 2012 +0200 @@ -11,7 +11,7 @@ &fts_parser_script }; -bool fts_parser_init(struct mail_user *user, +bool fts_parser_init(struct mail_user *user, bool require_short_utf8, const char *content_type, const char *content_disposition, struct fts_parser **parser_r) { @@ -20,8 +20,10 @@ for (i = 0; i < N_ELEMENTS(parsers); i++) { *parser_r = parsers[i]->try_init(user, content_type, content_disposition); - if (*parser_r != NULL) + if (*parser_r != NULL) { + (*parser_r)->require_short_utf8 = require_short_utf8; return TRUE; + } } return FALSE; } @@ -56,11 +58,15 @@ void fts_parser_more(struct fts_parser *parser, struct message_block *block) { + bool valid_utf8; + if (parser->v.more != NULL) parser->v.more(parser, block); - if (!uni_utf8_data_is_valid(block->data, block->size) || - data_has_nuls(block->data, block->size)) { + valid_utf8 = parser->require_short_utf8 ? + uni_utf8_short_data_is_valid(block->data, block->size) : + uni_utf8_data_is_valid(block->data, block->size); + if (!valid_utf8 || data_has_nuls(block->data, block->size)) { /* output isn't valid UTF-8. make it. */ if (parser->utf8_output == NULL) { parser->utf8_output = @@ -68,8 +74,14 @@ } else { buffer_set_used_size(parser->utf8_output, 0); } - (void)uni_utf8_get_valid_data(block->data, block->size, - parser->utf8_output); + if (parser->require_short_utf8) { + (void)uni_utf8_short_get_valid_data(block->data, + block->size, + parser->utf8_output); + } else { + (void)uni_utf8_get_valid_data(block->data, block->size, + parser->utf8_output); + } replace_nul_bytes(parser->utf8_output); block->data = parser->utf8_output->data; block->size = parser->utf8_output->used; diff -r 172295f5a78b -r 01550514f189 src/plugins/fts/fts-parser.h --- a/src/plugins/fts/fts-parser.h Tue Nov 27 03:48:15 2012 +0200 +++ b/src/plugins/fts/fts-parser.h Tue Nov 27 03:49:25 2012 +0200 @@ -15,12 +15,13 @@ struct fts_parser { struct fts_parser_vfuncs v; buffer_t *utf8_output; + bool require_short_utf8; }; extern struct fts_parser_vfuncs fts_parser_html; extern struct fts_parser_vfuncs fts_parser_script; -bool fts_parser_init(struct mail_user *user, +bool fts_parser_init(struct mail_user *user, bool require_short_utf8, const char *content_type, const char *content_disposition, struct fts_parser **parser_r); struct fts_parser *fts_parser_text_init(void); From dovecot at dovecot.org Tue Nov 27 03:49:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 03:49:46 +0200 Subject: dovecot-2.1: fts-solr: Marked as FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/339e654f371e changeset: 14808:339e654f371e user: Timo Sirainen date: Tue Nov 27 03:49:36 2012 +0200 description: fts-solr: Marked as FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 2 +- src/plugins/fts-solr/fts-backend-solr.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diffs (25 lines): diff -r 01550514f189 -r 339e654f371e src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Tue Nov 27 03:49:25 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Tue Nov 27 03:49:36 2012 +0200 @@ -810,7 +810,7 @@ struct fts_backend fts_backend_solr_old = { .name = "solr_old", - .flags = 0, + .flags = FTS_BACKEND_FLAG_BUILD_SHORT_UTF8, { fts_backend_solr_alloc, diff -r 01550514f189 -r 339e654f371e src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 03:49:25 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 03:49:36 2012 +0200 @@ -879,7 +879,8 @@ struct fts_backend fts_backend_solr = { .name = "solr", - .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH, + .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH | + FTS_BACKEND_FLAG_BUILD_SHORT_UTF8, { fts_backend_solr_alloc, From dovecot at dovecot.org Tue Nov 27 06:21:33 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 06:21:33 +0200 Subject: dovecot-2.1: doveadm: Include username in internal errors and "n... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/6f19c535110e changeset: 14809:6f19c535110e user: Timo Sirainen date: Tue Nov 27 06:19:10 2012 +0200 description: doveadm: Include username in internal errors and "no such user" errors when running via server. diffstat: src/doveadm/doveadm-mail-server.c | 27 ++++++++++++++++++++++----- 1 files changed, 22 insertions(+), 5 deletions(-) diffs (73 lines): diff -r 339e654f371e -r 6f19c535110e src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Tue Nov 27 03:49:36 2012 +0200 +++ b/src/doveadm/doveadm-mail-server.c Tue Nov 27 06:19:10 2012 +0200 @@ -22,6 +22,11 @@ #define DOVEADM_MAIL_SERVER_FAILED() \ (internal_failure || master_service_is_killed(master_service)) +struct doveadm_mail_server_cmd { + struct server_connection *conn; + char *username; +}; + static struct hash_table *servers; static pool_t server_pool; static struct doveadm_mail_cmd_context *cmd_ctx; @@ -80,16 +85,22 @@ static void doveadm_cmd_callback(enum server_cmd_reply reply, void *context) { - struct server_connection *conn = context; - struct doveadm_server *server; + struct doveadm_mail_server_cmd *servercmd = context; + struct doveadm_server *server = + server_connection_get_server(servercmd->conn); + const char *username = t_strdup(servercmd->username); + + i_free(servercmd->username); + i_free(servercmd); switch (reply) { case SERVER_CMD_REPLY_INTERNAL_FAILURE: + i_error("%s: Internal failure for %s", server->name, username); internal_failure = TRUE; master_service_stop(master_service); return; case SERVER_CMD_REPLY_UNKNOWN_USER: - i_error("No such user"); + i_error("%s: No such user: %s", server->name, username); if (cmd_ctx->exit_code == 0) cmd_ctx->exit_code = EX_NOUSER; break; @@ -100,8 +111,8 @@ break; } - server = server_connection_get_server(conn); if (array_count(&server->queue) > 0) { + struct server_connection *conn; char *const *usernamep = array_idx(&server->queue, 0); char *username = *usernamep; @@ -119,6 +130,7 @@ static void doveadm_mail_server_handle(struct server_connection *conn, const char *username) { + struct doveadm_mail_server_cmd *servercmd; string_t *cmd; unsigned int i; @@ -138,7 +150,12 @@ str_tabescape_write(cmd, cmd_ctx->full_args[i]); } str_append_c(cmd, '\n'); - server_connection_cmd(conn, str_c(cmd), doveadm_cmd_callback, conn); + + servercmd = i_new(struct doveadm_mail_server_cmd, 1); + servercmd->conn = conn; + servercmd->username = i_strdup(username); + server_connection_cmd(conn, str_c(cmd), + doveadm_cmd_callback, servercmd); } static void doveadm_server_flush_one(struct doveadm_server *server) From dovecot at dovecot.org Tue Nov 27 06:21:33 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 06:21:33 +0200 Subject: dovecot-2.1: doveadm: Don't disconnect from server too early whi... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/275a57b8dc70 changeset: 14810:275a57b8dc70 user: Timo Sirainen date: Tue Nov 27 06:19:57 2012 +0200 description: doveadm: Don't disconnect from server too early while there are still commands. diffstat: src/doveadm/server-connection.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 6f19c535110e -r 275a57b8dc70 src/doveadm/server-connection.c --- a/src/doveadm/server-connection.c Tue Nov 27 06:19:10 2012 +0200 +++ b/src/doveadm/server-connection.c Tue Nov 27 06:19:57 2012 +0200 @@ -269,8 +269,10 @@ server_connection_callback(conn, reply); } else i_error("doveadm server sent broken input"); - /* we're finished, close the connection */ - server_connection_destroy(&conn); + if (conn->callback == NULL) { + /* we're finished, close the connection */ + server_connection_destroy(&conn); + } break; } } From dovecot at dovecot.org Tue Nov 27 06:21:33 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 06:21:33 +0200 Subject: dovecot-2.1: doveadm-server: Avoid eating CPU when client discon... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/0dc3f56e6468 changeset: 14811:0dc3f56e6468 user: Timo Sirainen date: Tue Nov 27 06:20:44 2012 +0200 description: doveadm-server: Avoid eating CPU when client disconnects before authentication is done. diffstat: src/doveadm/client-connection.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (16 lines): diff -r 275a57b8dc70 -r 0dc3f56e6468 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Tue Nov 27 06:19:57 2012 +0200 +++ b/src/doveadm/client-connection.c Tue Nov 27 06:20:44 2012 +0200 @@ -239,8 +239,11 @@ const unsigned char *data; size_t size; - if ((line = i_stream_read_next_line(conn->input)) == NULL) + if ((line = i_stream_read_next_line(conn->input)) == NULL) { + if (conn->input->eof) + return -1; return 0; + } if (*conn->set->doveadm_password == '\0') { i_error("doveadm_password not set, " From dovecot at dovecot.org Tue Nov 27 06:21:33 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 06:21:33 +0200 Subject: dovecot-2.1: doveadm-server: Reset headers in print output betwe... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/fdc509644d05 changeset: 14812:fdc509644d05 user: Timo Sirainen date: Tue Nov 27 06:21:18 2012 +0200 description: doveadm-server: Reset headers in print output between running commands. diffstat: src/doveadm/client-connection.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 0dc3f56e6468 -r fdc509644d05 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Tue Nov 27 06:20:44 2012 +0200 +++ b/src/doveadm/client-connection.c Tue Nov 27 06:21:18 2012 +0200 @@ -139,6 +139,10 @@ o_stream_send(conn->output, "\n+\n", 3); } pool_unref(&ctx->pool); + + /* clear all headers */ + doveadm_print_deinit(); + doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); } static bool client_is_allowed_command(const struct doveadm_settings *set, From dovecot at dovecot.org Tue Nov 27 07:50:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 07:50:29 +0200 Subject: dovecot-2.1: Reversed recent "short utf8" changes. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/f584aae3b566 changeset: 14813:f584aae3b566 user: Timo Sirainen date: Tue Nov 27 07:50:06 2012 +0200 description: Reversed recent "short utf8" changes. Solr code needs to parse the UTF8 input explicitly anyway to encode the XML characters. And all the character checks were already done in it. diffstat: src/lib/unichar.c | 64 ----------------------------- src/lib/unichar.h | 7 --- src/plugins/fts-solr/fts-backend-solr-old.c | 2 +- src/plugins/fts-solr/fts-backend-solr.c | 3 +- src/plugins/fts/fts-api-private.h | 4 +- src/plugins/fts/fts-build-mail.c | 7 +-- src/plugins/fts/fts-parser.c | 24 ++-------- src/plugins/fts/fts-parser.h | 3 +- 8 files changed, 12 insertions(+), 102 deletions(-) diffs (227 lines): diff -r fdc509644d05 -r f584aae3b566 src/lib/unichar.c --- a/src/lib/unichar.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/lib/unichar.c Tue Nov 27 07:50:06 2012 +0200 @@ -420,67 +420,3 @@ return uni_utf8_find_invalid_pos(data, size, &i) == 0; } - -static int -uni_utf8_short_find_invalid_pos(const unsigned char *input, size_t size, - size_t *pos_r) -{ - size_t i, len; - - /* find the first invalid utf8 sequence */ - for (i = 0; i < size;) { - if (input[i] < 0x80) - i++; - else { - len = is_valid_utf8_seq(input + i, size-i); - if (unlikely(len == 0 || len > 4)) { - *pos_r = i; - return -1; - } - i += len; - } - } - return 0; -} - -bool uni_utf8_short_get_valid_data(const unsigned char *input, size_t size, - buffer_t *buf) -{ - size_t i, len; - - if (uni_utf8_short_find_invalid_pos(input, size, &i) == 0) - return TRUE; - - /* broken utf-8 input - skip the broken characters */ - while (i < size) { - if (input[i] < 0x80) { - buffer_append_c(buf, input[i++]); - continue; - } - - len = is_valid_utf8_seq(input + i, size-i); - if (len == 0 || len > 4) { - i += I_MAX(len, 1); - output_add_replacement_char(buf); - continue; - } - buffer_append(buf, input + i, len); - i += len; - } - return FALSE; -} - -bool uni_utf8_short_str_is_valid(const char *str) -{ - size_t i; - - return uni_utf8_short_find_invalid_pos((const unsigned char *)str, - strlen(str), &i) == 0; -} - -bool uni_utf8_short_data_is_valid(const unsigned char *data, size_t size) -{ - size_t i; - - return uni_utf8_find_invalid_pos(data, size, &i) == 0; -} diff -r fdc509644d05 -r f584aae3b566 src/lib/unichar.h --- a/src/lib/unichar.h Tue Nov 27 06:21:18 2012 +0200 +++ b/src/lib/unichar.h Tue Nov 27 07:50:06 2012 +0200 @@ -88,11 +88,4 @@ /* Returns TRUE if data contains only valid UTF-8 input. */ bool uni_utf8_data_is_valid(const unsigned char *data, size_t size); -/* Same as the non-short variants, but assume 5-byte and 6-byte UTF8 - sequences are illegal. */ -bool uni_utf8_short_get_valid_data(const unsigned char *input, size_t size, - buffer_t *buf); -bool uni_utf8_short_str_is_valid(const char *str); -bool uni_utf8_short_data_is_valid(const unsigned char *data, size_t size); - #endif diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Tue Nov 27 07:50:06 2012 +0200 @@ -810,7 +810,7 @@ struct fts_backend fts_backend_solr_old = { .name = "solr_old", - .flags = FTS_BACKEND_FLAG_BUILD_SHORT_UTF8, + .flags = 0, { fts_backend_solr_alloc, diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 07:50:06 2012 +0200 @@ -879,8 +879,7 @@ struct fts_backend fts_backend_solr = { .name = "solr", - .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH | - FTS_BACKEND_FLAG_BUILD_SHORT_UTF8, + .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH, { fts_backend_solr_alloc, diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts/fts-api-private.h --- a/src/plugins/fts/fts-api-private.h Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts/fts-api-private.h Tue Nov 27 07:50:06 2012 +0200 @@ -59,9 +59,7 @@ /* Send only fully indexable words rather than randomly sized blocks */ FTS_BACKEND_FLAG_BUILD_FULL_WORDS = 0x04, /* Fuzzy search works */ - FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08, - /* Don't allow 5-byte or 6-byte UTF8 sequences */ - FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 = 0x10 + FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08 }; struct fts_backend { diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts/fts-build-mail.c Tue Nov 27 07:50:06 2012 +0200 @@ -144,7 +144,6 @@ struct mail_storage *storage; const char *content_type; struct fts_backend_build_key key; - bool require_short_utf8; i_assert(ctx->body_parser == NULL); @@ -159,11 +158,9 @@ return FALSE; } - require_short_utf8 = (ctx->update_ctx->backend->flags & - FTS_BACKEND_FLAG_BUILD_SHORT_UTF8) != 0; - + storage = mailbox_get_storage(ctx->mail->box); - if (fts_parser_init(mail_storage_get_user(storage), require_short_utf8, + if (fts_parser_init(mail_storage_get_user(storage), content_type, ctx->content_disposition, &ctx->body_parser)) { /* extract text using the the returned parser */ diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts/fts-parser.c --- a/src/plugins/fts/fts-parser.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts/fts-parser.c Tue Nov 27 07:50:06 2012 +0200 @@ -11,7 +11,7 @@ &fts_parser_script }; -bool fts_parser_init(struct mail_user *user, bool require_short_utf8, +bool fts_parser_init(struct mail_user *user, const char *content_type, const char *content_disposition, struct fts_parser **parser_r) { @@ -20,10 +20,8 @@ for (i = 0; i < N_ELEMENTS(parsers); i++) { *parser_r = parsers[i]->try_init(user, content_type, content_disposition); - if (*parser_r != NULL) { - (*parser_r)->require_short_utf8 = require_short_utf8; + if (*parser_r != NULL) return TRUE; - } } return FALSE; } @@ -58,15 +56,11 @@ void fts_parser_more(struct fts_parser *parser, struct message_block *block) { - bool valid_utf8; - if (parser->v.more != NULL) parser->v.more(parser, block); - valid_utf8 = parser->require_short_utf8 ? - uni_utf8_short_data_is_valid(block->data, block->size) : - uni_utf8_data_is_valid(block->data, block->size); - if (!valid_utf8 || data_has_nuls(block->data, block->size)) { + if (!uni_utf8_data_is_valid(block->data, block->size) || + data_has_nuls(block->data, block->size)) { /* output isn't valid UTF-8. make it. */ if (parser->utf8_output == NULL) { parser->utf8_output = @@ -74,14 +68,8 @@ } else { buffer_set_used_size(parser->utf8_output, 0); } - if (parser->require_short_utf8) { - (void)uni_utf8_short_get_valid_data(block->data, - block->size, - parser->utf8_output); - } else { - (void)uni_utf8_get_valid_data(block->data, block->size, - parser->utf8_output); - } + (void)uni_utf8_get_valid_data(block->data, block->size, + parser->utf8_output); replace_nul_bytes(parser->utf8_output); block->data = parser->utf8_output->data; block->size = parser->utf8_output->used; diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts/fts-parser.h --- a/src/plugins/fts/fts-parser.h Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts/fts-parser.h Tue Nov 27 07:50:06 2012 +0200 @@ -15,13 +15,12 @@ struct fts_parser { struct fts_parser_vfuncs v; buffer_t *utf8_output; - bool require_short_utf8; }; extern struct fts_parser_vfuncs fts_parser_html; extern struct fts_parser_vfuncs fts_parser_script; -bool fts_parser_init(struct mail_user *user, bool require_short_utf8, +bool fts_parser_init(struct mail_user *user, const char *content_type, const char *content_disposition, struct fts_parser **parser_r); struct fts_parser *fts_parser_text_init(void); From dovecot at dovecot.org Tue Nov 27 08:08:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 08:08:55 +0200 Subject: dovecot-2.1: fts-solr: Removed deprecated waitFlush parameter fr... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/0f3972fa6dba changeset: 14814:0f3972fa6dba user: Timo Sirainen date: Tue Nov 27 08:08:50 2012 +0200 description: fts-solr: Removed deprecated waitFlush parameter from commit. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r f584aae3b566 -r 0f3972fa6dba src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 07:50:06 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 08:08:50 2012 +0200 @@ -352,8 +352,7 @@ visible to the following search */ if (ctx->expunges) fts_backend_solr_expunge_flush(ctx); - str = t_strdup_printf("", + str = t_strdup_printf("", ctx->documents_added ? "true" : "false"); if (solr_connection_post(solr_conn, str) < 0) ret = -1; From dovecot at dovecot.org Tue Nov 27 08:43:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 08:43:31 +0200 Subject: dovecot-2.1: auth: userdb passwd lookups should have been done v... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/8e5d9d88e250 changeset: 14815:8e5d9d88e250 user: Timo Sirainen date: Tue Nov 27 08:43:15 2012 +0200 description: auth: userdb passwd lookups should have been done via worker processes by default. diffstat: src/auth/userdb-passwd.c | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diffs (18 lines): diff -r 0f3972fa6dba -r 8e5d9d88e250 src/auth/userdb-passwd.c --- a/src/auth/userdb-passwd.c Tue Nov 27 08:08:50 2012 +0200 +++ b/src/auth/userdb-passwd.c Tue Nov 27 08:43:15 2012 +0200 @@ -216,11 +216,10 @@ module = p_new(pool, struct passwd_userdb_module, 1); module->module.cache_key = USER_CACHE_KEY; module->tmpl = userdb_template_build(pool, "passwd", args); + module->module.blocking = TRUE; - if (userdb_template_remove(module->tmpl, "blocking", &value)) { - module->module.blocking = value == NULL || - strcasecmp(value, "yes") == 0; - } + if (userdb_template_remove(module->tmpl, "blocking", &value)) + module->module.blocking = strcasecmp(value, "yes") == 0; /* FIXME: backwards compatibility */ if (!userdb_template_is_empty(module->tmpl)) i_warning("userdb passwd: Move templates args to override_fields setting"); From dovecot at dovecot.org Tue Nov 27 08:44:18 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 08:44:18 +0200 Subject: dovecot-2.0: auth: userdb passwd lookups should have been done v... Message-ID: details: http://hg.dovecot.org/dovecot-2.0/rev/74d9f61e224d changeset: 13108:74d9f61e224d user: Timo Sirainen date: Tue Nov 27 08:44:12 2012 +0200 description: auth: userdb passwd lookups should have been done via worker processes by default. diffstat: src/auth/userdb-passwd.c | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diffs (19 lines): diff -r 13d764bbde11 -r 74d9f61e224d src/auth/userdb-passwd.c --- a/src/auth/userdb-passwd.c Fri Oct 05 00:25:02 2012 +0300 +++ b/src/auth/userdb-passwd.c Tue Nov 27 08:44:12 2012 +0200 @@ -154,12 +154,11 @@ module = p_new(pool, struct passwd_userdb_module, 1); module->module.cache_key = USER_CACHE_KEY; module->tmpl = userdb_static_template_build(pool, "passwd", args); + module->module.blocking = TRUE; if (userdb_static_template_remove(module->tmpl, "blocking", - &value)) { - module->module.blocking = value == NULL || - strcasecmp(value, "yes") == 0; - } + &value)) + module->module.blocking = strcasecmp(value, "yes") == 0; return &module->module; } From dovecot at dovecot.org Tue Nov 27 09:53:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 09:53:52 +0200 Subject: dovecot-2.1: login: Reverted previous fix attempt for SSL_accept... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/174d7e974326 changeset: 14816:174d7e974326 user: Timo Sirainen date: Tue Nov 27 09:13:57 2012 +0200 description: login: Reverted previous fix attempt for SSL_accept() busy-looping. Apparently didn't work. Probably this patch doesn't help anything then, so just remove the extra complexity. diffstat: src/login-common/ssl-proxy-openssl.c | 17 ++++++----------- 1 files changed, 6 insertions(+), 11 deletions(-) diffs (63 lines): diff -r 8e5d9d88e250 -r 174d7e974326 src/login-common/ssl-proxy-openssl.c --- a/src/login-common/ssl-proxy-openssl.c Tue Nov 27 08:43:15 2012 +0200 +++ b/src/login-common/ssl-proxy-openssl.c Tue Nov 27 09:13:57 2012 +0200 @@ -394,9 +394,8 @@ return ssl_err2str(err, data, flags); } -static void -ssl_handle_error(struct ssl_proxy *proxy, int ret, bool remove_wrong_direction, - const char *func_name) +static void ssl_handle_error(struct ssl_proxy *proxy, int ret, + const char *func_name) { const char *errstr = NULL; int err; @@ -409,13 +408,9 @@ switch (err) { case SSL_ERROR_WANT_READ: ssl_set_io(proxy, SSL_ADD_INPUT); - if (remove_wrong_direction) - ssl_set_io(proxy, SSL_REMOVE_OUTPUT); break; case SSL_ERROR_WANT_WRITE: ssl_set_io(proxy, SSL_ADD_OUTPUT); - if (remove_wrong_direction) - ssl_set_io(proxy, SSL_REMOVE_INPUT); break; case SSL_ERROR_SYSCALL: /* eat up the error queue */ @@ -463,13 +458,13 @@ if (proxy->client_proxy) { ret = SSL_connect(proxy->ssl); if (ret != 1) { - ssl_handle_error(proxy, ret, TRUE, "SSL_connect()"); + ssl_handle_error(proxy, ret, "SSL_connect()"); return; } } else { ret = SSL_accept(proxy->ssl); if (ret != 1) { - ssl_handle_error(proxy, ret, TRUE, "SSL_accept()"); + ssl_handle_error(proxy, ret, "SSL_accept()"); return; } } @@ -496,7 +491,7 @@ sizeof(proxy->plainout_buf) - proxy->plainout_size); if (ret <= 0) { - ssl_handle_error(proxy, ret, FALSE, "SSL_read()"); + ssl_handle_error(proxy, ret, "SSL_read()"); break; } else { i_free_and_null(proxy->last_error); @@ -512,7 +507,7 @@ ret = SSL_write(proxy->ssl, proxy->sslout_buf, proxy->sslout_size); if (ret <= 0) - ssl_handle_error(proxy, ret, FALSE, "SSL_write()"); + ssl_handle_error(proxy, ret, "SSL_write()"); else { i_free_and_null(proxy->last_error); proxy->sslout_size -= ret; From dovecot at dovecot.org Tue Nov 27 09:53:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 09:53:53 +0200 Subject: dovecot-2.1: lib-index: Added v2.2+ forwards compatibility to do... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/55e84bb452e6 changeset: 14817:55e84bb452e6 user: Timo Sirainen date: Tue Nov 27 09:53:33 2012 +0200 description: lib-index: Added v2.2+ forwards compatibility to dovecot.index.cache diffstat: src/lib-index/mail-cache-private.h | 3 ++- src/lib-index/mail-cache-transaction.c | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diffs (36 lines): diff -r 174d7e974326 -r 55e84bb452e6 src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Tue Nov 27 09:13:57 2012 +0200 +++ b/src/lib-index/mail-cache-private.h Tue Nov 27 09:53:33 2012 +0200 @@ -47,7 +47,8 @@ compatibility. */ uint8_t version; uint8_t compat_sizeof_uoff_t; - uint8_t unused[2]; + uint8_t minor_version; + uint8_t unused; uint32_t indexid; uint32_t file_seq; diff -r 174d7e974326 -r 55e84bb452e6 src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Tue Nov 27 09:13:57 2012 +0200 +++ b/src/lib-index/mail-cache-transaction.c Tue Nov 27 09:53:33 2012 +0200 @@ -315,6 +315,9 @@ i_assert(cache->locked); + if (hdr->minor_version > 0) + return FALSE; /* this is record_count field in v2.2+ */ + offset = hdr->hole_offset; prev_offset = 0; while (offset != 0) { if (pread_full(cache->fd, &hole, sizeof(hole), offset) <= 0) { @@ -481,7 +484,8 @@ /* we can just set used_file_size back */ cache->hdr_modified = TRUE; cache->hdr_copy.used_file_size = offset; - } else if (size >= MAIL_CACHE_MIN_HOLE_SIZE) { + } else if (size >= MAIL_CACHE_MIN_HOLE_SIZE && + cache->hdr_copy.minor_version == 0) { /* set it up as a hole */ hole.next_offset = cache->hdr_copy.hole_offset; hole.size = size; From dovecot at dovecot.org Tue Nov 27 10:00:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 10:00:46 +0200 Subject: dovecot-2.2: NEWS updated Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2b18b4b92cd9 changeset: 15419:2b18b4b92cd9 user: Timo Sirainen date: Tue Nov 27 10:00:42 2012 +0200 description: NEWS updated diffstat: NEWS | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 25e8459431fc -r 2b18b4b92cd9 NEWS --- a/NEWS Tue Nov 27 00:44:34 2012 +0200 +++ b/NEWS Tue Nov 27 10:00:42 2012 +0200 @@ -6,6 +6,8 @@ * "doveadm auth" command was renamed to "doveadm auth test" * IMAP: ID command now advertises server name as Dovecot by default. It was already trivial to guess this from command replies. + * dovecot.index.cache files can be safely accessed only by v2.1.11+. + Older versions may think they're corrupted and delete them. + Implemented IMAP MOVE and BINARY extensions + Implemented IMAP CATENATE, URLAUTH and URLAUTH=BINARY extensions From dovecot at dovecot.org Tue Nov 27 10:03:08 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 10:03:08 +0200 Subject: dovecot-2.1: mailbox-alias plugin: Error handling fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/fd2d24171afd changeset: 14818:fd2d24171afd user: Timo Sirainen date: Tue Nov 27 10:02:59 2012 +0200 description: mailbox-alias plugin: Error handling fix. diffstat: src/plugins/mailbox-alias/mailbox-alias-plugin.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 55e84bb452e6 -r fd2d24171afd src/plugins/mailbox-alias/mailbox-alias-plugin.c --- a/src/plugins/mailbox-alias/mailbox-alias-plugin.c Tue Nov 27 09:53:33 2012 +0200 +++ b/src/plugins/mailbox-alias/mailbox-alias-plugin.c Tue Nov 27 10:02:59 2012 +0200 @@ -114,7 +114,7 @@ if (mailbox_symlink_exists(list, alias->new_vname, &existence) < 0) ret = -1; - if (existence == MAILBOX_SYMLINK_EXISTENCE_SYMLINK) + else if (existence == MAILBOX_SYMLINK_EXISTENCE_SYMLINK) return 1; } } From dovecot at dovecot.org Tue Nov 27 10:41:40 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 10:41:40 +0200 Subject: dovecot-2.1: mdbox: When rebuilding storage, fsck the map index ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/8770940057b9 changeset: 14819:8770940057b9 user: Timo Sirainen date: Tue Nov 27 10:39:26 2012 +0200 description: mdbox: When rebuilding storage, fsck the map index first to make sure it's valid. This fixes assert-crash when the map index contained records with UIDs in wrong order. diffstat: src/lib-storage/index/dbox-multi/mdbox-map.c | 12 +++++++----- src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 6 ++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diffs (49 lines): diff -r fd2d24171afd -r 8770940057b9 src/lib-storage/index/dbox-multi/mdbox-map.c --- a/src/lib-storage/index/dbox-multi/mdbox-map.c Tue Nov 27 10:02:59 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Tue Nov 27 10:39:26 2012 +0200 @@ -243,7 +243,8 @@ int mdbox_map_refresh(struct mdbox_map *map) { struct mail_index_view_sync_ctx *ctx; - bool delayed_expunges; + bool delayed_expunges, fscked; + int ret = 0; /* some open files may have read partially written mails. now that map syncing makes the new mails visible, we need to make sure the @@ -262,14 +263,15 @@ ctx = mail_index_view_sync_begin(map->view, MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); - if (mail_index_reset_fscked(map->view->index)) - mdbox_storage_set_corrupted(map->storage); + fscked = mail_index_reset_fscked(map->view->index); if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) { mail_storage_set_internal_error(MAP_STORAGE(map)); mail_index_reset_error(map->index); - return -1; + ret = -1; } - return 0; + if (fscked) + mdbox_storage_set_corrupted(map->storage); + return ret; } static void diff -r fd2d24171afd -r 8770940057b9 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Tue Nov 27 10:02:59 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Tue Nov 27 10:39:26 2012 +0200 @@ -836,6 +836,12 @@ if (mdbox_map_atomic_lock(ctx->atomic) < 0) return -1; + /* fsck the map just in case its UIDs are broken */ + if (mail_index_fsck(ctx->storage->map->index) < 0) { + mail_storage_set_internal_error(&ctx->storage->storage.storage); + return -1; + } + /* get old map header */ mail_index_get_header_ext(ctx->atomic->sync_view, ctx->storage->map->map_ext_id, From dovecot at dovecot.org Tue Nov 27 10:48:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 10:48:22 +0200 Subject: dovecot-2.1: fts_parser: Minor error logging improvement Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/a9e7f9333f57 changeset: 14820:a9e7f9333f57 user: Timo Sirainen date: Tue Nov 27 10:48:11 2012 +0200 description: fts_parser: Minor error logging improvement diffstat: src/plugins/fts/fts-parser-script.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (20 lines): diff -r 8770940057b9 -r a9e7f9333f57 src/plugins/fts/fts-parser-script.c --- a/src/plugins/fts/fts-parser-script.c Tue Nov 27 10:39:26 2012 +0200 +++ b/src/plugins/fts/fts-parser-script.c Tue Nov 27 10:48:11 2012 +0200 @@ -99,10 +99,13 @@ content->content_type = args[0]; content->extensions = (const void *)(args+1); } + if (!eof_seen) { + if (input->v_offset == 0) + i_error("parser script didn't send any data"); + else + i_error("parser script didn't send empty EOF line"); + } i_stream_destroy(&input); - - if (!eof_seen) - i_error("parser script didn't send empty EOF line"); return 0; } From dovecot at dovecot.org Tue Nov 27 10:49:50 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 10:49:50 +0200 Subject: dovecot-2.1: lib-storage: Added struct mail_user.nonexistent fla... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/fe1be6725304 changeset: 14821:fe1be6725304 user: Timo Sirainen date: Tue Nov 27 10:49:18 2012 +0200 description: lib-storage: Added struct mail_user.nonexistent flag, which is filled by userdb lookup. diffstat: src/lib-storage/mail-user.c | 19 +++++++++++-------- src/lib-storage/mail-user.h | 3 +++ 2 files changed, 14 insertions(+), 8 deletions(-) diffs (50 lines): diff -r a9e7f9333f57 -r fe1be6725304 src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Tue Nov 27 10:48:11 2012 +0200 +++ b/src/lib-storage/mail-user.c Tue Nov 27 10:49:18 2012 +0200 @@ -289,9 +289,6 @@ if (user->remote_ip != NULL) info.remote_ip = *user->remote_ip; - if (mail_user_auth_master_conn == NULL) - return 0; - userdb_pool = pool_alloconly_create("userdb lookup", 2048); ret = auth_master_user_lookup(mail_user_auth_master_conn, user->username, &info, userdb_pool, @@ -313,12 +310,18 @@ return user->_home != NULL ? 1 : 0; } - ret = mail_user_userdb_lookup_home(user); - if (ret < 0) + if (mail_user_auth_master_conn == NULL) { + /* no userdb connection. we can only use mail_home setting. */ + user->_home = user->set->mail_home; + } else if ((ret = mail_user_userdb_lookup_home(user)) < 0) { + /* userdb lookup failed */ return -1; - - if (ret > 0 && user->_home == NULL && *user->set->mail_home != '\0') { - /* no home in userdb, fallback to mail_home setting */ + } else if (ret == 0) { + /* user doesn't exist */ + user->nonexistent = TRUE; + } else if (user->_home == NULL && *user->set->mail_home != '\0') { + /* no home returned by userdb lookup, fallback to mail_home + setting. */ user->_home = user->set->mail_home; } user->home_looked_up = TRUE; diff -r a9e7f9333f57 -r fe1be6725304 src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Tue Nov 27 10:48:11 2012 +0200 +++ b/src/lib-storage/mail-user.h Tue Nov 27 10:49:18 2012 +0200 @@ -42,6 +42,9 @@ /* Module-specific contexts. See mail_storage_module_id. */ ARRAY_DEFINE(module_contexts, union mail_user_module_context *); + /* User doesn't exist (as reported by userdb lookup when looking + up home) */ + unsigned int nonexistent:1; /* Either home is set or there is no home for the user. */ unsigned int home_looked_up:1; /* User is an administrator. Allow operations not normally allowed From dovecot at dovecot.org Tue Nov 27 10:49:50 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 10:49:50 +0200 Subject: dovecot-2.1: lib-storage: Handle better when attempting to acces... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/7bca3be44b8f changeset: 14822:7bca3be44b8f user: Timo Sirainen date: Tue Nov 27 10:49:41 2012 +0200 description: lib-storage: Handle better when attempting to access shared mailboxes for nonexistent users. diffstat: src/lib-storage/index/shared/shared-storage.c | 18 ++++++++++-------- 1 files changed, 10 insertions(+), 8 deletions(-) diffs (29 lines): diff -r fe1be6725304 -r 7bca3be44b8f src/lib-storage/index/shared/shared-storage.c --- a/src/lib-storage/index/shared/shared-storage.c Tue Nov 27 10:49:18 2012 +0200 +++ b/src/lib-storage/index/shared/shared-storage.c Tue Nov 27 10:49:41 2012 +0200 @@ -257,15 +257,17 @@ user->unexpanded_set); owner->autocreated = TRUE; if (mail_user_init(owner, &error) < 0) { - mailbox_list_set_critical(list, - "Couldn't create namespace '%s' for user %s: %s", - ns->prefix, userdomain, error); - mail_user_unref(&owner); - return -1; - } - if (!var_has_key(storage->location, 'h', "home")) + if (!owner->nonexistent) { + mailbox_list_set_critical(list, + "Couldn't create namespace '%s' for user %s: %s", + ns->prefix, userdomain, error); + mail_user_unref(&owner); + return -1; + } + ret = 0; + } else if (!var_has_key(storage->location, 'h', "home")) { ret = 1; - else { + } else { /* we'll need to look up the user's home directory */ if ((ret = mail_user_get_home(owner, &tab[3].value)) < 0) { mailbox_list_set_critical(list, "Namespace '%s': " From dovecot at dovecot.org Tue Nov 27 23:06:44 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 27 Nov 2012 23:06:44 +0200 Subject: dovecot-2.1: fts-solr: XML-encode non-body fields as well Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/6a97faf3e500 changeset: 14823:6a97faf3e500 user: Timo Sirainen date: Tue Nov 27 23:06:38 2012 +0200 description: fts-solr: XML-encode non-body fields as well diffstat: src/plugins/fts-solr/fts-backend-solr.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7bca3be44b8f -r 6a97faf3e500 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 10:49:41 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 23:06:38 2012 +0200 @@ -305,7 +305,7 @@ } array_foreach_modifiable(&ctx->fields, field) { str_printfa(ctx->cmd, "", field->key); - str_append_str(ctx->cmd, field->value); + xml_encode_data(ctx->cmd, str_data(field->value), str_len(field->value)); str_append(ctx->cmd, ""); str_truncate(field->value, 0); } From dovecot at dovecot.org Wed Nov 28 01:43:34 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 01:43:34 +0200 Subject: dovecot-2.2: i_stream_default_stat(): If we can't know the size,... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fee60a91d586 changeset: 15420:fee60a91d586 user: Timo Sirainen date: Wed Nov 28 01:43:24 2012 +0200 description: i_stream_default_stat(): If we can't know the size, return -1 instead of assert crash. This fixes a crash saving to mdbox when input stream's size wasn't known. diffstat: src/lib/istream.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (21 lines): diff -r 2b18b4b92cd9 -r fee60a91d586 src/lib/istream.c --- a/src/lib/istream.c Tue Nov 27 10:00:42 2012 +0200 +++ b/src/lib/istream.c Wed Nov 28 01:43:24 2012 +0200 @@ -623,13 +623,13 @@ if (stream->parent == NULL) return 0; - if (exact && !stream->stream_size_passthrough) { - i_panic("istream %s: stat() doesn't support getting exact size", - i_stream_get_name(&stream->istream)); - } if (i_stream_stat(stream->parent, exact, &st) < 0) return -1; stream->statbuf = *st; + if (exact && !stream->stream_size_passthrough) { + /* exact size is not known, even if parent returned something */ + stream->statbuf.st_size = -1; + } return 0; } From dovecot at dovecot.org Wed Nov 28 01:44:58 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 01:44:58 +0200 Subject: dovecot-2.2: lib-fs: Don't crash when trying to initialize nonex... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aade5fa372a6 changeset: 15421:aade5fa372a6 user: Timo Sirainen date: Wed Nov 28 01:44:53 2012 +0200 description: lib-fs: Don't crash when trying to initialize nonexistent fs driver. diffstat: src/lib-fs/fs-api.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r fee60a91d586 -r aade5fa372a6 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Wed Nov 28 01:43:24 2012 +0200 +++ b/src/lib-fs/fs-api.c Wed Nov 28 01:44:53 2012 +0200 @@ -83,8 +83,8 @@ module_dir_init(fs_modules); module = module_dir_find(fs_modules, module_name); - fs_class = module_get_symbol(module, - t_strdup_printf("fs_class_%s", driver)); + fs_class = module == NULL ? NULL : + module_get_symbol(module, t_strdup_printf("fs_class_%s", driver)); if (fs_class != NULL) fs_class_register(fs_class); From dovecot at dovecot.org Wed Nov 28 01:52:42 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 01:52:42 +0200 Subject: dovecot-2.2: lib-storage: Don't assert-crash when fetching BINAR... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa8f2e0fcb21 changeset: 15422:aa8f2e0fcb21 user: Timo Sirainen date: Wed Nov 28 01:52:32 2012 +0200 description: lib-storage: Don't assert-crash when fetching BINARY contents for empty body. diffstat: src/lib-storage/index/index-mail-binary.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r aade5fa372a6 -r aa8f2e0fcb21 src/lib-storage/index/index-mail-binary.c --- a/src/lib-storage/index/index-mail-binary.c Wed Nov 28 01:44:53 2012 +0200 +++ b/src/lib-storage/index/index-mail-binary.c Wed Nov 28 01:52:32 2012 +0200 @@ -162,6 +162,11 @@ ctx->copy_start_offset = part_end_offset; return 0; } + if (part->body_size.physical_size == 0) { + /* no body */ + ctx->copy_start_offset = part_end_offset; + return 0; + } /* single part - write decoded data */ block = array_append_space(&ctx->blocks); From dovecot at dovecot.org Wed Nov 28 02:35:44 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 02:35:44 +0200 Subject: dovecot-2.2: iostream-rawlog: Buffered output kept logging the s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9bff748f3437 changeset: 15423:9bff748f3437 user: Timo Sirainen date: Wed Nov 28 02:35:15 2012 +0200 description: iostream-rawlog: Buffered output kept logging the same data over and over again. diffstat: src/lib/iostream-rawlog.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r aa8f2e0fcb21 -r 9bff748f3437 src/lib/iostream-rawlog.c --- a/src/lib/iostream-rawlog.c Wed Nov 28 01:52:32 2012 +0200 +++ b/src/lib/iostream-rawlog.c Wed Nov 28 02:35:15 2012 +0200 @@ -123,6 +123,7 @@ if (rawlog_write(rstream, rstream->buffer->data, rstream->buffer->used) < 0) break; + buffer_set_used_size(rstream->buffer, 0); } if (rawlog_write(rstream, data, pos) < 0) break; From dovecot at dovecot.org Wed Nov 28 02:35:44 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 02:35:44 +0200 Subject: dovecot-2.2: ostream-rawlog: Make sure rawlog logs only the data... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b10fbe79f619 changeset: 15424:b10fbe79f619 user: Timo Sirainen date: Wed Nov 28 02:35:37 2012 +0200 description: ostream-rawlog: Make sure rawlog logs only the data that got actually sent. diffstat: src/lib/ostream-rawlog.c | 19 +++++++++++++------ 1 files changed, 13 insertions(+), 6 deletions(-) diffs (34 lines): diff -r 9bff748f3437 -r b10fbe79f619 src/lib/ostream-rawlog.c --- a/src/lib/ostream-rawlog.c Wed Nov 28 02:35:15 2012 +0200 +++ b/src/lib/ostream-rawlog.c Wed Nov 28 02:35:37 2012 +0200 @@ -24,17 +24,24 @@ { struct rawlog_ostream *rstream = (struct rawlog_ostream *)stream; unsigned int i; - ssize_t ret; - - for (i = 0; i < iov_count; i++) { - iostream_rawlog_write(&rstream->riostream, - iov[i].iov_base, iov[i].iov_len); - } + ssize_t ret, bytes; if ((ret = o_stream_sendv(stream->parent, iov, iov_count)) < 0) { o_stream_copy_error_from_parent(stream); return -1; } + bytes = ret; + for (i = 0; i < iov_count && bytes > 0; i++) { + if (iov[i].iov_len < (size_t)bytes) { + iostream_rawlog_write(&rstream->riostream, + iov[i].iov_base, iov[i].iov_len); + bytes -= iov[i].iov_len; + } else { + iostream_rawlog_write(&rstream->riostream, + iov[i].iov_base, bytes); + break; + } + } stream->ostream.offset += ret; return ret; From dovecot at dovecot.org Wed Nov 28 03:33:02 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 03:33:02 +0200 Subject: dovecot-2.1: lib-auth: Added auth_master_get_socket_path() Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/755a2f08cde5 changeset: 14824:755a2f08cde5 user: Timo Sirainen date: Wed Nov 28 03:32:01 2012 +0200 description: lib-auth: Added auth_master_get_socket_path() diffstat: src/lib-auth/auth-master.c | 5 +++++ src/lib-auth/auth-master.h | 3 +++ 2 files changed, 8 insertions(+), 0 deletions(-) diffs (28 lines): diff -r 6a97faf3e500 -r 755a2f08cde5 src/lib-auth/auth-master.c --- a/src/lib-auth/auth-master.c Tue Nov 27 23:06:38 2012 +0200 +++ b/src/lib-auth/auth-master.c Wed Nov 28 03:32:01 2012 +0200 @@ -107,6 +107,11 @@ i_free(conn); } +const char *auth_master_get_socket_path(struct auth_master_connection *conn) +{ + return conn->auth_socket_path; +} + static void auth_request_lookup_abort(struct auth_master_connection *conn) { io_loop_stop(conn->ioloop); diff -r 6a97faf3e500 -r 755a2f08cde5 src/lib-auth/auth-master.h --- a/src/lib-auth/auth-master.h Tue Nov 27 23:06:38 2012 +0200 +++ b/src/lib-auth/auth-master.h Wed Nov 28 03:32:01 2012 +0200 @@ -27,6 +27,9 @@ auth_master_init(const char *auth_socket_path, enum auth_master_flags flags); void auth_master_deinit(struct auth_master_connection **conn); +/* Returns the auth_socket_path */ +const char *auth_master_get_socket_path(struct auth_master_connection *conn); + /* Do a USER lookup. Returns -1 = error, 0 = user not found, 1 = ok. When returning -1 and fields[0] isn't NULL, it contains an error message that should be shown to user. */ From dovecot at dovecot.org Wed Nov 28 03:33:02 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 03:33:02 +0200 Subject: dovecot-2.1: doveadm: Log better errors for failing passdb proxy... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/8bc9d40d77c4 changeset: 14825:8bc9d40d77c4 user: Timo Sirainen date: Wed Nov 28 03:32:31 2012 +0200 description: doveadm: Log better errors for failing passdb proxy lookups. diffstat: src/doveadm/doveadm-mail-server.c | 15 +++++++++++---- 1 files changed, 11 insertions(+), 4 deletions(-) diffs (45 lines): diff -r 755a2f08cde5 -r 8bc9d40d77c4 src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Wed Nov 28 03:32:01 2012 +0200 +++ b/src/doveadm/doveadm-mail-server.c Wed Nov 28 03:32:31 2012 +0200 @@ -177,7 +177,7 @@ struct auth_master_connection *auth_conn; struct auth_user_info info; pool_t pool; - const char *proxy_host, *const *fields; + const char *auth_socket_path, *proxy_host, *const *fields; unsigned int i; bool proxying; int ret; @@ -195,14 +195,15 @@ pool = pool_alloconly_create("auth lookup", 1024); auth_conn = mail_storage_service_get_auth_conn(ctx->storage_service); + auth_socket_path = auth_master_get_socket_path(auth_conn); ret = auth_master_pass_lookup(auth_conn, input->username, &info, pool, &fields); if (ret < 0) { *error_r = fields[0] != NULL ? t_strdup(fields[0]) : "passdb lookup failed"; - *error_r = t_strdup_printf("%s (to see if user is proxied, " + *error_r = t_strdup_printf("%s: %s (to see if user is proxied, " "because doveadm_proxy_port is set)", - *error_r); + auth_socket_path, *error_r); } else if (ret == 0) { /* user not found from passdb. it could be in userdb though, so just continue with the default host */ @@ -218,7 +219,13 @@ if (!proxying) ret = 0; else if (proxy_host == NULL) { - *error_r = "Proxy is missing destination host"; + *error_r = t_strdup_printf("%s: Proxy is missing destination host", + auth_socket_path); + if (strstr(auth_socket_path, "/auth-userdb") != NULL) { + *error_r = t_strdup_printf( + "%s (maybe set auth_socket_path=director-userdb)", + *error_r); + } ret = -1; } else { *host_r = t_strdup_printf("%s:%u", proxy_host, From dovecot at dovecot.org Wed Nov 28 03:56:17 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 03:56:17 +0200 Subject: dovecot-2.1: lib-index: Fixed assert-crash caused by recent cach... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/348e9bad84f0 changeset: 14827:348e9bad84f0 user: Timo Sirainen date: Wed Nov 28 03:56:06 2012 +0200 description: lib-index: Fixed assert-crash caused by recent cache file changes. diffstat: src/lib-index/mail-cache.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (46 lines): diff -r e62938129955 -r 348e9bad84f0 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Wed Nov 28 03:55:25 2012 +0200 +++ b/src/lib-index/mail-cache.c Wed Nov 28 03:56:06 2012 +0200 @@ -281,6 +281,7 @@ !MAIL_CACHE_IS_UNUSABLE(cache) && cache->hdr->file_seq != 0 ? cache->hdr->file_seq : 0; + cache->hdr = NULL; return -1; } } @@ -296,6 +297,7 @@ } else { i_assert(cache->hdr != NULL); } + i_assert(cache->hdr->file_seq != 0); if (offset + size > cache->mmap_length) return 0; @@ -365,8 +367,6 @@ return mail_cache_map_with_read(cache, offset, size, data_r); if (cache->file_cache != NULL) { - cache->hdr = NULL; - ret = file_cache_read(cache->file_cache, offset, size); if (ret < 0) { /* In case of ESTALE we'll simply fail without error @@ -378,6 +378,7 @@ offsets. */ if (errno != ESTALE) mail_cache_set_syscall_error(cache, "read()"); + cache->hdr = NULL; return -1; } @@ -385,7 +386,8 @@ &cache->mmap_length); *data_r = offset > cache->mmap_length ? NULL : CONST_PTR_OFFSET(data, offset); - return mail_cache_map_finish(cache, offset, size, data, TRUE); + return mail_cache_map_finish(cache, offset, size, + offset == 0 ? data : NULL, TRUE); } if (offset < cache->mmap_length && From dovecot at dovecot.org Wed Nov 28 03:56:17 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 03:56:17 +0200 Subject: dovecot-2.1: lib-index: Fixed memory leak with MAIL_INDEX_OPEN_F... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/e62938129955 changeset: 14826:e62938129955 user: Timo Sirainen date: Wed Nov 28 03:55:25 2012 +0200 description: lib-index: Fixed memory leak with MAIL_INDEX_OPEN_FLAG_SAVEONLY diffstat: src/lib-index/mail-cache.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 8bc9d40d77c4 -r e62938129955 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Wed Nov 28 03:32:31 2012 +0200 +++ b/src/lib-index/mail-cache.c Wed Nov 28 03:55:25 2012 +0200 @@ -552,6 +552,8 @@ mail_cache_file_close(cache); + if (cache->read_buf != NULL) + buffer_free(&cache->read_buf); hash_table_destroy(&cache->field_name_hash); pool_unref(&cache->field_pool); i_free(cache->field_file_map); From dovecot at dovecot.org Wed Nov 28 04:21:14 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 04:21:14 +0200 Subject: dovecot-2.1: stats: Fixed crash when handling multiple users wit... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/31ed1f5cc8f1 changeset: 14828:31ed1f5cc8f1 user: Timo Sirainen date: Wed Nov 28 04:21:03 2012 +0200 description: stats: Fixed crash when handling multiple users within same process. diffstat: src/plugins/stats/stats-plugin.c | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diffs (20 lines): diff -r 348e9bad84f0 -r 31ed1f5cc8f1 src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Wed Nov 28 03:56:06 2012 +0200 +++ b/src/plugins/stats/stats-plugin.c Wed Nov 28 04:21:03 2012 +0200 @@ -587,9 +587,13 @@ stats_global_user = user; } else if (stats_user_count == 1) { /* second user connection. we'll need to start doing - per-io callback tracking now. */ - stats_add_session(stats_global_user); - stats_global_user = NULL; + per-io callback tracking now. (we might have been doing it + also previously but just temporarily quickly dropped to + having 1 user, in which case stats_global_user=NULL) */ + if (stats_global_user != NULL) { + stats_add_session(stats_global_user); + stats_global_user = NULL; + } } stats_user_count++; From dovecot at dovecot.org Wed Nov 28 04:30:09 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 04:30:09 +0200 Subject: dovecot-2.1: stats: Log username and service for invalid UPDATE-... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c81f1ca3cda6 changeset: 14829:c81f1ca3cda6 user: Timo Sirainen date: Wed Nov 28 04:29:58 2012 +0200 description: stats: Log username and service for invalid UPDATE-SESSIONs diffstat: src/stats/mail-session.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (23 lines): diff -r 31ed1f5cc8f1 -r c81f1ca3cda6 src/stats/mail-session.c --- a/src/stats/mail-session.c Wed Nov 28 04:21:03 2012 +0200 +++ b/src/stats/mail-session.c Wed Nov 28 04:29:58 2012 +0200 @@ -255,13 +255,16 @@ return -1; if (mail_stats_parse(args+1, &stats, error_r) < 0) { - *error_r = t_strconcat("UPDATE-SESSION: ", *error_r, NULL); + *error_r = t_strdup_printf("UPDATE-SESSION %s %s: %s", + session->user->name, + session->service, *error_r); return -1; } if (!mail_stats_diff(&session->stats, &stats, &diff_stats, &error)) { - *error_r = t_strconcat("UPDATE-SESSION: stats shrank: ", - error, NULL); + *error_r = t_strdup_printf("UPDATE-SESSION %s %s: stats shrank: %s", + session->user->name, + session->service, error); return -1; } mail_session_refresh(session, &diff_stats); From dovecot at dovecot.org Wed Nov 28 06:44:09 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 06:44:09 +0200 Subject: dovecot-2.2: Make static analyzer happier Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fc890adce3d0 changeset: 15426:fc890adce3d0 user: Timo Sirainen date: Wed Nov 28 06:43:55 2012 +0200 description: Make static analyzer happier diffstat: src/imap-urlauth/imap-urlauth-worker.c | 2 +- src/lib-index/mail-transaction-log-file.c | 1 - 2 files changed, 1 insertions(+), 2 deletions(-) diffs (23 lines): diff -r cd93c3506efe -r fc890adce3d0 src/imap-urlauth/imap-urlauth-worker.c --- a/src/imap-urlauth/imap-urlauth-worker.c Wed Nov 28 06:42:40 2012 +0200 +++ b/src/imap-urlauth/imap-urlauth-worker.c Wed Nov 28 06:43:55 2012 +0200 @@ -766,7 +766,7 @@ static int client_ctrl_read_fds(struct client *client) { - unsigned char data; + unsigned char data = 0; ssize_t ret = 1; if (client->fd_in == -1) { diff -r cd93c3506efe -r fc890adce3d0 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Wed Nov 28 06:42:40 2012 +0200 +++ b/src/lib-index/mail-transaction-log-file.c Wed Nov 28 06:43:55 2012 +0200 @@ -1351,7 +1351,6 @@ } file->sync_offset += trans_size; - trans_size = 0; } if (file->mmap_base != NULL && !file->locked) { From dovecot at dovecot.org Wed Nov 28 06:44:09 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 06:44:09 +0200 Subject: dovecot-2.2: lib-index: Fixed randomly failing to write to dovec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cd93c3506efe changeset: 15425:cd93c3506efe user: Timo Sirainen date: Wed Nov 28 06:42:40 2012 +0200 description: lib-index: Fixed randomly failing to write to dovecot.index.cache file diffstat: src/lib-index/mail-cache-transaction.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b10fbe79f619 -r cd93c3506efe src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Wed Nov 28 02:35:37 2012 +0200 +++ b/src/lib-index/mail-cache-transaction.c Wed Nov 28 06:42:40 2012 +0200 @@ -365,7 +365,7 @@ { struct stat st; uint32_t write_offset = 0; - int ret; + int ret = 0; i_assert(!ctx->cache->locked); From dovecot at dovecot.org Wed Nov 28 06:51:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 06:51:25 +0200 Subject: dovecot-2.2: lib-storage: Fixed assert-crash with private indexe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3c5f674285a8 changeset: 15427:3c5f674285a8 user: Timo Sirainen date: Wed Nov 28 06:51:16 2012 +0200 description: lib-storage: Fixed assert-crash with private indexes when looking for first unseen msg. diffstat: src/lib-storage/index/index-status.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r fc890adce3d0 -r 3c5f674285a8 src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Wed Nov 28 06:43:55 2012 +0200 +++ b/src/lib-storage/index/index-status.c Wed Nov 28 06:51:16 2012 +0200 @@ -85,6 +85,8 @@ pvt_count = mail_index_view_get_messages_count(box->view_pvt); mail_index_lookup_first(box->view_pvt, 0, MAIL_SEEN, &pvt_seq); + if (pvt_seq == 0) + pvt_seq = pvt_count+1; for (; pvt_seq <= pvt_count; pvt_seq++) { pvt_rec = mail_index_lookup(box->view_pvt, pvt_seq); if ((pvt_rec->flags & MAIL_SEEN) == 0 && From dovecot at dovecot.org Wed Nov 28 07:25:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 07:25:28 +0200 Subject: dovecot-2.2: lib-storage: When updating flags in private index a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9107b794b7ad changeset: 15428:9107b794b7ad user: Timo Sirainen date: Wed Nov 28 07:24:01 2012 +0200 description: lib-storage: When updating flags in private index and nothing changes, don't increase modseq. diffstat: src/lib-storage/index/index-mail.c | 31 +++++++++++++++++++++++++++++-- 1 files changed, 29 insertions(+), 2 deletions(-) diffs (48 lines): diff -r 3c5f674285a8 -r 9107b794b7ad src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Nov 28 06:51:16 2012 +0200 +++ b/src/lib-storage/index/index-mail.c Wed Nov 28 07:24:01 2012 +0200 @@ -1655,6 +1655,34 @@ } } +static bool +index_mail_update_pvt_flags(struct mail *_mail, enum modify_type modify_type, + enum mail_flags pvt_flags) +{ + struct mail_private *mail = (struct mail_private *)_mail; + const struct mail_index_record *rec; + enum mail_flags old_pvt_flags; + + if (!index_mail_get_pvt(_mail)) + return FALSE; + if (pvt_flags == 0 && modify_type != MODIFY_REPLACE) + return FALSE; + + /* see if the flags actually change anything */ + rec = mail_index_lookup(_mail->transaction->view_pvt, mail->seq_pvt); + old_pvt_flags = rec->flags & mailbox_get_private_flags_mask(_mail->box); + + switch (modify_type) { + case MODIFY_ADD: + return (old_pvt_flags & pvt_flags) != pvt_flags; + case MODIFY_REPLACE: + return old_pvt_flags != pvt_flags; + case MODIFY_REMOVE: + return (old_pvt_flags & pvt_flags) != 0; + } + i_unreached(); +} + void index_mail_update_flags(struct mail *_mail, enum modify_type modify_type, enum mail_flags flags) { @@ -1672,8 +1700,7 @@ pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box); pvt_flags = flags & pvt_flags_mask; flags &= ~pvt_flags_mask; - if (index_mail_get_pvt(_mail) && - (pvt_flags != 0 || modify_type == MODIFY_REPLACE)) { + if (index_mail_update_pvt_flags(_mail, modify_type, pvt_flags)) { mail_index_update_flags(_mail->transaction->itrans_pvt, mail->seq_pvt, modify_type, pvt_flags); From dovecot at dovecot.org Wed Nov 28 07:25:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 07:25:28 +0200 Subject: dovecot-2.2: lib-storage: Notify about \Seen flag changes in pri... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e123e31de340 changeset: 15429:e123e31de340 user: Timo Sirainen date: Wed Nov 28 07:25:18 2012 +0200 description: lib-storage: Notify about \Seen flag changes in private index. Some earlier "don't send unnecessary flag changes" optimizations caused this. diffstat: src/lib-storage/index/index-sync-private.h | 4 +++- src/lib-storage/index/index-sync-pvt.c | 27 ++++++++++++++++++++++++--- src/lib-storage/index/index-sync.c | 18 ++++++++++++------ 3 files changed, 39 insertions(+), 10 deletions(-) diffs (102 lines): diff -r 9107b794b7ad -r e123e31de340 src/lib-storage/index/index-sync-private.h --- a/src/lib-storage/index/index-sync-private.h Wed Nov 28 07:24:01 2012 +0200 +++ b/src/lib-storage/index/index-sync-private.h Wed Nov 28 07:25:18 2012 +0200 @@ -22,6 +22,8 @@ void index_sync_search_results_update(struct index_mailbox_sync_context *ctx); void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx); -int index_storage_mailbox_sync_pvt(struct mailbox *box); +int index_storage_mailbox_sync_pvt(struct mailbox *box, + ARRAY_TYPE(seq_range) *flag_updates, + ARRAY_TYPE(seq_range) *hidden_updates); #endif diff -r 9107b794b7ad -r e123e31de340 src/lib-storage/index/index-sync-pvt.c --- a/src/lib-storage/index/index-sync-pvt.c Wed Nov 28 07:24:01 2012 +0200 +++ b/src/lib-storage/index/index-sync-pvt.c Wed Nov 28 07:25:18 2012 +0200 @@ -201,9 +201,13 @@ return ret; } -int index_storage_mailbox_sync_pvt(struct mailbox *box) +int index_storage_mailbox_sync_pvt(struct mailbox *box, + ARRAY_TYPE(seq_range) *flag_updates, + ARRAY_TYPE(seq_range) *hidden_updates) { struct mail_index_view_sync_ctx *view_sync_ctx; + struct mail_index_view_sync_rec sync_rec; + uint32_t seq1, seq2; enum mail_flags pvt_flags; bool delayed_expunges; int ret; @@ -220,8 +224,25 @@ return -1; /* sync the private view */ - view_sync_ctx = mail_index_view_sync_begin(box->view_pvt, - MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + view_sync_ctx = mail_index_view_sync_begin(box->view_pvt, 0); + while (mail_index_view_sync_next(view_sync_ctx, &sync_rec)) { + if (sync_rec.type != MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS) + continue; + + /* *_updates contains box->view sequences (not view_pvt + sequences) */ + if (mail_index_lookup_seq_range(box->view, + sync_rec.uid1, sync_rec.uid2, + &seq1, &seq2)) { + if (!sync_rec.hidden) { + seq_range_array_add_range(flag_updates, + seq1, seq2); + } else { + seq_range_array_add_range(hidden_updates, + seq1, seq2); + } + } + } if (mail_index_view_sync_commit(&view_sync_ctx, &delayed_expunges) < 0) return -1; return 0; diff -r 9107b794b7ad -r e123e31de340 src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Wed Nov 28 07:24:01 2012 +0200 +++ b/src/lib-storage/index/index-sync.c Wed Nov 28 07:25:18 2012 +0200 @@ -162,7 +162,11 @@ break; } } +} +static void +index_view_sync_cleanup_updates(struct index_mailbox_sync_context *ctx) +{ /* remove expunged messages from flag updates */ if (ctx->expunges != NULL) { seq_range_array_remove_seq_range(&ctx->flag_updates, @@ -210,6 +214,14 @@ } index_view_sync_recs_get(ctx); index_sync_search_results_expunge(ctx); + + /* sync private index if needed. it doesn't use box->view, so it + doesn't matter if it's called at _sync_init() or _sync_deinit(). + however we also need to know if any private flags have changed + since last sync, so we need to call it before _sync_next() calls. */ + (void)index_storage_mailbox_sync_pvt(box, &ctx->flag_updates, + &ctx->hidden_updates); + index_view_sync_cleanup_updates(ctx); return &ctx->ctx; } @@ -384,12 +396,6 @@ if (status_r != NULL) status_r->sync_delayed_expunges = delayed_expunges; - /* sync private index if needed. do this after real sync to make sure - that all the new messages are added to the private index, so their - flags can be updated. */ - if (ret == 0) - (void)index_storage_mailbox_sync_pvt(_ctx->box); - /* update search results after private index is updated */ index_sync_search_results_update(ctx); From dovecot at dovecot.org Wed Nov 28 09:14:49 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 28 Nov 2012 09:14:49 +0200 Subject: dovecot-2.2: dsync: Don't crash when syncing multiple namespaces. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22799e820158 changeset: 15430:22799e820158 user: Timo Sirainen date: Wed Nov 28 09:14:33 2012 +0200 description: dsync: Don't crash when syncing multiple namespaces. diffstat: src/doveadm/dsync/dsync-mailbox-tree.c | 13 +++++++------ 1 files changed, 7 insertions(+), 6 deletions(-) diffs (37 lines): diff -r e123e31de340 -r 22799e820158 src/doveadm/dsync/dsync-mailbox-tree.c --- a/src/doveadm/dsync/dsync-mailbox-tree.c Wed Nov 28 07:25:18 2012 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree.c Wed Nov 28 09:14:33 2012 +0200 @@ -301,7 +301,9 @@ return 0; old_node = hash_table_lookup(tree->guid_hash, guid); - if (old_node != NULL) { + if (old_node == NULL) + hash_table_insert(tree->guid_hash, guid, node); + else if (old_node != node) { i_error("Duplicate mailbox GUID %s " "for mailboxes %s and %s", guid_128_to_string(node->mailbox_guid), @@ -309,7 +311,6 @@ dsync_mailbox_node_get_full_name(tree, node)); return -1; } - hash_table_insert(tree->guid_hash, guid, node); return 0; } @@ -320,10 +321,10 @@ const char *name; int ret = 0; - i_assert(!hash_table_is_created(tree->guid_hash)); - - hash_table_create(&tree->guid_hash, tree->pool, 0, - guid_128_hash, guid_128_cmp); + if (!hash_table_is_created(tree->guid_hash)) { + hash_table_create(&tree->guid_hash, tree->pool, 0, + guid_128_hash, guid_128_cmp); + } iter = dsync_mailbox_tree_iter_init(tree); while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { if (dsync_mailbox_tree_guid_hash_add(tree, node) < 0) From dovecot at dovecot.org Thu Nov 29 01:57:50 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 01:57:50 +0200 Subject: dovecot-2.2: lib-http: http_url_parse() now returns the http_url... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c9f6deb65d7b changeset: 15431:c9f6deb65d7b user: Timo Sirainen date: Thu Nov 29 01:57:40 2012 +0200 description: lib-http: http_url_parse() now returns the http_url allocated from given memory pool. diffstat: src/lib-http/http-client-request.c | 3 ++- src/lib-http/http-url.c | 6 +++--- src/lib-http/http-url.h | 2 +- src/lib-http/test-http-url.c | 8 +++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diffs (81 lines): diff -r 22799e820158 -r c9f6deb65d7b src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Wed Nov 28 09:14:33 2012 +0200 +++ b/src/lib-http/http-client-request.c Thu Nov 29 01:57:40 2012 +0200 @@ -320,7 +320,8 @@ unsigned int newport; /* parse URL */ - if (http_url_parse(location, NULL, 0, &url, &error) < 0) { + if (http_url_parse(location, NULL, 0, + pool_datastack_create(), &url, &error) < 0) { http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, t_strdup_printf("Invalid redirect location: %s", error)); return; diff -r 22799e820158 -r c9f6deb65d7b src/lib-http/http-url.c --- a/src/lib-http/http-url.c Wed Nov 28 09:14:33 2012 +0200 +++ b/src/lib-http/http-url.c Thu Nov 29 01:57:40 2012 +0200 @@ -209,7 +209,7 @@ /* Public API */ int http_url_parse(const char *url, struct http_url *base, - enum http_url_parse_flags flags, + enum http_url_parse_flags flags, pool_t pool, struct http_url **url_r, const char **error_r) { struct http_url_parser url_parser; @@ -219,9 +219,9 @@ i_assert((flags & HTTP_URL_PARSE_SCHEME_EXTERNAL) == 0 || base == NULL); memset(&url_parser, '\0', sizeof(url_parser)); - uri_parser_init(&url_parser.parser, pool_datastack_create(), url); + uri_parser_init(&url_parser.parser, pool, url); - url_parser.url = t_new(struct http_url, 1); + url_parser.url = p_new(pool, struct http_url, 1); url_parser.base = base; url_parser.flags = flags; diff -r 22799e820158 -r c9f6deb65d7b src/lib-http/http-url.h --- a/src/lib-http/http-url.h Wed Nov 28 09:14:33 2012 +0200 +++ b/src/lib-http/http-url.h Thu Nov 29 01:57:40 2012 +0200 @@ -34,7 +34,7 @@ }; int http_url_parse(const char *url, struct http_url *base, - enum http_url_parse_flags flags, + enum http_url_parse_flags flags, pool_t pool, struct http_url **url_r, const char **error_r); /* diff -r 22799e820158 -r c9f6deb65d7b src/lib-http/test-http-url.c --- a/src/lib-http/test-http-url.c Wed Nov 28 09:14:33 2012 +0200 +++ b/src/lib-http/test-http-url.c Thu Nov 29 01:57:40 2012 +0200 @@ -269,7 +269,7 @@ test_begin(t_strdup_printf("http url valid [%d]", i)); if (urlb->host_name == NULL) urlb = NULL; - if (http_url_parse(url, urlb, flags, &urlp, &error) < 0) + if (http_url_parse(url, urlb, flags, pool_datastack_create(), &urlp, &error) < 0) urlp = NULL; test_out_reason(t_strdup_printf("http_url_parse(%s)", @@ -396,7 +396,8 @@ test_begin(t_strdup_printf("http url invalid [%d]", i)); - if (http_url_parse(url, urlb, flags, &urlp, &error) < 0) + if (http_url_parse(url, urlb, flags, + pool_datastack_create(), &urlp, &error) < 0) urlp = NULL; test_out_reason(t_strdup_printf("parse %s", url), urlp == NULL, error); @@ -435,7 +436,8 @@ test_begin(t_strdup_printf("http url parse/create [%d]", i)); if (http_url_parse - (url, NULL, HTTP_URL_ALLOW_FRAGMENT_PART, &urlp, &error) < 0) + (url, NULL, HTTP_URL_ALLOW_FRAGMENT_PART, + pool_datastack_create(), &urlp, &error) < 0) urlp = NULL; test_out_reason(t_strdup_printf("parse %s", url), urlp != NULL, error); if (urlp != NULL) { From dovecot at dovecot.org Thu Nov 29 02:02:19 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 02:02:19 +0200 Subject: dovecot-2.2: lib-http: http_url_parse() didn't allocate all stri... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/12a31e13fcab changeset: 15432:12a31e13fcab user: Timo Sirainen date: Thu Nov 29 02:01:56 2012 +0200 description: lib-http: http_url_parse() didn't allocate all strings from the given pool. diffstat: src/lib-http/http-url.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (24 lines): diff -r c9f6deb65d7b -r 12a31e13fcab src/lib-http/http-url.c --- a/src/lib-http/http-url.c Thu Nov 29 01:57:40 2012 +0200 +++ b/src/lib-http/http-url.c Thu Nov 29 02:01:56 2012 +0200 @@ -94,7 +94,7 @@ } if (ret > 0 && url != NULL) { - url->host_name = auth.host_literal; + url->host_name = p_strdup(parser->pool, auth.host_literal); url->host_ip = auth.host_ip; url->have_host_ip = auth.have_host_ip; url->port = auth.port; @@ -166,9 +166,9 @@ } if (url != NULL) - url->path = str_c(fullpath); + url->path = p_strdup(parser->pool, str_c(fullpath)); } else if (relative && url != NULL) { - url->path = base->path; + url->path = p_strdup(parser->pool, base->path); } /* [ "?" query ] */ From dovecot at dovecot.org Thu Nov 29 02:13:04 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 02:13:04 +0200 Subject: dovecot-2.2: lib-http: http-url.h always requires net.h to compi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dfd7a3ea9df3 changeset: 15433:dfd7a3ea9df3 user: Timo Sirainen date: Thu Nov 29 02:12:54 2012 +0200 description: lib-http: http-url.h always requires net.h to compile, #include it diffstat: src/lib-http/http-url.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 12a31e13fcab -r dfd7a3ea9df3 src/lib-http/http-url.h --- a/src/lib-http/http-url.h Thu Nov 29 02:01:56 2012 +0200 +++ b/src/lib-http/http-url.h Thu Nov 29 02:12:54 2012 +0200 @@ -1,6 +1,8 @@ #ifndef HTTP_URL_H #define HTTP_URL_H +#include "net.h" + struct http_url { /* server */ const char *host_name; From dovecot at dovecot.org Thu Nov 29 02:30:26 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 02:30:26 +0200 Subject: dovecot-2.2: uri-util.h always requires net.h to compile, #inclu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/309c9fe1f0a9 changeset: 15434:309c9fe1f0a9 user: Timo Sirainen date: Thu Nov 29 02:30:15 2012 +0200 description: uri-util.h always requires net.h to compile, #include it diffstat: src/lib/uri-util.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r dfd7a3ea9df3 -r 309c9fe1f0a9 src/lib/uri-util.h --- a/src/lib/uri-util.h Thu Nov 29 02:12:54 2012 +0200 +++ b/src/lib/uri-util.h Thu Nov 29 02:30:15 2012 +0200 @@ -1,6 +1,8 @@ #ifndef URI_UTIL_H #define URI_UTIL_H +#include "net.h" + /* * Generic URI parsing. */ From dovecot at dovecot.org Thu Nov 29 06:26:44 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 06:26:44 +0200 Subject: dovecot-2.2: JSON parser supports now nested objects and parsing... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/86572582647e changeset: 15435:86572582647e user: Timo Sirainen date: Thu Nov 29 06:26:29 2012 +0200 description: JSON parser supports now nested objects and parsing from non-blocking input stream. diffstat: src/auth/db-dict.c | 14 +- src/lib/Makefile.am | 1 + src/lib/json-parser.c | 295 ++++++++++++++++++++++++++++++++------------ src/lib/json-parser.h | 19 +- src/lib/test-json-parser.c | 98 ++++++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 7 files changed, 335 insertions(+), 94 deletions(-) diffs (truncated from 688 to 300 lines): diff -r 309c9fe1f0a9 -r 86572582647e src/auth/db-dict.c --- a/src/auth/db-dict.c Thu Nov 29 02:30:15 2012 +0200 +++ b/src/auth/db-dict.c Thu Nov 29 06:26:29 2012 +0200 @@ -5,6 +5,7 @@ #include "settings.h" #include "dict.h" #include "json-parser.h" +#include "istream.h" #include "str.h" #include "auth-request.h" #include "auth-worker-client.h" @@ -122,6 +123,7 @@ db_dict_value_iter_init(struct dict_connection *conn, const char *value) { struct db_dict_value_iter *iter; + struct istream *input; i_assert(strcmp(conn->set.value_format, "json") == 0); @@ -129,7 +131,9 @@ value types are supported. */ iter = i_new(struct db_dict_value_iter, 1); iter->key = str_new(default_pool, 64); - iter->parser = json_parser_init((const void *)value, strlen(value)); + input = i_stream_create_from_data(value, strlen(value)); + iter->parser = json_parser_init(input); + i_stream_unref(&input); return iter; } @@ -139,7 +143,7 @@ enum json_type type; const char *value; - if (!json_parse_next(iter->parser, &type, &value)) + if (json_parse_next(iter->parser, &type, &value) < 0) return FALSE; if (type != JSON_TYPE_OBJECT_KEY) { iter->error = "Object expected"; @@ -152,10 +156,14 @@ str_truncate(iter->key, 0); str_append(iter->key, value); - if (!json_parse_next(iter->parser, &type, &value)) { + if (json_parse_next(iter->parser, &type, &value) < 0) { iter->error = "Missing value"; return FALSE; } + if (type == JSON_TYPE_OBJECT) { + iter->error = "Nested objects not supported"; + return FALSE; + } *key_r = str_c(iter->key); *value_r = value; return TRUE; diff -r 309c9fe1f0a9 -r 86572582647e src/lib/Makefile.am --- a/src/lib/Makefile.am Thu Nov 29 02:30:15 2012 +0200 +++ b/src/lib/Makefile.am Thu Nov 29 06:26:29 2012 +0200 @@ -275,6 +275,7 @@ test-istream-crlf.c \ test-istream-seekable.c \ test-istream-tee.c \ + test-json-parser.c \ test-llist.c \ test-mempool-alloconly.c \ test-network.c \ diff -r 309c9fe1f0a9 -r 86572582647e src/lib/json-parser.c --- a/src/lib/json-parser.c Thu Nov 29 02:30:15 2012 +0200 +++ b/src/lib/json-parser.c Thu Nov 29 06:26:29 2012 +0200 @@ -2,12 +2,14 @@ #include "lib.h" #include "str.h" +#include "istream.h" #include "hex-dec.h" #include "unichar.h" #include "json-parser.h" enum json_state { JSON_STATE_ROOT = 0, + JSON_STATE_OBJECT_OPEN, JSON_STATE_OBJECT_KEY, JSON_STATE_OBJECT_COLON, JSON_STATE_OBJECT_VALUE, @@ -16,22 +18,81 @@ }; struct json_parser { - const unsigned char *data, *end; + struct istream *input; + uoff_t highwater_offset; + + const unsigned char *start, *end, *data; const char *error; string_t *value; enum json_state state; + unsigned int nested_object_count; }; -struct json_parser * -json_parser_init(const unsigned char *data, unsigned int len) +static int +json_try_parse_next(struct json_parser *parser, enum json_type *type_r, + const char **value_r); + +static int json_parser_read_more(struct json_parser *parser) +{ + uoff_t cur_highwater = parser->input->v_offset + + i_stream_get_data_size(parser->input); + size_t size; + ssize_t ret; + + i_assert(parser->highwater_offset <= cur_highwater); + + if (parser->error != NULL) + return -1; + + if (parser->highwater_offset == cur_highwater) { + ret = i_stream_read(parser->input); + if (ret == -2) { + parser->error = "Token too large"; + return -1; + } + if (ret <= 0) + return ret; + + cur_highwater = parser->input->v_offset + + i_stream_get_data_size(parser->input); + i_assert(parser->highwater_offset < cur_highwater); + parser->highwater_offset = cur_highwater; + } + + parser->start = parser->data = i_stream_get_data(parser->input, &size); + parser->end = parser->start + size; + i_assert(size > 0); + return 1; +} + +static void json_parser_update_input_pos(struct json_parser *parser) +{ + size_t size; + + if (parser->data == parser->start) + return; + + i_stream_skip(parser->input, parser->data - parser->start); + parser->start = parser->data = i_stream_get_data(parser->input, &size); + parser->end = parser->start + size; + if (size > 0) { + /* we skipped over some data and there's still data left. + no need to read() the next time. */ + parser->highwater_offset = 0; + } else { + parser->highwater_offset = parser->input->v_offset; + } +} + +struct json_parser *json_parser_init(struct istream *input) { struct json_parser *parser; parser = i_new(struct json_parser, 1); - parser->data = data; - parser->end = data + len; + parser->input = input; parser->value = str_new(default_pool, 128); + i_stream_ref(input); return parser; } @@ -41,12 +102,22 @@ *_parser = NULL; - if (parser->error == NULL && parser->data == parser->end && - parser->state != JSON_STATE_ROOT && - parser->state != JSON_STATE_DONE) - parser->error = "Missing '}'"; - - *error_r = parser->error; + if (parser->error != NULL) { + /* actual parser error */ + *error_r = parser->error; + } else if (parser->input->stream_errno != 0) { + *error_r = t_strdup_printf("read(%s) failed: %m", + i_stream_get_name(parser->input)); + } else if (parser->data == parser->end && + !i_stream_have_bytes_left(parser->input) && + parser->state != JSON_STATE_ROOT && + parser->state != JSON_STATE_DONE) { + *error_r = "Missing '}'"; + } else { + *error_r = NULL; + } + + i_stream_unref(&parser->input); str_free(&parser->value); i_free(parser); return *error_r != NULL ? -1 : 0; @@ -62,34 +133,35 @@ case '\n': break; default: + json_parser_update_input_pos(parser); return TRUE; } } + json_parser_update_input_pos(parser); return FALSE; } static int json_parse_string(struct json_parser *parser, const char **value_r) { - const unsigned char *p; - if (*parser->data != '"') return -1; + parser->data++; str_truncate(parser->value, 0); - for (p = parser->data + 1; p < parser->end; p++) { - if (*p == '"') { - parser->data = p + 1; + for (; parser->data != parser->end; parser->data++) { + if (*parser->data == '"') { + parser->data++; *value_r = str_c(parser->value); return 0; } - if (*p != '\\') - str_append_c(parser->value, *p); + if (*parser->data != '\\') + str_append_c(parser->value, *parser->data); else { - switch (*++p) { + switch (*++parser->data) { case '"': case '\\': case '/': - str_append_c(parser->value, *p); + str_append_c(parser->value, *parser->data); break; case 'b': str_append_c(parser->value, '\b'); @@ -107,11 +179,11 @@ str_append_c(parser->value, '\t'); break; case 'u': - if (parser->end - p < 4) + if (parser->end - parser->data < 4) return -1; - uni_ucs4_to_utf8_c(hex2dec(p, 4), + uni_ucs4_to_utf8_c(hex2dec(parser->data, 4), parser->value); - p += 3; + parser->data += 3; break; default: return -1; @@ -122,99 +194,131 @@ } static int -json_parse_digits(struct json_parser *parser, const unsigned char **_p) +json_parse_digits(struct json_parser *parser) { - const unsigned char *p = *_p; - - if (p >= parser->end || *p < '0' || *p > '9') + if (parser->data == parser->end || + *parser->data < '0' || *parser->data > '9') return -1; - for (; p < parser->end && *p >= '0' && *p <= '9'; p++) - str_append_c(parser->value, *p++); - *_p = p; + while (parser->data != parser->end && + *parser->data >= '0' && *parser->data <= '9') + str_append_c(parser->value, *parser->data++); return 0; } -static int json_parse_int(struct json_parser *parser, const unsigned char **_p) +static int json_parse_int(struct json_parser *parser) { - const unsigned char *p = *_p; - - if (*p == '-') { - str_append_c(parser->value, *p++); - if (p == parser->end) + if (*parser->data == '-') { + str_append_c(parser->value, *parser->data++); + if (parser->data == parser->end) return -1; } - if (*p == '0') - str_append_c(parser->value, *p++); + if (*parser->data == '0') + str_append_c(parser->value, *parser->data++); From dovecot at dovecot.org Thu Nov 29 07:31:26 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 07:31:26 +0200 Subject: dovecot-2.2: JSON parser fixes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9a31c44c1184 changeset: 15436:9a31c44c1184 user: Timo Sirainen date: Thu Nov 29 07:30:23 2012 +0200 description: JSON parser fixes diffstat: src/lib/json-parser.c | 89 +++++++++++++++++++++++++++------------------ src/lib/test-json-parser.c | 9 +++- 2 files changed, 59 insertions(+), 39 deletions(-) diffs (238 lines): diff -r 86572582647e -r 9a31c44c1184 src/lib/json-parser.c --- a/src/lib/json-parser.c Thu Nov 29 06:26:29 2012 +0200 +++ b/src/lib/json-parser.c Thu Nov 29 07:30:23 2012 +0200 @@ -148,11 +148,11 @@ parser->data++; str_truncate(parser->value, 0); - for (; parser->data != parser->end; parser->data++) { + for (; parser->data != parser->end; parser->data++) { if (*parser->data == '"') { parser->data++; *value_r = str_c(parser->value); - return 0; + return 1; } if (*parser->data != '\\') str_append_c(parser->value, *parser->data); @@ -179,8 +179,12 @@ str_append_c(parser->value, '\t'); break; case 'u': - if (parser->end - parser->data < 4) - return -1; + parser->data++; + if (parser->end - parser->data < 4) { + /* wait for more data */ + parser->data = parser->end; + return 0; + } uni_ucs4_to_utf8_c(hex2dec(parser->data, 4), parser->value); parser->data += 3; @@ -190,64 +194,69 @@ } } } - return -1; + return 0; } static int json_parse_digits(struct json_parser *parser) { - if (parser->data == parser->end || - *parser->data < '0' || *parser->data > '9') + if (parser->data == parser->end) + return 0; + if (*parser->data < '0' || *parser->data > '9') return -1; while (parser->data != parser->end && *parser->data >= '0' && *parser->data <= '9') str_append_c(parser->value, *parser->data++); - return 0; + return 1; } static int json_parse_int(struct json_parser *parser) { + int ret; + if (*parser->data == '-') { str_append_c(parser->value, *parser->data++); if (parser->data == parser->end) - return -1; + return 0; } if (*parser->data == '0') str_append_c(parser->value, *parser->data++); else { - if (json_parse_digits(parser) < 0) - return -1; + if ((ret = json_parse_digits(parser)) <= 0) + return ret; } - return 0; + return 1; } static int json_parse_number(struct json_parser *parser, const char **value_r) { + int ret; + str_truncate(parser->value, 0); - if (json_parse_int(parser) < 0) - return -1; + if ((ret = json_parse_int(parser)) <= 0) + return ret; if (parser->data != parser->end && *parser->data == '.') { /* frac */ str_append_c(parser->value, *parser->data++); - if (json_parse_digits(parser) < 0) - return -1; + if ((ret = json_parse_digits(parser)) <= 0) + return ret; } if (parser->data != parser->end && (*parser->data == 'e' || *parser->data == 'E')) { /* exp */ str_append_c(parser->value, *parser->data++); if (parser->data == parser->end) - return -1; + return 0; if (*parser->data == '+' || *parser->data == '-') str_append_c(parser->value, *parser->data++); - if (json_parse_digits(parser) < 0) - return -1; + if ((ret = json_parse_digits(parser)) <= 0) + return ret; } if (parser->data == parser->end && !parser->input->eof) - return -1; + return 0; *value_r = str_c(parser->value); - return 0; + return 1; } static int json_parse_atom(struct json_parser *parser, const char *atom) @@ -256,16 +265,17 @@ avail = parser->end - parser->data; if (avail < len) { - if (memcmp(parser->data, atom, avail) == 0) { - /* everything matches so far, but we need more data */ - parser->data += avail; - } - return -1; + if (memcmp(parser->data, atom, avail) != 0) + return -1; + + /* everything matches so far, but we need more data */ + parser->data += avail; + return 0; } if (memcmp(parser->data, atom, len) != 0) return -1; parser->data += len; - return 0; + return 1; } static int @@ -290,6 +300,8 @@ json_try_parse_next(struct json_parser *parser, enum json_type *type_r, const char **value_r) { + int ret; + if (!json_parse_whitespace(parser)) return -1; @@ -304,11 +316,7 @@ json_parser_update_input_pos(parser); return json_try_parse_next(parser, type_r, value_r); case JSON_STATE_OBJECT_VALUE: - if (json_parse_string(parser, value_r) == 0) - *type_r = JSON_TYPE_STRING; - else if (json_parse_number(parser, value_r) == 0) - *type_r = JSON_TYPE_NUMBER; - else if (*parser->data == '[') { + if (*parser->data == '[') { parser->error = "Arrays not supported"; return -1; } else if (*parser->data == '{') { @@ -318,19 +326,28 @@ json_parser_update_input_pos(parser); *type_r = JSON_TYPE_OBJECT; return 0; - } else if (json_parse_atom(parser, "true") == 0) { + } + if ((ret = json_parse_string(parser, value_r)) >= 0) { + *type_r = JSON_TYPE_STRING; + } else if ((ret = json_parse_number(parser, value_r)) >= 0) { + *type_r = JSON_TYPE_NUMBER; + } else if ((ret = json_parse_atom(parser, "true")) >= 0) { *type_r = JSON_TYPE_TRUE; *value_r = "true"; - } else if (json_parse_atom(parser, "false") == 0) { + } else if ((ret = json_parse_atom(parser, "false")) >= 0) { *type_r = JSON_TYPE_FALSE; *value_r = "false"; - } else if (json_parse_atom(parser, "null") == 0) { + } else if ((ret = json_parse_atom(parser, "null")) >= 0) { *type_r = JSON_TYPE_NULL; *value_r = NULL; } else { parser->error = "Invalid data as value"; return -1; } + if (ret == 0) { + i_assert(parser->data == parser->end); + return -1; + } parser->state = parser->state == JSON_STATE_ROOT ? JSON_STATE_DONE : JSON_STATE_OBJECT_VALUE_NEXT; @@ -341,7 +358,7 @@ parser->state = JSON_STATE_OBJECT_KEY; /* fall through */ case JSON_STATE_OBJECT_KEY: - if (json_parse_string(parser, value_r) < 0) { + if (json_parse_string(parser, value_r) <= 0) { parser->error = "Expected string as object key"; return -1; } diff -r 86572582647e -r 9a31c44c1184 src/lib/test-json-parser.c --- a/src/lib/test-json-parser.c Thu Nov 29 06:26:29 2012 +0200 +++ b/src/lib/test-json-parser.c Thu Nov 29 07:30:23 2012 +0200 @@ -20,7 +20,8 @@ " \"sub2\":-12.456,\n" " \"sub3\":12.456e9,\n" " \"sub4\":0.456e-789" - "}" + "}," + "\"key9\": \"\\\\\\\"\\b\\f\\n\\r\\t\\u0001\uffff\"" "}\n"; static struct { @@ -53,7 +54,9 @@ { JSON_TYPE_NUMBER, "12.456e9" }, { JSON_TYPE_OBJECT_KEY, "sub4" }, { JSON_TYPE_NUMBER, "0.456e-789" }, - { JSON_TYPE_OBJECT_END, NULL } + { JSON_TYPE_OBJECT_END, NULL }, + { JSON_TYPE_OBJECT_KEY, "key9" }, + { JSON_TYPE_STRING, "\\\"\b\f\n\r\t\001\xef\xbf\xbf" } }; static void test_json_parser_success(bool full_size) @@ -93,6 +96,6 @@ void test_json_parser(void) { + test_json_parser_success(FALSE); test_json_parser_success(TRUE); - test_json_parser_success(FALSE); } From dovecot at dovecot.org Thu Nov 29 07:52:58 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 07:52:58 +0200 Subject: dovecot-2.2: JSON parser: Added support for reading string value... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/014be18f7130 changeset: 15437:014be18f7130 user: Timo Sirainen date: Thu Nov 29 07:52:51 2012 +0200 description: JSON parser: Added support for reading string value as istream. diffstat: src/lib/Makefile.am | 2 + src/lib/istream-jsonstr.c | 179 +++++++++++++++++++++++++++++++++++++++++++++ src/lib/istream-jsonstr.h | 7 + src/lib/json-parser.c | 103 +++++++++++++++++++++++++ src/lib/json-parser.h | 6 + src/lib/test-json-parser.c | 49 ++++++++++- 6 files changed, 340 insertions(+), 6 deletions(-) diffs (truncated from 475 to 300 lines): diff -r 9a31c44c1184 -r 014be18f7130 src/lib/Makefile.am --- a/src/lib/Makefile.am Thu Nov 29 07:30:23 2012 +0200 +++ b/src/lib/Makefile.am Thu Nov 29 07:52:51 2012 +0200 @@ -62,6 +62,7 @@ istream-crlf.c \ istream-data.c \ istream-file.c \ + istream-jsonstr.c \ istream-limit.c \ istream-mmap.c \ istream-rawlog.c \ @@ -187,6 +188,7 @@ istream-chain.h \ istream-concat.h \ istream-crlf.h \ + istream-jsonstr.h \ istream-private.h \ istream-rawlog.h \ istream-seekable.h \ diff -r 9a31c44c1184 -r 014be18f7130 src/lib/istream-jsonstr.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-jsonstr.c Thu Nov 29 07:52:51 2012 +0200 @@ -0,0 +1,179 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "hex-dec.h" +#include "unichar.h" +#include "istream-private.h" +#include "istream-jsonstr.h" + +#define MAX_UTF8_LEN 6 + +struct jsonstr_istream { + struct istream_private istream; + + /* The end '"' was found */ + unsigned int str_end:1; +}; + +static int +i_stream_jsonstr_read_parent(struct jsonstr_istream *jstream, + unsigned int min_bytes) +{ + struct istream_private *stream = &jstream->istream; + size_t size, avail; + ssize_t ret; + + size = i_stream_get_data_size(stream->parent); + while (size < min_bytes) { + ret = i_stream_read(stream->parent); + if (ret <= 0 && (ret != -2 || stream->skip == 0)) { + stream->istream.stream_errno = + stream->parent->stream_errno; + stream->istream.eof = stream->parent->eof; + return ret; + } + size = i_stream_get_data_size(stream->parent); + } + + if (!i_stream_try_alloc(stream, size, &avail)) + return -2; + return 1; +} + +static int +i_stream_json_unescape(const unsigned char *src, unsigned char *dest, + unsigned int *src_size_r, unsigned int *dest_size_r) +{ + switch (*src) { + case '"': + case '\\': + case '/': + *dest = *src; + break; + case 'b': + *dest = '\b'; + break; + case 'f': + *dest = '\f'; + break; + case 'n': + *dest = '\n'; + break; + case 'r': + *dest = '\r'; + break; + case 't': + *dest = '\t'; + break; + case 'u': { + buffer_t buf; + + buffer_create_from_data(&buf, dest, MAX_UTF8_LEN); + uni_ucs4_to_utf8_c(hex2dec(src+1, 4), &buf); + *src_size_r = 5; + *dest_size_r = buf.used; + return 0; + } + default: + return -1; + } + *src_size_r = 1; + *dest_size_r = 1; + return 0; +} + +static ssize_t i_stream_jsonstr_read(struct istream_private *stream) +{ + struct jsonstr_istream *jstream = (struct jsonstr_istream *)stream; + const unsigned char *data; + unsigned int srcskip, destskip, extra; + size_t i, dest, size; + ssize_t ret; + + if (jstream->str_end) { + stream->istream.eof = TRUE; + return -1; + } + + ret = i_stream_jsonstr_read_parent(jstream, 1); + if (ret <= 0) + return ret; + + /* @UNSAFE */ + dest = stream->pos; + extra = 0; + + data = i_stream_get_data(stream->parent, &size); + for (i = 0; i < size && dest < stream->buffer_size; ) { + if (data[i] == '"') { + jstream->str_end = TRUE; + if (dest == stream->pos) { + stream->istream.eof = TRUE; + return -1; + } + break; + } else if (data[i] == '\\') { + if (i+1 == size) { + /* not enough input for \x */ + extra = 1; + break; + } + if ((data[i+1] == 'u' && i+1+4 >= size)) { + /* not enough input for \u0000 */ + extra = 5; + break; + } + if (data[i+1] == 'u' && stream->buffer_size - dest < MAX_UTF8_LEN) { + /* UTF8 output is max. 6 chars */ + if (dest == stream->pos) + return -2; + break; + } + i++; + if (i_stream_json_unescape(data + i, + stream->w_buffer + dest, + &srcskip, &destskip) < 0) { + /* invalid string */ + stream->istream.stream_errno = EINVAL; + return -1; + } + i += srcskip; + i_assert(i <= size); + dest += destskip; + i_assert(dest <= stream->buffer_size); + } else { + stream->w_buffer[dest++] = data[i]; + i++; + } + } + i_stream_skip(stream->parent, i); + + ret = dest - stream->pos; + if (ret == 0) { + /* not enough input */ + i_assert(extra > 0); + ret = i_stream_jsonstr_read_parent(jstream, i+extra+1); + if (ret <= 0) + return ret; + return i_stream_jsonstr_read(stream); + } + i_assert(ret > 0); + stream->pos = dest; + return ret; +} + +struct istream *i_stream_create_jsonstr(struct istream *input) +{ + struct jsonstr_istream *dstream; + + dstream = i_new(struct jsonstr_istream, 1); + dstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + dstream->istream.read = i_stream_jsonstr_read; + + dstream->istream.istream.readable_fd = FALSE; + dstream->istream.istream.blocking = input->blocking; + dstream->istream.istream.seekable = FALSE; + return i_stream_create(&dstream->istream, input, + i_stream_get_fd(input)); +} diff -r 9a31c44c1184 -r 014be18f7130 src/lib/istream-jsonstr.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-jsonstr.h Thu Nov 29 07:52:51 2012 +0200 @@ -0,0 +1,7 @@ +#ifndef ISTREAM_JSONSTR_H +#define ISTREAM_JSONSTR_H + +/* Parse input until '"' is reached. Unescape JSON \x codes. */ +struct istream *i_stream_create_jsonstr(struct istream *input); + +#endif diff -r 9a31c44c1184 -r 014be18f7130 src/lib/json-parser.c --- a/src/lib/json-parser.c Thu Nov 29 07:30:23 2012 +0200 +++ b/src/lib/json-parser.c Thu Nov 29 07:52:51 2012 +0200 @@ -5,6 +5,7 @@ #include "istream.h" #include "hex-dec.h" #include "unichar.h" +#include "istream-jsonstr.h" #include "json-parser.h" enum json_state { @@ -14,6 +15,7 @@ JSON_STATE_OBJECT_COLON, JSON_STATE_OBJECT_VALUE, JSON_STATE_OBJECT_VALUE_NEXT, + JSON_STATE_STRINPUT_FINISH, JSON_STATE_DONE }; @@ -24,6 +26,7 @@ const unsigned char *start, *end, *data; const char *error; string_t *value; + struct istream *strinput; enum json_state state; unsigned int nested_object_count; @@ -197,6 +200,37 @@ return 0; } +static int json_skip_string(struct json_parser *parser) +{ + for (; parser->data != parser->end; parser->data++) { + if (*parser->data == '"') { + parser->data++; + return 0; + } + if (*parser->data == '\\') { + switch (*++parser->data) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + break; + case 'u': + if (parser->end - parser->data < 4) + return -1; + parser->data += 3; + break; + default: + return -1; + } + } + } + return -1; +} + static int json_parse_digits(struct json_parser *parser) { @@ -385,6 +419,11 @@ parser->data++; json_parser_update_input_pos(parser); return json_try_parse_next(parser, type_r, value_r); + case JSON_STATE_STRINPUT_FINISH: + if (json_skip_string(parser) < 0) + return -1; + parser->state = JSON_STATE_OBJECT_VALUE_NEXT; + return json_try_parse_next(parser, type_r, value_r); case JSON_STATE_DONE: parser->error = "Unexpected data at the end"; return -1; @@ -398,6 +437,8 @@ { int ret; + i_assert(parser->strinput == NULL); + *value_r = NULL; while ((ret = json_parser_read_more(parser)) > 0) { @@ -412,3 +453,65 @@ From dovecot at dovecot.org Thu Nov 29 08:36:01 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 08:36:01 +0200 Subject: dovecot-2.1: doveadm: Don't truncate printed output that contain... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/6fde42ec5c81 changeset: 14830:6fde42ec5c81 user: Timo Sirainen date: Thu Nov 29 08:35:51 2012 +0200 description: doveadm: Don't truncate printed output that contain NUL bytes. doveadm fetch text output especially could have been truncated. diffstat: src/doveadm/doveadm-print-flow.c | 2 +- src/doveadm/doveadm-print-pager.c | 2 +- src/doveadm/doveadm-print-tab.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diffs (36 lines): diff -r c81f1ca3cda6 -r 6fde42ec5c81 src/doveadm/doveadm-print-flow.c --- a/src/doveadm/doveadm-print-flow.c Wed Nov 28 04:29:58 2012 +0200 +++ b/src/doveadm/doveadm-print-flow.c Thu Nov 29 08:35:51 2012 +0200 @@ -63,7 +63,7 @@ if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) printf("%s=", hdr->title); } - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); if (size == 0) { flow_next_hdr(); ctx->streaming = FALSE; diff -r c81f1ca3cda6 -r 6fde42ec5c81 src/doveadm/doveadm-print-pager.c --- a/src/doveadm/doveadm-print-pager.c Wed Nov 28 04:29:58 2012 +0200 +++ b/src/doveadm/doveadm-print-pager.c Thu Nov 29 08:35:51 2012 +0200 @@ -56,7 +56,7 @@ ctx->streaming = TRUE; printf("%s:\n", hdr->title); } - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); if (size == 0) { pager_next_hdr(); ctx->streaming = FALSE; diff -r c81f1ca3cda6 -r 6fde42ec5c81 src/doveadm/doveadm-print-tab.c --- a/src/doveadm/doveadm-print-tab.c Wed Nov 28 04:29:58 2012 +0200 +++ b/src/doveadm/doveadm-print-tab.c Thu Nov 29 08:35:51 2012 +0200 @@ -51,7 +51,7 @@ } if (ctx.header_idx > 0) printf("\t"); - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); } static void doveadm_print_tab_flush(void) From dovecot at dovecot.org Thu Nov 29 08:45:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 08:45:39 +0200 Subject: dovecot-2.1: Released v2.1.11. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/741d800a192f changeset: 14831:741d800a192f user: Timo Sirainen date: Thu Nov 29 08:41:21 2012 +0200 description: Released v2.1.11. diffstat: NEWS | 19 +++++++++++++++++++ configure.in | 2 +- 2 files changed, 20 insertions(+), 1 deletions(-) diffs (36 lines): diff -r 6fde42ec5c81 -r 741d800a192f NEWS --- a/NEWS Thu Nov 29 08:35:51 2012 +0200 +++ b/NEWS Thu Nov 29 08:41:21 2012 +0200 @@ -1,3 +1,22 @@ +v2.1.11 2012-11-29 Timo Sirainen + + * lmtp/lda: dovecot.index.cache file is no longer fully mapped to + memory, allowing mail deliveries to work even if the file is huge. + * auth: userdb passwd lookups are now done by auth worker processes + instead of auth master process (as it was documented, but + accidentally didn't work that way). + + + lmtp: lmtp_rcpt_check_quota=yes setting checks quota on RCPT TO. + - lmtp: After successful proxying RCPT TO, the next one to a + nonexistent user gave tempfail error instead of "user not found". + - lmtp proxy: Fixed hanging if remote server was down. + - imap: Fixed crash when SEARCH contained multiple KEYWORD parameters. + - doveadm: Various fixes to handling doveadm-server connections. + - -i parameter for Dovecot tools didn't work correctly. + - director was somewhat broken in v2.1.10. This version also includes + various reliability enhancements. + - auth: passdb imap was broken in v2.1.10. + v2.1.10 2012-09-18 Timo Sirainen + imap: Implemented THREAD=ORDEREDSUBJECT extension. diff -r 6fde42ec5c81 -r 741d800a192f configure.in --- a/configure.in Thu Nov 29 08:35:51 2012 +0200 +++ b/configure.in Thu Nov 29 08:41:21 2012 +0200 @@ -1,5 +1,5 @@ AC_PREREQ([2.59]) -AC_INIT([Dovecot],[2.1.10],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.1.11],[dovecot at dovecot.org]) AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) From dovecot at dovecot.org Thu Nov 29 08:45:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 08:45:39 +0200 Subject: dovecot-2.1: Added tag 2.1.11 for changeset 741d800a192f Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/15ec2d078a17 changeset: 14832:15ec2d078a17 user: Timo Sirainen date: Thu Nov 29 08:41:21 2012 +0200 description: Added tag 2.1.11 for changeset 741d800a192f diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 741d800a192f -r 15ec2d078a17 .hgtags --- a/.hgtags Thu Nov 29 08:41:21 2012 +0200 +++ b/.hgtags Thu Nov 29 08:41:21 2012 +0200 @@ -87,3 +87,4 @@ 7e5f36fd989d27a2fb48250adbab8fa54ddb6083 2.1.8 bc86680293d256d5a8009690caeb73ab2e34e359 2.1.9 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 +741d800a192fa23572bb14196df2a8917cf20614 2.1.11 From dovecot at dovecot.org Thu Nov 29 08:45:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 08:45:39 +0200 Subject: dovecot-2.1: Added signature for changeset 741d800a192f Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b101277a3d26 changeset: 14833:b101277a3d26 user: Timo Sirainen date: Thu Nov 29 08:41:28 2012 +0200 description: Added signature for changeset 741d800a192f diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 15ec2d078a17 -r b101277a3d26 .hgsigs --- a/.hgsigs Thu Nov 29 08:41:21 2012 +0200 +++ b/.hgsigs Thu Nov 29 08:41:28 2012 +0200 @@ -50,3 +50,4 @@ 7e5f36fd989d27a2fb48250adbab8fa54ddb6083 0 iEYEABECAAYFAk/yVakACgkQyUhSUUBVismekwCfSEVQjd6fwdChjd53LSt03b4UWKoAoIxd/IjLatTISlHm44iiQwzRKByo bc86680293d256d5a8009690caeb73ab2e34e359 0 iEYEABECAAYFAlAZaTUACgkQyUhSUUBVisnTAACfU1pB34RrXEyLnpnL4Ee/oeNBYcoAnRWxTqx870Efjwf+eBPzafO0C/NU 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT +741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy From dovecot at dovecot.org Thu Nov 29 09:38:57 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 09:38:57 +0200 Subject: dovecot-2.2: JSON parser: Added json_parse_skip_next() to skip o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4f7e0be9cc35 changeset: 15438:4f7e0be9cc35 user: Timo Sirainen date: Thu Nov 29 09:38:34 2012 +0200 description: JSON parser: Added json_parse_skip_next() to skip over unwanted values. diffstat: src/lib/json-parser.c | 148 ++++++++++++++++++++++++++++---------------- src/lib/json-parser.h | 4 +- src/lib/test-json-parser.c | 17 ++++- 3 files changed, 110 insertions(+), 59 deletions(-) diffs (truncated from 356 to 300 lines): diff -r 014be18f7130 -r 4f7e0be9cc35 src/lib/json-parser.c --- a/src/lib/json-parser.c Thu Nov 29 07:52:51 2012 +0200 +++ b/src/lib/json-parser.c Thu Nov 29 09:38:34 2012 +0200 @@ -15,7 +15,7 @@ JSON_STATE_OBJECT_COLON, JSON_STATE_OBJECT_VALUE, JSON_STATE_OBJECT_VALUE_NEXT, - JSON_STATE_STRINPUT_FINISH, + JSON_STATE_SKIP_STRING, JSON_STATE_DONE }; @@ -30,12 +30,10 @@ enum json_state state; unsigned int nested_object_count; + unsigned int nested_skip_count; + bool skipping; }; -static int -json_try_parse_next(struct json_parser *parser, enum json_type *type_r, - const char **value_r); - static int json_parser_read_more(struct json_parser *parser) { uoff_t cur_highwater = parser->input->v_offset + @@ -144,12 +142,51 @@ return FALSE; } -static int json_parse_string(struct json_parser *parser, const char **value_r) +static int json_skip_string(struct json_parser *parser) +{ + for (; parser->data != parser->end; parser->data++) { + if (*parser->data == '"') { + parser->data++; + json_parser_update_input_pos(parser); + return 1; + } + if (*parser->data == '\\') { + switch (*++parser->data) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + break; + case 'u': + if (parser->end - parser->data < 4) + return -1; + parser->data += 3; + break; + default: + return -1; + } + } + } + json_parser_update_input_pos(parser); + return 0; +} + +static int json_parse_string(struct json_parser *parser, bool allow_skip, + const char **value_r) { if (*parser->data != '"') return -1; parser->data++; + if (parser->skipping && allow_skip) { + *value_r = NULL; + return json_skip_string(parser); + } + str_truncate(parser->value, 0); for (; parser->data != parser->end; parser->data++) { if (*parser->data == '"') { @@ -200,37 +237,6 @@ return 0; } -static int json_skip_string(struct json_parser *parser) -{ - for (; parser->data != parser->end; parser->data++) { - if (*parser->data == '"') { - parser->data++; - return 0; - } - if (*parser->data == '\\') { - switch (*++parser->data) { - case '"': - case '\\': - case '/': - case 'b': - case 'f': - case 'n': - case 'r': - case 't': - break; - case 'u': - if (parser->end - parser->data < 4) - return -1; - parser->data += 3; - break; - default: - return -1; - } - } - } - return -1; -} - static int json_parse_digits(struct json_parser *parser) { @@ -313,8 +319,7 @@ } static int -json_parse_object_close(struct json_parser *parser, enum json_type *type_r, - const char **value_r) +json_parse_object_close(struct json_parser *parser, enum json_type *type_r) { parser->data++; json_parser_update_input_pos(parser); @@ -323,17 +328,22 @@ /* closing a nested object */ parser->nested_object_count--; parser->state = JSON_STATE_OBJECT_VALUE_NEXT; + if (parser->nested_skip_count > 0) { + parser->nested_skip_count--; + return 0; + } *type_r = JSON_TYPE_OBJECT_END; - return 0; + return 1; } parser->state = JSON_STATE_DONE; - return json_try_parse_next(parser, type_r, value_r); + return 0; } static int json_try_parse_next(struct json_parser *parser, enum json_type *type_r, const char **value_r) { + bool skipping = parser->skipping; int ret; if (!json_parse_whitespace(parser)) @@ -348,7 +358,7 @@ parser->data++; parser->state = JSON_STATE_OBJECT_OPEN; json_parser_update_input_pos(parser); - return json_try_parse_next(parser, type_r, value_r); + return 0; case JSON_STATE_OBJECT_VALUE: if (*parser->data == '[') { parser->error = "Arrays not supported"; @@ -358,10 +368,15 @@ parser->state = JSON_STATE_OBJECT_OPEN; parser->nested_object_count++; json_parser_update_input_pos(parser); + + if (parser->skipping) { + parser->nested_skip_count++; + return 0; + } *type_r = JSON_TYPE_OBJECT; - return 0; + return 1; } - if ((ret = json_parse_string(parser, value_r)) >= 0) { + if ((ret = json_parse_string(parser, TRUE, value_r)) >= 0) { *type_r = JSON_TYPE_STRING; } else if ((ret = json_parse_number(parser, value_r)) >= 0) { *type_r = JSON_TYPE_NUMBER; @@ -380,6 +395,12 @@ } if (ret == 0) { i_assert(parser->data == parser->end); + if (parser->skipping && *type_r == JSON_TYPE_STRING) { + /* a large string that we want to skip over. */ + json_parser_update_input_pos(parser); + parser->state = JSON_STATE_SKIP_STRING; + return 0; + } return -1; } parser->state = parser->state == JSON_STATE_ROOT ? @@ -388,11 +409,11 @@ break; case JSON_STATE_OBJECT_OPEN: if (*parser->data == '}') - return json_parse_object_close(parser, type_r, value_r); + return json_parse_object_close(parser, type_r); parser->state = JSON_STATE_OBJECT_KEY; /* fall through */ case JSON_STATE_OBJECT_KEY: - if (json_parse_string(parser, value_r) <= 0) { + if (json_parse_string(parser, FALSE, value_r) <= 0) { parser->error = "Expected string as object key"; return -1; } @@ -407,10 +428,14 @@ parser->data++; parser->state = JSON_STATE_OBJECT_VALUE; json_parser_update_input_pos(parser); - return json_try_parse_next(parser, type_r, value_r); + return 0; case JSON_STATE_OBJECT_VALUE_NEXT: + if (parser->skipping && parser->nested_skip_count == 0) { + /* we skipped over the previous value */ + parser->skipping = FALSE; + } if (*parser->data == '}') - return json_parse_object_close(parser, type_r, value_r); + return json_parse_object_close(parser, type_r); if (*parser->data != ',') { parser->error = "Expected ',' or '}' after object value"; return -1; @@ -418,18 +443,18 @@ parser->state = JSON_STATE_OBJECT_KEY; parser->data++; json_parser_update_input_pos(parser); - return json_try_parse_next(parser, type_r, value_r); - case JSON_STATE_STRINPUT_FINISH: - if (json_skip_string(parser) < 0) + return 0; + case JSON_STATE_SKIP_STRING: + if (json_skip_string(parser) <= 0) return -1; parser->state = JSON_STATE_OBJECT_VALUE_NEXT; - return json_try_parse_next(parser, type_r, value_r); + return 0; case JSON_STATE_DONE: parser->error = "Unexpected data at the end"; return -1; } json_parser_update_input_pos(parser); - return 0; + return skipping ? 0 : 1; } int json_parse_next(struct json_parser *parser, enum json_type *type_r, @@ -442,7 +467,9 @@ *value_r = NULL; while ((ret = json_parser_read_more(parser)) > 0) { - if (json_try_parse_next(parser, type_r, value_r) == 0) + while ((ret = json_try_parse_next(parser, type_r, value_r)) == 0) + ; + if (ret > 0) break; if (parser->data != parser->end) return -1; @@ -454,6 +481,16 @@ return ret; } +void json_parse_skip_next(struct json_parser *parser) +{ + i_assert(!parser->skipping); + i_assert(parser->strinput == NULL); + i_assert(parser->state == JSON_STATE_OBJECT_COLON || + parser->state == JSON_STATE_OBJECT_VALUE); + + parser->skipping = TRUE; +} + static void json_strinput_destroyed(struct json_parser *parser) { i_assert(parser->strinput != NULL); @@ -484,7 +521,7 @@ parser->data++; json_parser_update_input_pos(parser); - parser->state = JSON_STATE_STRINPUT_FINISH; + parser->state = JSON_STATE_SKIP_STRING; parser->strinput = i_stream_create_jsonstr(parser->input); i_stream_set_destroy_callback(parser->strinput, json_strinput_destroyed, parser); @@ -498,6 +535,7 @@ { int ret; + i_assert(!parser->skipping); i_assert(parser->strinput == NULL); i_assert(parser->state == JSON_STATE_OBJECT_COLON || parser->state == JSON_STATE_OBJECT_VALUE); diff -r 014be18f7130 -r 4f7e0be9cc35 src/lib/json-parser.h --- a/src/lib/json-parser.h Thu Nov 29 07:52:51 2012 +0200 +++ b/src/lib/json-parser.h Thu Nov 29 09:38:34 2012 +0200 @@ -25,7 +25,9 @@ non-blocking and needs more input, -1 if input stream is at EOF. */ int json_parse_next(struct json_parser *parser, enum json_type *type_r, const char **value_r); From dovecot at dovecot.org Thu Nov 29 10:29:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 10:29:31 +0200 Subject: dovecot-2.2: JSON parser: Added support for arrays. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f456fffeec5c changeset: 15439:f456fffeec5c user: Timo Sirainen date: Thu Nov 29 10:29:19 2012 +0200 description: JSON parser: Added support for arrays. The parser should be complete now. diffstat: src/lib/json-parser.c | 143 +++++++++++++++++++++++++++++++++----------- src/lib/json-parser.h | 6 +- src/lib/test-json-parser.c | 24 ++++++- 3 files changed, 133 insertions(+), 40 deletions(-) diffs (truncated from 344 to 300 lines): diff -r 4f7e0be9cc35 -r f456fffeec5c src/lib/json-parser.c --- a/src/lib/json-parser.c Thu Nov 29 09:38:34 2012 +0200 +++ b/src/lib/json-parser.c Thu Nov 29 10:29:19 2012 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "str.h" #include "istream.h" #include "hex-dec.h" @@ -14,8 +15,12 @@ JSON_STATE_OBJECT_KEY, JSON_STATE_OBJECT_COLON, JSON_STATE_OBJECT_VALUE, - JSON_STATE_OBJECT_VALUE_NEXT, - JSON_STATE_SKIP_STRING, + JSON_STATE_OBJECT_SKIP_STRING, + JSON_STATE_OBJECT_NEXT, + JSON_STATE_ARRAY_OPEN, + JSON_STATE_ARRAY_VALUE, + JSON_STATE_ARRAY_SKIP_STRING, + JSON_STATE_ARRAY_NEXT, JSON_STATE_DONE }; @@ -29,7 +34,7 @@ struct istream *strinput; enum json_state state; - unsigned int nested_object_count; + ARRAY(enum json_state) nesting; unsigned int nested_skip_count; bool skipping; }; @@ -93,6 +98,7 @@ parser = i_new(struct json_parser, 1); parser->input = input; parser->value = str_new(default_pool, 128); + i_array_init(&parser->nesting, 8); i_stream_ref(input); return parser; } @@ -111,7 +117,6 @@ i_stream_get_name(parser->input)); } else if (parser->data == parser->end && !i_stream_have_bytes_left(parser->input) && - parser->state != JSON_STATE_ROOT && parser->state != JSON_STATE_DONE) { *error_r = "Missing '}'"; } else { @@ -119,6 +124,7 @@ } i_stream_unref(&parser->input); + array_free(&parser->nesting); str_free(&parser->value); i_free(parser); return *error_r != NULL ? -1 : 0; @@ -318,25 +324,55 @@ return 1; } -static int -json_parse_object_close(struct json_parser *parser, enum json_type *type_r) +static int json_parse_denest(struct json_parser *parser) { + const enum json_state *nested_states; + unsigned count; + parser->data++; json_parser_update_input_pos(parser); - if (parser->nested_object_count > 0) { - /* closing a nested object */ - parser->nested_object_count--; - parser->state = JSON_STATE_OBJECT_VALUE_NEXT; - if (parser->nested_skip_count > 0) { - parser->nested_skip_count--; - return 0; - } - *type_r = JSON_TYPE_OBJECT_END; - return 1; + nested_states = array_get(&parser->nesting, &count); + if (count == 0) { + /* closing root */ + parser->state = JSON_STATE_DONE; + return 0; } - parser->state = JSON_STATE_DONE; - return 0; + + /* closing a nested object */ + if (count == 1) { + /* we're back to root */ + parser->state = JSON_STATE_OBJECT_NEXT; + } else { + /* back to previous nested object */ + parser->state = nested_states[count-2] == JSON_STATE_OBJECT_OPEN ? + JSON_STATE_OBJECT_NEXT : JSON_STATE_ARRAY_NEXT; + } + array_delete(&parser->nesting, count-1, 1); + + if (parser->nested_skip_count > 0) { + parser->nested_skip_count--; + return 0; + } + return 1; +} + +static int +json_parse_close_object(struct json_parser *parser, enum json_type *type_r) +{ + if (json_parse_denest(parser) == 0) + return 0; + *type_r = JSON_TYPE_OBJECT_END; + return 1; +} + +static int +json_parse_close_array(struct json_parser *parser, enum json_type *type_r) +{ + if (json_parse_denest(parser) == 0) + return 0; + *type_r = JSON_TYPE_ARRAY_END; + return 1; } static int @@ -360,13 +396,11 @@ json_parser_update_input_pos(parser); return 0; case JSON_STATE_OBJECT_VALUE: - if (*parser->data == '[') { - parser->error = "Arrays not supported"; - return -1; - } else if (*parser->data == '{') { + case JSON_STATE_ARRAY_VALUE: + if (*parser->data == '{') { parser->data++; parser->state = JSON_STATE_OBJECT_OPEN; - parser->nested_object_count++; + array_append(&parser->nesting, &parser->state, 1); json_parser_update_input_pos(parser); if (parser->skipping) { @@ -375,7 +409,20 @@ } *type_r = JSON_TYPE_OBJECT; return 1; + } else if (*parser->data == '[') { + parser->data++; + parser->state = JSON_STATE_ARRAY_OPEN; + array_append(&parser->nesting, &parser->state, 1); + json_parser_update_input_pos(parser); + + if (parser->skipping) { + parser->nested_skip_count++; + return 0; + } + *type_r = JSON_TYPE_ARRAY; + return 1; } + if ((ret = json_parse_string(parser, TRUE, value_r)) >= 0) { *type_r = JSON_TYPE_STRING; } else if ((ret = json_parse_number(parser, value_r)) >= 0) { @@ -398,18 +445,19 @@ if (parser->skipping && *type_r == JSON_TYPE_STRING) { /* a large string that we want to skip over. */ json_parser_update_input_pos(parser); - parser->state = JSON_STATE_SKIP_STRING; + parser->state = parser->state == JSON_STATE_OBJECT_VALUE ? + JSON_STATE_OBJECT_SKIP_STRING : + JSON_STATE_ARRAY_SKIP_STRING; return 0; } return -1; } - parser->state = parser->state == JSON_STATE_ROOT ? - JSON_STATE_DONE : - JSON_STATE_OBJECT_VALUE_NEXT; + parser->state = parser->state == JSON_STATE_OBJECT_VALUE ? + JSON_STATE_OBJECT_NEXT : JSON_STATE_ARRAY_NEXT; break; case JSON_STATE_OBJECT_OPEN: if (*parser->data == '}') - return json_parse_object_close(parser, type_r); + return json_parse_close_object(parser, type_r); parser->state = JSON_STATE_OBJECT_KEY; /* fall through */ case JSON_STATE_OBJECT_KEY: @@ -429,13 +477,13 @@ parser->state = JSON_STATE_OBJECT_VALUE; json_parser_update_input_pos(parser); return 0; - case JSON_STATE_OBJECT_VALUE_NEXT: + case JSON_STATE_OBJECT_NEXT: if (parser->skipping && parser->nested_skip_count == 0) { /* we skipped over the previous value */ parser->skipping = FALSE; } if (*parser->data == '}') - return json_parse_object_close(parser, type_r); + return json_parse_close_object(parser, type_r); if (*parser->data != ',') { parser->error = "Expected ',' or '}' after object value"; return -1; @@ -444,10 +492,32 @@ parser->data++; json_parser_update_input_pos(parser); return 0; - case JSON_STATE_SKIP_STRING: + case JSON_STATE_ARRAY_OPEN: + if (*parser->data == ']') + return json_parse_close_array(parser, type_r); + parser->state = JSON_STATE_ARRAY_VALUE; + return 0; + case JSON_STATE_ARRAY_NEXT: + if (parser->skipping && parser->nested_skip_count == 0) { + /* we skipped over the previous value */ + parser->skipping = FALSE; + } + if (*parser->data == ']') + return json_parse_close_array(parser, type_r); + if (*parser->data != ',') { + parser->error = "Expected ',' or '}' after array value"; + return -1; + } + parser->state = JSON_STATE_ARRAY_VALUE; + parser->data++; + json_parser_update_input_pos(parser); + return 0; + case JSON_STATE_OBJECT_SKIP_STRING: + case JSON_STATE_ARRAY_SKIP_STRING: if (json_skip_string(parser) <= 0) return -1; - parser->state = JSON_STATE_OBJECT_VALUE_NEXT; + parser->state = parser->state == JSON_STATE_OBJECT_SKIP_STRING ? + JSON_STATE_OBJECT_NEXT : JSON_STATE_ARRAY_NEXT; return 0; case JSON_STATE_DONE: parser->error = "Unexpected data at the end"; @@ -486,7 +556,8 @@ i_assert(!parser->skipping); i_assert(parser->strinput == NULL); i_assert(parser->state == JSON_STATE_OBJECT_COLON || - parser->state == JSON_STATE_OBJECT_VALUE); + parser->state == JSON_STATE_OBJECT_VALUE || + parser->state == JSON_STATE_ARRAY_VALUE); parser->skipping = TRUE; } @@ -521,7 +592,8 @@ parser->data++; json_parser_update_input_pos(parser); - parser->state = JSON_STATE_SKIP_STRING; + parser->state = parser->state == JSON_STATE_OBJECT_VALUE ? + JSON_STATE_OBJECT_SKIP_STRING : JSON_STATE_ARRAY_SKIP_STRING; parser->strinput = i_stream_create_jsonstr(parser->input); i_stream_set_destroy_callback(parser->strinput, json_strinput_destroyed, parser); @@ -538,7 +610,8 @@ i_assert(!parser->skipping); i_assert(parser->strinput == NULL); i_assert(parser->state == JSON_STATE_OBJECT_COLON || - parser->state == JSON_STATE_OBJECT_VALUE); + parser->state == JSON_STATE_OBJECT_VALUE || + parser->state == JSON_STATE_ARRAY_VALUE); *input_r = NULL; diff -r 4f7e0be9cc35 -r f456fffeec5c src/lib/json-parser.h --- a/src/lib/json-parser.h Thu Nov 29 09:38:34 2012 +0200 +++ b/src/lib/json-parser.h Thu Nov 29 10:29:19 2012 +0200 @@ -9,6 +9,9 @@ /* } (not returned for the root object) */ JSON_TYPE_OBJECT_END, + JSON_TYPE_ARRAY, + JSON_TYPE_ARRAY_END, + JSON_TYPE_STRING, JSON_TYPE_NUMBER, JSON_TYPE_TRUE, @@ -16,8 +19,7 @@ JSON_TYPE_NULL }; -/* Parse JSON tokens from the input stream. Currently arrays aren't - supported. */ +/* Parse JSON tokens from the input stream. */ struct json_parser *json_parser_init(struct istream *input); int json_parser_deinit(struct json_parser **parser, const char **error_r); diff -r 4f7e0be9cc35 -r f456fffeec5c src/lib/test-json-parser.c --- a/src/lib/test-json-parser.c Thu Nov 29 09:38:34 2012 +0200 +++ b/src/lib/test-json-parser.c Thu Nov 29 10:29:19 2012 +0200 @@ -15,7 +15,7 @@ From dovecot at dovecot.org Thu Nov 29 12:44:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 29 Nov 2012 12:44:05 +0200 Subject: dovecot-2.1: Makefile: dovecot-config was missing quotes in LIBD... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/f77e87298a66 changeset: 14834:f77e87298a66 user: Timo Sirainen date: Thu Nov 29 12:43:54 2012 +0200 description: Makefile: dovecot-config was missing quotes in LIBDOVECOT_LOGIN diffstat: Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b101277a3d26 -r f77e87298a66 Makefile.am --- a/Makefile.am Thu Nov 29 08:41:28 2012 +0200 +++ b/Makefile.am Thu Nov 29 12:43:54 2012 +0200 @@ -63,7 +63,7 @@ grep -v '^LIBDOVECOT_.*_INCLUDE' dovecot-config | \ grep -v '^LIBDOVECOT.*_DEPS' | sed \ -e "s|^\(LIBDOVECOT\)=.*$$|\1='-L$(pkglibdir) -ldovecot'|" \ - -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login $(SSL_LIBS)|" \ + -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1='-ldovecot-login $(SSL_LIBS)'|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ From dovecot at dovecot.org Fri Nov 30 03:14:21 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 30 Nov 2012 03:14:21 +0200 Subject: dovecot-2.1: expire plugin: Don't crash when doveadm is run with... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/75dc4cb4bfe0 changeset: 14835:75dc4cb4bfe0 user: Timo Sirainen date: Fri Nov 30 03:14:10 2012 +0200 description: expire plugin: Don't crash when doveadm is run with proxying diffstat: src/doveadm/client-connection.c | 1 + src/doveadm/doveadm-mail.h | 2 ++ src/plugins/expire/doveadm-expire.c | 5 ++++- 3 files changed, 7 insertions(+), 1 deletions(-) diffs (38 lines): diff -r f77e87298a66 -r 75dc4cb4bfe0 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Thu Nov 29 12:43:54 2012 +0200 +++ b/src/doveadm/client-connection.c Fri Nov 30 03:14:10 2012 +0200 @@ -57,6 +57,7 @@ ctx = doveadm_mail_cmd_init(cmd, set); ctx->full_args = (const void *)(argv + 1); + ctx->proxying = TRUE; ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | diff -r f77e87298a66 -r 75dc4cb4bfe0 src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Thu Nov 29 12:43:54 2012 +0200 +++ b/src/doveadm/doveadm-mail.h Fri Nov 30 03:14:10 2012 +0200 @@ -62,6 +62,8 @@ /* if non-zero, exit with this code */ int exit_code; + /* This command is being called by a remote doveadm client. */ + unsigned int proxying:1; /* We're handling only a single user */ unsigned int iterate_single_user:1; /* We're going through all users (not set for wildcard usernames) */ diff -r f77e87298a66 -r 75dc4cb4bfe0 src/plugins/expire/doveadm-expire.c --- a/src/plugins/expire/doveadm-expire.c Thu Nov 29 12:43:54 2012 +0200 +++ b/src/plugins/expire/doveadm-expire.c Fri Nov 30 03:14:10 2012 +0200 @@ -374,7 +374,10 @@ if (expire_dict == NULL) return; - if (ctx->iterate_single_user) { + /* doveadm proxying uses expire database only locally. the remote + doveadm handles each user one at a time (even though + iterate_single_user=FALSE) */ + if (ctx->iterate_single_user || ctx->proxying) { if (doveadm_debug) { i_debug("expire: Iterating only a single user, " "ignoring expire database"); From dovecot at dovecot.org Fri Nov 30 09:01:56 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 30 Nov 2012 09:01:56 +0200 Subject: dovecot-2.1: Released v2.1.12. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/75bfda4a7c6c changeset: 14836:75bfda4a7c6c user: Timo Sirainen date: Fri Nov 30 08:56:44 2012 +0200 description: Released v2.1.12. diffstat: NEWS | 4 ++++ configure.in | 2 +- 2 files changed, 5 insertions(+), 1 deletions(-) diffs (21 lines): diff -r 75dc4cb4bfe0 -r 75bfda4a7c6c NEWS --- a/NEWS Fri Nov 30 03:14:10 2012 +0200 +++ b/NEWS Fri Nov 30 08:56:44 2012 +0200 @@ -1,3 +1,7 @@ +v2.1.12 2012-11-30 Timo Sirainen + + - dovecot-config in v2.1.11 caused build problems with Pigeonhole + v2.1.11 2012-11-29 Timo Sirainen * lmtp/lda: dovecot.index.cache file is no longer fully mapped to diff -r 75dc4cb4bfe0 -r 75bfda4a7c6c configure.in --- a/configure.in Fri Nov 30 03:14:10 2012 +0200 +++ b/configure.in Fri Nov 30 08:56:44 2012 +0200 @@ -1,5 +1,5 @@ AC_PREREQ([2.59]) -AC_INIT([Dovecot],[2.1.11],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.1.12],[dovecot at dovecot.org]) AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) From dovecot at dovecot.org Fri Nov 30 09:01:56 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 30 Nov 2012 09:01:56 +0200 Subject: dovecot-2.1: Added tag 2.1.12 for changeset 75bfda4a7c6c Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/23ed55ce16d0 changeset: 14837:23ed55ce16d0 user: Timo Sirainen date: Fri Nov 30 08:56:44 2012 +0200 description: Added tag 2.1.12 for changeset 75bfda4a7c6c diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 75bfda4a7c6c -r 23ed55ce16d0 .hgtags --- a/.hgtags Fri Nov 30 08:56:44 2012 +0200 +++ b/.hgtags Fri Nov 30 08:56:44 2012 +0200 @@ -88,3 +88,4 @@ bc86680293d256d5a8009690caeb73ab2e34e359 2.1.9 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 741d800a192fa23572bb14196df2a8917cf20614 2.1.11 +75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 From dovecot at dovecot.org Fri Nov 30 09:01:56 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 30 Nov 2012 09:01:56 +0200 Subject: dovecot-2.1: Added signature for changeset 75bfda4a7c6c Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b60872b87d26 changeset: 14838:b60872b87d26 user: Timo Sirainen date: Fri Nov 30 08:56:55 2012 +0200 description: Added signature for changeset 75bfda4a7c6c diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 23ed55ce16d0 -r b60872b87d26 .hgsigs --- a/.hgsigs Fri Nov 30 08:56:44 2012 +0200 +++ b/.hgsigs Fri Nov 30 08:56:55 2012 +0200 @@ -51,3 +51,4 @@ bc86680293d256d5a8009690caeb73ab2e34e359 0 iEYEABECAAYFAlAZaTUACgkQyUhSUUBVisnTAACfU1pB34RrXEyLnpnL4Ee/oeNBYcoAnRWxTqx870Efjwf+eBPzafO0C/NU 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT 741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy +75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99