From dovecot at dovecot.org Sat Dec 1 02:41:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 01 Dec 2012 02:41:46 +0200 Subject: dovecot-2.2: lib-http: Don't free connection immediately in payl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9cab24687819 changeset: 15440:9cab24687819 user: Timo Sirainen date: Sat Dec 01 02:41:34 2012 +0200 description: lib-http: Don't free connection immediately in payload-destroyed callback. This caused a crash in http_client_connection_return_response() after the stream was unrefed. diffstat: src/lib-http/http-client-connection.c | 17 +++++++++++------ 1 files changed, 11 insertions(+), 6 deletions(-) diffs (40 lines): diff -r f456fffeec5c -r 9cab24687819 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Thu Nov 29 10:29:19 2012 +0200 +++ b/src/lib-http/http-client-connection.c Sat Dec 01 02:41:34 2012 +0200 @@ -293,6 +293,16 @@ http_client_connection_input, &conn->conn); } +static void +http_client_payload_destroyed_timeout(struct http_client_connection *conn) +{ + if (conn->close_indicated) { + http_client_connection_server_close(&conn); + return; + } + http_client_connection_input(&conn->conn); +} + static void http_client_payload_destroyed(struct http_client_connection *conn) { i_assert(conn->incoming_payload != NULL); @@ -310,18 +320,13 @@ http_client_request_finish(&conn->pending_request); conn->pending_request = NULL; - if (conn->close_indicated) { - http_client_connection_server_close(&conn); - return; - } - /* input stream may have pending input. make sure input handler gets called (but don't do it directly, since we get get here somewhere from the API user's code, which we can't really know what state it is in). this call also triggers sending a new request if necessary. */ conn->to_input = - timeout_add_short(0, http_client_connection_input, &conn->conn); + timeout_add_short(0, http_client_payload_destroyed_timeout, conn); } static bool From dovecot at dovecot.org Sat Dec 1 06:55:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 01 Dec 2012 06:55:05 +0200 Subject: dovecot-2.2: Added json_append_escaped() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/009287402687 changeset: 15441:009287402687 user: Timo Sirainen date: Sat Dec 01 06:54:55 2012 +0200 description: Added json_append_escaped() diffstat: src/lib/json-parser.c | 35 +++++++++++++++++++++++++++++++++++ src/lib/json-parser.h | 3 +++ 2 files changed, 38 insertions(+), 0 deletions(-) diffs (53 lines): diff -r 9cab24687819 -r 009287402687 src/lib/json-parser.c --- a/src/lib/json-parser.c Sat Dec 01 02:41:34 2012 +0200 +++ b/src/lib/json-parser.c Sat Dec 01 06:54:55 2012 +0200 @@ -626,3 +626,38 @@ } return ret; } + +void json_append_escaped(string_t *dest, const char *src) +{ + for (; *src != '\0'; src++) { + switch (*src) { + case '\b': + str_append(dest, "\\b"); + break; + case '\f': + str_append(dest, "\\f"); + break; + case '\n': + str_append(dest, "\\n"); + break; + case '\r': + str_append(dest, "\\r"); + break; + case '\t': + str_append(dest, "\\t"); + break; + case '"': + str_append(dest, "\\\""); + break; + case '\\': + str_append(dest, "\\\\"); + break; + default: + if (*src < 32) + str_printfa(dest, "\\u%04x", *src); + else + str_append_c(dest, *src); + break; + } + } +} diff -r 9cab24687819 -r 009287402687 src/lib/json-parser.h --- a/src/lib/json-parser.h Sat Dec 01 02:41:34 2012 +0200 +++ b/src/lib/json-parser.h Sat Dec 01 06:54:55 2012 +0200 @@ -36,4 +36,7 @@ int json_parse_next_stream(struct json_parser *parser, struct istream **input_r); +/* Append data to already opened JSON string. */ +void json_append_escaped(string_t *dest, const char *src); + #endif From dovecot at dovecot.org Sat Dec 1 06:56:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 01 Dec 2012 06:56:06 +0200 Subject: dovecot-2.2: lib-fs: Added fs-test to easily test FS API calls. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cea9a269311a changeset: 15442:cea9a269311a user: Timo Sirainen date: Sat Dec 01 06:56:02 2012 +0200 description: lib-fs: Added fs-test to easily test FS API calls. diffstat: .hgignore | 1 + src/lib-fs/Makefile.am | 12 +++ src/lib-fs/fs-test.c | 155 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 0 deletions(-) diffs (197 lines): diff -r 009287402687 -r cea9a269311a .hgignore --- a/.hgignore Sat Dec 01 06:54:55 2012 +0200 +++ b/.hgignore Sat Dec 01 06:56:02 2012 +0200 @@ -77,6 +77,7 @@ src/ipc/ipc src/lib/unicodemap.c src/lib/UnicodeData.txt +src/lib-fs/fs-test src/lib-dict/dict-drivers-register.c src/lib-sql/sql-drivers-register.c src/lib-storage/register/mail-storage-register.c diff -r 009287402687 -r cea9a269311a src/lib-fs/Makefile.am --- a/src/lib-fs/Makefile.am Sat Dec 01 06:54:55 2012 +0200 +++ b/src/lib-fs/Makefile.am Sat Dec 01 06:56:02 2012 +0200 @@ -1,5 +1,7 @@ noinst_LTLIBRARIES = libfs.la +noinst_PROGRAMS = fs-test + AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -DMODULE_DIR=\""$(moduledir)"\" @@ -18,5 +20,15 @@ fs-sis-common.h \ ostream-cmp.h +fs_test_SOURCES = fs-test.c +fs_test_LDFLAGS = -export-dynamic +fs_test_DEPENDENCIES = \ + libfs.la \ + ../lib-http/libhttp.la \ + ../lib-ssl-iostream/libssl_iostream.la \ + ../lib-dns/libdns.la \ + ../lib/liblib.la +fs_test_LDADD = $(fs_test_DEPENDENCIES) $(MODULE_LIBS) + pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) diff -r 009287402687 -r cea9a269311a src/lib-fs/fs-test.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fs/fs-test.c Sat Dec 01 06:56:02 2012 +0200 @@ -0,0 +1,155 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "fs-api.h" + +#include + +static const struct fs_settings fs_set = { + .temp_dir = "/tmp" +}; + +static void fs_test_file_get(struct fs *fs, const char *path) +{ + struct fs_file *file; + struct istream *input; + const unsigned char *data; + size_t size; + ssize_t ret; + + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + input = fs_read_stream(file, IO_BLOCK_SIZE); + while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) { + fwrite(data, 1, size, stdout); + i_stream_skip(input, size); + } + i_stream_unref(&input); + fs_file_deinit(&file); +} + +static void +fs_test_file_put(struct fs *fs, const char *src_path, const char *dest_path) +{ + struct fs_file *file; + struct istream *input; + struct ostream *output; + off_t ret; + + if (dest_path == NULL) + i_fatal("dest path missing"); + + file = fs_file_init(fs, dest_path, FS_OPEN_MODE_REPLACE); + output = fs_write_stream(file); + input = i_stream_create_file(src_path, IO_BLOCK_SIZE); + if ((ret = o_stream_send_istream(output, input)) < 0) { + if (output->stream_errno != 0) + i_error("write(%s) failed: %m", dest_path); + else + i_error("read(%s) failed: %m", src_path); + } else { + printf("%"PRIuUOFF_T" bytes written\n", ret); + } + i_stream_destroy(&input); + if (fs_write_stream_finish(file, &output) < 0) { + i_error("fs_write_stream_finish() failed: %s", + fs_file_last_error(file)); + } + fs_file_deinit(&file); +} + +static void +fs_test_file_copy(struct fs *fs, const char *src_path, const char *dest_path) +{ + struct fs_file *src_file, *dest_file; + + if (dest_path == NULL) + i_fatal("dest path missing"); + + src_file = fs_file_init(fs, src_path, FS_OPEN_MODE_READONLY); + dest_file = fs_file_init(fs, dest_path, FS_OPEN_MODE_REPLACE); + if (fs_copy(src_file, dest_file) < 0) { + i_error("fs_copy(%s, %s) failed: %s", + src_path, dest_path, fs_last_error(fs)); + } + fs_file_deinit(&src_file); + fs_file_deinit(&dest_file); +} + +static void fs_test_file_stat(struct fs *fs, const char *path) +{ + struct fs_file *file; + struct stat st; + + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + if (fs_stat(file, &st) < 0) { + i_error("fs_stat(%s) failed: %s", + path, fs_file_last_error(file)); + } else { + printf("%s size=%lld\n", path, (long long)st.st_size); + } + fs_file_deinit(&file); +} + +static void fs_test_file_delete(struct fs *fs, const char *path) +{ + struct fs_file *file; + + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + if (fs_delete(file) < 0) { + i_error("fs_delete(%s) failed: %s", + path, fs_file_last_error(file)); + } + fs_file_deinit(&file); +} + +static void fs_test_file_iter(struct fs *fs, const char *path) +{ + struct fs_iter *iter; + const char *fname; + + iter = fs_iter_init(fs, path); + while ((fname = fs_iter_next(iter)) != NULL) + printf("%s\n", fname); + if (fs_iter_deinit(&iter) < 0) { + i_error("fs_iter_deinit(%s) failed: %s", + path, fs_last_error(fs)); + } +} + +int main(int argc, char *argv[]) +{ + struct ioloop *ioloop; + struct fs *fs; + const char *error; + + lib_init(); + ioloop = io_loop_create(); + + if (argc < 5) + i_fatal("Usage: "); + if (fs_init(argv[1], argv[2], &fs_set, &fs, &error) < 0) + i_fatal("fs_init() failed: %s", error); + + if (strcmp(argv[3], "get") == 0) + fs_test_file_get(fs, argv[4]); + else if (strcmp(argv[3], "put") == 0) + fs_test_file_put(fs, argv[4], argv[5]); + else if (strcmp(argv[3], "copy") == 0) + fs_test_file_copy(fs, argv[4], argv[5]); + else if (strcmp(argv[3], "stat") == 0) + fs_test_file_stat(fs, argv[4]); + else if (strcmp(argv[3], "delete") == 0) + fs_test_file_delete(fs, argv[4]); + else if (strcmp(argv[3], "iter") == 0) + fs_test_file_iter(fs, argv[4]); + else + i_fatal("Unknown command: %s", argv[3]); + + fs_deinit(&fs); + io_loop_destroy(&ioloop); + lib_deinit(); + return 0; +} From dovecot at dovecot.org Sun Dec 2 02:35:16 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 02 Dec 2012 02:35:16 +0200 Subject: dovecot-2.2: Makefile: Reordered directories to fix build failure Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d168fdd4aa8d changeset: 15443:d168fdd4aa8d user: Timo Sirainen date: Sun Dec 02 02:35:05 2012 +0200 description: Makefile: Reordered directories to fix build failure diffstat: src/Makefile.am | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diffs (28 lines): diff -r cea9a269311a -r d168fdd4aa8d src/Makefile.am --- a/src/Makefile.am Sat Dec 01 06:56:02 2012 +0200 +++ b/src/Makefile.am Sun Dec 02 02:35:05 2012 +0200 @@ -1,18 +1,18 @@ LIBDOVECOT_SUBDIRS = \ lib-test \ lib \ + lib-settings \ lib-auth \ + lib-master \ lib-charset \ lib-dns \ + lib-dict \ + lib-ssl-iostream + lib-http \ lib-fs \ lib-mail \ lib-imap \ - lib-imap-storage \ - lib-master \ - lib-dict \ - lib-settings \ - lib-ssl-iostream \ - lib-http + lib-imap-storage SUBDIRS = \ $(LIBDOVECOT_SUBDIRS) \ From dovecot at dovecot.org Mon Dec 3 02:22:45 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 03 Dec 2012 02:22:45 +0200 Subject: dovecot-2.2: Makefile: Added missing \ Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d1570c02ca21 changeset: 15444:d1570c02ca21 user: Timo Sirainen date: Mon Dec 03 02:21:54 2012 +0200 description: Makefile: Added missing \ diffstat: src/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d168fdd4aa8d -r d1570c02ca21 src/Makefile.am --- a/src/Makefile.am Sun Dec 02 02:35:05 2012 +0200 +++ b/src/Makefile.am Mon Dec 03 02:21:54 2012 +0200 @@ -7,7 +7,7 @@ lib-charset \ lib-dns \ lib-dict \ - lib-ssl-iostream + lib-ssl-iostream \ lib-http \ lib-fs \ lib-mail \ From dovecot at dovecot.org Tue Dec 4 05:39:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 05:39:46 +0200 Subject: dovecot-2.2: lib-fs: Comment update Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8f0af55ee4b6 changeset: 15445:8f0af55ee4b6 user: Timo Sirainen date: Tue Dec 04 05:39:36 2012 +0200 description: lib-fs: Comment update diffstat: src/lib-fs/fs-api.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d1570c02ca21 -r 8f0af55ee4b6 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Mon Dec 03 02:21:54 2012 +0200 +++ b/src/lib-fs/fs-api.h Tue Dec 04 05:39:36 2012 +0200 @@ -153,7 +153,7 @@ directories are created automatically. Returns 0 if ok, -1 if error occurred. */ int fs_copy(struct fs_file *src, struct fs_file *dest); -/* Try to finish asynchronous fs_copy() */ +/* Try to finish asynchronous fs_copy(). Returns the same as fs_copy(). */ int fs_copy_finish_async(struct fs_file *dest); /* Atomically rename a file. Destination parent directories are created automatically. Returns 0 if ok, -1 if error occurred. */ From dovecot at dovecot.org Tue Dec 4 07:04:42 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 07:04:42 +0200 Subject: dovecot-2.2: lib-fs: Added a default fs_write() implementation u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bde65904d980 changeset: 15446:bde65904d980 user: Timo Sirainen date: Tue Dec 04 07:04:32 2012 +0200 description: lib-fs: Added a default fs_write() implementation using the fs_write_stream*(). diffstat: src/lib-fs/fs-api-private.h | 1 + src/lib-fs/fs-api.c | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletions(-) diffs (61 lines): diff -r 8f0af55ee4b6 -r bde65904d980 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Tue Dec 04 05:39:36 2012 +0200 +++ b/src/lib-fs/fs-api-private.h Tue Dec 04 07:04:32 2012 +0200 @@ -64,6 +64,7 @@ struct fs *fs; struct ostream *output; char *path; + bool write_pending; }; struct fs_lock { diff -r 8f0af55ee4b6 -r bde65904d980 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Tue Dec 04 05:39:36 2012 +0200 +++ b/src/lib-fs/fs-api.c Tue Dec 04 07:04:32 2012 +0200 @@ -4,6 +4,7 @@ #include "array.h" #include "module-dir.h" #include "str.h" +#include "ostream.h" #include "fs-api-private.h" static struct module *fs_modules = NULL; @@ -208,7 +209,37 @@ int fs_write(struct fs_file *file, const void *data, size_t size) { - return file->fs->v.write(file, data, size); + struct ostream *output; + ssize_t ret; + int err; + + if (file->fs->v.write != NULL) + return file->fs->v.write(file, data, size); + + /* backend didn't bother to implement write(), but we can do it with + streams. */ + if (!file->write_pending) { + output = fs_write_stream(file); + if ((ret = o_stream_send(output, data, size)) < 0) { + err = errno; + fs_set_error(file->fs, "fs_write(%s) failed: %m", + o_stream_get_name(output)); + fs_write_stream_abort(file, &output); + errno = err; + return -1; + } + i_assert((size_t)ret == size); + ret = fs_write_stream_finish(file, &output); + } else { + ret = fs_write_stream_finish_async(file); + } + if (ret == 0) { + fs_set_error_async(file->fs); + file->write_pending = TRUE; + return -1; + } + file->write_pending = FALSE; + return ret < 0 ? -1 : 0; } struct ostream *fs_write_stream(struct fs_file *file) From dovecot at dovecot.org Tue Dec 4 07:04:58 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 07:04:58 +0200 Subject: dovecot-2.2: lib-http: Increased initial memory pool size Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4aabd11948d1 changeset: 15447:4aabd11948d1 user: Timo Sirainen date: Tue Dec 04 07:04:54 2012 +0200 description: lib-http: Increased initial memory pool size diffstat: src/lib-http/http-response-parser.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r bde65904d980 -r 4aabd11948d1 src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Tue Dec 04 07:04:32 2012 +0200 +++ b/src/lib-http/http-response-parser.c Tue Dec 04 07:04:54 2012 +0200 @@ -78,7 +78,7 @@ str_truncate(parser->strbuf, 0); if (parser->response_pool != NULL) pool_unref(&parser->response_pool); - parser->response_pool = pool_alloconly_create("http_response", 2048); + parser->response_pool = pool_alloconly_create("http_response", 4096); parser->response = p_new(parser->response_pool, struct http_response, 1); parser->response->date = (time_t)-1; p_array_init(&parser->response->headers, parser->response_pool, 32); From dovecot at dovecot.org Tue Dec 4 07:18:14 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 07:18:14 +0200 Subject: dovecot-2.2: lib-fs: Added a default fs_read() implementation us... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/674e45904706 changeset: 15448:674e45904706 user: Timo Sirainen date: Tue Dec 04 07:18:03 2012 +0200 description: lib-fs: Added a default fs_read() implementation using the fs_read_stream(). diffstat: src/lib-fs/fs-api-private.h | 2 ++ src/lib-fs/fs-api.c | 28 +++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletions(-) diffs (57 lines): diff -r 4aabd11948d1 -r 674e45904706 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Tue Dec 04 07:04:54 2012 +0200 +++ b/src/lib-fs/fs-api-private.h Tue Dec 04 07:18:03 2012 +0200 @@ -64,6 +64,8 @@ struct fs *fs; struct ostream *output; char *path; + + struct istream *pending_read_input; bool write_pending; }; diff -r 4aabd11948d1 -r 674e45904706 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Tue Dec 04 07:04:54 2012 +0200 +++ b/src/lib-fs/fs-api.c Tue Dec 04 07:18:03 2012 +0200 @@ -4,6 +4,7 @@ #include "array.h" #include "module-dir.h" #include "str.h" +#include "istream.h" #include "ostream.h" #include "fs-api-private.h" @@ -199,7 +200,32 @@ ssize_t fs_read(struct fs_file *file, void *buf, size_t size) { - return file->fs->v.read(file, buf, size); + const unsigned char *data; + size_t data_size; + ssize_t ret; + + if (file->fs->v.read != NULL) + return file->fs->v.read(file, buf, size); + + /* backend didn't bother to implement read(), but we can do it with + streams. */ + if (file->pending_read_input == NULL) + file->pending_read_input = fs_read_stream(file, size+1); + ret = i_stream_read_data(file->pending_read_input, + &data, &data_size, size-1); + if (ret == 0) { + fs_set_error_async(file->fs); + return -1; + } + if (ret < 0 && file->pending_read_input->stream_errno != 0) { + fs_set_error(file->fs, "read(%s) failed: %m", + i_stream_get_name(file->pending_read_input)); + } else { + ret = I_MIN(size, data_size); + memcpy(buf, data, ret); + } + i_stream_unref(&file->pending_read_input); + return ret; } struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size) From dovecot at dovecot.org Tue Dec 4 11:33:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 11:33:22 +0200 Subject: dovecot-2.2: lib-storage: Set name for the "binary istream". Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b3f14d2d2019 changeset: 15449:b3f14d2d2019 user: Timo Sirainen date: Tue Dec 04 11:33:16 2012 +0200 description: lib-storage: Set name for the "binary istream". diffstat: src/lib-storage/index/index-mail-binary.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 674e45904706 -r b3f14d2d2019 src/lib-storage/index/index-mail-binary.c --- a/src/lib-storage/index/index-mail-binary.c Tue Dec 04 07:18:03 2012 +0200 +++ b/src/lib-storage/index/index-mail-binary.c Tue Dec 04 11:33:16 2012 +0200 @@ -381,6 +381,9 @@ cache->input = i_streams_merge(blocks_get_streams(&ctx), IO_BLOCK_SIZE, fd_callback, _mail); + i_stream_set_name(cache->input, t_strdup_printf( + "", + _mail->box->vname, _mail->uid)); if (blocks_count_lines(&ctx, cache->input) < 0) { mail_storage_set_critical(_mail->box->storage, "read(%s) failed: %m", From dovecot at dovecot.org Tue Dec 4 11:40:48 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 11:40:48 +0200 Subject: dovecot-2.2: lib-mail: quoted-printable decoder istream shouldn'... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4d382ee358fb changeset: 15450:4d382ee358fb user: Timo Sirainen date: Tue Dec 04 11:40:35 2012 +0200 description: lib-mail: quoted-printable decoder istream shouldn't treat trailing "=" as error. diffstat: src/lib-mail/istream-qp-decoder.c | 24 +++++++++++++++++++----- 1 files changed, 19 insertions(+), 5 deletions(-) diffs (42 lines): diff -r b3f14d2d2019 -r 4d382ee358fb src/lib-mail/istream-qp-decoder.c --- a/src/lib-mail/istream-qp-decoder.c Tue Dec 04 11:33:16 2012 +0200 +++ b/src/lib-mail/istream-qp-decoder.c Tue Dec 04 11:40:35 2012 +0200 @@ -70,6 +70,8 @@ { struct qp_decoder_istream *bstream = (struct qp_decoder_istream *)stream; + const unsigned char *data; + size_t size; size_t pre_count, post_count; int ret; size_t prev_size = 0; @@ -77,12 +79,24 @@ do { ret = i_stream_read_parent(stream, &prev_size); if (ret <= 0) { - if (ret < 0 && stream->istream.stream_errno == 0 && - i_stream_get_data_size(stream->parent) > 0) { - /* qp input with a partial block */ - stream->istream.stream_errno = EINVAL; + if (ret != -1 || stream->istream.stream_errno != 0) + return 0; + + data = i_stream_get_data(stream->parent, &size); + if (size == 0) + return -1; + + if (size == 1 && data[0] == '=') { + /* ends with "=". normally this would be + followed by LF, but it's not really an + error even without. */ + i_stream_skip(stream->parent, 1); + stream->istream.eof = TRUE; + return -1; } - return ret; + /* qp input with a partial block */ + stream->istream.stream_errno = EINVAL; + return -1; } /* encode as many blocks as fits into destination buffer */ From dovecot at dovecot.org Tue Dec 4 13:02:58 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 13:02:58 +0200 Subject: dovecot-2.2: imap: Fixed handling FETCH BINARY for broken content. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9c1791d1834f changeset: 15451:9c1791d1834f user: Timo Sirainen date: Tue Dec 04 13:02:22 2012 +0200 description: imap: Fixed handling FETCH BINARY for broken content. diffstat: src/doveadm/doveadm-mail.c | 1 + src/imap/cmd-fetch.c | 4 ++-- src/imap/imap-commands-util.c | 1 + src/imap/imap-fetch-body.c | 23 ++++++++++++++++++----- src/lib-storage/index/index-mail-binary.c | 16 ++++++++++++---- src/lib-storage/mail-error.h | 5 ++++- 6 files changed, 38 insertions(+), 12 deletions(-) diffs (131 lines): diff -r 4d382ee358fb -r 9c1791d1834f src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Tue Dec 04 11:40:35 2012 +0200 +++ b/src/doveadm/doveadm-mail.c Tue Dec 04 13:02:22 2012 +0200 @@ -48,6 +48,7 @@ case MAIL_ERROR_NOTPOSSIBLE: case MAIL_ERROR_EXISTS: case MAIL_ERROR_CONVERSION: + case MAIL_ERROR_INVALIDDATA: exit_code = DOVEADM_EX_NOTPOSSIBLE; break; case MAIL_ERROR_PARAMS: diff -r 4d382ee358fb -r 9c1791d1834f src/imap/cmd-fetch.c --- a/src/imap/cmd-fetch.c Tue Dec 04 11:40:35 2012 +0200 +++ b/src/imap/cmd-fetch.c Tue Dec 04 13:02:22 2012 +0200 @@ -195,8 +195,8 @@ errstr = mailbox_get_last_error(cmd->client->mailbox, &error); if (error == MAIL_ERROR_CONVERSION) { /* BINARY found unsupported Content-Transfer-Encoding */ - tagged_reply = "NO ["IMAP_RESP_CODE_UNKNOWN_CTE"] " - "Unknown Content-Transfer-Encoding."; + tagged_reply = t_strdup_printf( + "NO ["IMAP_RESP_CODE_UNKNOWN_CTE"] %s", errstr); } else { /* We never want to reply NO to FETCH requests, BYE is preferrable (see imap-ml for reasons). */ diff -r 4d382ee358fb -r 9c1791d1834f src/imap/imap-commands-util.c --- a/src/imap/imap-commands-util.c Tue Dec 04 11:40:35 2012 +0200 +++ b/src/imap/imap-commands-util.c Tue Dec 04 13:02:22 2012 +0200 @@ -137,6 +137,7 @@ resp_code = IMAP_RESP_CODE_INUSE; break; case MAIL_ERROR_CONVERSION: + case MAIL_ERROR_INVALIDDATA: break; } if (resp_code == NULL || *error_string == '[') diff -r 4d382ee358fb -r 9c1791d1834f src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Tue Dec 04 11:40:35 2012 +0200 +++ b/src/imap/imap-fetch-body.c Tue Dec 04 13:02:22 2012 +0200 @@ -135,8 +135,16 @@ return 1; } - if (imap_msgpart_open(mail, body->msgpart, &result) < 0) - return -1; + if (imap_msgpart_open(mail, body->msgpart, &result) < 0) { + if (!body->binary || + mailbox_get_last_mail_error(mail->box) != MAIL_ERROR_INVALIDDATA) + return -1; + /* tried to do BINARY fetch for a MIME part with broken + content */ + str = get_prefix(&ctx->state, body, (uoff_t)-1, FALSE); + o_stream_nsend(ctx->client->output, str_data(str), str_len(str)); + return 1; + } ctx->state.cur_input = result.input; ctx->state.cur_size = result.size; ctx->state.cur_size_field = result.size_field; @@ -161,8 +169,13 @@ return 1; } - if (imap_msgpart_size(mail, body->msgpart, &size) < 0) - return -1; + if (imap_msgpart_size(mail, body->msgpart, &size) < 0) { + if (mailbox_get_last_mail_error(mail->box) != MAIL_ERROR_INVALIDDATA) + return -1; + /* tried to do BINARY.SIZE fetch for a MIME part with broken + content */ + size = 0; + } str = t_str_new(128); if (ctx->state.cur_first) @@ -406,7 +419,7 @@ ctx->name = p_strdup(ctx->pool, get_body_name(body)); if (body->binary_size) { imap_fetch_add_handler(ctx, IMAP_FETCH_HANDLER_FLAG_WANT_DEINIT, - "NIL", fetch_binary_size, body); + "0", fetch_binary_size, body); } else { imap_fetch_add_handler(ctx, IMAP_FETCH_HANDLER_FLAG_WANT_DEINIT, "NIL", fetch_body_msgpart, body); diff -r 4d382ee358fb -r 9c1791d1834f src/lib-storage/index/index-mail-binary.c --- a/src/lib-storage/index/index-mail-binary.c Tue Dec 04 11:40:35 2012 +0200 +++ b/src/lib-storage/index/index-mail-binary.c Tue Dec 04 13:02:22 2012 +0200 @@ -119,7 +119,8 @@ if (cte == MESSAGE_CTE_UNKNOWN) { mail_storage_set_error(ctx->mail->box->storage, - MAIL_ERROR_CONVERSION, "Unknown CTE"); + MAIL_ERROR_CONVERSION, + "Unknown Content-Transfer-Encoding."); return -1; } @@ -385,9 +386,16 @@ "", _mail->box->vname, _mail->uid)); if (blocks_count_lines(&ctx, cache->input) < 0) { - mail_storage_set_critical(_mail->box->storage, - "read(%s) failed: %m", - i_stream_get_name(cache->input)); + if (cache->input->stream_errno == EINVAL) { + /* MIME part contains invalid data */ + mail_storage_set_error(_mail->box->storage, + MAIL_ERROR_INVALIDDATA, + "Invalid data in MIME part"); + } else { + mail_storage_set_critical(_mail->box->storage, + "read(%s) failed: %m", + i_stream_get_name(cache->input)); + } mail_storage_free_binary_cache(_mail->box->storage); binary_streams_free(&ctx); return -1; diff -r 4d382ee358fb -r 9c1791d1834f src/lib-storage/mail-error.h --- a/src/lib-storage/mail-error.h Tue Dec 04 11:40:35 2012 +0200 +++ b/src/lib-storage/mail-error.h Tue Dec 04 13:02:22 2012 +0200 @@ -43,7 +43,10 @@ MAIL_ERROR_INUSE, /* Can't do the requested data conversion (e.g. IMAP BINARY's UNKNOWN-CTE code) */ - MAIL_ERROR_CONVERSION + MAIL_ERROR_CONVERSION, + /* Can't do the requested data conversion because the original data + isn't valid. */ + MAIL_ERROR_INVALIDDATA }; /* Convert errno to mail_error and an error string. Returns TRUE if successful, From dovecot at dovecot.org Tue Dec 4 13:05:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 13:05:05 +0200 Subject: dovecot-2.2: imap: Don't crash in IDLE if remote IP isn't known ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/266e24f1c78c changeset: 15452:266e24f1c78c user: Timo Sirainen date: Tue Dec 04 13:04:59 2012 +0200 description: imap: Don't crash in IDLE if remote IP isn't known (PREAUTH) diffstat: src/imap/cmd-idle.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 9c1791d1834f -r 266e24f1c78c src/imap/cmd-idle.c --- a/src/imap/cmd-idle.c Tue Dec 04 13:02:22 2012 +0200 +++ b/src/imap/cmd-idle.c Tue Dec 04 13:04:59 2012 +0200 @@ -212,7 +212,8 @@ time, but this can be avoided by using a properly configured Dovecot proxy. we'll also try to avoid this by not doing it for the commonly used intranet IP ranges. */ - client_hash = remote_ip_is_usable(ctx->client->user->remote_ip) ? + client_hash = ctx->client->user->remote_ip != NULL && + remote_ip_is_usable(ctx->client->user->remote_ip) ? net_ip_hash(ctx->client->user->remote_ip) : crc32_str(ctx->client->user->username); interval -= (time(NULL) + client_hash) % interval; From dovecot at dovecot.org Tue Dec 4 14:10:41 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Dec 2012 14:10:41 +0200 Subject: dovecot-2.2: lib-mail: Detect errors in quoted-printable input. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0be51d94b0d9 changeset: 15453:0be51d94b0d9 user: Timo Sirainen date: Tue Dec 04 14:10:34 2012 +0200 description: lib-mail: Detect errors in quoted-printable input. diffstat: src/lib-mail/istream-qp-decoder.c | 27 +++++++------- src/lib-mail/message-decoder.c | 10 ++-- src/lib-mail/message-header-decode.c | 8 ++- src/lib-mail/quoted-printable.c | 36 +++++++++++++++--- src/lib-mail/quoted-printable.h | 15 +++++-- src/lib-mail/test-istream-qp-decoder.c | 3 +- src/lib-mail/test-message-decoder.c | 5 +- src/lib-mail/test-quoted-printable.c | 62 ++++++++++++++++++++++++++------- 8 files changed, 115 insertions(+), 51 deletions(-) diffs (truncated from 361 to 300 lines): diff -r 266e24f1c78c -r 0be51d94b0d9 src/lib-mail/istream-qp-decoder.c --- a/src/lib-mail/istream-qp-decoder.c Tue Dec 04 13:04:59 2012 +0200 +++ b/src/lib-mail/istream-qp-decoder.c Tue Dec 04 14:10:34 2012 +0200 @@ -35,12 +35,13 @@ } static int -i_stream_qp_try_decode_block(struct qp_decoder_istream *bstream) +i_stream_qp_try_decode_block(struct qp_decoder_istream *bstream, bool eof) { struct istream_private *stream = &bstream->istream; const unsigned char *data; size_t size, avail, buffer_avail, pos; buffer_t buf; + int ret; data = i_stream_get_data(stream->parent, &size); if (size == 0) @@ -59,7 +60,12 @@ buffer_create_from_data(&buf, stream->w_buffer + stream->pos, buffer_avail); - quoted_printable_decode(data, size, &pos, &buf); + ret = !eof ? quoted_printable_decode(data, size, &pos, &buf) : + quoted_printable_decode_final(data, size, &pos, &buf); + if (ret < 0) { + stream->istream.stream_errno = EINVAL; + return -1; + } stream->pos += buf.used; i_stream_skip(stream->parent, pos); @@ -70,8 +76,6 @@ { struct qp_decoder_istream *bstream = (struct qp_decoder_istream *)stream; - const unsigned char *data; - size_t size; size_t pre_count, post_count; int ret; size_t prev_size = 0; @@ -82,26 +86,21 @@ if (ret != -1 || stream->istream.stream_errno != 0) return 0; - data = i_stream_get_data(stream->parent, &size); - if (size == 0) - return -1; - - if (size == 1 && data[0] == '=') { - /* ends with "=". normally this would be - followed by LF, but it's not really an - error even without. */ - i_stream_skip(stream->parent, 1); + ret = i_stream_qp_try_decode_block(bstream, TRUE); + if (ret == 0) { + /* ended with =[whitespace] but without LF */ stream->istream.eof = TRUE; return -1; } /* qp input with a partial block */ + i_assert(ret < 0); stream->istream.stream_errno = EINVAL; return -1; } /* encode as many blocks as fits into destination buffer */ pre_count = stream->pos - stream->skip; - while ((ret = i_stream_qp_try_decode_block(bstream)) > 0) ; + while ((ret = i_stream_qp_try_decode_block(bstream, FALSE)) > 0) ; post_count = stream->pos - stream->skip; } while (ret == 0 && pre_count == post_count); diff -r 266e24f1c78c -r 0be51d94b0d9 src/lib-mail/message-decoder.c --- a/src/lib-mail/message-decoder.c Tue Dec 04 13:04:59 2012 +0200 +++ b/src/lib-mail/message-decoder.c Tue Dec 04 14:10:34 2012 +0200 @@ -281,12 +281,12 @@ case MESSAGE_CTE_QP: buffer_set_used_size(ctx->buf, 0); if (ctx->encoding_buf->used != 0) { - quoted_printable_decode(ctx->encoding_buf->data, - ctx->encoding_buf->used, - &pos, ctx->buf); + (void)quoted_printable_decode(ctx->encoding_buf->data, + ctx->encoding_buf->used, + &pos, ctx->buf); } else { - quoted_printable_decode(input->data, input->size, - &pos, ctx->buf); + (void)quoted_printable_decode(input->data, input->size, + &pos, ctx->buf); } data = ctx->buf->data; size = ctx->buf->used; diff -r 266e24f1c78c -r 0be51d94b0d9 src/lib-mail/message-header-decode.c --- a/src/lib-mail/message-header-decode.c Tue Dec 04 13:04:59 2012 +0200 +++ b/src/lib-mail/message-header-decode.c Tue Dec 04 14:10:34 2012 +0200 @@ -36,9 +36,11 @@ switch (data[start_pos[0]+1]) { case 'q': case 'Q': - quoted_printable_q_decode(data + start_pos[1] + 1, - start_pos[2] - start_pos[1] - 1, - decodebuf); + if (quoted_printable_q_decode(data + start_pos[1] + 1, + start_pos[2] - start_pos[1] - 1, + decodebuf) < 0) { + /* we skipped over some invalid data */ + } break; case 'b': case 'B': diff -r 266e24f1c78c -r 0be51d94b0d9 src/lib-mail/quoted-printable.c --- a/src/lib-mail/quoted-printable.c Tue Dec 04 13:04:59 2012 +0200 +++ b/src/lib-mail/quoted-printable.c Tue Dec 04 14:10:34 2012 +0200 @@ -27,11 +27,13 @@ return -1; } -void quoted_printable_decode(const unsigned char *src, size_t src_size, - size_t *src_pos_r, buffer_t *dest) +static int +quoted_printable_decode_full(const unsigned char *src, size_t src_size, + size_t *src_pos_r, buffer_t *dest, bool eof) { char hexbuf[3]; size_t src_pos, pos, next; + bool errors = FALSE; int ret; hexbuf[2] = '\0'; @@ -64,11 +66,15 @@ continue; } if (ret < 0) { - /* unknown yet if this is end of line */ + /* '=' was followed only by whitespace */ break; } - if (src_pos+2 >= src_size) + if (src_pos+2 >= src_size) { + /* '=' was followed by non-whitespace */ + if (eof) + errors = TRUE; break; + } /* = */ hexbuf[0] = src[src_pos+1]; @@ -79,6 +85,7 @@ next = src_pos + 1; } else { /* non-hex data, show as-is */ + errors = TRUE; next = src_pos; } } @@ -91,15 +98,28 @@ buffer_append(dest, src + next, src_pos - next); next = src_pos; } - *src_pos_r = next; + return errors ? -1 : 0; } -void quoted_printable_q_decode(const unsigned char *src, size_t src_size, - buffer_t *dest) +int quoted_printable_decode(const unsigned char *src, size_t src_size, + size_t *src_pos_r, buffer_t *dest) +{ + return quoted_printable_decode_full(src, src_size, src_pos_r, dest, FALSE); +} + +int quoted_printable_decode_final(const unsigned char *src, size_t src_size, + size_t *src_pos_r, buffer_t *dest) +{ + return quoted_printable_decode_full(src, src_size, src_pos_r, dest, TRUE); +} + +int quoted_printable_q_decode(const unsigned char *src, size_t src_size, + buffer_t *dest) { char hexbuf[3]; size_t src_pos, next; + bool errors = FALSE; hexbuf[2] = '\0'; @@ -129,8 +149,10 @@ next = src_pos+1; } else { /* non-hex data, show as-is */ + errors = TRUE; next = src_pos; } } buffer_append(dest, src + next, src_size - next); + return errors ? -1 : 0; } diff -r 266e24f1c78c -r 0be51d94b0d9 src/lib-mail/quoted-printable.h --- a/src/lib-mail/quoted-printable.h Tue Dec 04 13:04:59 2012 +0200 +++ b/src/lib-mail/quoted-printable.h Tue Dec 04 14:10:34 2012 +0200 @@ -2,14 +2,19 @@ #define QUOTED_PRINTABLE_H /* Translates quoted printable data into binary. dest must be at least the - size of src, and may be same as src. Decoding errors are ignored. + size of src, and may be same as src. Returns 0 if input was valid, -1 if + there were some decoding errors (which were skipped over). This function may be called multiple times for parsing the same stream. src_pos is updated to first non-translated character in src. */ -void quoted_printable_decode(const unsigned char *src, size_t src_size, - size_t *src_pos_r, buffer_t *dest); +int quoted_printable_decode(const unsigned char *src, size_t src_size, + size_t *src_pos_r, buffer_t *dest); +/* Like quoted_printable_decode(), but handle src as the final block. + This allows src to end without LF. */ +int quoted_printable_decode_final(const unsigned char *src, size_t src_size, + size_t *src_pos_r, buffer_t *dest); /* Decode MIME "Q" encoding. */ -void quoted_printable_q_decode(const unsigned char *src, size_t src_size, - buffer_t *dest); +int quoted_printable_q_decode(const unsigned char *src, size_t src_size, + buffer_t *dest); #endif diff -r 266e24f1c78c -r 0be51d94b0d9 src/lib-mail/test-istream-qp-decoder.c --- a/src/lib-mail/test-istream-qp-decoder.c Tue Dec 04 13:04:59 2012 +0200 +++ b/src/lib-mail/test-istream-qp-decoder.c Tue Dec 04 14:10:34 2012 +0200 @@ -10,7 +10,8 @@ const char *output; } tests[] = { { "p=C3=A4=C3=A4t=C3=B6s", "p??t?s" }, - { "p=c3=a4=c3=a4t=c3=b6s", "p??t?s" }, + { "p=c3=a4=c3=a4t=c3=b6s= ", "p??t?s" }, + { "p=c3=a4=c3=a4t=c3=b6s= \n", "p??t?s" }, { "p=c3=a4= \t \n=c3=\r\n=a4t= \r\n=c3=b6s", "p??t?s" }, }; diff -r 266e24f1c78c -r 0be51d94b0d9 src/lib-mail/test-message-decoder.c --- a/src/lib-mail/test-message-decoder.c Tue Dec 04 13:04:59 2012 +0200 +++ b/src/lib-mail/test-message-decoder.c Tue Dec 04 14:10:34 2012 +0200 @@ -16,13 +16,14 @@ buffer_append(dest, data, size); } -void quoted_printable_decode(const unsigned char *src, size_t src_size, - size_t *src_pos_r, buffer_t *dest) +int quoted_printable_decode(const unsigned char *src, size_t src_size, + size_t *src_pos_r, buffer_t *dest) { while (src_size > 0 && src[src_size-1] == ' ') src_size--; buffer_append(dest, src, src_size); *src_pos_r = src_size; + return 0; } int charset_to_utf8_begin(const char *charset ATTR_UNUSED, diff -r 266e24f1c78c -r 0be51d94b0d9 src/lib-mail/test-quoted-printable.c --- a/src/lib-mail/test-quoted-printable.c Tue Dec 04 13:04:59 2012 +0200 +++ b/src/lib-mail/test-quoted-printable.c Tue Dec 04 14:10:34 2012 +0200 @@ -10,34 +10,38 @@ const char *input; const char *output; int end_skip; + int ret; }; static void test_quoted_printable_decode(void) { static struct test_quoted_printable_decode_data data[] = { - { "foo \r\nbar=", "foo\r\nbar", 1 }, - { "foo\t=\nbar", "foo\tbar", 0 }, - { "foo = \n=01", "foo \001", 0 }, - { "foo =\t\r\nbar", "foo bar", 0 }, - { "foo =\r\n=01", "foo \001", 0 }, - { "foo \nbar=", "foo\r\nbar", 1 }, - { "=0A=0D ", "\n\r", 2 }, - { "foo_bar", "foo_bar", 0 }, - { "foo=", "foo", 1 }, - { "foo=A", "foo", 2 }, - { "foo=Ax", "foo=Ax", 0 }, - { "foo=Ax=xy", "foo=Ax=xy", 0 } + { "foo \r\nbar=", "foo\r\nbar", 1, 0 }, + { "foo\t=\nbar", "foo\tbar", 0, 0 }, + { "foo = \n=01", "foo \001", 0, 0 }, + { "foo =\t\r\nbar", "foo bar", 0, 0 }, + { "foo =\r\n=01", "foo \001", 0, 0 }, + { "foo \nbar=", "foo\r\nbar", 1, 0 }, + { "=0A=0D ", "\n\r", 2, 0 }, + { "foo_bar", "foo_bar", 0, 0 }, + { "foo=", "foo", 1, 0 }, + { "foo= ", "foo", 3, 0 }, + { "foo=A", "foo", 2, 0 }, + { "foo=Ax", "foo=Ax", 0, -1 }, + { "foo=Ax=xy", "foo=Ax=xy", 0, -1 } }; From dovecot at dovecot.org Wed Dec 5 07:57:47 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 05 Dec 2012 07:57:47 +0200 Subject: dovecot-2.2: lib-http: Parser didn't always fully skip over the ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b61b94be996e changeset: 15454:b61b94be996e user: Timo Sirainen date: Wed Dec 05 07:57:35 2012 +0200 description: lib-http: Parser didn't always fully skip over the previous request's payload. diffstat: src/lib-http/http-response-parser.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (17 lines): diff -r 0be51d94b0d9 -r b61b94be996e src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Tue Dec 04 14:10:34 2012 +0200 +++ b/src/lib-http/http-response-parser.c Wed Dec 05 07:57:35 2012 +0200 @@ -379,10 +379,10 @@ i_assert(parser->state == HTTP_RESPONSE_PARSE_STATE_INIT); - if (!payload->eof) { - while ((ret=i_stream_read(payload)) > 0) { + if (i_stream_have_bytes_left(payload)) { + do { i_stream_skip(payload, i_stream_get_data_size(payload)); - } + } while ((ret=i_stream_read(payload)) > 0); if (ret == 0) return 0; if (ret < 0 && !payload->eof) { From dovecot at dovecot.org Fri Dec 7 05:54:04 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 07 Dec 2012 05:54:04 +0200 Subject: dovecot-2.2: ostream-buffer: Fixed o_stream_sendv() with multipl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/67448d683720 changeset: 15455:67448d683720 user: Timo Sirainen date: Fri Dec 07 05:53:53 2012 +0200 description: ostream-buffer: Fixed o_stream_sendv() with multiple iovs. Patch by Stephan Bosch diffstat: src/lib/ostream-buffer.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (16 lines): diff -r b61b94be996e -r 67448d683720 src/lib/ostream-buffer.c --- a/src/lib/ostream-buffer.c Wed Dec 05 07:57:35 2012 +0200 +++ b/src/lib/ostream-buffer.c Fri Dec 07 05:53:53 2012 +0200 @@ -40,11 +40,11 @@ n = I_MIN(left, iov[i].iov_len); buffer_write(bstream->buf, stream->ostream.offset, iov[i].iov_base, n); + stream->ostream.offset += n; ret += n; if (n != iov[i].iov_len) break; } - stream->ostream.offset += ret; return ret; } From dovecot at dovecot.org Sat Dec 8 09:18:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 08 Dec 2012 09:18:54 +0200 Subject: dovecot-2.2: lib-fs: Added FS_PROPERTY_DIRECTORIES. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/959afc4f76eb changeset: 15456:959afc4f76eb user: Timo Sirainen date: Sat Dec 08 09:18:43 2012 +0200 description: lib-fs: Added FS_PROPERTY_DIRECTORIES. This is a hint for fs API users if they should call fs_delete() for directories. diffstat: src/lib-fs/fs-api.h | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 67448d683720 -r 959afc4f76eb src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Fri Dec 07 05:53:53 2012 +0200 +++ b/src/lib-fs/fs-api.h Sat Dec 08 09:18:43 2012 +0200 @@ -16,7 +16,10 @@ FS_PROPERTY_ITER = 0x20, /* Iteration always returns all of the files (instead of possibly slightly out of date view) */ - FS_PROPERTY_RELIABLEITER= 0x40 + FS_PROPERTY_RELIABLEITER= 0x40, + /* Backend uses directories, which aren't automatically deleted + when its children are deleted. */ + FS_PROPERTY_DIRECTORIES = 0x80 }; enum fs_open_mode { From dovecot at dovecot.org Sat Dec 8 09:35:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 08 Dec 2012 09:35:55 +0200 Subject: dovecot-2.1: fts-solr: Fixed memory leak Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/cfa6fc9d2856 changeset: 14839:cfa6fc9d2856 user: Timo Sirainen date: Sat Dec 08 09:35:33 2012 +0200 description: fts-solr: Fixed memory leak diffstat: src/plugins/fts-solr/solr-connection.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b60872b87d26 -r cfa6fc9d2856 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Fri Nov 30 08:56:55 2012 +0200 +++ b/src/plugins/fts-solr/solr-connection.c Sat Dec 08 09:35:33 2012 +0200 @@ -192,6 +192,7 @@ curl_slist_free_all(conn->headers_post); curl_multi_cleanup(conn->curlm); curl_easy_cleanup(conn->curl); + XML_ParserFree(conn->xml_parser); i_free(conn->last_sent_url); i_free(conn->url); i_free(conn); From dovecot at dovecot.org Sat Dec 8 09:37:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 08 Dec 2012 09:37:29 +0200 Subject: dovecot-2.2: lib-storage: Copy MAIL_STORAGE_CLASS_FLAG_NO_ROOT t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3c7cef3ffff9 changeset: 15457:3c7cef3ffff9 user: Timo Sirainen date: Sat Dec 08 09:36:54 2012 +0200 description: lib-storage: Copy MAIL_STORAGE_CLASS_FLAG_NO_ROOT to MAILBOX_LIST_FLAG_NO_MAIL_FILES diffstat: src/lib-storage/mail-storage.c | 2 ++ src/lib-storage/mailbox-list.h | 4 +++- 2 files changed, 5 insertions(+), 1 deletions(-) diffs (26 lines): diff -r 959afc4f76eb -r 3c7cef3ffff9 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Sat Dec 08 09:18:43 2012 +0200 +++ b/src/lib-storage/mail-storage.c Sat Dec 08 09:36:54 2012 +0200 @@ -349,6 +349,8 @@ /* first storage for namespace */ if (mail_storage_is_mailbox_file(storage_class)) list_flags |= MAILBOX_LIST_FLAG_MAILBOX_FILES; + if ((storage_class->class_flags & MAIL_STORAGE_CLASS_FLAG_NO_ROOT) != 0) + list_flags |= MAILBOX_LIST_FLAG_NO_MAIL_FILES; if (mailbox_list_create(list_set.layout, ns, &list_set, list_flags, &list, error_r) < 0) { *error_r = t_strdup_printf("Mailbox list driver %s: %s", diff -r 959afc4f76eb -r 3c7cef3ffff9 src/lib-storage/mailbox-list.h --- a/src/lib-storage/mailbox-list.h Sat Dec 08 09:18:43 2012 +0200 +++ b/src/lib-storage/mailbox-list.h Sat Dec 08 09:36:54 2012 +0200 @@ -31,7 +31,9 @@ MAILBOX_LIST_FLAG_MAILBOX_FILES = 0x01, /* Namespace already has a mailbox list, don't assign this mailbox list to it. */ - MAILBOX_LIST_FLAG_SECONDARY = 0x02 + MAILBOX_LIST_FLAG_SECONDARY = 0x02, + /* There are no mail files, only index and/or control files. */ + MAILBOX_LIST_FLAG_NO_MAIL_FILES = 0x04 }; enum mailbox_info_flags { From dovecot at dovecot.org Sat Dec 8 09:37:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 08 Dec 2012 09:37:29 +0200 Subject: dovecot-2.2: layout=index: If MAILBOX_LIST_FLAG_NO_MAIL_FILES is... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0851ffee594b changeset: 15458:0851ffee594b user: Timo Sirainen date: Sat Dec 08 09:37:21 2012 +0200 description: layout=index: If MAILBOX_LIST_FLAG_NO_MAIL_FILES is set, don't try to delete any mail files. diffstat: src/lib-storage/list/mailbox-list-index-backend.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 3c7cef3ffff9 -r 0851ffee594b src/lib-storage/list/mailbox-list-index-backend.c --- a/src/lib-storage/list/mailbox-list-index-backend.c Sat Dec 08 09:36:54 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index-backend.c Sat Dec 08 09:37:21 2012 +0200 @@ -424,7 +424,9 @@ if (ret <= 0) return ret; - if ((_list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) != 0) { + if ((_list->flags & MAILBOX_LIST_FLAG_NO_MAIL_FILES) != 0) { + ret = 0; + } else if ((_list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) != 0) { ret = mailbox_list_delete_mailbox_file(_list, name, path); } else { ret = mailbox_list_delete_mailbox_nonrecursive(_list, name, From dovecot at dovecot.org Sat Dec 8 10:43:34 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 08 Dec 2012 10:43:34 +0200 Subject: dovecot-2.1: imap: If mailbox doesn't have read ACL, hide APPEND... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/f74f1b14975d changeset: 14840:f74f1b14975d user: Timo Sirainen date: Sat Dec 08 10:43:21 2012 +0200 description: imap: If mailbox doesn't have read ACL, hide APPENDUID/COPYUID resp-codes. diffstat: src/imap/cmd-append.c | 2 +- src/imap/cmd-copy.c | 6 ++++-- src/lib-storage/mail-storage.h | 3 +++ src/plugins/acl/acl-mailbox.c | 17 ++++++++++++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diffs (88 lines): diff -r cfa6fc9d2856 -r f74f1b14975d src/imap/cmd-append.c --- a/src/imap/cmd-append.c Sat Dec 08 09:35:33 2012 +0200 +++ b/src/imap/cmd-append.c Sat Dec 08 10:43:21 2012 +0200 @@ -290,7 +290,7 @@ msg = t_str_new(256); save_count = seq_range_count(&changes.saved_uids); - if (save_count == 0) { + if (save_count == 0 || changes.no_read_perm) { /* not supported by backend (virtual) */ str_append(msg, "OK Append completed."); } else { diff -r cfa6fc9d2856 -r f74f1b14975d src/imap/cmd-copy.c --- a/src/imap/cmd-copy.c Sat Dec 08 09:35:33 2012 +0200 +++ b/src/imap/cmd-copy.c Sat Dec 08 10:43:21 2012 +0200 @@ -125,8 +125,10 @@ else if (copy_count == 0) { str_append(msg, "OK No messages copied."); pool_unref(&changes.pool); - } else if (seq_range_count(&changes.saved_uids) == 0) { - /* not supported by backend (virtual) */ + } else if (seq_range_count(&changes.saved_uids) == 0 || + changes.no_read_perm) { + /* not supported by backend (virtual) or no read permissions + for mailbox */ str_append(msg, "OK Copy completed."); pool_unref(&changes.pool); } else { diff -r cfa6fc9d2856 -r f74f1b14975d src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Sat Dec 08 09:35:33 2012 +0200 +++ b/src/lib-storage/mail-storage.h Sat Dec 08 10:43:21 2012 +0200 @@ -268,6 +268,9 @@ /* TRUE if anything actually changed with this commit */ bool changed; + /* User doesn't have read ACL for the mailbox, so don't show the + uid_validity / saved_uids. */ + bool no_read_perm; }; struct mailbox_sync_rec { diff -r cfa6fc9d2856 -r f74f1b14975d src/plugins/acl/acl-mailbox.c --- a/src/plugins/acl/acl-mailbox.c Sat Dec 08 09:35:33 2012 +0200 +++ b/src/plugins/acl/acl-mailbox.c Sat Dec 08 10:43:21 2012 +0200 @@ -21,6 +21,7 @@ struct acl_object *aclobj; bool skip_acl_checks; bool acl_enabled; + bool no_read_right; }; struct acl_transaction_context { @@ -408,13 +409,19 @@ { struct acl_mailbox *abox = ACL_CONTEXT(ctx->box); void *at = ACL_CONTEXT(ctx); + int ret; if (at != NULL) { abox->module_ctx.super.transaction_rollback(ctx); return -1; } - return abox->module_ctx.super.transaction_commit(ctx, changes_r); + ret = abox->module_ctx.super.transaction_commit(ctx, changes_r); + if (abox->no_read_right) { + /* don't allow IMAP client to see what UIDs the messages got */ + changes_r->no_read_perm = TRUE; + } + return ret; } static int acl_mailbox_exists(struct mailbox *box, bool auto_boxes, @@ -473,6 +480,14 @@ } return -1; } + if (open_right != ACL_STORAGE_RIGHT_READ) { + ret = acl_object_have_right(abox->aclobj, + idx_arr[ACL_STORAGE_RIGHT_READ]); + if (ret < 0) + return -1; + if (ret == 0) + abox->no_read_right = TRUE; + } return 0; } From dovecot at dovecot.org Sat Dec 8 10:48:30 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 08 Dec 2012 10:48:30 +0200 Subject: dovecot-2.1: doveadm: table formatter supports now writing by st... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c1e47738e7c3 changeset: 14841:c1e47738e7c3 user: Timo Sirainen date: Sat Dec 08 10:48:23 2012 +0200 description: doveadm: table formatter supports now writing by streaming. This fixes it to work with doveadm server, which sometimes may use streaming. diffstat: src/doveadm/doveadm-print-table.c | 17 ++++++++++++++--- 1 files changed, 14 insertions(+), 3 deletions(-) diffs (56 lines): diff -r f74f1b14975d -r c1e47738e7c3 src/doveadm/doveadm-print-table.c --- a/src/doveadm/doveadm-print-table.c Sat Dec 08 10:43:21 2012 +0200 +++ b/src/doveadm/doveadm-print-table.c Sat Dec 08 10:48:23 2012 +0200 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "str.h" #include "doveadm-print-private.h" #include @@ -24,6 +25,7 @@ pool_t pool; ARRAY_DEFINE(headers, struct doveadm_print_table_header); ARRAY_TYPE(const_string) buffered_values; + string_t *stream; unsigned int hdr_idx; unsigned int columns; @@ -179,10 +181,17 @@ } static void -doveadm_print_table_print_stream(const unsigned char *value ATTR_UNUSED, - size_t size ATTR_UNUSED) +doveadm_print_table_print_stream(const unsigned char *value, size_t size) { - i_fatal("table formatter doesn't support multi-line values"); + if (memchr(value, '\n', size) != NULL) + i_fatal("table formatter doesn't support multi-line values"); + + if (size != 0) + str_append_n(ctx->stream, value, size); + else { + doveadm_print_table_print(str_c(ctx->stream)); + str_truncate(ctx->stream, 0); + } } static void doveadm_print_table_flush(void) @@ -199,6 +208,7 @@ pool = pool_alloconly_create("doveadm print table", 2048); ctx = p_new(pool, struct doveadm_print_table_context, 1); ctx->pool = pool; + ctx->stream = str_new(default_pool, 128); p_array_init(&ctx->headers, pool, 16); i_array_init(&ctx->buffered_values, 64); ctx->columns = DEFAULT_COLUMNS; @@ -211,6 +221,7 @@ static void doveadm_print_table_deinit(void) { + str_free(&ctx->stream); array_free(&ctx->buffered_values); pool_unref(&ctx->pool); ctx = NULL; From dovecot at dovecot.org Sat Dec 8 11:12:20 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 08 Dec 2012 11:12:20 +0200 Subject: dovecot-2.1: lib-master: Ignore mountpoints under /Volumes by de... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/657cb4c35a0e changeset: 14842:657cb4c35a0e user: Timo Sirainen date: Sat Dec 08 11:12:10 2012 +0200 description: lib-master: Ignore mountpoints under /Volumes by default. diffstat: src/lib-master/mountpoint-list.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r c1e47738e7c3 -r 657cb4c35a0e src/lib-master/mountpoint-list.c --- a/src/lib-master/mountpoint-list.c Sat Dec 08 10:48:23 2012 +0200 +++ b/src/lib-master/mountpoint-list.c Sat Dec 08 11:12:10 2012 +0200 @@ -57,6 +57,7 @@ "/proc", "/var/run", "/run", + "/Volumes", /* OSX */ NULL }; From dovecot at dovecot.org Sat Dec 8 11:13:57 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 08 Dec 2012 11:13:57 +0200 Subject: dovecot-2.1: lib-master: Ignore /Volumes and /private/tmp mountp... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/265adb00eacd changeset: 14843:265adb00eacd user: Timo Sirainen date: Sat Dec 08 11:13:51 2012 +0200 description: lib-master: Ignore /Volumes and /private/tmp mountpoints with OSX (only). diffstat: src/lib-master/mountpoint-list.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 657cb4c35a0e -r 265adb00eacd src/lib-master/mountpoint-list.c --- a/src/lib-master/mountpoint-list.c Sat Dec 08 11:12:10 2012 +0200 +++ b/src/lib-master/mountpoint-list.c Sat Dec 08 11:13:51 2012 +0200 @@ -57,7 +57,10 @@ "/proc", "/var/run", "/run", - "/Volumes", /* OSX */ +#ifdef __APPLE__ + "/Volumes", + "/private/tmp", +#endif NULL }; From dovecot at dovecot.org Tue Dec 11 08:08:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Dec 2012 08:08:23 +0200 Subject: dovecot-2.2: quota: Fixed quota calculation with mailbox_move(). Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4a110cf281aa changeset: 15459:4a110cf281aa user: Timo Sirainen date: Tue Dec 11 08:08:12 2012 +0200 description: quota: Fixed quota calculation with mailbox_move(). diffstat: src/plugins/quota/quota-storage.c | 29 ++++++++++++++--------------- 1 files changed, 14 insertions(+), 15 deletions(-) diffs (65 lines): diff -r 0851ffee594b -r 4a110cf281aa src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Sat Dec 08 09:37:21 2012 +0200 +++ b/src/plugins/quota/quota-storage.c Tue Dec 11 08:08:12 2012 +0200 @@ -161,13 +161,22 @@ MODULE_CONTEXT_SET_SELF(mail, quota_mail_module, qmail); } -static int quota_check(struct mailbox_transaction_context *t, struct mail *mail) +static int quota_check(struct mail_save_context *ctx) { + struct mailbox_transaction_context *t = ctx->transaction; struct quota_transaction_context *qt = QUOTA_CONTEXT(t); int ret; bool too_large; - ret = quota_try_alloc(qt, mail, &too_large); + if (ctx->moving) { + /* the mail is being moved. the quota won't increase (after + the following expunge), so allow this even if user is + currently over quota */ + quota_alloc(qt, ctx->dest_mail); + return 0; + } + + ret = quota_try_alloc(qt, ctx->dest_mail, &too_large); if (ret > 0) return 0; else if (ret == 0) { @@ -206,12 +215,7 @@ quota */ return 0; } - if (ctx->moving) { - /* the mail is being moved. the quota won't increase, so allow - this even if user is currently over quota */ - return 0; - } - return quota_check(t, ctx->dest_mail); + return quota_check(ctx); } static int @@ -223,7 +227,7 @@ uoff_t size; int ret; - if (i_stream_get_size(input, TRUE, &size) > 0 && !ctx->moving) { + if (!ctx->moving && i_stream_get_size(input, TRUE, &size) > 0) { /* Input size is known, check for quota immediately. This check isn't perfect, especially because input stream's linefeeds may contain CR+LFs while physical message would @@ -267,12 +271,7 @@ if (qbox->module_ctx.super.save_finish(ctx) < 0) return -1; - if (ctx->moving) { - /* the mail is being moved. the quota won't increase, so allow - this even if user is currently over quota */ - return 0; - } - return quota_check(ctx->transaction, ctx->dest_mail); + return quota_check(ctx); } static void quota_mailbox_sync_cleanup(struct quota_mailbox *qbox) From dovecot at dovecot.org Tue Dec 11 20:06:20 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Dec 2012 20:06:20 +0200 Subject: dovecot-2.1: virtual plugin: Don't fail if mailbox patterns don'... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/f13f2725882e changeset: 14844:f13f2725882e user: Timo Sirainen date: Tue Dec 11 20:04:10 2012 +0200 description: virtual plugin: Don't fail if mailbox patterns don't match anything. diffstat: src/plugins/virtual/virtual-config.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (28 lines): diff -r 265adb00eacd -r f13f2725882e src/plugins/virtual/virtual-config.c --- a/src/plugins/virtual/virtual-config.c Sat Dec 08 11:13:51 2012 +0200 +++ b/src/plugins/virtual/virtual-config.c Tue Dec 11 20:04:10 2012 +0200 @@ -27,6 +27,7 @@ char sep; bool have_wildcards; + bool have_mailbox_defines; }; static struct mail_search_args * @@ -169,6 +170,7 @@ bbox->name++; ctx->mbox->save_bbox = bbox; } + ctx->have_mailbox_defines = TRUE; array_append(&ctx->mbox->backend_boxes, &bbox, 1); return 0; } @@ -419,7 +421,7 @@ if (ret == 0 && ctx.have_wildcards) ret = virtual_config_expand_wildcards(&ctx); - if (ret == 0 && array_count(&mbox->backend_boxes) == 0) { + if (ret == 0 && !ctx.have_mailbox_defines) { mail_storage_set_critical(storage, "%s: No mailboxes defined", path); ret = -1; From dovecot at dovecot.org Wed Dec 12 11:34:58 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Dec 2012 11:34:58 +0200 Subject: dovecot-2.2: lib-http: Added support for chunked input/output st... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7a4d8cd0e079 changeset: 15460:7a4d8cd0e079 user: Stephan Bosch date: Wed Dec 12 11:34:09 2012 +0200 description: lib-http: Added support for chunked input/output streams and some bugfixes. diffstat: src/lib-http/http-client-connection.c | 10 +- src/lib-http/http-client-private.h | 4 +- src/lib-http/http-client-request.c | 48 ++++++- src/lib-http/http-response-parser.c | 1 - src/lib-http/http-transfer-chunked.c | 202 ++++++++++++++++++++++++--- src/lib-http/http-transfer.h | 4 + src/lib-http/test-http-client.c | 13 +- src/lib-http/test-http-transfer.c | 243 ++++++++++++++++++++++++++++++++- 8 files changed, 478 insertions(+), 47 deletions(-) diffs (truncated from 816 to 300 lines): diff -r 4a110cf281aa -r 7a4d8cd0e079 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Tue Dec 11 08:08:12 2012 +0200 +++ b/src/lib-http/http-client-connection.c Wed Dec 12 11:34:09 2012 +0200 @@ -62,7 +62,7 @@ bool http_client_connection_is_ready(struct http_client_connection *conn) { return (conn->connected && !conn->output_locked && - array_count(&conn->request_wait_list) < + !conn->close_indicated && array_count(&conn->request_wait_list) < conn->client->set.max_pipelined_requests); } @@ -428,6 +428,10 @@ return; } + /* Got some response; cancel response timeout */ + if (conn->to_response != NULL) + timeout_remove(&conn->to_response); + /* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21 Section 7.2: @@ -438,8 +442,6 @@ */ if (req->payload_sync && response->status == 100) { conn->payload_continue = TRUE; - if (conn->to_response != NULL) - timeout_remove(&conn->to_response); http_client_connection_debug(conn, "Got expected 100-continue response"); if (http_client_request_send_more(req) < 0) { @@ -469,7 +471,7 @@ if (!aborted) { if (response->status / 100 == 3) { /* redirect */ - http_client_request_redirect(req, response->location); + http_client_request_redirect(req, response->status, response->location); } else { /* response for application */ if (!http_client_connection_return_response(conn, req, response)) diff -r 4a110cf281aa -r 7a4d8cd0e079 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Tue Dec 11 08:08:12 2012 +0200 +++ b/src/lib-http/http-client-private.h Wed Dec 12 11:34:09 2012 +0200 @@ -57,7 +57,7 @@ string_t *headers; struct istream *input; - uoff_t input_size; + uoff_t input_size, input_offset; unsigned int attempts; unsigned int redirects; @@ -198,7 +198,7 @@ void http_client_request_error(struct http_client_request *req, unsigned int status, const char *error); void http_client_request_redirect(struct http_client_request *req, - const char *location); + unsigned int status, const char *location); void http_client_request_finish(struct http_client_request **_req); struct connection_list *http_client_connection_list_init(void); diff -r 4a110cf281aa -r 7a4d8cd0e079 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Dec 11 08:08:12 2012 +0200 +++ b/src/lib-http/http-client-request.c Wed Dec 12 11:34:09 2012 +0200 @@ -137,6 +137,7 @@ req->input = input; if (i_stream_get_size(input, TRUE, &req->input_size) <= 0) i_unreached(); //FIXME + req->input_offset = input->v_offset; /* prepare request payload sync using 100 Continue response from server */ if (req->input_size > 0 && sync) { @@ -175,7 +176,6 @@ i_error("stream input size changed"); //FIXME return -1; } - i_stream_unref(&req->input); req->state = HTTP_REQUEST_STATE_WAITING; conn->output_locked = FALSE; http_client_request_debug(req, "Sent all payload"); @@ -313,7 +313,7 @@ } void http_client_request_redirect(struct http_client_request *req, - const char *location) + unsigned int status, const char *location) { struct http_url *url; const char *error; @@ -341,6 +341,18 @@ return; } + /* rewind payload stream */ + if (req->input != NULL && req->input_size > 0 && status != 303) { + if (req->input->v_offset != req->input_offset && !req->input->seekable) { + http_client_request_error(req, + HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Redirect failed: Cannot resend payload; stream is not seekable"); + return; + } else { + i_stream_seek(req->input, req->input_offset); + } + } + newport = (url->have_port ? url->port : (url->have_ssl ? 443 : 80)); http_client_request_debug(req, "Redirecting to http://%s:%u%s", @@ -354,6 +366,26 @@ req->target = p_strdup(req->pool, url->path); req->ssl = url->have_ssl; + /* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21 + Section-7.4.4 + + -> A 303 `See Other' redirect status response is handled a bit differently. + Basically, the response content is located elsewhere, but the original + (POST) request is handled already. + */ + if (status == 303 && strcasecmp(req->method, "HEAD") != 0 && + strcasecmp(req->method, "GET") != 0) { + // FIXME: should we provide the means to skip this step? The original + // request was already handled at this point. + req->method = p_strdup(req->pool, "GET"); + + /* drop payload */ + if (req->input != NULL) + i_stream_unref(&req->input); + req->input_size = 0; + req->input_offset = 0; + } + /* resubmit */ req->client->pending_requests--; req->state = HTTP_REQUEST_STATE_NEW; @@ -364,6 +396,18 @@ { http_client_request_debug(req, "Resubmitting request"); + /* rewind payload stream */ + if (req->input != NULL && req->input_size > 0) { + if (req->input->v_offset != req->input_offset && !req->input->seekable) { + http_client_request_error(req, + HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Resubmission failed: Cannot resend payload; stream is not seekable"); + return; + } else { + i_stream_seek(req->input, req->input_offset); + } + } + req->conn = NULL; req->peer = NULL; req->state = HTTP_REQUEST_STATE_QUEUED; diff -r 4a110cf281aa -r 7a4d8cd0e079 src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Tue Dec 11 08:08:12 2012 +0200 +++ b/src/lib-http/http-response-parser.c Wed Dec 12 11:34:09 2012 +0200 @@ -476,7 +476,6 @@ / transfer-extension ; [FIXME] transfer-extension = token *( OWS ";" OWS transfer-parameter ) */ - // FIXME: parse this directly when the header is parsed http_parser_init(&hparser, (const unsigned char *)parser->transfer_encoding, strlen(parser->transfer_encoding)); diff -r 4a110cf281aa -r 7a4d8cd0e079 src/lib-http/http-transfer-chunked.c --- a/src/lib-http/http-transfer-chunked.c Tue Dec 11 08:08:12 2012 +0200 +++ b/src/lib-http/http-transfer-chunked.c Wed Dec 12 11:34:09 2012 +0200 @@ -132,25 +132,25 @@ /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21; Section 4.1: - chunked-body = *chunk - last-chunk - trailer-part - CRLF + chunked-body = *chunk + last-chunk + trailer-part + CRLF - chunk = chunk-size [ chunk-ext ] CRLF - chunk-data CRLF - chunk-size = 1*HEXDIG - last-chunk = 1*("0") [ chunk-ext ] CRLF + chunk = chunk-size [ chunk-ext ] CRLF + chunk-data CRLF + chunk-size = 1*HEXDIG + last-chunk = 1*("0") [ chunk-ext ] CRLF - chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) - chunk-ext-name = token - chunk-ext-val = token / quoted-str-nf - chunk-data = 1*OCTET ; a sequence of chunk-size octets - trailer-part = *( header-field CRLF ) + chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) + chunk-ext-name = token + chunk-ext-val = token / quoted-str-nf + chunk-data = 1*OCTET ; a sequence of chunk-size octets + trailer-part = *( header-field CRLF ) - quoted-str-nf = DQUOTE *( qdtext-nf / quoted-pair ) DQUOTE - ; like quoted-string, but disallowing line folding - qdtext-nf = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text + quoted-str-nf = DQUOTE *( qdtext-nf / quoted-pair ) DQUOTE + ; like quoted-string, but disallowing line folding + qdtext-nf = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) */ @@ -202,7 +202,7 @@ if (*tcstream->cur != '"') { tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN; break; - } + } tcstream->cur++; tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING; if (tcstream->cur >= tcstream->end) @@ -219,11 +219,11 @@ } else if ((ret=http_transfer_chunked_skip_qdtext(tcstream)) <= 0) { if (ret < 0) tcstream->error = "Invalid chunked extension value"; - return ret; + return ret; } else if (*tcstream->cur == '\\') { tcstream->cur++; tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE; - if (tcstream->cur >= tcstream->end) + if (tcstream->cur >= tcstream->end) return 0; break; } else { @@ -243,7 +243,7 @@ return -1; } tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING; - if (tcstream->cur >= tcstream->end) + if (tcstream->cur >= tcstream->end) return 0; break; case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN: @@ -258,7 +258,7 @@ tcstream->state = HTTP_CHUNKED_PARSE_STATE_LF; if (*tcstream->cur == '\r') { tcstream->cur++; - if (tcstream->cur >= tcstream->end) + if (tcstream->cur >= tcstream->end) return 0; } /* fall through */ @@ -281,7 +281,7 @@ tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_LF; if (*tcstream->cur == '\r') { tcstream->cur++; - if (tcstream->cur >= tcstream->end) + if (tcstream->cur >= tcstream->end) return 0; } /* fall through */ @@ -307,6 +307,7 @@ static int http_transfer_chunked_parse_next( struct http_transfer_chunked_istream *tcstream) { + struct istream_private *stream = &tcstream->istream; struct istream *input = tcstream->istream.parent; size_t size; int ret; @@ -316,8 +317,10 @@ tcstream->cur = tcstream->begin; tcstream->end = tcstream->cur + size; - if ((ret=http_transfer_chunked_parse(tcstream)) < 0) + if ((ret=http_transfer_chunked_parse(tcstream)) < 0) { + stream->istream.stream_errno = EIO; return -1; + } i_stream_skip(input, tcstream->cur - tcstream->begin); @@ -329,9 +332,23 @@ } i_assert(ret != -2); + + if (ret < 0) { + if ( stream->parent->eof && stream->parent->stream_errno == 0 ) { + /* unexpected EOF */ + tcstream->error = "Unexpected end of payload"; + stream->istream.stream_errno = EIO; + } else { + /* parent stream error */ + tcstream->error = "Stream error"; + stream->istream.stream_errno = stream->parent->stream_errno; + } + } return ret; } From dovecot at dovecot.org Wed Dec 12 15:21:56 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Dec 2012 15:21:56 +0200 Subject: dovecot-2.2: lib-http: Fixed sending HTTP request's payload. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2125cf15b26d changeset: 15461:2125cf15b26d user: Timo Sirainen date: Wed Dec 12 15:21:45 2012 +0200 description: lib-http: Fixed sending HTTP request's payload. diffstat: src/lib-http/http-client-request.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7a4d8cd0e079 -r 2125cf15b26d src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Wed Dec 12 11:34:09 2012 +0200 +++ b/src/lib-http/http-client-request.c Wed Dec 12 15:21:45 2012 +0200 @@ -171,7 +171,7 @@ ret = -1; o_stream_set_max_buffer_size(output, (size_t)-1); - if (req->input->eof) { + if (!i_stream_have_bytes_left(req->input)) { if (req->input->v_offset != req->input_size) { i_error("stream input size changed"); //FIXME return -1; From dovecot at dovecot.org Thu Dec 13 12:13:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Dec 2012 12:13:52 +0200 Subject: dovecot-2.1: Added stat_first_parent() helper function. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/40f9e2e339a6 changeset: 14845:40f9e2e339a6 user: Timo Sirainen date: Thu Dec 13 12:13:42 2012 +0200 description: Added stat_first_parent() helper function. diffstat: src/lib/mkdir-parents.c | 20 ++++++++++++++++++++ src/lib/mkdir-parents.h | 8 ++++++++ 2 files changed, 28 insertions(+), 0 deletions(-) diffs (50 lines): diff -r f13f2725882e -r 40f9e2e339a6 src/lib/mkdir-parents.c --- a/src/lib/mkdir-parents.c Tue Dec 11 20:04:10 2012 +0200 +++ b/src/lib/mkdir-parents.c Thu Dec 13 12:13:42 2012 +0200 @@ -125,3 +125,23 @@ { return mkdir_parents_chown(path, mode, (uid_t)-1, (gid_t)-1); } + +int stat_first_parent(const char *path, const char **root_dir_r, + struct stat *st_r) +{ + const char *p; + + while (stat(path, st_r) < 0) { + if (errno != ENOENT || strcmp(path, "/") == 0) { + *root_dir_r = path; + return -1; + } + p = strrchr(path, '/'); + if (p == NULL) + path = "/"; + else + path = t_strdup_until(path, p); + } + *root_dir_r = path; + return 0; +} diff -r f13f2725882e -r 40f9e2e339a6 src/lib/mkdir-parents.h --- a/src/lib/mkdir-parents.h Tue Dec 11 20:04:10 2012 +0200 +++ b/src/lib/mkdir-parents.h Thu Dec 13 12:13:42 2012 +0200 @@ -1,6 +1,8 @@ #ifndef MKDIR_PARENTS_H #define MKDIR_PARENTS_H +#include + /* Create path and all the directories under it if needed. Permissions for existing directories isn't changed. Returns 0 if ok. If directory already exists, returns -1 with errno=EEXIST. */ @@ -20,4 +22,10 @@ int mkdir_chgrp(const char *path, mode_t mode, gid_t gid, const char *gid_origin); +/* stat() the path or its first parent that exists. Returns 0 if ok, -1 if + failed. root_dir is set to the last stat()ed directory (on success and + on failure). */ +int stat_first_parent(const char *path, const char **root_dir_r, + struct stat *st_r); + #endif From dovecot at dovecot.org Thu Dec 13 12:15:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Dec 2012 12:15:06 +0200 Subject: dovecot-2.1: lib-storage: Use stat_first_parent() instead of doi... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c0ad6afa4128 changeset: 14846:c0ad6afa4128 user: Timo Sirainen date: Thu Dec 13 12:14:03 2012 +0200 description: lib-storage: Use stat_first_parent() instead of doing it ourself. diffstat: src/lib-storage/mailbox-list.c | 26 +++----------------------- 1 files changed, 3 insertions(+), 23 deletions(-) diffs (44 lines): diff -r 40f9e2e339a6 -r c0ad6afa4128 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Thu Dec 13 12:13:42 2012 +0200 +++ b/src/lib-storage/mailbox-list.c Thu Dec 13 12:14:03 2012 +0200 @@ -765,27 +765,6 @@ } } -static int -mailbox_list_stat_parent(const char *path, const char **root_dir_r, - struct stat *st_r, const char **error_r) -{ - const char *p; - - while (stat(path, st_r) < 0) { - if (errno != ENOENT || strcmp(path, "/") == 0) { - *error_r = t_strdup_printf("stat(%s) failed: %m", path); - return -1; - } - p = strrchr(path, '/'); - if (p == NULL) - path = "/"; - else - path = t_strdup_until(path, p); - } - *root_dir_r = path; - return 0; -} - static const char * get_expanded_path(const char *unexpanded_start, const char *unexpanded_stop, const char *expanded_full) @@ -873,9 +852,10 @@ /* up to this directory get the permissions from the first parent directory that exists, if it has setgid bit enabled. */ - if (mailbox_list_stat_parent(expanded, &root_dir, &st, - error_r) < 0) + if (stat_first_parent(expanded, &root_dir, &st) < 0) { + *error_r = t_strdup_printf("stat(%s) failed: %m", root_dir); return -1; + } if ((st.st_mode & S_ISGID) != 0 && root_dir != expanded) { if (mkdir_parents_chgrp(expanded, st.st_mode, (gid_t)-1, root_dir) < 0 && From dovecot at dovecot.org Thu Dec 13 12:15:07 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Dec 2012 12:15:07 +0200 Subject: dovecot-2.1: dict-file: Automatically mkdir missing parent direc... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/3c6a13c0d525 changeset: 14847:3c6a13c0d525 user: Timo Sirainen date: Thu Dec 13 12:14:57 2012 +0200 description: dict-file: Automatically mkdir missing parent directories if they don't exist. diffstat: src/lib-dict/dict-file.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 files changed, 39 insertions(+), 0 deletions(-) diffs (70 lines): diff -r c0ad6afa4128 -r 3c6a13c0d525 src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Thu Dec 13 12:14:03 2012 +0200 +++ b/src/lib-dict/dict-file.c Thu Dec 13 12:14:57 2012 +0200 @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "hash.h" +#include "mkdir-parents.h" #include "file-lock.h" #include "file-dotlock.h" #include "nfs-workarounds.h" @@ -424,6 +425,33 @@ return fd_copy_stat_permissions(&src_st, dest_fd, dest_path); } +static int file_dict_mkdir(struct file_dict *dict) +{ + const char *path, *p, *root; + struct stat st; + mode_t mode = 0700; + + p = strrchr(dict->path, '/'); + if (p == NULL) + return 0; + path = t_strdup_until(dict->path, p); + + if (stat_first_parent(path, &root, &st) < 0) { + i_error("stat(%s) failed: %m", root); + return -1; + } + if ((st.st_mode & S_ISGID) != 0) { + /* preserve parent's permissions when it has setgid bit */ + mode = st.st_mode; + } + + if (mkdir_parents(path, mode) < 0) { + i_error("mkdir_parents(%s) failed: %m", path); + return -1; + } + return 0; +} + static int file_dict_lock(struct file_dict *dict, struct file_lock **lock_r) { @@ -435,6 +463,11 @@ if (dict->fd == -1) { /* quota file doesn't exist yet, we need to create it */ dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + if (dict->fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict) < 0) + return -1; + dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + } if (dict->fd == -1) { i_error("creat(%s) failed: %m", dict->path); return -1; @@ -485,6 +518,12 @@ case FILE_LOCK_METHOD_DOTLOCK: fd = file_dotlock_open(&file_dict_dotlock_settings, dict->path, 0, &dotlock); + if (fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict) < 0) + return -1; + fd = file_dotlock_open(&file_dict_dotlock_settings, + dict->path, 0, &dotlock); + } if (fd == -1) { i_error("file dict commit: file_dotlock_open(%s) failed: %m", dict->path); From dovecot at dovecot.org Thu Dec 13 12:24:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Dec 2012 12:24:39 +0200 Subject: dovecot-2.2: dbox: dbox-alt-root symlink now points to the alt r... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b1267f135417 changeset: 15462:b1267f135417 user: Timo Sirainen date: Thu Dec 13 12:24:32 2012 +0200 description: dbox: dbox-alt-root symlink now points to the alt root directory, not to mailboxes dir. Existing mailboxes/ symlinks are silently changed to new ones. This also means that accessing the dbox with an older version will log a warning about the change. diffstat: src/lib-storage/index/dbox-common/dbox-storage.c | 20 +++++++++++++++----- 1 files changed, 15 insertions(+), 5 deletions(-) diffs (49 lines): diff -r 2125cf15b26d -r b1267f135417 src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Wed Dec 12 15:21:45 2012 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Thu Dec 13 12:24:32 2012 +0200 @@ -30,8 +30,8 @@ } static bool -dbox_alt_path_has_changed(const char *root_dir, - const char *alt_path, const char *alt_symlink_path) +dbox_alt_path_has_changed(const char *root_dir, const char *alt_path, + const char *alt_path2, const char *alt_symlink_path) { const char *linkpath; @@ -47,6 +47,13 @@ "but currently no ALT path set", root_dir, linkpath); return TRUE; } else if (strcmp(linkpath, alt_path) != 0) { + if (strcmp(linkpath, alt_path2) == 0) { + /* FIXME: for backwards compatibility. old versions + created the symlink to mailboxes/ directory, which + was fine with sdbox, but didn't even exist with + mdbox. we'll silently replace the symlink. */ + return TRUE; + } i_warning("dbox %s: Original ALT=%s, " "but currently ALT=%s", root_dir, linkpath, alt_path); return TRUE; @@ -56,14 +63,17 @@ static void dbox_verify_alt_path(struct mailbox_list *list) { - const char *root_dir, *alt_symlink_path, *alt_path; + const char *root_dir, *alt_symlink_path, *alt_path, *alt_path2; root_dir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_DIR); alt_symlink_path = t_strconcat(root_dir, "/"DBOX_ALT_SYMLINK_NAME, NULL); + (void)mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_ALT_DIR, + &alt_path); (void)mailbox_list_get_root_path(list, MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX, - &alt_path); - if (!dbox_alt_path_has_changed(root_dir, alt_path, alt_symlink_path)) + &alt_path2); + if (!dbox_alt_path_has_changed(root_dir, alt_path, alt_path2, + alt_symlink_path)) return; /* unlink/create the current alt path symlink */ From dovecot at dovecot.org Fri Dec 14 11:23:04 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Dec 2012 11:23:04 +0200 Subject: dovecot-2.2: dsync: Fixed hangs with remote dsyncing Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f9e8cbb2792d changeset: 15463:f9e8cbb2792d user: Timo Sirainen date: Fri Dec 14 11:17:19 2012 +0200 description: dsync: Fixed hangs with remote dsyncing diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (28 lines): diff -r b1267f135417 -r f9e8cbb2792d src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Thu Dec 13 12:24:32 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Fri Dec 14 11:17:19 2012 +0200 @@ -143,7 +143,11 @@ static int dsync_ibc_stream_read_mail_stream(struct dsync_ibc_stream *ibc) { - if (i_stream_read(ibc->mail_input) < 0) { + while (i_stream_read(ibc->mail_input) > 0) { + i_stream_skip(ibc->mail_input, + i_stream_get_data_size(ibc->mail_input)); + } + if (ibc->mail_input->eof) { if (ibc->mail_input->stream_errno != 0) { errno = ibc->mail_input->stream_errno; i_error("dsync(%s): read() failed: %m", ibc->name); @@ -153,10 +157,10 @@ /* finished reading the mail stream */ i_assert(ibc->mail_input->eof); i_stream_seek(ibc->mail_input, 0); + ibc->has_pending_data = TRUE; ibc->mail_input = NULL; return 1; } - i_stream_skip(ibc->mail_input, i_stream_get_data_size(ibc->mail_input)); return 0; } From dovecot at dovecot.org Fri Dec 14 11:23:04 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Dec 2012 11:23:04 +0200 Subject: dovecot-2.2: dsync: Added debugging and rawlogging support. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/af3b86a16ac3 changeset: 15464:af3b86a16ac3 user: Timo Sirainen date: Fri Dec 14 11:20:04 2012 +0200 description: dsync: Added debugging and rawlogging support. diffstat: src/doveadm/dsync/doveadm-dsync.c | 58 +++++++++++++++++++++++++------- src/doveadm/dsync/dsync-brain-mails.c | 27 +++++++++++++++ src/doveadm/dsync/dsync-brain-private.h | 1 + src/doveadm/dsync/dsync-brain.c | 24 +++++++++++++ src/doveadm/dsync/dsync-brain.h | 3 +- src/doveadm/dsync/dsync-ibc-stream.c | 26 +++++--------- src/doveadm/dsync/dsync-ibc.h | 4 +- 7 files changed, 111 insertions(+), 32 deletions(-) diffs (truncated from 376 to 300 lines): diff -r f9e8cbb2792d -r af3b86a16ac3 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Fri Dec 14 11:17:19 2012 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Fri Dec 14 11:20:04 2012 +0200 @@ -4,6 +4,10 @@ #include "lib-signals.h" #include "array.h" #include "execv-const.h" +#include "fd-set-nonblock.h" +#include "istream.h" +#include "ostream.h" +#include "iostream-rawlog.h" #include "str.h" #include "var-expand.h" #include "settings-parser.h" @@ -23,13 +27,13 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+dEfl:m:n:s:" +#define DSYNC_COMMON_GETOPT_ARGS "+dEfm:n:r:Rs:" struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; enum dsync_brain_sync_type sync_type; const char *mailbox, *namespace_prefix; - const char *state_input; + const char *state_input, *rawlog_path; const char *remote_name; const char *local_location; @@ -312,6 +316,23 @@ return get_ssh_cmd_args(ctx, host, login, username); } +static struct dsync_ibc * +cmd_dsync_icb_stream_init(struct dsync_cmd_context *ctx, + const char *name, const char *temp_prefix) +{ + struct istream *input; + struct ostream *output; + + fd_set_nonblock(ctx->fd_in, TRUE); + fd_set_nonblock(ctx->fd_out, TRUE); + + input = i_stream_create_fd(ctx->fd_in, (size_t)-1, FALSE); + output = o_stream_create_fd(ctx->fd_out, (size_t)-1, FALSE); + if (ctx->rawlog_path != NULL) + iostream_rawlog_create_path(ctx->rawlog_path, &input, &output); + return dsync_ibc_init_stream(input, output, name, temp_prefix); +} + static int cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) { @@ -335,22 +356,20 @@ else { string_t *temp_prefix = t_str_new(64); mail_user_set_get_temp_prefix(temp_prefix, user->set); - ibc = dsync_ibc_init_stream(ctx->fd_in, ctx->fd_out, - ctx->remote_name, - str_c(temp_prefix)); - } - - if (doveadm_debug || doveadm_verbose) { - // FIXME + ibc = cmd_dsync_icb_stream_init(ctx, ctx->remote_name, + str_c(temp_prefix)); } brain_flags = DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS | DSYNC_BRAIN_FLAG_SEND_GUID_REQUESTS; + if (ctx->reverse_backup) brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; else if (ctx->backup) brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; + if (doveadm_debug) + brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; brain = dsync_brain_master_init(user, ibc, sync_ns, ctx->sync_type, brain_flags, ctx->state_input == NULL ? "" : @@ -376,6 +395,11 @@ dsync_ibc_deinit(&ibc2); if (ctx->io_err != NULL) io_remove(&ctx->io_err); + if (ctx->fd_in != -1) { + if (ctx->fd_out != ctx->fd_in) + i_close_fd(&ctx->fd_out); + i_close_fd(&ctx->fd_in); + } if (ctx->fd_err != -1) i_close_fd(&ctx->fd_err); return ret; @@ -482,6 +506,9 @@ case 'n': ctx->namespace_prefix = optarg; break; + case 'r': + ctx->rawlog_path = optarg; + break; case 'R': if (!ctx->backup) return FALSE; @@ -527,9 +554,10 @@ } static int -cmd_dsync_server_run(struct doveadm_mail_cmd_context *_ctx ATTR_UNUSED, +cmd_dsync_server_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) { + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; struct dsync_ibc *ibc; struct dsync_brain *brain; string_t *temp_prefix; @@ -542,8 +570,7 @@ temp_prefix = t_str_new(64); mail_user_set_get_temp_prefix(temp_prefix, user->set); - ibc = dsync_ibc_init_stream(STDIN_FILENO, STDOUT_FILENO, - "local", str_c(temp_prefix)); + ibc = cmd_dsync_icb_stream_init(ctx, "local", str_c(temp_prefix)); brain = dsync_brain_slave_init(user, ibc); io_loop_run(current_ioloop); @@ -567,6 +594,9 @@ if (str_to_uint(optarg, &ctx->lock_timeout) < 0) i_error("Invalid -l parameter: %s", optarg); break; + case 'r': + ctx->rawlog_path = optarg; + break; case 'n': ctx->namespace_prefix = optarg; break; @@ -581,10 +611,12 @@ struct dsync_cmd_context *ctx; ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); - ctx->ctx.getopt_args = "El:n:"; + ctx->ctx.getopt_args = "El:n:r:"; ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg; ctx->ctx.v.run = cmd_dsync_server_run; ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; + ctx->fd_in = STDIN_FILENO; + ctx->fd_out = STDOUT_FILENO; return &ctx->ctx; } diff -r f9e8cbb2792d -r af3b86a16ac3 src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Fri Dec 14 11:17:19 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Fri Dec 14 11:20:04 2012 +0200 @@ -8,6 +8,15 @@ #include "dsync-mailbox-export.h" #include "dsync-brain-private.h" +static const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1] = { + "mailbox", + "changes", + "mail_requests", + "mails", + "recv_last_common", + "done" +}; + static bool dsync_brain_master_sync_recv_mailbox(struct dsync_brain *brain) { const struct dsync_mailbox *dsync_box; @@ -182,6 +191,10 @@ dsync_brain_sync_half_finished(brain); return TRUE; } + if (brain->debug) { + i_debug("brain %c: import mail uid %u guid %s", + brain->master_brain ? 'M' : 'S', mail->uid, mail->guid); + } dsync_mailbox_import_mail(brain->box_importer, mail); if (mail->input != NULL) i_stream_unref(&mail->input); @@ -236,6 +249,13 @@ i_assert(brain->box != NULL); + if (brain->debug) { + i_debug("brain %c: in box '%s' recv_state=%s send_state=%s", + brain->master_brain ? 'M' : 'S', + mailbox_get_vname(brain->box), + dsync_box_state_names[brain->box_recv_state], + dsync_box_state_names[brain->box_send_state]); + } switch (brain->box_recv_state) { case DSYNC_BOX_STATE_MAILBOX: changed = dsync_brain_master_sync_recv_mailbox(brain); @@ -282,5 +302,12 @@ case DSYNC_BOX_STATE_DONE: break; } + if (brain->debug) { + i_debug("brain %c: out box '%s' recv_state=%s send_state=%s changed=%d", + brain->master_brain ? 'M' : 'S', + brain->box == NULL ? "" : mailbox_get_vname(brain->box), + dsync_box_state_names[brain->box_recv_state], + dsync_box_state_names[brain->box_send_state], changed); + } return changed; } diff -r f9e8cbb2792d -r af3b86a16ac3 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Fri Dec 14 11:17:19 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-private.h Fri Dec 14 11:20:04 2012 +0200 @@ -80,6 +80,7 @@ unsigned int mails_have_guids:1; unsigned int backup_send:1; unsigned int backup_recv:1; + unsigned int debug:1; unsigned int changes_during_sync:1; unsigned int failed:1; }; diff -r f9e8cbb2792d -r af3b86a16ac3 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Fri Dec 14 11:17:19 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Fri Dec 14 11:20:04 2012 +0200 @@ -8,6 +8,20 @@ #include "dsync-ibc.h" #include "dsync-brain-private.h" +static const char *dsync_state_names[DSYNC_STATE_DONE+1] = { + "recv_handshake", + "send_last_common", + "recv_last_common", + "send_mailbox_tree", + "send_mailbox_tree_deletes", + "recv_mailbox_tree", + "recv_mailbox_tree_deletes", + "master_send_mailbox", + "slave_recv_mailbox", + "sync_mails", + "done" +}; + static void dsync_brain_run_io(void *context) { struct dsync_brain *brain = context; @@ -62,6 +76,7 @@ (flags & DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS) != 0; brain->backup_send = (flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0; brain->backup_recv = (flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0; + brain->debug = (flags & DSYNC_BRAIN_FLAG_DEBUG) != 0; } struct dsync_brain * @@ -244,6 +259,10 @@ if (brain->failed) return FALSE; + if (brain->debug) { + i_debug("brain %c: in state=%s", brain->master_brain ? 'M' : 'S', + dsync_state_names[brain->state]); + } switch (brain->state) { case DSYNC_STATE_SLAVE_RECV_HANDSHAKE: changed = dsync_brain_slave_recv_handshake(brain); @@ -284,6 +303,11 @@ ret = FALSE; break; } + if (brain->debug) { + i_debug("brain %c: out state=%s changed=%d", + brain->master_brain ? 'M' : 'S', + dsync_state_names[brain->state], changed); + } *changed_r = changed; return brain->failed ? FALSE : ret; diff -r f9e8cbb2792d -r af3b86a16ac3 src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Fri Dec 14 11:17:19 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain.h Fri Dec 14 11:20:04 2012 +0200 @@ -9,7 +9,8 @@ DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS = 0x01, DSYNC_BRAIN_FLAG_SEND_GUID_REQUESTS = 0x02, DSYNC_BRAIN_FLAG_BACKUP_SEND = 0x04, - DSYNC_BRAIN_FLAG_BACKUP_RECV = 0x08 + DSYNC_BRAIN_FLAG_BACKUP_RECV = 0x08, + DSYNC_BRAIN_FLAG_DEBUG = 0x10 }; enum dsync_brain_sync_type { diff -r f9e8cbb2792d -r af3b86a16ac3 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Dec 14 11:17:19 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Fri Dec 14 11:20:04 2012 +0200 @@ -60,7 +60,7 @@ { NULL, '\0', NULL, NULL }, { .name = "handshake", .chr = 'H', - .optional_keys = "sync_ns_prefix sync_type " + .optional_keys = "sync_ns_prefix sync_type debug " "mails_have_guids send_guid_requests backup_send backup_recv" }, { .name = "mailbox_state", @@ -267,11 +267,6 @@ { From dovecot at dovecot.org Fri Dec 14 11:46:00 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Dec 2012 11:46:00 +0200 Subject: dovecot-2.2: lib-storage: Fixed vname -> storage name conversion... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c6275c34d158 changeset: 15465:c6275c34d158 user: Timo Sirainen date: Fri Dec 14 11:45:54 2012 +0200 description: lib-storage: Fixed vname -> storage name conversion for some nonexistent mailbox names. For example prefix=INBOX/ name=INBOX-foo would have converted the name to "" instead of keeping "INBOX-foo". diffstat: src/lib-storage/mailbox-list.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r af3b86a16ac3 -r c6275c34d158 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Fri Dec 14 11:20:04 2012 +0200 +++ b/src/lib-storage/mailbox-list.c Fri Dec 14 11:45:54 2012 +0200 @@ -513,6 +513,7 @@ if (strncmp(ns->prefix, storage_name, prefix_len) == 0) storage_name += prefix_len; else if (strncmp(ns->prefix, storage_name, prefix_len-1) == 0 && + strlen(storage_name) == prefix_len-1 && ns->prefix[prefix_len-1] == mail_namespace_get_sep(ns)) { /* trying to access the namespace prefix itself */ storage_name = ""; From dovecot at dovecot.org Fri Dec 14 12:32:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Dec 2012 12:32:54 +0200 Subject: dovecot-2.2: dsync: Never try to create INBOX via temporary name... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9a707794ee54 changeset: 15466:9a707794ee54 user: Timo Sirainen date: Fri Dec 14 12:03:55 2012 +0200 description: dsync: Never try to create INBOX via temporary name and rename. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (26 lines): diff -r c6275c34d158 -r 9a707794ee54 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Fri Dec 14 11:45:54 2012 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Fri Dec 14 12:03:55 2012 +0200 @@ -634,8 +634,12 @@ local_node2 = hash_table_lookup(ctx->local_tree->guid_hash, guid_p); /* FIXME: kludge to avoid problems where one of the mailboxes - doesn't exist yet */ + doesn't exist yet. they seem to somewhat unnecessarily try to create + temporary mailboxes and later rename them. this definitely doesn't + work with INBOX. */ if (local_node2 == NULL && + (strcmp(local_node1->name, "INBOX") != 0 || + local_node1->parent->parent != NULL) && remote_node2->existence == DSYNC_MAILBOX_NODE_EXISTS && !dsync_mailbox_node_is_dir(remote_node2) && ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { @@ -643,6 +647,8 @@ return TRUE; } if (remote_node1 == NULL && + (strcmp(remote_node2->name, "INBOX") != 0 || + remote_node2->parent->parent != NULL) && local_node1->existence == DSYNC_MAILBOX_NODE_EXISTS && !dsync_mailbox_node_is_dir(local_node1) && ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { From dovecot at dovecot.org Fri Dec 14 15:35:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Dec 2012 15:35:25 +0200 Subject: dovecot-2.2: dsync: Fixes to creating/renaming mailboxes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/467cf7e5a616 changeset: 15467:467cf7e5a616 user: Timo Sirainen date: Fri Dec 14 15:32:43 2012 +0200 description: dsync: Fixes to creating/renaming mailboxes This gets rid of unnecessary mailbox renames when they should have simply been created. The algorithm appears to work, but again really should be fully thought out later to figure out why exactly it works. I only wanted to get it working now for other tests.. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 82 ++++++++++++---------------- 1 files changed, 35 insertions(+), 47 deletions(-) diffs (132 lines): diff -r 9a707794ee54 -r 467cf7e5a616 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Fri Dec 14 12:03:55 2012 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Fri Dec 14 15:32:43 2012 +0200 @@ -381,6 +381,7 @@ /* we're modifying a local tree. remember this change. */ new_name = dsync_mailbox_node_get_full_name(tree, node); + i_assert(strcmp(old_name, "INBOX") != 0); change = array_append_space(&ctx->changes); change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME; change->ns = node->ns; @@ -412,7 +413,9 @@ const char *name, *other_name; /* move/rename node in the tree, so that its position/name is identical - to other_node (in other_tree) */ + to other_node (in other_tree). temp_node's name is changed to + temporary name (i.e. it assumes that node's name becomes temp_node's + original name) */ other_tree = tree == ctx->local_tree ? ctx->remote_tree : ctx->local_tree; @@ -442,6 +445,7 @@ /* we're modifying a local tree. remember this change. */ other_name = dsync_mailbox_node_get_full_name(other_tree, other_node); + i_assert(strcmp(name, "INBOX") != 0); change = array_append_space(&ctx->changes); change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME; change->ns = node->ns; @@ -578,6 +582,22 @@ sync_rename_node(ctx, ctx->remote_tree, remote_node2, remote_node1, local_node1); return TRUE; + } else if (node_has_parent(local_node1, local_node2)) { + /* node2 is a parent of node1, but it should be + vice versa */ + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node1, local_node2->parent); + return TRUE; + } else if (node_has_parent(local_node2, local_node1)) { + /* node1 is a parent of node2, but it should be + vice versa */ + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node2, local_node1->parent); + return TRUE; + } else if (local_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node2, remote_node2->parent); + return TRUE; } else { /* local : 1A, 2B remote: 2A -> (2B) @@ -594,33 +614,23 @@ sync_rename_node(ctx, ctx->local_tree, local_node1, local_node2, remote_node2); return TRUE; + } else if (node_has_parent(remote_node1, remote_node2)) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node1, remote_node2->parent); + return TRUE; + } else if (node_has_parent(remote_node2, remote_node1)) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node2, remote_node1->parent); + return TRUE; + } else if (remote_node2->existence == DSYNC_MAILBOX_NODE_EXISTS) { + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node1, local_node1->parent); + return TRUE; } } return FALSE; } -static void -add_missing_mailbox(struct dsync_mailbox_tree_sync_ctx *ctx, - struct dsync_mailbox_tree *tree, - const struct dsync_mailbox_node *src) -{ - struct dsync_mailbox_node *node; - - node = sync_node_new(tree, &tree->root.first_child, &tree->root, src); - sync_rename_node_to_temp(ctx, tree, node, node->parent); - - node->existence = DSYNC_MAILBOX_NODE_EXISTS; - node->uid_validity = src->uid_validity; - memcpy(node->mailbox_guid, src->mailbox_guid, - sizeof(node->mailbox_guid)); - if (tree == ctx->local_tree) { - sync_add_create_change(ctx, node, - dsync_mailbox_node_get_full_name(tree, node)); - } - if (dsync_mailbox_tree_guid_hash_add(tree, node) < 0) - i_unreached(); -} - static bool sync_rename_conflict(struct dsync_mailbox_tree_sync_ctx *ctx, struct dsync_mailbox_node *local_node1, struct dsync_mailbox_node *remote_node2) @@ -633,30 +643,8 @@ guid_p = remote_node2->mailbox_guid; local_node2 = hash_table_lookup(ctx->local_tree->guid_hash, guid_p); - /* FIXME: kludge to avoid problems where one of the mailboxes - doesn't exist yet. they seem to somewhat unnecessarily try to create - temporary mailboxes and later rename them. this definitely doesn't - work with INBOX. */ - if (local_node2 == NULL && - (strcmp(local_node1->name, "INBOX") != 0 || - local_node1->parent->parent != NULL) && - remote_node2->existence == DSYNC_MAILBOX_NODE_EXISTS && - !dsync_mailbox_node_is_dir(remote_node2) && - ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { - add_missing_mailbox(ctx, ctx->local_tree, remote_node2); - return TRUE; - } - if (remote_node1 == NULL && - (strcmp(remote_node2->name, "INBOX") != 0 || - remote_node2->parent->parent != NULL) && - local_node1->existence == DSYNC_MAILBOX_NODE_EXISTS && - !dsync_mailbox_node_is_dir(local_node1) && - ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { - add_missing_mailbox(ctx, ctx->remote_tree, local_node1); - return TRUE; - } - - if (remote_node1 != NULL || local_node2 != NULL) { + if ((remote_node1 != NULL && remote_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) || + (local_node2 != NULL && local_node2->existence == DSYNC_MAILBOX_NODE_EXISTS)) { /* conflicting name, rename the one with lower timestamp */ return sync_rename_lower_ts(ctx, local_node1, remote_node1, local_node2, remote_node2); From dovecot at dovecot.org Fri Dec 14 15:35:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Dec 2012 15:35:25 +0200 Subject: dovecot-2.2: dsync: Mark mailbox tree memory pool as "growing" t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9f332e29836d changeset: 15468:9f332e29836d user: Timo Sirainen date: Fri Dec 14 15:34:00 2012 +0200 description: dsync: Mark mailbox tree memory pool as "growing" to avoid warnings. diffstat: src/doveadm/dsync/dsync-mailbox-tree.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 467cf7e5a616 -r 9f332e29836d src/doveadm/dsync/dsync-mailbox-tree.c --- a/src/doveadm/dsync/dsync-mailbox-tree.c Fri Dec 14 15:32:43 2012 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree.c Fri Dec 14 15:34:00 2012 +0200 @@ -24,7 +24,7 @@ i_assert(sep != '\0'); - pool = pool_alloconly_create("dsync mailbox tree", 4096); + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox tree", 4096); tree = p_new(pool, struct dsync_mailbox_tree, 1); tree->pool = pool; tree->sep = tree->sep_str[0] = sep; From dovecot at dovecot.org Fri Dec 14 15:35:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Dec 2012 15:35:25 +0200 Subject: dovecot-2.2: dsync: Added -a parameter to sync all namespaces, n... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/05d83c4df130 changeset: 15469:05d83c4df130 user: Timo Sirainen date: Fri Dec 14 15:34:56 2012 +0200 description: dsync: Added -a parameter to sync all namespaces, not just the default one. diffstat: src/doveadm/dsync/doveadm-dsync.c | 10 +++++++++- src/doveadm/dsync/dsync-brain-mailbox-tree.c | 6 ++++++ src/doveadm/dsync/dsync-brain-private.h | 1 + src/doveadm/dsync/dsync-brain.c | 2 ++ src/doveadm/dsync/dsync-brain.h | 3 ++- src/doveadm/dsync/dsync-ibc-stream.c | 6 +++++- 6 files changed, 25 insertions(+), 3 deletions(-) diffs (137 lines): diff -r 9f332e29836d -r 05d83c4df130 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Fri Dec 14 15:34:00 2012 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Fri Dec 14 15:34:56 2012 +0200 @@ -27,7 +27,7 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+dEfm:n:r:Rs:" +#define DSYNC_COMMON_GETOPT_ARGS "+adEfm:n:r:Rs:" struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; @@ -44,6 +44,7 @@ unsigned int lock_timeout; unsigned int lock:1; + unsigned int sync_all_namespaces:1; unsigned int default_replica_location:1; unsigned int backup:1; unsigned int reverse_backup:1; @@ -362,6 +363,8 @@ brain_flags = DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS | DSYNC_BRAIN_FLAG_SEND_GUID_REQUESTS; + if (ctx->sync_all_namespaces) + brain_flags |= DSYNC_BRAIN_FLAG_SYNC_ALL_NAMESPACES; if (ctx->reverse_backup) brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; @@ -459,6 +462,8 @@ run_cmd(ctx, remote_cmd_args); ctx->remote = TRUE; } + if (ctx->sync_all_namespaces && !ctx->remote) + i_fatal("-a parameter requires syncing with remote host"); return 0; } @@ -490,6 +495,9 @@ struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; switch (c) { + case 'a': + ctx->sync_all_namespaces = TRUE; + break; case 'd': ctx->default_replica_location = TRUE; break; diff -r 9f332e29836d -r 05d83c4df130 src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Fri Dec 14 15:34:00 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Fri Dec 14 15:34:56 2012 +0200 @@ -16,6 +16,8 @@ { if (brain->sync_ns == ns) return TRUE; + if (brain->sync_all_namespaces) + return TRUE; return brain->sync_ns == NULL && strcmp(ns->unexpanded_set->location, @@ -138,6 +140,10 @@ unsigned int part_len; char ns_sep = mail_namespace_get_sep(ns); + if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 && + strcmp(name_parts[0], "INBOX") == 0 && name_parts[1] == NULL) + return TRUE; + for (; *name_parts != NULL && *prefix != '\0'; name_parts++) { part = *name_parts; part_len = strlen(part); diff -r 9f332e29836d -r 05d83c4df130 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Fri Dec 14 15:34:00 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-private.h Fri Dec 14 15:34:56 2012 +0200 @@ -81,6 +81,7 @@ unsigned int backup_send:1; unsigned int backup_recv:1; unsigned int debug:1; + unsigned int sync_all_namespaces:1; unsigned int changes_during_sync:1; unsigned int failed:1; }; diff -r 9f332e29836d -r 05d83c4df130 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Fri Dec 14 15:34:00 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Fri Dec 14 15:34:56 2012 +0200 @@ -77,6 +77,8 @@ brain->backup_send = (flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0; brain->backup_recv = (flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0; brain->debug = (flags & DSYNC_BRAIN_FLAG_DEBUG) != 0; + brain->sync_all_namespaces = + (flags & DSYNC_BRAIN_FLAG_SYNC_ALL_NAMESPACES) != 0; } struct dsync_brain * diff -r 9f332e29836d -r 05d83c4df130 src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Fri Dec 14 15:34:00 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain.h Fri Dec 14 15:34:56 2012 +0200 @@ -10,7 +10,8 @@ DSYNC_BRAIN_FLAG_SEND_GUID_REQUESTS = 0x02, DSYNC_BRAIN_FLAG_BACKUP_SEND = 0x04, DSYNC_BRAIN_FLAG_BACKUP_RECV = 0x08, - DSYNC_BRAIN_FLAG_DEBUG = 0x10 + DSYNC_BRAIN_FLAG_DEBUG = 0x10, + DSYNC_BRAIN_FLAG_SYNC_ALL_NAMESPACES = 0x20 }; enum dsync_brain_sync_type { diff -r 9f332e29836d -r 05d83c4df130 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Dec 14 15:34:00 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Fri Dec 14 15:34:56 2012 +0200 @@ -60,7 +60,7 @@ { NULL, '\0', NULL, NULL }, { .name = "handshake", .chr = 'H', - .optional_keys = "sync_ns_prefix sync_type debug " + .optional_keys = "sync_ns_prefix sync_type debug sync_all_namespaces " "mails_have_guids send_guid_requests backup_send backup_recv" }, { .name = "mailbox_state", @@ -541,6 +541,8 @@ dsync_serializer_encode_add(encoder, "backup_recv", ""); if ((set->brain_flags & DSYNC_BRAIN_FLAG_DEBUG) != 0) dsync_serializer_encode_add(encoder, "debug", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_SYNC_ALL_NAMESPACES) != 0) + dsync_serializer_encode_add(encoder, "sync_all_namespaces", ""); dsync_serializer_encode_finish(&encoder, str); dsync_ibc_stream_send_string(ibc, str); @@ -599,6 +601,8 @@ set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; if (dsync_deserializer_decode_try(decoder, "debug", &value)) set->brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; + if (dsync_deserializer_decode_try(decoder, "sync_all_namespaces", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_SYNC_ALL_NAMESPACES; *set_r = set; return DSYNC_IBC_RECV_RET_OK; From dovecot at dovecot.org Sat Dec 15 13:01:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:01:29 +0200 Subject: dovecot-2.2: dsync: Error handling fix when mailbox is unexpecte... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/01ba0511d04d changeset: 15470:01ba0511d04d user: Timo Sirainen date: Sat Dec 15 12:31:56 2012 +0200 description: dsync: Error handling fix when mailbox is unexpectedly lost. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 25 ++++++++++++++++++------- src/doveadm/dsync/dsync-ibc-stream.c | 2 +- 2 files changed, 19 insertions(+), 8 deletions(-) diffs (65 lines): diff -r 05d83c4df130 -r 01ba0511d04d src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Fri Dec 14 15:34:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sat Dec 15 12:31:56 2012 +0200 @@ -530,10 +530,24 @@ } } +static void +dsync_brain_slave_send_mailbox_lost(struct dsync_brain *brain, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_mailbox delete_box; + + memset(&delete_box, 0, sizeof(delete_box)); + memcpy(delete_box.mailbox_guid, dsync_box->mailbox_guid, + sizeof(delete_box.mailbox_guid)); + t_array_init(&delete_box.cache_fields, 0); + delete_box.mailbox_lost = TRUE; + dsync_ibc_send_mailbox(brain->ibc, &delete_box); +} + bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain) { const struct dsync_mailbox *dsync_box; - struct dsync_mailbox local_dsync_box, delete_box; + struct dsync_mailbox local_dsync_box; struct mailbox *box; int ret; @@ -553,8 +567,9 @@ } if (box == NULL) { /* mailbox was probably deleted/renamed during sync */ - //FIXME: in case it wasn't, do error handling + //FIXME: verify this from log, and if not log an error. brain->changes_during_sync = TRUE; + dsync_brain_slave_send_mailbox_lost(brain, dsync_box); return TRUE; } if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { @@ -573,11 +588,7 @@ return TRUE; } /* another process just deleted this mailbox? */ - memset(&delete_box, 0, sizeof(delete_box)); - memcpy(delete_box.mailbox_guid, dsync_box->mailbox_guid, - sizeof(delete_box.mailbox_guid)); - delete_box.mailbox_lost = TRUE; - dsync_ibc_send_mailbox(brain->ibc, &delete_box); + dsync_brain_slave_send_mailbox_lost(brain, dsync_box); return TRUE; } i_assert(local_dsync_box.uid_validity != 0); diff -r 05d83c4df130 -r 01ba0511d04d src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Dec 14 15:34:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 12:31:56 2012 +0200 @@ -81,7 +81,7 @@ }, { .name = "mailbox", .chr = 'B', - .required_keys = "mailbox_guid uid_validity uid_next " + .required_keys = "mailbox_lost mailbox_guid uid_validity uid_next " "messages_count first_recent_uid highest_modseq", .optional_keys = "cache_fields" }, From dovecot at dovecot.org Sat Dec 15 13:01:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:01:29 +0200 Subject: dovecot-2.2: dsync: Added back support for syncing only one mail... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/88ac919f8afe changeset: 15471:88ac919f8afe user: Timo Sirainen date: Sat Dec 15 13:01:09 2012 +0200 description: dsync: Added back support for syncing only one mailbox (-m parameter) diffstat: src/doveadm/dsync/doveadm-dsync.c | 2 +- src/doveadm/dsync/dsync-brain-mailbox-tree.c | 5 ++--- src/doveadm/dsync/dsync-brain-private.h | 1 + src/doveadm/dsync/dsync-brain.c | 5 ++++- src/doveadm/dsync/dsync-brain.h | 2 +- src/doveadm/dsync/dsync-ibc-pipe.c | 1 + src/doveadm/dsync/dsync-ibc-stream.c | 6 +++++- src/doveadm/dsync/dsync-ibc.h | 2 ++ src/doveadm/dsync/dsync-mailbox-tree-fill.c | 7 ++++--- src/doveadm/dsync/dsync-mailbox-tree.h | 5 +++-- 10 files changed, 24 insertions(+), 12 deletions(-) diffs (206 lines): diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Sat Dec 15 13:01:09 2012 +0200 @@ -373,7 +373,7 @@ if (doveadm_debug) brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; - brain = dsync_brain_master_init(user, ibc, sync_ns, + brain = dsync_brain_master_init(user, ibc, sync_ns, ctx->mailbox, ctx->sync_type, brain_flags, ctx->state_input == NULL ? "" : ctx->state_input); diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Sat Dec 15 13:01:09 2012 +0200 @@ -76,14 +76,14 @@ /* fill the local mailbox tree */ if (brain->sync_ns != NULL) { if (dsync_mailbox_tree_fill(brain->local_mailbox_tree, - brain->sync_ns) < 0) + brain->sync_ns, brain->sync_box) < 0) brain->failed = TRUE; } else { for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { if (!dsync_brain_want_namespace(brain, ns)) continue; if (dsync_mailbox_tree_fill(brain->local_mailbox_tree, - ns) < 0) + ns, brain->sync_box) < 0) brain->failed = TRUE; } } @@ -92,7 +92,6 @@ dsync_mailbox_tree_iter_init(brain->local_mailbox_tree); } - void dsync_brain_send_mailbox_tree(struct dsync_brain *brain) { struct dsync_mailbox_node *node; diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-private.h Sat Dec 15 13:01:09 2012 +0200 @@ -46,6 +46,7 @@ struct mail_user *user; struct dsync_ibc *ibc; struct mail_namespace *sync_ns; + char *sync_box; enum dsync_brain_sync_type sync_type; char hierarchy_sep; diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Sat Dec 15 13:01:09 2012 +0200 @@ -83,7 +83,7 @@ struct dsync_brain * dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, - struct mail_namespace *sync_ns, + struct mail_namespace *sync_ns, const char *sync_box, enum dsync_brain_sync_type sync_type, enum dsync_brain_flags flags, const char *state) @@ -99,6 +99,7 @@ brain->sync_type = sync_type; if (sync_ns != NULL) brain->sync_ns = sync_ns; + brain->sync_box = p_strdup(brain->pool, sync_box); brain->master_brain = TRUE; dsync_brain_set_flags(brain, flags); @@ -120,6 +121,7 @@ memset(&ibc_set, 0, sizeof(ibc_set)); ibc_set.sync_ns_prefix = sync_ns == NULL ? NULL : sync_ns->prefix; + ibc_set.sync_box = sync_box; ibc_set.sync_type = sync_type; /* reverse the backup direction for the slave */ ibc_set.brain_flags = flags & ~(DSYNC_BRAIN_FLAG_BACKUP_SEND | @@ -183,6 +185,7 @@ brain->sync_ns = mail_namespace_find(brain->user->namespaces, ibc_set->sync_ns_prefix); } + brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box); i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); brain->sync_type = ibc_set->sync_type; dsync_brain_set_flags(brain, ibc_set->brain_flags); diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain.h Sat Dec 15 13:01:09 2012 +0200 @@ -28,7 +28,7 @@ struct dsync_brain * dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, - struct mail_namespace *sync_ns, + struct mail_namespace *sync_ns, const char *sync_box, enum dsync_brain_sync_type sync_type, enum dsync_brain_flags flags, const char *state); diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-ibc-pipe.c --- a/src/doveadm/dsync/dsync-ibc-pipe.c Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-pipe.c Sat Dec 15 13:01:09 2012 +0200 @@ -157,6 +157,7 @@ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_HANDSHAKE); item->u.set = *set; item->u.set.sync_ns_prefix = p_strdup(item->pool, set->sync_ns_prefix); + item->u.set.sync_box = p_strdup(item->pool, set->sync_box); } static enum dsync_ibc_recv_ret diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 13:01:09 2012 +0200 @@ -60,7 +60,7 @@ { NULL, '\0', NULL, NULL }, { .name = "handshake", .chr = 'H', - .optional_keys = "sync_ns_prefix sync_type debug sync_all_namespaces " + .optional_keys = "sync_ns_prefix sync_box sync_type debug sync_all_namespaces " "mails_have_guids send_guid_requests backup_send backup_recv" }, { .name = "mailbox_state", @@ -514,6 +514,8 @@ dsync_serializer_encode_add(encoder, "sync_ns_prefix", set->sync_ns_prefix); } + if (set->sync_box != NULL) + dsync_serializer_encode_add(encoder, "sync_box", set->sync_box); sync_type[0] = sync_type[1] = '\0'; switch (set->sync_type) { @@ -574,6 +576,8 @@ if (dsync_deserializer_decode_try(decoder, "sync_ns_prefix", &value)) set->sync_ns_prefix = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "sync_box", &value)) + set->sync_box = p_strdup(pool, value); if (dsync_deserializer_decode_try(decoder, "sync_type", &value)) { switch (value[0]) { case 'f': diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-ibc.h --- a/src/doveadm/dsync/dsync-ibc.h Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc.h Sat Dec 15 13:01:09 2012 +0200 @@ -31,6 +31,8 @@ struct dsync_ibc_settings { /* if non-NULL, sync only this namespace */ const char *sync_ns_prefix; + /* if non-NULL, sync only this mailbox name */ + const char *sync_box; enum dsync_brain_sync_type sync_type; enum dsync_brain_flags brain_flags; diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-mailbox-tree-fill.c --- a/src/doveadm/dsync/dsync-mailbox-tree-fill.c Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-fill.c Sat Dec 15 13:01:09 2012 +0200 @@ -170,7 +170,7 @@ } int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree, - struct mail_namespace *ns) + struct mail_namespace *ns, const char *box_name) { const enum mailbox_list_iter_flags list_flags = MAILBOX_LIST_ITER_NO_AUTO_BOXES; @@ -181,6 +181,7 @@ struct mailbox_list_iterate_context *iter; struct dsync_mailbox_node *node; const struct mailbox_info *info; + const char *list_pattern = box_name != NULL ? box_name : "*"; int ret = 0; i_assert(mail_namespace_get_sep(ns) == tree->sep); @@ -195,7 +196,7 @@ } /* first add all of the existing mailboxes */ - iter = mailbox_list_iter_init(ns->list, "*", list_flags); + iter = mailbox_list_iter_init(ns->list, list_pattern, list_flags); while ((info = mailbox_list_iter_next(iter)) != NULL) { if (dsync_mailbox_tree_add(tree, info) < 0) ret = -1; @@ -206,7 +207,7 @@ } /* add subscriptions */ - iter = mailbox_list_iter_init(ns->list, "*", subs_list_flags); + iter = mailbox_list_iter_init(ns->list, list_pattern, subs_list_flags); while ((info = mailbox_list_iter_next(iter)) != NULL) { if (dsync_mailbox_tree_add_node(tree, info, &node) < 0) ret = -1; diff -r 01ba0511d04d -r 88ac919f8afe src/doveadm/dsync/dsync-mailbox-tree.h --- a/src/doveadm/dsync/dsync-mailbox-tree.h Sat Dec 15 12:31:56 2012 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree.h Sat Dec 15 13:01:09 2012 +0200 @@ -121,9 +121,10 @@ void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, const struct dsync_mailbox_node *src); -/* Add nodes to tree from the given namespace. */ +/* Add nodes to tree from the given namespace. If box_name is non-NULL, + add only that mailbox to the tree. */ int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree, - struct mail_namespace *ns); + struct mail_namespace *ns, const char *box_name); /* Return all known deleted mailboxes and directories. */ const struct dsync_mailbox_delete * From dovecot at dovecot.org Sat Dec 15 13:25:59 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:25:59 +0200 Subject: dovecot-2.2: maildir: If mailbox_update() doesn't affect dovecot... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0e88b2518e0f changeset: 15472:0e88b2518e0f user: Timo Sirainen date: Sat Dec 15 13:25:34 2012 +0200 description: maildir: If mailbox_update() doesn't affect dovecot-uidlist file, don't lock it. diffstat: src/lib-storage/index/maildir/maildir-storage.c | 25 ++++++++++++++----------- 1 files changed, 14 insertions(+), 11 deletions(-) diffs (44 lines): diff -r 88ac919f8afe -r 0e88b2518e0f src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Sat Dec 15 13:01:09 2012 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.c Sat Dec 15 13:25:34 2012 +0200 @@ -420,7 +420,7 @@ { struct maildir_mailbox *mbox = (struct maildir_mailbox *)box; struct maildir_uidlist *uidlist; - int ret; + int ret = 0; if (!box->opened) { if (mailbox_open(box) < 0) @@ -428,18 +428,21 @@ } uidlist = mbox->uidlist; - if (maildir_uidlist_lock(uidlist) <= 0) - return -1; + if (update->uid_validity != 0 || update->min_next_uid != 0 || + !guid_128_is_empty(update->mailbox_guid)) { + if (maildir_uidlist_lock(uidlist) <= 0) + return -1; - if (!guid_128_is_empty(update->mailbox_guid)) - maildir_uidlist_set_mailbox_guid(uidlist, update->mailbox_guid); - if (update->uid_validity != 0) - maildir_uidlist_set_uid_validity(uidlist, update->uid_validity); - if (update->min_next_uid != 0) { - maildir_uidlist_set_next_uid(uidlist, update->min_next_uid, - FALSE); + if (!guid_128_is_empty(update->mailbox_guid)) + maildir_uidlist_set_mailbox_guid(uidlist, update->mailbox_guid); + if (update->uid_validity != 0) + maildir_uidlist_set_uid_validity(uidlist, update->uid_validity); + if (update->min_next_uid != 0) { + maildir_uidlist_set_next_uid(uidlist, update->min_next_uid, + FALSE); + } + ret = maildir_uidlist_update(uidlist); } - ret = maildir_uidlist_update(uidlist); if (ret == 0) ret = index_storage_mailbox_update(box, update); maildir_uidlist_unlock(uidlist); From dovecot at dovecot.org Sat Dec 15 13:25:59 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:25:59 +0200 Subject: dovecot-2.2: mbox: If mailbox_update() doesn't affect the mbox f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/646c5d0a60b5 changeset: 15473:646c5d0a60b5 user: Timo Sirainen date: Sat Dec 15 13:25:49 2012 +0200 description: mbox: If mailbox_update() doesn't affect the mbox file, don't sync it. diffstat: src/lib-storage/index/mbox/mbox-storage.c | 14 +++++++++----- 1 files changed, 9 insertions(+), 5 deletions(-) diffs (30 lines): diff -r 0e88b2518e0f -r 646c5d0a60b5 src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Sat Dec 15 13:25:34 2012 +0200 +++ b/src/lib-storage/index/mbox/mbox-storage.c Sat Dec 15 13:25:49 2012 +0200 @@ -491,17 +491,21 @@ mbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update) { struct mbox_mailbox *mbox = (struct mbox_mailbox *)box; - int ret; + int ret = 0; if (!box->opened) { if (mailbox_open(box) < 0) return -1; } - mbox->sync_hdr_update = update; - ret = mbox_sync(mbox, MBOX_SYNC_HEADER | MBOX_SYNC_FORCE_SYNC | - MBOX_SYNC_REWRITE); - mbox->sync_hdr_update = NULL; + if (update->uid_validity != 0 || update->min_next_uid != 0) { + mbox->sync_hdr_update = update; + ret = mbox_sync(mbox, MBOX_SYNC_HEADER | MBOX_SYNC_FORCE_SYNC | + MBOX_SYNC_REWRITE); + mbox->sync_hdr_update = NULL; + } + if (ret == 0) + ret = index_storage_mailbox_update(box, update); return ret; } From dovecot at dovecot.org Sat Dec 15 13:26:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:26:53 +0200 Subject: dovecot-2.2: mbox: Mailbox GUID lookup unnecessarily always sync... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b765a09d8c6c changeset: 15474:b765a09d8c6c user: Timo Sirainen date: Sat Dec 15 13:26:48 2012 +0200 description: mbox: Mailbox GUID lookup unnecessarily always synced mbox. diffstat: src/lib-storage/index/mbox/mbox-storage.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 646c5d0a60b5 -r b765a09d8c6c src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Sat Dec 15 13:25:49 2012 +0200 +++ b/src/lib-storage/index/mbox/mbox-storage.c Sat Dec 15 13:26:48 2012 +0200 @@ -613,6 +613,9 @@ "Mailbox GUIDs are not permanent without index files"); return -1; } + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + if (!guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { /* we have the GUID */ } else if (mbox_file_open(mbox) < 0) From dovecot at dovecot.org Sat Dec 15 13:28:09 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:28:09 +0200 Subject: dovecot-2.2: mbox: Mailbox GUID generation re-opened the mailbox... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/507a4a57d7cd changeset: 15475:507a4a57d7cd user: Timo Sirainen date: Sat Dec 15 13:28:04 2012 +0200 description: mbox: Mailbox GUID generation re-opened the mailbox using wrong name, possibly crashing. diffstat: src/lib-storage/index/mbox/mbox-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b765a09d8c6c -r 507a4a57d7cd src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Sat Dec 15 13:26:48 2012 +0200 +++ b/src/lib-storage/index/mbox/mbox-storage.c Sat Dec 15 13:28:04 2012 +0200 @@ -631,7 +631,7 @@ int ret; i_assert(mbox->mbox_lock_type == F_UNLCK); - box2 = mailbox_alloc(mbox->box.list, mbox->box.name, 0); + box2 = mailbox_alloc(mbox->box.list, mbox->box.vname, 0); ret = mailbox_sync(box2, 0); mbox2 = (struct mbox_mailbox *)box2; memcpy(guid_r, mbox2->mbox_hdr.mailbox_guid, GUID_128_SIZE); From dovecot at dovecot.org Sat Dec 15 13:30:09 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:30:09 +0200 Subject: dovecot-2.2: mbox: Removed assert that caused dsync to crash. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fdf755b505c3 changeset: 15476:fdf755b505c3 user: Timo Sirainen date: Sat Dec 15 13:30:02 2012 +0200 description: mbox: Removed assert that caused dsync to crash. This is valid: 1. dsync opened an internal transaction and locked mbox 2. dsync opened an external transaction, which didn't lock mbox 3. dsync committed internal transaction 4. dsync locks mbox with external transaction diffstat: src/lib-storage/index/mbox/mbox-storage.c | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (12 lines): diff -r 507a4a57d7cd -r fdf755b505c3 src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Sat Dec 15 13:28:04 2012 +0200 +++ b/src/lib-storage/index/mbox/mbox-storage.c Sat Dec 15 13:30:02 2012 +0200 @@ -730,8 +730,6 @@ i_assert(mbox->box.transaction_count > 0 || mbox->external_transactions > 0 || mbox->mbox_lock_type == F_UNLCK); - i_assert(mbox->external_transactions == 0 || - mbox->mbox_lock_type == F_WRLCK); } else { /* mailbox opened with MAILBOX_FLAG_KEEP_LOCKED */ i_assert(mbox->mbox_lock_type == F_WRLCK); From dovecot at dovecot.org Sat Dec 15 13:32:48 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:32:48 +0200 Subject: dovecot-2.2: dsync: Don't start reading a mailbox until we know ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5a40e43bbe23 changeset: 15477:5a40e43bbe23 user: Timo Sirainen date: Sat Dec 15 13:31:31 2012 +0200 description: dsync: Don't start reading a mailbox until we know we actually need to sync it. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 41 +++++++++++++++++++------------- src/doveadm/dsync/dsync-brain-mails.c | 2 + src/doveadm/dsync/dsync-brain-private.h | 1 + 3 files changed, 27 insertions(+), 17 deletions(-) diffs (129 lines): diff -r fdf755b505c3 -r 5a40e43bbe23 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Sat Dec 15 13:30:02 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sat Dec 15 13:31:31 2012 +0200 @@ -100,16 +100,13 @@ } } -static int +static void dsync_brain_sync_mailbox_init(struct dsync_brain *brain, struct mailbox *box, const struct dsync_mailbox *local_dsync_box, bool wait_for_remote_box) { - enum dsync_mailbox_exporter_flags exporter_flags = 0; const struct dsync_mailbox_state *state; - uint32_t last_common_uid, highest_wanted_uid; - uint64_t last_common_modseq; i_assert(brain->box_importer == NULL); i_assert(brain->box_exporter == NULL); @@ -127,28 +124,36 @@ memset(&brain->remote_dsync_box, 0, sizeof(brain->remote_dsync_box)); state = dsync_mailbox_state_find(brain, local_dsync_box->mailbox_guid); - if (state != NULL) { + if (state != NULL) brain->mailbox_state = *state; - last_common_uid = state->last_common_uid; - last_common_modseq = state->last_common_modseq; - } else { + else { memset(&brain->mailbox_state, 0, sizeof(brain->mailbox_state)); memcpy(brain->mailbox_state.mailbox_guid, local_dsync_box->mailbox_guid, sizeof(brain->mailbox_state.mailbox_guid)); brain->mailbox_state.last_uidvalidity = local_dsync_box->uid_validity; - last_common_uid = 0; - last_common_modseq = 0; } +} +int dsync_brain_sync_mailbox_open(struct dsync_brain *brain) +{ + enum dsync_mailbox_exporter_flags exporter_flags = 0; + uint32_t last_common_uid, highest_wanted_uid; + uint64_t last_common_modseq; + + i_assert(brain->log_scan == NULL); + + last_common_uid = brain->mailbox_state.last_common_uid; + last_common_modseq = brain->mailbox_state.last_common_modseq; highest_wanted_uid = last_common_uid == 0 ? (uint32_t)-1 : last_common_uid; - if (dsync_transaction_log_scan_init(box->view, highest_wanted_uid, + if (dsync_transaction_log_scan_init(brain->box->view, + highest_wanted_uid, last_common_modseq, &brain->log_scan) < 0) { i_error("Failed to read transaction log for mailbox %s", - mailbox_get_vname(box)); + mailbox_get_vname(brain->box)); brain->failed = TRUE; return -1; } @@ -159,8 +164,9 @@ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS; brain->box_exporter = brain->backup_recv ? NULL : - dsync_mailbox_export_init(box, brain->log_scan, last_common_uid, - last_common_modseq, exporter_flags); + dsync_mailbox_export_init(brain->box, brain->log_scan, + last_common_uid, last_common_modseq, + exporter_flags); return 0; } @@ -173,6 +179,7 @@ uint64_t last_common_modseq; i_assert(brain->box_importer == NULL); + i_assert(brain->log_scan != NULL); i_assert(memcmp(brain->local_dsync_box.mailbox_guid, remote_dsync_box->mailbox_guid, @@ -391,7 +398,7 @@ /* start exporting this mailbox (wait for remote to start importing) */ dsync_ibc_send_mailbox(brain->ibc, &dsync_box); - (void)dsync_brain_sync_mailbox_init(brain, box, &dsync_box, TRUE); + dsync_brain_sync_mailbox_init(brain, box, &dsync_box, TRUE); brain->state = DSYNC_STATE_SYNC_MAILS; } @@ -606,8 +613,8 @@ } /* start export/import */ - if (dsync_brain_sync_mailbox_init(brain, box, &local_dsync_box, - FALSE) == 0) + dsync_brain_sync_mailbox_init(brain, box, &local_dsync_box, FALSE); + if (dsync_brain_sync_mailbox_open(brain) == 0) dsync_brain_sync_mailbox_init_remote(brain, dsync_box); brain->state = DSYNC_STATE_SYNC_MAILS; diff -r fdf755b505c3 -r 5a40e43bbe23 src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Sat Dec 15 13:30:02 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Sat Dec 15 13:31:31 2012 +0200 @@ -53,6 +53,8 @@ dsync_brain_sync_mailbox_deinit(brain); return TRUE; } + if (dsync_brain_sync_mailbox_open(brain) < 0) + return TRUE; dsync_brain_sync_mailbox_init_remote(brain, dsync_box); dsync_brain_sync_init_box_states(brain); return TRUE; diff -r fdf755b505c3 -r 5a40e43bbe23 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Sat Dec 15 13:30:02 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain-private.h Sat Dec 15 13:31:31 2012 +0200 @@ -108,6 +108,7 @@ void dsync_brain_master_send_mailbox(struct dsync_brain *brain); bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain); +int dsync_brain_sync_mailbox_open(struct dsync_brain *brain); void dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, const struct dsync_mailbox *remote_dsync_box); bool dsync_brain_sync_mails(struct dsync_brain *brain); From dovecot at dovecot.org Sat Dec 15 13:43:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:43:54 +0200 Subject: dovecot-2.2: i_stream_next_line(): Don't reset stream_errno if s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b277f0bb18a2 changeset: 15478:b277f0bb18a2 user: Timo Sirainen date: Sat Dec 15 13:43:44 2012 +0200 description: i_stream_next_line(): Don't reset stream_errno if stream is already closed. diffstat: src/lib/istream.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 5a40e43bbe23 -r b277f0bb18a2 src/lib/istream.c --- a/src/lib/istream.c Sat Dec 15 13:31:31 2012 +0200 +++ b/src/lib/istream.c Sat Dec 15 13:43:44 2012 +0200 @@ -363,7 +363,8 @@ const unsigned char *pos; if (_stream->skip >= _stream->pos) { - stream->stream_errno = 0; + if (!unlikely(stream->closed)) + stream->stream_errno = 0; return NULL; } From dovecot at dovecot.org Sat Dec 15 13:48:27 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:48:27 +0200 Subject: dovecot-2.2: i_stream_close(): Set stream_errno to EPIPE instead... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/984a5b4fa793 changeset: 15479:984a5b4fa793 user: Timo Sirainen date: Sat Dec 15 13:48:22 2012 +0200 description: i_stream_close(): Set stream_errno to EPIPE instead of ENOENT. ENOENT implies that this stream pointed to a file that never existed. EPIPE isn't perfect either, but probably clearer than the alternatives. diffstat: src/lib/istream.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b277f0bb18a2 -r 984a5b4fa793 src/lib/istream.c --- a/src/lib/istream.c Sat Dec 15 13:43:44 2012 +0200 +++ b/src/lib/istream.c Sat Dec 15 13:48:22 2012 +0200 @@ -75,7 +75,7 @@ stream->closed = TRUE; if (stream->stream_errno == 0) - stream->stream_errno = ENOENT; + stream->stream_errno = EPIPE; } void i_stream_set_init_buffer_size(struct istream *stream, size_t size) From dovecot at dovecot.org Sat Dec 15 13:55:47 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:55:47 +0200 Subject: dovecot-2.2: dsync: Set doveadm exit codes properly on errors. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/79bcf862bf1c changeset: 15480:79bcf862bf1c user: Timo Sirainen date: Sat Dec 15 13:55:33 2012 +0200 description: dsync: Set doveadm exit codes properly on errors. diffstat: src/doveadm/dsync/doveadm-dsync.c | 35 +++++++++++++++++++++++++++-------- 1 files changed, 27 insertions(+), 8 deletions(-) diffs (87 lines): diff -r 984a5b4fa793 -r 79bcf862bf1c src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Sat Dec 15 13:48:22 2012 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Sat Dec 15 13:55:33 2012 +0200 @@ -244,6 +244,7 @@ struct setting_parser_context *set_parser; const char *set_line, *path1, *path2; bool brain1_running, brain2_running, changed1, changed2; + int ret; i_assert(ctx->local_location != NULL); @@ -255,16 +256,23 @@ set_line = t_strconcat("mail_location=", ctx->local_location, NULL); if (settings_parse_line(set_parser, set_line) < 0) i_unreached(); - if (mail_storage_service_next(ctx->ctx.storage_service, - ctx->ctx.cur_service_user, &user2) < 0) - i_fatal("User init failed"); + ret = mail_storage_service_next(ctx->ctx.storage_service, + ctx->ctx.cur_service_user, &user2); + if (ret < 0) { + ctx->ctx.exit_code = ret == -1 ? EX_TEMPFAIL : EX_CONFIG; + mail_user_unref(&user2); + return -1; + } user2->admin = TRUE; if (mail_namespaces_get_root_sep(user->namespaces) != mail_namespaces_get_root_sep(user2->namespaces)) { - i_fatal("Mail locations must use the same " + i_error("Mail locations must use the same " "virtual mailbox hierarchy separator " "(specify separator for the default namespace)"); + ctx->ctx.exit_code = EX_CONFIG; + mail_user_unref(&user2); + return -1; } if (mailbox_list_get_root_path(user->namespaces->list, MAILBOX_LIST_PATH_TYPE_MAILBOX, @@ -273,8 +281,11 @@ MAILBOX_LIST_PATH_TYPE_MAILBOX, &path2) && strcmp(path1, path2) == 0) { - i_fatal("Both source and destination mail_location " + i_error("Both source and destination mail_location " "points to same directory: %s", path1); + ctx->ctx.exit_code = EX_CONFIG; + mail_user_unref(&user2); + return -1; } brain2 = dsync_brain_slave_init(user2, ibc2); @@ -291,7 +302,11 @@ brain2_running = dsync_brain_run(brain2, &changed2); } mail_user_unref(&user2); - return dsync_brain_deinit(&brain2); + if (dsync_brain_deinit(&brain2) < 0) { + ctx->ctx.exit_code = EX_TEMPFAIL; + return -1; + } + return 0; } static void @@ -380,7 +395,7 @@ if (!ctx->remote) { if (cmd_dsync_run_local(ctx, user, brain, ibc2) < 0) - _ctx->exit_code = EX_TEMPFAIL; + ret = -1; } else { cmd_dsync_run_remote(user); } @@ -584,7 +599,11 @@ io_loop_run(current_ioloop); dsync_ibc_deinit(&ibc); - return dsync_brain_deinit(&brain); + if (dsync_brain_deinit(&brain) < 0) { + _ctx->exit_code = EX_TEMPFAIL; + return -1; + } + return 0; } static bool From dovecot at dovecot.org Sat Dec 15 13:56:21 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 13:56:21 +0200 Subject: dovecot-2.2: dsync: Fixed remote syncing due to recent changes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1204e8bae160 changeset: 15481:1204e8bae160 user: Timo Sirainen date: Sat Dec 15 13:56:16 2012 +0200 description: dsync: Fixed remote syncing due to recent changes diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (15 lines): diff -r 79bcf862bf1c -r 1204e8bae160 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 13:55:33 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 13:56:16 2012 +0200 @@ -81,9 +81,9 @@ }, { .name = "mailbox", .chr = 'B', - .required_keys = "mailbox_lost mailbox_guid uid_validity uid_next " + .required_keys = "mailbox_guid uid_validity uid_next " "messages_count first_recent_uid highest_modseq", - .optional_keys = "cache_fields" + .optional_keys = "mailbox_lost cache_fields" }, { .name = "mail_change", .chr = 'C', From dovecot at dovecot.org Sat Dec 15 14:25:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 14:25:31 +0200 Subject: dovecot-2.2: dsync: Fixed hang when sending mail stream to remot... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6f940b2bfbe6 changeset: 15482:6f940b2bfbe6 user: Timo Sirainen date: Sat Dec 15 14:24:41 2012 +0200 description: dsync: Fixed hang when sending mail stream to remote dsync. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 11 +++++++++-- 1 files changed, 9 insertions(+), 2 deletions(-) diffs (42 lines): diff -r 1204e8bae160 -r 6f940b2bfbe6 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 13:56:16 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 14:24:41 2012 +0200 @@ -243,6 +243,7 @@ struct ostream *output = ibc->output; int ret; + o_stream_cork(ibc->output); if ((ret = o_stream_flush(output)) < 0) ret = 1; else if (ibc->mail_output != NULL) { @@ -253,6 +254,7 @@ if (!dsync_ibc_is_send_queue_full(&ibc->ibc)) ibc->ibc.io_callback(ibc->ibc.io_context); + o_stream_uncork(ibc->output); return ret; } @@ -1368,7 +1370,11 @@ ibc->mail_output_last = '\0'; ibc->mail_output = mail->input; i_stream_ref(ibc->mail_output); - (void)dsync_ibc_stream_send_mail_stream(ibc); + if (dsync_ibc_stream_send_mail_stream(ibc) == 0) { + /* flush callback isn't being called while output + stream is corked */ + o_stream_uncork(ibc->output); + } } } @@ -1471,7 +1477,8 @@ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; o_stream_uncork(ibc->output); - o_stream_cork(ibc->output); + if (ibc->mail_output == NULL) + o_stream_cork(ibc->output); } static bool dsync_ibc_stream_is_send_queue_full(struct dsync_ibc *_ibc) From dovecot at dovecot.org Sat Dec 15 14:28:01 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 14:28:01 +0200 Subject: dovecot-2.2: dsync: Minor (probably unnecessary) fix for i_strea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fc6873897e8b changeset: 15483:fc6873897e8b user: Timo Sirainen date: Sat Dec 15 14:27:56 2012 +0200 description: dsync: Minor (probably unnecessary) fix for i_stream_read() API usage. Avoid the first read returning -2. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 6f940b2bfbe6 -r fc6873897e8b src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 14:24:41 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Sat Dec 15 14:27:56 2012 +0200 @@ -143,10 +143,10 @@ static int dsync_ibc_stream_read_mail_stream(struct dsync_ibc_stream *ibc) { - while (i_stream_read(ibc->mail_input) > 0) { + do { i_stream_skip(ibc->mail_input, i_stream_get_data_size(ibc->mail_input)); - } + } while (i_stream_read(ibc->mail_input) > 0); if (ibc->mail_input->eof) { if (ibc->mail_input->stream_errno != 0) { errno = ibc->mail_input->stream_errno; From dovecot at dovecot.org Sat Dec 15 14:37:32 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 14:37:32 +0200 Subject: dovecot-2.2: maildir: Fixed a crash caused by previous mailbox_u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eed88e231043 changeset: 15484:eed88e231043 user: Timo Sirainen date: Sat Dec 15 14:37:20 2012 +0200 description: maildir: Fixed a crash caused by previous mailbox_update() change diffstat: src/lib-storage/index/maildir/maildir-storage.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (29 lines): diff -r fc6873897e8b -r eed88e231043 src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Sat Dec 15 14:27:56 2012 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.c Sat Dec 15 14:37:20 2012 +0200 @@ -420,6 +420,7 @@ { struct maildir_mailbox *mbox = (struct maildir_mailbox *)box; struct maildir_uidlist *uidlist; + bool locked = FALSE; int ret = 0; if (!box->opened) { @@ -433,6 +434,7 @@ if (maildir_uidlist_lock(uidlist) <= 0) return -1; + locked = TRUE; if (!guid_128_is_empty(update->mailbox_guid)) maildir_uidlist_set_mailbox_guid(uidlist, update->mailbox_guid); if (update->uid_validity != 0) @@ -445,7 +447,8 @@ } if (ret == 0) ret = index_storage_mailbox_update(box, update); - maildir_uidlist_unlock(uidlist); + if (locked) + maildir_uidlist_unlock(uidlist); return ret; } From dovecot at dovecot.org Sat Dec 15 15:33:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Dec 2012 15:33:25 +0200 Subject: dovecot-2.2: lib-storage: Refresh private index before syncing t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/58fdb8ded6b7 changeset: 15485:58fdb8ded6b7 user: Timo Sirainen date: Sat Dec 15 15:33:12 2012 +0200 description: lib-storage: Refresh private index before syncing to make sure we see latest changes. diffstat: src/lib-storage/index/index-sync-pvt.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r eed88e231043 -r 58fdb8ded6b7 src/lib-storage/index/index-sync-pvt.c --- a/src/lib-storage/index/index-sync-pvt.c Sat Dec 15 14:37:20 2012 +0200 +++ b/src/lib-storage/index/index-sync-pvt.c Sat Dec 15 15:33:12 2012 +0200 @@ -116,7 +116,8 @@ int ret; /* open a view for the latest version of the index */ - if (mail_index_refresh(box->index) < 0) { + if (mail_index_refresh(box->index) < 0 || + mail_index_refresh(box->index_pvt) < 0) { mailbox_set_index_error(box); return -1; } From dovecot at dovecot.org Tue Dec 18 10:44:42 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 10:44:42 +0200 Subject: dovecot-2.2: lib-storage: Fixed assert-crash when trying to fetc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cde6da31e6ec changeset: 15486:cde6da31e6ec user: Timo Sirainen date: Tue Dec 18 10:44:28 2012 +0200 description: lib-storage: Fixed assert-crash when trying to fetch a binary section for empty message. diffstat: src/lib-storage/index/index-mail-binary.c | 13 +++++++++---- 1 files changed, 9 insertions(+), 4 deletions(-) diffs (30 lines): diff -r 58fdb8ded6b7 -r cde6da31e6ec src/lib-storage/index/index-mail-binary.c --- a/src/lib-storage/index/index-mail-binary.c Sat Dec 15 15:33:12 2012 +0200 +++ b/src/lib-storage/index/index-mail-binary.c Tue Dec 18 10:44:28 2012 +0200 @@ -344,8 +344,8 @@ i_assert(ret == -1); if (full_input->stream_errno != 0) return -1; - i_assert(!i_stream_have_bytes_left(cur_block->input)); - i_assert(block_idx+1 == block_count); + i_assert(block_count == 0 || !i_stream_have_bytes_left(cur_block->input)); + i_assert(block_count == 0 || block_idx+1 == block_count); return 0; } @@ -380,8 +380,13 @@ cache->orig_physical_pos = part->physical_pos; cache->include_hdr = include_hdr; - cache->input = i_streams_merge(blocks_get_streams(&ctx), - IO_BLOCK_SIZE, fd_callback, _mail); + if (array_count(&ctx.blocks) != 0) { + cache->input = i_streams_merge(blocks_get_streams(&ctx), + IO_BLOCK_SIZE, + fd_callback, _mail); + } else { + cache->input = i_stream_create_from_data("", 0); + } i_stream_set_name(cache->input, t_strdup_printf( "", _mail->box->vname, _mail->uid)); From dovecot at dovecot.org Tue Dec 18 18:54:37 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 18:54:37 +0200 Subject: dovecot-2.2: uri-util: Control characters weren't properly escaped. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ce258aa11821 changeset: 15487:ce258aa11821 user: Timo Sirainen date: Tue Dec 18 18:54:31 2012 +0200 description: uri-util: Control characters weren't properly escaped. e.g. "%0a" is correct, "% a" is not. diffstat: src/lib/uri-util.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r cde6da31e6ec -r ce258aa11821 src/lib/uri-util.c --- a/src/lib/uri-util.c Tue Dec 18 10:44:28 2012 +0200 +++ b/src/lib/uri-util.c Tue Dec 18 18:54:31 2012 +0200 @@ -763,7 +763,7 @@ while (*p != '\0') { if ((*p & 0x80) != 0 || (esc_table[*p] & esc_mask) == 0 || strchr(esc_extra, (char)*p) != NULL) { - str_printfa(out, "%%%2x", *p); + str_printfa(out, "%%%02x", *p); } else { str_append_c(out, *p); } From dovecot at dovecot.org Tue Dec 18 18:55:09 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 18:55:09 +0200 Subject: dovecot-2.2: lib-http: Fixed a crash when http_client_request_su... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/57d552496340 changeset: 15488:57d552496340 user: Timo Sirainen date: Tue Dec 18 18:55:00 2012 +0200 description: lib-http: Fixed a crash when http_client_request_submit() failed immediately diffstat: src/lib-http/http-client-request.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (14 lines): diff -r ce258aa11821 -r 57d552496340 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Dec 18 18:54:31 2012 +0200 +++ b/src/lib-http/http-client-request.c Tue Dec 18 18:55:00 2012 +0200 @@ -153,9 +153,9 @@ http_client_request_debug(req, "Submitted"); host = http_client_host_get(req->client, req->hostname); - http_client_host_submit_request(host, req); req->state = HTTP_REQUEST_STATE_QUEUED; req->client->pending_requests++; + http_client_host_submit_request(host, req); } int http_client_request_send_more(struct http_client_request *req) From dovecot at dovecot.org Tue Dec 18 18:57:13 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 18:57:13 +0200 Subject: dovecot-2.2: auth: When auto-loading auth mechanisms from plugin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4eebc8959849 changeset: 15489:4eebc8959849 user: Timo Sirainen date: Tue Dec 18 18:56:59 2012 +0200 description: auth: When auto-loading auth mechanisms from plugins, generate the name better. diffstat: src/auth/mech.c | 17 ++++++++++++++++- 1 files changed, 16 insertions(+), 1 deletions(-) diffs (41 lines): diff -r 57d552496340 -r 4eebc8959849 src/auth/mech.c --- a/src/auth/mech.c Tue Dec 18 18:55:00 2012 +0200 +++ b/src/auth/mech.c Tue Dec 18 18:56:59 2012 +0200 @@ -7,6 +7,7 @@ #include "passdb.h" #include +#include static struct mech_module_list *mech_modules; @@ -112,6 +113,20 @@ reg->modules = list; } +static const char *mech_get_plugin_name(const char *name) +{ + string_t *str = t_str_new(32); + + str_append(str, "mech_"); + for (; *name != '\0'; name++) { + if (*name == '-') + str_append_c(str, '_'); + else + str_append_c(str, i_tolower(*name)); + } + return str_c(str); +} + struct mechanisms_register * mech_register_init(const struct auth_settings *set) { @@ -139,7 +154,7 @@ mech = mech_module_find(name); if (mech == NULL) { /* maybe it's a plugin. try to load it. */ - auth_module_load(t_strconcat("mech_", name, NULL)); + auth_module_load(mech_get_plugin_name(name)); mech = mech_module_find(name); } if (mech == NULL) From dovecot at dovecot.org Tue Dec 18 18:57:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 18:57:46 +0200 Subject: dovecot-2.1: auth: When auto-loading auth mechanisms from plugin... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/4ec13ce76f6e changeset: 14848:4ec13ce76f6e user: Timo Sirainen date: Tue Dec 18 18:57:41 2012 +0200 description: auth: When auto-loading auth mechanisms from plugins, generate the name better. diffstat: src/auth/mech.c | 17 ++++++++++++++++- 1 files changed, 16 insertions(+), 1 deletions(-) diffs (41 lines): diff -r 3c6a13c0d525 -r 4ec13ce76f6e src/auth/mech.c --- a/src/auth/mech.c Thu Dec 13 12:14:57 2012 +0200 +++ b/src/auth/mech.c Tue Dec 18 18:57:41 2012 +0200 @@ -7,6 +7,7 @@ #include "passdb.h" #include +#include static struct mech_module_list *mech_modules; @@ -112,6 +113,20 @@ reg->modules = list; } +static const char *mech_get_plugin_name(const char *name) +{ + string_t *str = t_str_new(32); + + str_append(str, "mech_"); + for (; *name != '\0'; name++) { + if (*name == '-') + str_append_c(str, '_'); + else + str_append_c(str, i_tolower(*name)); + } + return str_c(str); +} + struct mechanisms_register * mech_register_init(const struct auth_settings *set) { @@ -139,7 +154,7 @@ mech = mech_module_find(name); if (mech == NULL) { /* maybe it's a plugin. try to load it. */ - auth_module_load(t_strconcat("mech_", name, NULL)); + auth_module_load(mech_get_plugin_name(name)); mech = mech_module_find(name); } if (mech == NULL) From dovecot at dovecot.org Tue Dec 18 20:48:45 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 20:48:45 +0200 Subject: dovecot-2.1: indexer-worker: Don't assert-crash with some mailbo... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/04704d1eb6c3 changeset: 14849:04704d1eb6c3 user: Timo Sirainen date: Tue Dec 18 20:48:36 2012 +0200 description: indexer-worker: Don't assert-crash with some mailbox names in some configurations. Virtual mailbox name shouldn't be accessed directly using mailbox_list_*() functions, since they expect a storage name. In some configurations this could have caused assert-crashes with mailbox names that contained invalid characters. diffstat: src/indexer/master-connection.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 4ec13ce76f6e -r 04704d1eb6c3 src/indexer/master-connection.c --- a/src/indexer/master-connection.c Tue Dec 18 18:57:41 2012 +0200 +++ b/src/indexer/master-connection.c Tue Dec 18 20:48:36 2012 +0200 @@ -129,10 +129,10 @@ return -1; } - path = mailbox_list_get_path(ns->list, mailbox, + path = mailbox_list_get_path(ns->list, NULL, MAILBOX_LIST_PATH_TYPE_INDEX); if (*path == '\0') { - i_info("Indexes disabled for Mailbox %s, skipping", mailbox); + i_info("Indexes disabled for mailbox %s, skipping", mailbox); return 0; } From dovecot at dovecot.org Tue Dec 18 21:15:24 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 21:15:24 +0200 Subject: dovecot-2.1: lib-index: If sure mail_index_alloc_cache_destroy_u... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b53ab1c11843 changeset: 14850:b53ab1c11843 user: Timo Sirainen date: Tue Dec 18 21:15:19 2012 +0200 description: lib-index: If sure mail_index_alloc_cache_destroy_unrefed() closes indexes it alone keeps open. This makes sure that all index files are closed within process when mailbox is deleted. diffstat: src/lib-index/mail-index-alloc-cache.c | 15 +++++++++++++-- 1 files changed, 13 insertions(+), 2 deletions(-) diffs (47 lines): diff -r 04704d1eb6c3 -r b53ab1c11843 src/lib-index/mail-index-alloc-cache.c --- a/src/lib-index/mail-index-alloc-cache.c Tue Dec 18 20:48:36 2012 +0200 +++ b/src/lib-index/mail-index-alloc-cache.c Tue Dec 18 21:15:19 2012 +0200 @@ -22,6 +22,7 @@ struct mail_index *index; char *mailbox_path; int refcount; + bool referenced; dev_t index_dir_dev; ino_t index_dir_ino; @@ -58,7 +59,7 @@ static void mail_index_alloc_cache_list_free(struct mail_index_alloc_cache_list *list) { - if (list->index->open_count > 0) + if (list->referenced) mail_index_close(list->index); mail_index_free(&list->index); i_free(list->mailbox_path); @@ -166,6 +167,15 @@ } else { if (rec->refcount == 0) seen_ref0 = TRUE; + if (all && rec->index->open_count == 1 && + rec->referenced) { + /* we're the only one keeping this index open. + we might be here, because the caller is + deleting this mailbox and wants its indexes + to be closed. so close it. */ + rec->referenced = FALSE; + mail_index_close(rec->index); + } list = &(*list)->next; } } @@ -228,8 +238,9 @@ list->index_dir_dev = st.st_dev; } } - if (list != NULL) { + if (list != NULL && !list->referenced) { /* keep it referenced for ourself */ + list->referenced = TRUE; index->open_count++; } } From dovecot at dovecot.org Tue Dec 18 21:20:45 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 21:20:45 +0200 Subject: dovecot-2.1: example-config: Added missing dovecot-dict-auth.con... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/310b3a77a9d1 changeset: 14851:310b3a77a9d1 user: Timo Sirainen date: Tue Dec 18 21:20:38 2012 +0200 description: example-config: Added missing dovecot-dict-auth.conf.ext diffstat: doc/example-config/Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b53ab1c11843 -r 310b3a77a9d1 doc/example-config/Makefile.am --- a/doc/example-config/Makefile.am Tue Dec 18 21:15:19 2012 +0200 +++ b/doc/example-config/Makefile.am Tue Dec 18 21:20:38 2012 +0200 @@ -13,6 +13,7 @@ example_DATA = \ dovecot.conf \ dovecot-db.conf.ext \ + dovecot-dict-auth.conf.ext \ dovecot-dict-sql.conf.ext \ dovecot-ldap.conf.ext \ dovecot-sql.conf.ext From dovecot at dovecot.org Tue Dec 18 21:38:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 21:38:11 +0200 Subject: dovecot-2.1: config: Notify process creation success to master o... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/bc11033ef035 changeset: 14852:bc11033ef035 user: Timo Sirainen date: Tue Dec 18 21:37:57 2012 +0200 description: config: Notify process creation success to master only after parsing config file diffstat: src/config/main.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 310b3a77a9d1 -r bc11033ef035 src/config/main.c --- a/src/config/main.c Tue Dec 18 21:20:38 2012 +0200 +++ b/src/config/main.c Tue Dec 18 21:37:57 2012 +0200 @@ -26,13 +26,17 @@ restrict_access_by_env(NULL, FALSE); restrict_access_allow_coredumps(TRUE); - master_service_init_finish(master_service); config_parse_load_modules(); path = master_service_get_config_path(master_service); if (config_parse_file(path, TRUE, "", &error) <= 0) i_fatal("%s", error); + /* notify about our success only after successfully parsing the + config file, so if the parsing fails, master won't immediately + just recreate this process (and fail again and so on). */ + master_service_init_finish(master_service); + master_service_run(master_service, client_connected); config_connections_destroy_all(); From dovecot at dovecot.org Tue Dec 18 21:45:13 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 21:45:13 +0200 Subject: dovecot-2.1: raw storage: Make sure "from envelope" isn't return... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/579984fdb6e5 changeset: 14853:579984fdb6e5 user: Timo Sirainen date: Tue Dec 18 21:45:08 2012 +0200 description: raw storage: Make sure "from envelope" isn't returned as NULL. diffstat: src/lib-storage/index/raw/raw-mail.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r bc11033ef035 -r 579984fdb6e5 src/lib-storage/index/raw/raw-mail.c --- a/src/lib-storage/index/raw/raw-mail.c Tue Dec 18 21:37:57 2012 +0200 +++ b/src/lib-storage/index/raw/raw-mail.c Tue Dec 18 21:45:08 2012 +0200 @@ -101,7 +101,8 @@ switch (field) { case MAIL_FETCH_FROM_ENVELOPE: - *value_r = mbox->envelope_sender; + *value_r = mbox->envelope_sender != NULL ? + mbox->envelope_sender : ""; return 0; case MAIL_FETCH_UIDL_FILE_NAME: *value_r = mbox->have_filename ? From dovecot at dovecot.org Tue Dec 18 22:07:45 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 22:07:45 +0200 Subject: dovecot-2.1: lib-index: Make sure a corrupted mail_cache_header_... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/2f848393f78e changeset: 14854:2f848393f78e user: Timo Sirainen date: Tue Dec 18 22:05:55 2012 +0200 description: lib-index: Make sure a corrupted mail_cache_header_fields.size doesn't cause crashes. diffstat: src/lib-index/mail-cache-fields.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (31 lines): diff -r 579984fdb6e5 -r 2f848393f78e src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Tue Dec 18 21:45:08 2012 +0200 +++ b/src/lib-index/mail-cache-fields.c Tue Dec 18 22:05:55 2012 +0200 @@ -206,7 +206,7 @@ const struct mail_cache_header_fields *field_hdr; struct mail_cache_header_fields tmp_field_hdr; const void *data; - uint32_t offset = 0, next_offset; + uint32_t offset = 0, next_offset, field_hdr_size; unsigned int next_count = 0; bool invalidate = FALSE; int ret; @@ -276,14 +276,16 @@ cache->need_compress_file_seq = cache->hdr->file_seq; if (field_hdr_r != NULL) { + /* detect corrupted size later */ + field_hdr_size = I_MAX(field_hdr->size, sizeof(*field_hdr)); if (cache->file_cache != NULL && invalidate) { /* if this isn't the first header in file and we hadn't read this before, we can't trust that the cached data is valid */ file_cache_invalidate(cache->file_cache, offset, - field_hdr->size); + field_hdr_size); } - ret = mail_cache_map(cache, offset, field_hdr->size, &data); + ret = mail_cache_map(cache, offset, field_hdr_size, &data); if (ret < 0) return -1; if (ret == 0) { From dovecot at dovecot.org Tue Dec 18 22:07:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 22:07:46 +0200 Subject: dovecot-2.1: lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY was buggy ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/bded819417d9 changeset: 14855:bded819417d9 user: Timo Sirainen date: Tue Dec 18 22:07:36 2012 +0200 description: lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY was buggy when reading data near end of dovecot.index.cache. We assumed that we read as much as we requested, even if the file was smaller. diffstat: src/lib-index/mail-cache.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 2f848393f78e -r bded819417d9 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Dec 18 22:05:55 2012 +0200 +++ b/src/lib-index/mail-cache.c Tue Dec 18 22:07:36 2012 +0200 @@ -346,11 +346,12 @@ buffer_set_used_size(cache->read_buf, ret); cache->read_offset = offset; - cache->mmap_length = offset + size; + cache->mmap_length = offset + cache->read_buf->used; *data_r = data; hdr_data = offset == 0 ? *data_r : NULL; - return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE); + return mail_cache_map_finish(cache, offset, + cache->read_buf->used, hdr_data, TRUE); } int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, From dovecot at dovecot.org Tue Dec 18 22:14:00 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Dec 2012 22:14:00 +0200 Subject: dovecot-2.1: auth: Added CLEAR as yet another alias to PLAIN/CLE... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/bde8963a3774 changeset: 14856:bde8963a3774 user: Timo Sirainen date: Tue Dec 18 22:13:48 2012 +0200 description: auth: Added CLEAR as yet another alias to PLAIN/CLEARTEXT password scheme. This apparently is used by Sun LDAP server. diffstat: src/auth/password-scheme.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r bded819417d9 -r bde8963a3774 src/auth/password-scheme.c --- a/src/auth/password-scheme.c Tue Dec 18 22:07:36 2012 +0200 +++ b/src/auth/password-scheme.c Tue Dec 18 22:13:48 2012 +0200 @@ -817,6 +817,7 @@ { "SSHA256", PW_ENCODING_BASE64, 0, ssha256_verify, ssha256_generate }, { "SSHA512", PW_ENCODING_BASE64, 0, ssha512_verify, ssha512_generate }, { "PLAIN", PW_ENCODING_NONE, 0, NULL, plain_generate }, + { "CLEAR", PW_ENCODING_NONE, 0, NULL, plain_generate }, { "CLEARTEXT", PW_ENCODING_NONE, 0, NULL, plain_generate }, { "PLAIN-TRUNC", PW_ENCODING_NONE, 0, plain_trunc_verify, plain_generate }, { "CRAM-MD5", PW_ENCODING_HEX, CRAM_MD5_CONTEXTLEN, From dovecot at dovecot.org Wed Dec 19 14:40:13 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Dec 2012 14:40:13 +0200 Subject: dovecot-2.2: lib-http: Host entries were added to hash table usi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0c9a4af8eaf8 changeset: 15490:0c9a4af8eaf8 user: Timo Sirainen date: Wed Dec 19 14:39:55 2012 +0200 description: lib-http: Host entries were added to hash table using wrong hostname pointer. diffstat: src/lib-http/http-client-host.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 4eebc8959849 -r 0c9a4af8eaf8 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Tue Dec 18 18:56:59 2012 +0200 +++ b/src/lib-http/http-client-host.c Wed Dec 19 14:39:55 2012 +0200 @@ -274,7 +274,7 @@ host->name = i_strdup(hostname); i_array_init(&host->ports, 4); - hash_table_insert(client->hosts, hostname, host); + hash_table_insert(client->hosts, host->name, host); http_client_host_debug(host, "Host created"); } From dovecot at dovecot.org Wed Dec 19 14:41:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Dec 2012 14:41:11 +0200 Subject: dovecot-2.2: json-parser: Fixed infinite looping in some situati... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a1b814081f0c changeset: 15491:a1b814081f0c user: Timo Sirainen date: Wed Dec 19 14:41:01 2012 +0200 description: json-parser: Fixed infinite looping in some situations. diffstat: src/lib/json-parser.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 0c9a4af8eaf8 -r a1b814081f0c src/lib/json-parser.c --- a/src/lib/json-parser.c Wed Dec 19 14:39:55 2012 +0200 +++ b/src/lib/json-parser.c Wed Dec 19 14:41:01 2012 +0200 @@ -546,7 +546,8 @@ /* parsing probably failed because there wasn't enough input. reset the error and try reading more. */ parser->error = NULL; - + parser->highwater_offset = parser->input->v_offset + + i_stream_get_data_size(parser->input); } return ret; } @@ -623,6 +624,8 @@ /* parsing probably failed because there wasn't enough input. reset the error and try reading more. */ parser->error = NULL; + parser->highwater_offset = parser->input->v_offset + + i_stream_get_data_size(parser->input); } return ret; } From dovecot at dovecot.org Wed Dec 19 15:01:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Dec 2012 15:01:06 +0200 Subject: dovecot-2.1: lib-index: If map_with_read gets disabled, reset th... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/32ce915e046a changeset: 14857:32ce915e046a user: Timo Sirainen date: Wed Dec 19 15:00:45 2012 +0200 description: lib-index: If map_with_read gets disabled, reset the cache buffers properly. diffstat: src/lib-index/mail-cache-compress.c | 8 +++++++- src/lib-index/mail-cache.c | 1 + 2 files changed, 8 insertions(+), 1 deletions(-) diffs (29 lines): diff -r bde8963a3774 -r 32ce915e046a src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Tue Dec 18 22:13:48 2012 +0200 +++ b/src/lib-index/mail-cache-compress.c Wed Dec 19 15:00:45 2012 +0200 @@ -462,7 +462,13 @@ return 0; /* compression isn't very efficient with small read()s */ - cache->map_with_read = FALSE; + if (cache->map_with_read) { + cache->map_with_read = FALSE; + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + } if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) { /* we're using dotlocking, cache file creation itself creates diff -r bde8963a3774 -r 32ce915e046a src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Dec 18 22:13:48 2012 +0200 +++ b/src/lib-index/mail-cache.c Wed Dec 19 15:00:45 2012 +0200 @@ -394,6 +394,7 @@ if (offset < cache->mmap_length && size <= cache->mmap_length - offset) { /* already mapped */ + i_assert(cache->mmap_base != NULL); *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); return 1; } From dovecot at dovecot.org Wed Dec 19 16:36:59 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Dec 2012 16:36:59 +0200 Subject: dovecot-2.1: lib-index: Fixed sequence lookup of newly created m... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/7f4835cff04a changeset: 14858:7f4835cff04a user: lion at ubuntu.ubuntu-domain date: Fri Dec 07 08:50:35 2012 +0400 description: lib-index: Fixed sequence lookup of newly created mails in transaction view. diffstat: src/lib-index/mail-index-transaction-view.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 32ce915e046a -r 7f4835cff04a src/lib-index/mail-index-transaction-view.c --- a/src/lib-index/mail-index-transaction-view.c Wed Dec 19 15:00:45 2012 +0200 +++ b/src/lib-index/mail-index-transaction-view.c Fri Dec 07 08:50:35 2012 +0400 @@ -203,7 +203,7 @@ if (first_uid <= rec->uid) break; } - if (seq > tview->t->last_new_seq) { + if (seq > tview->t->last_new_seq || rec->uid > last_uid) { /* no messages in range */ return; } From dovecot at dovecot.org Wed Dec 19 16:41:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Dec 2012 16:41:29 +0200 Subject: dovecot-2.1: lib-index: Optimized single sequence lookup of newl... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c9e6ad8a42db changeset: 14859:c9e6ad8a42db user: lion at ubuntu.ubuntu-domain date: Fri Dec 07 08:50:35 2012 +0400 description: lib-index: Optimized single sequence lookup of newly created mails in transaction view. diffstat: src/lib-index/mail-index-transaction-view.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 7f4835cff04a -r c9e6ad8a42db src/lib-index/mail-index-transaction-view.c --- a/src/lib-index/mail-index-transaction-view.c Fri Dec 07 08:50:35 2012 +0400 +++ b/src/lib-index/mail-index-transaction-view.c Fri Dec 07 08:50:35 2012 +0400 @@ -208,6 +208,12 @@ return; } *first_seq_r = seq; + + if (rec->uid == last_uid) { + /* one seq in range */ + *last_seq_r = seq; + return; + } } seq = tview->t->last_new_seq; From dovecot at dovecot.org Wed Dec 19 23:34:36 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Dec 2012 23:34:36 +0200 Subject: dovecot-2.2: lib-http: Compile fix for previous change. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cd093d05037e changeset: 15492:cd093d05037e user: Timo Sirainen date: Wed Dec 19 23:34:21 2012 +0200 description: lib-http: Compile fix for previous change. diffstat: src/lib-http/http-client-host.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r a1b814081f0c -r cd093d05037e src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Wed Dec 19 14:41:01 2012 +0200 +++ b/src/lib-http/http-client-host.c Wed Dec 19 23:34:21 2012 +0200 @@ -274,7 +274,8 @@ host->name = i_strdup(hostname); i_array_init(&host->ports, 4); - hash_table_insert(client->hosts, host->name, host); + hostname = host->name; + hash_table_insert(client->hosts, hostname, host); http_client_host_debug(host, "Host created"); } From pigeonhole at rename-it.nl Sat Dec 22 23:27:42 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 22 Dec 2012 22:27:42 +0100 Subject: dovecot-2.1-pigeonhole: lib-sieve: Prevent passing NULL sender t... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/fc0bf6b1cb6b changeset: 1668:fc0bf6b1cb6b user: Stephan Bosch date: Sat Dec 22 22:27:34 2012 +0100 description: lib-sieve: Prevent passing NULL sender to raw mail storage when active message is substituted. diffstat: src/lib-sieve/sieve-config.h | 2 ++ src/lib-sieve/sieve-message.c | 5 ++++- 2 files changed, 6 insertions(+), 1 deletions(-) diffs (33 lines): diff -r b56711807edc -r fc0bf6b1cb6b src/lib-sieve/sieve-config.h --- a/src/lib-sieve/sieve-config.h Mon Nov 26 21:16:54 2012 +0100 +++ b/src/lib-sieve/sieve-config.h Sat Dec 22 22:27:34 2012 +0100 @@ -11,4 +11,6 @@ #define SIEVE_SCRIPT_FILEEXT "sieve" #define SIEVE_BINARY_FILEEXT "svbin" +#define DEFAULT_ENVELOPE_SENDER "MAILER-DAEMON" + #endif diff -r b56711807edc -r fc0bf6b1cb6b src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Mon Nov 26 21:16:54 2012 +0100 +++ b/src/lib-sieve/sieve-message.c Sat Dec 22 22:27:34 2012 +0100 @@ -357,6 +357,7 @@ struct sieve_message_version *version; struct mailbox_header_lookup_ctx *headers_ctx; struct mailbox *box = NULL; + const char *sender; int ret; if ( msgctx->raw_mail_user == NULL ) { @@ -367,8 +368,10 @@ } i_stream_seek(input, 0); + sender = sieve_message_get_sender(msgctx); + sender = (sender == NULL ? DEFAULT_ENVELOPE_SENDER : sender ); ret = raw_mailbox_alloc_stream(msgctx->raw_mail_user, input, (time_t)-1, - sieve_message_get_sender(msgctx), &box); + sender, &box); if ( ret < 0 ) { sieve_sys_error(msgctx->svinst, "can't open substituted mail as raw: %s", From pigeonhole at rename-it.nl Wed Dec 26 12:52:12 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 26 Dec 2012 11:52:12 +0100 Subject: dovecot-2.1-pigeonhole: Fixed compile on Mageia Linux. Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/fe5f2738d538 changeset: 1669:fe5f2738d538 user: Stephan Bosch date: Wed Dec 26 11:52:03 2012 +0100 description: Fixed compile on Mageia Linux. Added LIBDOVECOT_STORAGE to library dependencies in src/lib-sieve/Makefile.am. diffstat: src/lib-sieve/Makefile.am | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r fc0bf6b1cb6b -r fe5f2738d538 src/lib-sieve/Makefile.am --- a/src/lib-sieve/Makefile.am Sat Dec 22 22:27:34 2012 +0100 +++ b/src/lib-sieve/Makefile.am Wed Dec 26 11:52:03 2012 +0100 @@ -70,8 +70,8 @@ $(extdir)/vnd.dovecot/duplicate/libsieve_ext_duplicate.la \ $(unfinished_plugins) -libdovecot_sieve_la_DEPENDENCIES = $(plugins) $(LIBDOVECOT_LDA_DEPS) -libdovecot_sieve_la_LIBADD = $(plugins) $(LIBDOVECOT) $(LIBDOVECOT_LDA) +libdovecot_sieve_la_DEPENDENCIES = $(plugins) $(LIBDOVECOT_LDA_DEPS) $(LIBDOVECOT_STORAGE_DEPS) $(LIBDOVECOT_DEPS) +libdovecot_sieve_la_LIBADD = $(plugins) $(LIBDOVECOT_LDA) $(LIBDOVECOT_STORAGE) $(LIBDOVECOT) libdovecot_sieve_la_SOURCES = \ rfc2822.c \ From dovecot at dovecot.org Fri Dec 28 17:02:30 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Dec 2012 17:02:30 +0200 Subject: dovecot-2.2: configure: Use -lrt for check_gettime() only if nee... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac63858cbe5f changeset: 15493:ac63858cbe5f user: Timo Sirainen date: Fri Dec 28 17:02:18 2012 +0200 description: configure: Use -lrt for check_gettime() only if needed. diffstat: configure.ac | 6 +----- 1 files changed, 1 insertions(+), 5 deletions(-) diffs (17 lines): diff -r cd093d05037e -r ac63858cbe5f configure.ac --- a/configure.ac Wed Dec 19 23:34:21 2012 +0200 +++ b/configure.ac Fri Dec 28 17:02:18 2012 +0200 @@ -415,12 +415,8 @@ AC_CHECK_TYPES([struct sockpeercred]) -AC_CHECK_LIB(rt, clock_gettime, [ +AC_SEARCH_LIBS(clock_gettime, rt, [ AC_DEFINE(HAVE_CLOCK_GETTIME,, Define if you have the clock_gettime function) - LIBS="$LIBS -lrt" -], [ - # OpenBSD - AC_CHECK_FUNCS(clock_gettime) ]) dnl strtoimax and strtoumax are macros in HP-UX, so inttypes.h must be included