From dovecot at dovecot.org Tue Sep 1 13:26:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 01 Sep 2015 13:26:13 +0000 Subject: dovecot-2.2: lib-imap-client: If connect() fails immediately, lo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e157d13efac9 changeset: 19055:e157d13efac9 user: Timo Sirainen date: Tue Sep 01 16:25:11 2015 +0300 description: lib-imap-client: If connect() fails immediately, log an error and retry the next IP. Only if all IPs fail return a full failure. This is mainly intended to skip IPv6 addresses when IPv6 connectivity doesn't work. diffstat: src/lib-imap-client/imapc-connection.c | 23 +++++++++++++++++------ 1 files changed, 17 insertions(+), 6 deletions(-) diffs (37 lines): diff -r c0e61e4f47ec -r e157d13efac9 src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Tue Sep 01 00:11:37 2015 +0300 +++ b/src/lib-imap-client/imapc-connection.c Tue Sep 01 16:25:11 2015 +0300 @@ -1564,16 +1564,27 @@ static void imapc_connection_connect_next_ip(struct imapc_connection *conn) { const struct ip_addr *ip; + unsigned int i; int fd; i_assert(conn->client->set.max_idle_time > 0); - conn->prev_connect_idx = (conn->prev_connect_idx+1) % conn->ips_count; - ip = &conn->ips[conn->prev_connect_idx]; - fd = net_connect_ip(ip, conn->client->set.port, NULL); - if (fd == -1) { - imapc_connection_set_disconnected(conn); - return; + for (i = 0;;) { + conn->prev_connect_idx = (conn->prev_connect_idx+1) % conn->ips_count; + ip = &conn->ips[conn->prev_connect_idx]; + fd = net_connect_ip(ip, conn->client->set.port, NULL); + if (fd != -1) + break; + + /* failed to connect to one of the IPs immediately + (e.g. IPv6 address without connectivity). try all IPs + before failing completely. */ + i_error("net_connect_ip(%s:%u) failed: %m", + net_ip2addr(ip), conn->client->set.port); + if (++i == conn->ips_count) { + imapc_connection_set_disconnected(conn); + return; + } } conn->fd = fd; conn->input = conn->raw_input = i_stream_create_fd(fd, (size_t)-1, FALSE); From dovecot at dovecot.org Wed Sep 2 14:17:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:17:41 +0000 Subject: dovecot-2.2: lib-sql: sql_result.free() should never be reached ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/97ff15e9a7fd changeset: 19056:97ff15e9a7fd user: Timo Sirainen date: Wed Sep 02 17:16:41 2015 +0300 description: lib-sql: sql_result.free() should never be reached from the query callback. This code was probably added before sql_result refcounting. diffstat: src/lib-sql/driver-cassandra.c | 8 +------- src/lib-sql/driver-pgsql.c | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diffs (36 lines): diff -r e157d13efac9 -r 97ff15e9a7fd src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Tue Sep 01 16:25:11 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Wed Sep 02 17:16:41 2015 +0300 @@ -431,13 +431,7 @@ struct cassandra_db *db = (struct cassandra_db *)_result->db; struct cassandra_result *result = (struct cassandra_result *)_result; - if (result->api.callback) { - /* we're coming here from a user's sql_result_free() that's - being called from a callback. we'll do this later, - so ignore. */ - return; - } - + i_assert(!result->api.callback); i_assert(db->cur_result == result); i_assert(result->callback == NULL); diff -r e157d13efac9 -r 97ff15e9a7fd src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Tue Sep 01 16:25:11 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Wed Sep 02 17:16:41 2015 +0300 @@ -328,13 +328,7 @@ struct pgsql_result *result = (struct pgsql_result *)_result; bool success; - if (result->api.callback) { - /* we're coming here from a user's sql_result_free() that's - being called from a callback. we'll do this later, - so ignore. */ - return; - } - + i_assert(!result->api.callback); i_assert(db->cur_result == result); i_assert(result->callback == NULL); From dovecot at dovecot.org Wed Sep 2 14:20:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:20:09 +0000 Subject: dovecot-2.2: lib-sql: Debugging help - Added assert before clear... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0770ab82703d changeset: 19057:0770ab82703d user: Timo Sirainen date: Wed Sep 02 17:19:08 2015 +0300 description: lib-sql: Debugging help - Added assert before clearing sql_result.callback. If result is unrefed too many times, this still allows accessing the callback from a debugger. diffstat: src/lib-sql/driver-cassandra.c | 3 ++- src/lib-sql/driver-pgsql.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diffs (36 lines): diff -r 97ff15e9a7fd -r 0770ab82703d src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Wed Sep 02 17:16:41 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Wed Sep 02 17:19:08 2015 +0300 @@ -470,12 +470,13 @@ result->callback(&result->api, result->context); } T_END; result->api.callback = FALSE; - result->callback = NULL; free_result = db->sync_result != &result->api; if (db->ioloop != NULL) io_loop_stop(db->ioloop); + i_assert(!free_result || result->api.refcount > 0); + result->callback = NULL; if (free_result) sql_result_unref(&result->api); } diff -r 97ff15e9a7fd -r 0770ab82703d src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Wed Sep 02 17:16:41 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Wed Sep 02 17:19:08 2015 +0300 @@ -387,12 +387,13 @@ result->callback(&result->api, result->context); } T_END; result->api.callback = FALSE; - result->callback = NULL; free_result = db->sync_result != &result->api; if (db->ioloop != NULL) io_loop_stop(db->ioloop); + i_assert(!free_result || result->api.refcount > 0); + result->callback = NULL; if (free_result) sql_result_unref(&result->api); } From dovecot at dovecot.org Wed Sep 2 14:21:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:21:26 +0000 Subject: dovecot-2.2: lib-sql: Mark Cassandra driver as pooled. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3cf6441adf12 changeset: 19058:3cf6441adf12 user: Timo Sirainen date: Wed Sep 02 17:20:02 2015 +0300 description: lib-sql: Mark Cassandra driver as pooled. Otherwise all the asynchronous operations will assert-crash, since all the auto-connecting code is in the sqlpool code. diffstat: src/lib-sql/driver-cassandra.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 0770ab82703d -r 3cf6441adf12 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Wed Sep 02 17:19:08 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Wed Sep 02 17:20:02 2015 +0300 @@ -943,7 +943,7 @@ const struct sql_db driver_cassandra_db = { .name = "cassandra", - .flags = 0, + .flags = SQL_DB_FLAG_POOLED, .v = { driver_cassandra_init_v, From dovecot at dovecot.org Wed Sep 2 14:24:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:24:49 +0000 Subject: dovecot-2.2: dict-sql: Code cleanup - separated SQL query buildi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/46eb03d95feb changeset: 19060:46eb03d95feb user: Timo Sirainen date: Wed Sep 02 17:23:45 2015 +0300 description: dict-sql: Code cleanup - separated SQL query building function from sending it. diffstat: src/lib-dict/dict-sql.c | 98 ++++++++++++++++++++++++++++-------------------- 1 files changed, 57 insertions(+), 41 deletions(-) diffs (123 lines): diff -r 80f840616abb -r 46eb03d95feb src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 17:21:06 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 17:23:45 2015 +0300 @@ -341,7 +341,9 @@ return NULL; } -static bool sql_dict_iterate_next_query(struct sql_dict_iterate_context *ctx) +static bool +sql_dict_iterate_build_next_query(struct sql_dict_iterate_context *ctx, + string_t *query) { struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; const struct dict_sql_map *map; @@ -357,54 +359,68 @@ if (ctx->result != NULL) sql_result_unref(ctx->result); - T_BEGIN { - string_t *query = t_str_new(256); + str_append(query, "SELECT "); + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + str_printfa(query, "%s,", map->value_field); - str_append(query, "SELECT "); - if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) - str_printfa(query, "%s,", map->value_field); + /* get all missing fields */ + sql_fields = array_get(&map->sql_fields, &count); + i = array_count(&values); + if (i == count) { + /* we always want to know the last field since we're + iterating its children */ + i_assert(i > 0); + i--; + } + for (; i < count; i++) + str_printfa(query, "%s,", sql_fields[i]); + str_truncate(query, str_len(query)-1); - /* get all missing fields */ - sql_fields = array_get(&map->sql_fields, &count); - i = array_count(&values); - if (i == count) { - /* we always want to know the last field since we're - iterating its children */ - i_assert(i > 0); - i--; + str_printfa(query, " FROM %s", map->table); + + if ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0) + recurse_type = SQL_DICT_RECURSE_FULL; + else if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) + recurse_type = SQL_DICT_RECURSE_NONE; + else + recurse_type = SQL_DICT_RECURSE_ONE; + sql_dict_where_build(dict, map, &values, + ctx->paths[ctx->path_idx][0], + recurse_type, query); + + if ((ctx->flags & DICT_ITERATE_FLAG_SORT_BY_KEY) != 0) { + str_append(query, " ORDER BY "); + for (i = 0; i < count; i++) { + str_printfa(query, "%s", sql_fields[i]); + if (i < count-1) + str_append_c(query, ','); } - for (; i < count; i++) - str_printfa(query, "%s,", sql_fields[i]); - str_truncate(query, str_len(query)-1); - - str_printfa(query, " FROM %s", map->table); - - if ((ctx->flags & DICT_ITERATE_FLAG_RECURSE) != 0) - recurse_type = SQL_DICT_RECURSE_FULL; - else if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) - recurse_type = SQL_DICT_RECURSE_NONE; - else - recurse_type = SQL_DICT_RECURSE_ONE; - sql_dict_where_build(dict, map, &values, - ctx->paths[ctx->path_idx][0], - recurse_type, query); - - if ((ctx->flags & DICT_ITERATE_FLAG_SORT_BY_KEY) != 0) { - str_append(query, " ORDER BY "); - for (i = 0; i < count; i++) { - str_printfa(query, "%s", sql_fields[i]); - if (i < count-1) - str_append_c(query, ','); - } - } else if ((ctx->flags & DICT_ITERATE_FLAG_SORT_BY_VALUE) != 0) - str_printfa(query, " ORDER BY %s", map->value_field); - ctx->result = sql_query_s(dict->db, str_c(query)); - } T_END; + } else if ((ctx->flags & DICT_ITERATE_FLAG_SORT_BY_VALUE) != 0) + str_printfa(query, " ORDER BY %s", map->value_field); ctx->map = map; return TRUE; } +static bool sql_dict_iterate_next_query(struct sql_dict_iterate_context *ctx) +{ + struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; + bool ret; + + T_BEGIN { + string_t *query = t_str_new(256); + + ret = sql_dict_iterate_build_next_query(ctx, query); + if (!ret) { + /* failed */ + } else { + ctx->result = sql_query_s(dict->db, str_c(query)); + } + } T_END; + return ret; +} + + static struct dict_iterate_context * sql_dict_iterate_init(struct dict *_dict, const char *const *paths, enum dict_iterate_flags flags) From dovecot at dovecot.org Wed Sep 2 14:24:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:24:49 +0000 Subject: dovecot-2.2: lib-dict: Code cleanup - give name for enum dict_pr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/80f840616abb changeset: 19059:80f840616abb user: Timo Sirainen date: Wed Sep 02 17:21:06 2015 +0300 description: lib-dict: Code cleanup - give name for enum dict_protocol_cmd/reply diffstat: src/lib-dict/dict-client.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 3cf6441adf12 -r 80f840616abb src/lib-dict/dict-client.h --- a/src/lib-dict/dict-client.h Wed Sep 02 17:20:02 2015 +0300 +++ b/src/lib-dict/dict-client.h Wed Sep 02 17:21:06 2015 +0300 @@ -10,7 +10,7 @@ #define DICT_CLIENT_MAX_LINE_LENGTH (64*1024) -enum { +enum dict_protocol_cmd { /* */ DICT_PROTOCOL_CMD_HELLO = 'H', @@ -28,7 +28,7 @@ DICT_PROTOCOL_CMD_ATOMIC_INC = 'A' /* */ }; -enum { +enum dict_protocol_reply { DICT_PROTOCOL_REPLY_OK = 'O', /* */ DICT_PROTOCOL_REPLY_NOTFOUND = 'N', DICT_PROTOCOL_REPLY_FAIL = 'F', From dovecot at dovecot.org Wed Sep 2 14:28:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:28:15 +0000 Subject: dovecot-2.2: dict-sql: Fixed memory leak when committing/rollbac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c444f10cef9 changeset: 19061:5c444f10cef9 user: Timo Sirainen date: Wed Sep 02 17:26:08 2015 +0300 description: dict-sql: Fixed memory leak when committing/rollbacking unchanged transaction. diffstat: src/lib-dict/dict-sql.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (25 lines): diff -r 46eb03d95feb -r 5c444f10cef9 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 17:23:45 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 17:26:08 2015 +0300 @@ -546,7 +546,10 @@ if (ctx->failed) { sql_transaction_rollback(&ctx->sql_ctx); ret = -1; - } else if (_ctx->changed) { + } else if (!_ctx->changed) { + /* nothing changed, no need to commit */ + sql_transaction_rollback(&ctx->sql_ctx); + } else { if (sql_transaction_commit_s(&ctx->sql_ctx, &error) < 0) { i_error("sql dict: commit failed: %s", error); ret = -1; @@ -576,8 +579,7 @@ struct sql_dict_transaction_context *ctx = (struct sql_dict_transaction_context *)_ctx; - if (_ctx->changed) - sql_transaction_rollback(&ctx->sql_ctx); + sql_transaction_rollback(&ctx->sql_ctx); if (ctx->inc_row_pool != NULL) pool_unref(&ctx->inc_row_pool); From dovecot at dovecot.org Wed Sep 2 14:28:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:28:17 +0000 Subject: dovecot-2.2: dict-sql: Code cleanup - use a common sql_dict_tran... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cb6d6d861ffa changeset: 19062:cb6d6d861ffa user: Timo Sirainen date: Wed Sep 02 17:27:09 2015 +0300 description: dict-sql: Code cleanup - use a common sql_dict_transaction_free() diffstat: src/lib-dict/dict-sql.c | 19 ++++++++++--------- 1 files changed, 10 insertions(+), 9 deletions(-) diffs (43 lines): diff -r 5c444f10cef9 -r cb6d6d861ffa src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 17:26:08 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 17:27:09 2015 +0300 @@ -529,6 +529,14 @@ return &ctx->ctx; } +static void sql_dict_transaction_free(struct sql_dict_transaction_context *ctx) +{ + if (ctx->inc_row_pool != NULL) + pool_unref(&ctx->inc_row_pool); + i_free(ctx->prev_inc_key); + i_free(ctx); +} + static int sql_dict_transaction_commit(struct dict_transaction_context *_ctx, bool async ATTR_UNUSED, @@ -564,10 +572,7 @@ } } } - if (ctx->inc_row_pool != NULL) - pool_unref(&ctx->inc_row_pool); - i_free(ctx->prev_inc_key); - i_free(ctx); + sql_dict_transaction_free(ctx); if (callback != NULL) callback(ret, context); @@ -580,11 +585,7 @@ (struct sql_dict_transaction_context *)_ctx; sql_transaction_rollback(&ctx->sql_ctx); - - if (ctx->inc_row_pool != NULL) - pool_unref(&ctx->inc_row_pool); - i_free(ctx->prev_inc_key); - i_free(ctx); + sql_dict_transaction_free(ctx); } struct dict_sql_build_query_field { From dovecot at dovecot.org Wed Sep 2 14:29:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:29:46 +0000 Subject: dovecot-2.2: dict-sql: Code cleanup - added sql_dict_transaction... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dd083fddda5e changeset: 19063:dd083fddda5e user: Timo Sirainen date: Wed Sep 02 17:28:41 2015 +0300 description: dict-sql: Code cleanup - added sql_dict_transaction_has_nonexistent() diffstat: src/lib-dict/dict-sql.c | 23 +++++++++++++++-------- 1 files changed, 15 insertions(+), 8 deletions(-) diffs (40 lines): diff -r cb6d6d861ffa -r dd083fddda5e src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 17:27:09 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 17:28:41 2015 +0300 @@ -537,6 +537,19 @@ i_free(ctx); } +static bool +sql_dict_transaction_has_nonexistent(struct sql_dict_transaction_context *ctx) +{ + struct sql_dict_inc_row *inc_row; + + for (inc_row = ctx->inc_row; inc_row != NULL; inc_row = inc_row->prev) { + i_assert(inc_row->rows != UINT_MAX); + if (inc_row->rows == 0) + return TRUE; + } + return FALSE; +} + static int sql_dict_transaction_commit(struct dict_transaction_context *_ctx, bool async ATTR_UNUSED, @@ -562,14 +575,8 @@ i_error("sql dict: commit failed: %s", error); ret = -1; } else { - while (ctx->inc_row != NULL) { - i_assert(ctx->inc_row->rows != UINT_MAX); - if (ctx->inc_row->rows == 0) { - ret = 0; - break; - } - ctx->inc_row = ctx->inc_row->prev; - } + if (sql_dict_transaction_has_nonexistent(ctx)) + ret = 0; } } sql_dict_transaction_free(ctx); From dovecot at dovecot.org Wed Sep 2 14:35:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:35:46 +0000 Subject: dovecot-2.2: lib-dict: Added async API for lookup and iteration. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d40d7f24ffcf changeset: 19064:d40d7f24ffcf user: Timo Sirainen date: Wed Sep 02 17:34:43 2015 +0300 description: lib-dict: Added async API for lookup and iteration. diffstat: src/lib-dict/dict-client.c | 3 ++- src/lib-dict/dict-file.c | 3 ++- src/lib-dict/dict-fs.c | 1 + src/lib-dict/dict-memcached-ascii.c | 3 ++- src/lib-dict/dict-memcached.c | 1 + src/lib-dict/dict-private.h | 8 ++++++++ src/lib-dict/dict-redis.c | 3 ++- src/lib-dict/dict-sql.c | 3 ++- src/lib-dict/dict.c | 30 ++++++++++++++++++++++++++++++ src/lib-dict/dict.h | 29 +++++++++++++++++++++++++++-- 10 files changed, 77 insertions(+), 7 deletions(-) diffs (229 lines): diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict-client.c Wed Sep 02 17:34:43 2015 +0300 @@ -852,6 +852,7 @@ client_dict_set, client_dict_unset, client_dict_append, - client_dict_atomic_inc + client_dict_atomic_inc, + NULL } }; diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict-file.c Wed Sep 02 17:34:43 2015 +0300 @@ -650,6 +650,7 @@ dict_transaction_memory_set, dict_transaction_memory_unset, dict_transaction_memory_append, - dict_transaction_memory_atomic_inc + dict_transaction_memory_atomic_inc, + NULL } }; diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict-fs.c --- a/src/lib-dict/dict-fs.c Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict-fs.c Wed Sep 02 17:34:43 2015 +0300 @@ -284,6 +284,7 @@ dict_transaction_memory_set, dict_transaction_memory_unset, NULL, + NULL, NULL } }; diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict-memcached-ascii.c --- a/src/lib-dict/dict-memcached-ascii.c Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict-memcached-ascii.c Wed Sep 02 17:34:43 2015 +0300 @@ -663,6 +663,7 @@ dict_transaction_memory_set, dict_transaction_memory_unset, dict_transaction_memory_append, - dict_transaction_memory_atomic_inc + dict_transaction_memory_atomic_inc, + NULL } }; diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict-memcached.c --- a/src/lib-dict/dict-memcached.c Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict-memcached.c Wed Sep 02 17:34:43 2015 +0300 @@ -390,6 +390,7 @@ NULL, NULL, NULL, + NULL, NULL } }; diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict-private.h --- a/src/lib-dict/dict-private.h Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict-private.h Wed Sep 02 17:34:43 2015 +0300 @@ -35,6 +35,9 @@ const char *key, const char *value); void (*atomic_inc)(struct dict_transaction_context *ctx, const char *key, long long diff); + + void (*lookup_async)(struct dict *dict, const char *key, + dict_lookup_callback_t *callback, void *context); }; struct dict { @@ -45,6 +48,11 @@ struct dict_iterate_context { struct dict *dict; + + dict_iterate_callback_t *async_callback; + void *async_context; + + unsigned int has_more:1; }; struct dict_transaction_context { diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict-redis.c Wed Sep 02 17:34:43 2015 +0300 @@ -801,6 +801,7 @@ redis_set, redis_unset, redis_append, - redis_atomic_inc + redis_atomic_inc, + NULL } }; diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 17:34:43 2015 +0300 @@ -946,7 +946,8 @@ sql_dict_set, sql_dict_unset, sql_dict_append, - sql_dict_atomic_inc + sql_dict_atomic_inc, + NULL } }; diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict.c --- a/src/lib-dict/dict.c Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict.c Wed Sep 02 17:34:43 2015 +0300 @@ -119,6 +119,23 @@ return dict->v.lookup(dict, pool, key, value_r); } +void dict_lookup_async(struct dict *dict, const char *key, + dict_lookup_callback_t *callback, void *context) +{ + if (dict->v.lookup_async == NULL) { + struct dict_lookup_result result; + + memset(&result, 0, sizeof(result)); + result.ret = dict_lookup(dict, pool_datastack_create(), + key, &result.value); + if (result.ret < 0) + result.error = "Lookup failed"; + callback(&result, context); + return; + } + return dict->v.lookup_async(dict, key, callback, context); +} + struct dict_iterate_context * dict_iterate_init(struct dict *dict, const char *path, enum dict_iterate_flags flags) @@ -154,6 +171,19 @@ ctx->dict->v.iterate(ctx, key_r, value_r); } +void dict_iterate_set_async_callback(struct dict_iterate_context *ctx, + dict_iterate_callback_t *callback, + void *context) +{ + ctx->async_callback = callback; + ctx->async_context = context; +} + +bool dict_iterate_has_more(struct dict_iterate_context *ctx) +{ + return ctx->has_more; +} + int dict_iterate_deinit(struct dict_iterate_context **_ctx) { struct dict_iterate_context *ctx = *_ctx; diff -r dd083fddda5e -r d40d7f24ffcf src/lib-dict/dict.h --- a/src/lib-dict/dict.h Wed Sep 02 17:28:41 2015 +0300 +++ b/src/lib-dict/dict.h Wed Sep 02 17:34:43 2015 +0300 @@ -5,6 +5,7 @@ #define DICT_PATH_SHARED "shared/" struct dict; +struct dict_iterate_context; enum dict_iterate_flags { /* Recurse to all the sub-hierarchies (e.g. iterating "foo/" will @@ -18,7 +19,9 @@ DICT_ITERATE_FLAG_NO_VALUE = 0x08, /* Don't recurse at all. This is basically the same as dict_lookup(), but it'll return all the rows instead of only the first one. */ - DICT_ITERATE_FLAG_EXACT_KEY = 0x10 + DICT_ITERATE_FLAG_EXACT_KEY = 0x10, + /* Perform iteration asynchronously. */ + DICT_ITERATE_FLAG_ASYNC = 0x20 }; enum dict_data_type { @@ -35,6 +38,15 @@ const char *home_dir; }; +struct dict_lookup_result { + int ret; + const char *value; + const char *error; +}; + +typedef void dict_lookup_callback_t(const struct dict_lookup_result *result, + void *context); +typedef void dict_iterate_callback_t(void *context); typedef void dict_transaction_commit_callback_t(int ret, void *context); void dict_driver_register(struct dict *driver); @@ -55,7 +67,7 @@ struct dict **dict_r, const char **error_r); /* Close dictionary. */ void dict_deinit(struct dict **dict); -/* Wait for all pending asynchronous transaction commits to finish. +/* Wait for all pending asynchronous operations to finish. Returns 0 if ok, -1 if error. */ int dict_wait(struct dict *dict); @@ -63,6 +75,8 @@ Returns 1 if found, 0 if not found and -1 if lookup failed. */ int dict_lookup(struct dict *dict, pool_t pool, const char *key, const char **value_r); +void dict_lookup_async(struct dict *dict, const char *key, + dict_lookup_callback_t *callback, void *context); /* Iterate through all values in a path. flag indicates how iteration is carried out */ @@ -72,6 +86,17 @@ struct dict_iterate_context * dict_iterate_init_multiple(struct dict *dict, const char *const *paths, enum dict_iterate_flags flags); +/* Set async callback. Note that if dict_iterate_init() already did all the + work, this callback may never be called. So after dict_iterate_init() you + should call dict_iterate() in any case to see if all the results are + already available. */ +void dict_iterate_set_async_callback(struct dict_iterate_context *ctx, + dict_iterate_callback_t *callback, + void *context); +/* If dict_iterate() returns FALSE, the iteration may be finished or if this + is an async iteration it may be waiting for more data. If this function + returns TRUE, the dict callback is called again with more data. */ +bool dict_iterate_has_more(struct dict_iterate_context *ctx); bool dict_iterate(struct dict_iterate_context *ctx, const char **key_r, const char **value_r); /* Returns 0 = ok, -1 = iteration failed */ From dovecot at dovecot.org Wed Sep 2 14:37:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:37:48 +0000 Subject: dovecot-2.2: dict: Use the new async APIs for everything. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3de8de46f4a8 changeset: 19065:3de8de46f4a8 user: Timo Sirainen date: Wed Sep 02 17:36:47 2015 +0300 description: dict: Use the new async APIs for everything. If the dict backend supports async operations, this means that dict service can now be configured with client_count>1. diffstat: src/dict/dict-commands.c | 366 ++++++++++++++++++++++++++++---------------- src/dict/dict-commands.h | 3 + src/dict/dict-connection.c | 52 +++++- src/dict/dict-connection.h | 7 +- 4 files changed, 286 insertions(+), 142 deletions(-) diffs (truncated from 683 to 300 lines): diff -r d40d7f24ffcf -r 3de8de46f4a8 src/dict/dict-commands.c --- a/src/dict/dict-commands.c Wed Sep 02 17:34:43 2015 +0300 +++ b/src/dict/dict-commands.c Wed Sep 02 17:36:47 2015 +0300 @@ -14,87 +14,159 @@ #define DICT_OUTPUT_OPTIMAL_SIZE 1024 -struct dict_client_cmd { - int cmd; - int (*func)(struct dict_connection *conn, const char *line); +struct dict_cmd_func { + enum dict_protocol_cmd cmd; + int (*func)(struct dict_connection_cmd *cmd, const char *line); }; -static int cmd_lookup(struct dict_connection *conn, const char *line) +struct dict_connection_cmd { + const struct dict_cmd_func *cmd; + struct dict_connection *conn; + char *reply; + + struct dict_iterate_context *iter; + enum dict_iterate_flags iter_flags; + + struct dict_connection_transaction *trans; +}; + +static void dict_connection_cmd_output_more(struct dict_connection_cmd *cmd); + +static void dict_connection_cmd_free(struct dict_connection_cmd *cmd) { - const char *reply; - const char *value; - int ret; + if (cmd->iter != NULL) + (void)dict_iterate_deinit(&cmd->iter); + array_delete(&cmd->conn->cmds, 0, 1); + i_free(cmd->reply); - if (conn->iter_ctx != NULL) { - i_error("dict client: LOOKUP: Can't lookup while iterating"); - return -1; - } - - /* */ - ret = dict_lookup(conn->dict, pool_datastack_create(), line, &value); - if (ret > 0) { - reply = t_strdup_printf("%c%s\n", - DICT_PROTOCOL_REPLY_OK, str_tabescape(value)); - o_stream_nsend_str(conn->output, reply); - } else { - reply = t_strdup_printf("%c\n", ret == 0 ? - DICT_PROTOCOL_REPLY_NOTFOUND : - DICT_PROTOCOL_REPLY_FAIL); - o_stream_nsend_str(conn->output, reply); - } - return 0; + dict_connection_continue_input(cmd->conn); + i_free(cmd); } -static int cmd_iterate_flush(struct dict_connection *conn) +static void dict_connection_cmd_remove(struct dict_connection_cmd *cmd) +{ + struct dict_connection_cmd *const *cmds; + unsigned int i, count; + + cmds = array_get(&cmd->conn->cmds, &count); + for (i = 0; i < count; i++) { + if (cmds[i] == cmd) { + dict_connection_cmd_free(cmd); + return; + } + } + i_unreached(); +} + +static void dict_connection_cmds_flush(struct dict_connection *conn) +{ + struct dict_connection_cmd *cmd, *const *first_cmdp; + + while (array_count(&conn->cmds) > 0) { + first_cmdp = array_idx(&conn->cmds, 0); + cmd = *first_cmdp; + + if (cmd->reply == NULL) { + /* command not finished yet */ + break; + } + + o_stream_nsend_str(conn->output, cmd->reply); + dict_connection_cmd_remove(cmd); + } +} + +void dict_connection_cmds_free(struct dict_connection *conn) +{ + struct dict_connection_cmd *const *first_cmdp; + + while (array_count(&conn->cmds) > 0) { + first_cmdp = array_idx(&conn->cmds, 0); + dict_connection_cmd_remove(*first_cmdp); + } +} + +static void +cmd_lookup_callback(const struct dict_lookup_result *result, void *context) +{ + struct dict_connection_cmd *cmd = context; + + if (result->ret > 0) { + cmd->reply = i_strdup_printf("%c%s\n", + DICT_PROTOCOL_REPLY_OK, str_tabescape(result->value)); + } else if (result->ret == 0) { + cmd->reply = i_strdup_printf("%c\n", DICT_PROTOCOL_REPLY_NOTFOUND); + } else { + i_error("%s", result->error); + cmd->reply = i_strdup_printf("%c\n", DICT_PROTOCOL_REPLY_FAIL); + } + dict_connection_cmds_flush(cmd->conn); +} + +static int cmd_lookup(struct dict_connection_cmd *cmd, const char *line) +{ + /* */ + dict_lookup_async(cmd->conn->dict, line, cmd_lookup_callback, cmd); + return 1; +} + +static int cmd_iterate_flush(struct dict_connection_cmd *cmd) { string_t *str; const char *key, *value; str = t_str_new(256); - o_stream_cork(conn->output); - while (dict_iterate(conn->iter_ctx, &key, &value)) { + o_stream_cork(cmd->conn->output); + while (dict_iterate(cmd->iter, &key, &value)) { str_truncate(str, 0); str_append_c(str, DICT_PROTOCOL_REPLY_OK); str_append_tabescaped(str, key); str_append_c(str, '\t'); - if ((conn->iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + if ((cmd->iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) str_append_tabescaped(str, value); str_append_c(str, '\n'); - o_stream_nsend(conn->output, str_data(str), str_len(str)); + o_stream_nsend(cmd->conn->output, str_data(str), str_len(str)); - if (o_stream_get_buffer_used_size(conn->output) > + if (o_stream_get_buffer_used_size(cmd->conn->output) > DICT_OUTPUT_OPTIMAL_SIZE) { - if (o_stream_flush(conn->output) <= 0) { - /* continue later */ - o_stream_uncork(conn->output); + if (o_stream_flush(cmd->conn->output) <= 0) { + /* continue later when there's more space + in output buffer */ + o_stream_uncork(cmd->conn->output); + o_stream_set_flush_pending(cmd->conn->output, TRUE); return 0; } /* flushed everything, continue */ } } - - /* finished iterating */ - o_stream_unset_flush_callback(conn->output); + if (dict_iterate_has_more(cmd->iter)) { + /* wait for the next iteration callback */ + return 0; + } str_truncate(str, 0); - if (dict_iterate_deinit(&conn->iter_ctx) < 0) + if (dict_iterate_deinit(&cmd->iter) < 0) str_append_c(str, DICT_PROTOCOL_REPLY_FAIL); str_append_c(str, '\n'); - o_stream_nsend(conn->output, str_data(str), str_len(str)); - o_stream_uncork(conn->output); + o_stream_uncork(cmd->conn->output); + + cmd->reply = i_strdup(str_c(str)); + dict_connection_cmds_flush(cmd->conn); return 1; } -static int cmd_iterate(struct dict_connection *conn, const char *line) +static void cmd_iterate_callback(void *context) +{ + struct dict_connection_cmd *cmd = context; + + dict_connection_cmd_output_more(cmd); +} + +static int cmd_iterate(struct dict_connection_cmd *cmd, const char *line) { const char *const *args; unsigned int flags; - if (conn->iter_ctx != NULL) { - i_error("dict client: ITERATE: Already iterating"); - return -1; - } - args = t_strsplit_tabescaped(line); if (str_array_length(args) < 2 || str_to_uint(args[0], &flags) < 0) { @@ -103,12 +175,12 @@ } /* */ - conn->iter_ctx = dict_iterate_init_multiple(conn->dict, args+1, flags); - conn->iter_flags = flags; - - o_stream_set_flush_callback(conn->output, cmd_iterate_flush, conn); - (void)cmd_iterate_flush(conn); - return 0; + flags |= DICT_ITERATE_FLAG_ASYNC; + cmd->iter = dict_iterate_init_multiple(cmd->conn->dict, args+1, flags); + cmd->iter_flags = flags; + dict_iterate_set_async_callback(cmd->iter, cmd_iterate_callback, cmd); + dict_connection_cmd_output_more(cmd); + return 1; } static struct dict_connection_transaction * @@ -134,6 +206,8 @@ const struct dict_connection_transaction *transactions; unsigned int i, count; + i_assert(trans->ctx == NULL); + transactions = array_get(&conn->transactions, &count); for (i = 0; i < count; i++) { if (&transactions[i] == trans) { @@ -143,7 +217,7 @@ } } -static int cmd_begin(struct dict_connection *conn, const char *line) +static int cmd_begin(struct dict_connection_cmd *cmd, const char *line) { struct dict_connection_transaction *trans; unsigned int id; @@ -152,19 +226,19 @@ i_error("dict client: Invalid transaction ID %s", line); return -1; } - if (dict_connection_transaction_lookup(conn, id) != NULL) { + if (dict_connection_transaction_lookup(cmd->conn, id) != NULL) { i_error("dict client: Transaction ID %u already exists", id); return -1; } - if (!array_is_created(&conn->transactions)) - i_array_init(&conn->transactions, 4); + if (!array_is_created(&cmd->conn->transactions)) + i_array_init(&cmd->conn->transactions, 4); /* */ - trans = array_append_space(&conn->transactions); + trans = array_append_space(&cmd->conn->transactions); trans->id = id; - trans->conn = conn; - trans->ctx = dict_transaction_begin(conn->dict); + trans->conn = cmd->conn; + trans->ctx = dict_transaction_begin(cmd->conn->dict); return 0; } @@ -187,41 +261,9 @@ return 0; } -static int cmd_commit(struct dict_connection *conn, const char *line) +static void +cmd_commit_finish(struct dict_connection_cmd *cmd, int ret, bool async) { - struct dict_connection_transaction *trans; - char chr; - int ret; - - if (conn->iter_ctx != NULL) { - i_error("dict client: COMMIT: Can't commit while iterating"); - return -1; - } - - if (dict_connection_transaction_lookup_parse(conn, line, &trans) < 0) - return -1; - - ret = dict_transaction_commit(&trans->ctx); - switch (ret) { - case 1: - chr = DICT_PROTOCOL_REPLY_OK; - break; - case 0: - chr = DICT_PROTOCOL_REPLY_NOTFOUND; - break; From dovecot at dovecot.org Wed Sep 2 14:38:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:38:25 +0000 Subject: dovecot-2.2: dict-sql: Added support for async operations. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0d04ac4d43ca changeset: 19066:0d04ac4d43ca user: Timo Sirainen date: Wed Sep 02 17:37:16 2015 +0300 description: dict-sql: Added support for async operations. diffstat: src/lib-dict/dict-sql.c | 162 ++++++++++++++++++++++++++++++++++++++++------- 1 files changed, 136 insertions(+), 26 deletions(-) diffs (277 lines): diff -r 3de8de46f4a8 -r 0d04ac4d43ca src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 17:36:47 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 17:37:16 2015 +0300 @@ -63,6 +63,9 @@ pool_t inc_row_pool; struct sql_dict_inc_row *inc_row; + dict_transaction_commit_callback_t *async_callback; + void *async_context; + unsigned int failed:1; }; @@ -106,6 +109,12 @@ pool_unref(&dict->pool); } +static int sql_dict_wait(struct dict *dict ATTR_UNUSED) +{ + /* FIXME: lib-sql doesn't support this yet */ + return 0; +} + static bool dict_sql_map_match(const struct dict_sql_map *map, const char *path, ARRAY_TYPE(const_string) *values, unsigned int *pat_len_r, @@ -266,32 +275,44 @@ } } +static int sql_lookup_get_query(struct sql_dict *dict, const char *key, + string_t *query) +{ + const struct dict_sql_map *map; + ARRAY_TYPE(const_string) values; + + map = sql_dict_find_map(dict, key, &values); + if (map == NULL) { + i_error("sql dict lookup: Invalid/unmapped key: %s", key); + return -1; + } + str_printfa(query, "SELECT %s FROM %s", + map->value_field, map->table); + sql_dict_where_build(dict, map, &values, key[0], + SQL_DICT_RECURSE_NONE, query); + return 0; +} + static int sql_dict_lookup(struct dict *_dict, pool_t pool, const char *key, const char **value_r) { struct sql_dict *dict = (struct sql_dict *)_dict; - const struct dict_sql_map *map; - ARRAY_TYPE(const_string) values; - struct sql_result *result; + struct sql_result *result = NULL; int ret; - map = sql_dict_find_map(dict, key, &values); - if (map == NULL) { - i_error("sql dict lookup: Invalid/unmapped key: %s", key); - *value_r = NULL; - return 0; - } - T_BEGIN { string_t *query = t_str_new(256); - str_printfa(query, "SELECT %s FROM %s", - map->value_field, map->table); - sql_dict_where_build(dict, map, &values, key[0], - SQL_DICT_RECURSE_NONE, query); - result = sql_query_s(dict->db, str_c(query)); + ret = sql_lookup_get_query(dict, key, query); + if (ret == 0) + result = sql_query_s(dict->db, str_c(query)); } T_END; + if (ret < 0) { + *value_r = NULL; + return -1; + } + ret = sql_result_next_row(result); if (ret <= 0) { if (ret < 0) { @@ -308,6 +329,48 @@ return ret; } +struct sql_dict_lookup_context { + dict_lookup_callback_t *callback; + void *context; +}; + +static void +sql_dict_lookup_async_callback(struct sql_result *sql_result, + struct sql_dict_lookup_context *ctx) +{ + struct dict_lookup_result result; + + memset(&result, 0, sizeof(result)); + result.ret = sql_result_next_row(sql_result); + if (result.ret < 0) + result.error = sql_result_get_error(sql_result); + else if (result.ret > 0) + result.value = sql_result_get_field_value(sql_result, 0); + ctx->callback(&result, ctx->context); + + i_free(ctx); +} + +static void +sql_dict_lookup_async(struct dict *_dict, const char *key, + dict_lookup_callback_t *callback, void *context) +{ + struct sql_dict *dict = (struct sql_dict *)_dict; + struct sql_dict_lookup_context *ctx; + + T_BEGIN { + string_t *query = t_str_new(256); + + if (sql_lookup_get_query(dict, key, query) == 0) { + ctx = i_new(struct sql_dict_lookup_context, 1); + ctx->callback = callback; + ctx->context = context; + sql_query(dict->db, str_c(query), + sql_dict_lookup_async_callback, ctx); + } + } T_END; +} + static const struct dict_sql_map * sql_dict_iterate_find_next_map(struct sql_dict_iterate_context *ctx, ARRAY_TYPE(const_string) *values) @@ -356,8 +419,10 @@ if (map == NULL) return FALSE; - if (ctx->result != NULL) + if (ctx->result != NULL) { sql_result_unref(ctx->result); + ctx->result = NULL; + } str_append(query, "SELECT "); if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) @@ -402,6 +467,15 @@ return TRUE; } +static void sql_dict_iterate_callback(struct sql_result *result, + struct sql_dict_iterate_context *ctx) +{ + sql_result_ref(result); + ctx->result = result; + if (ctx->ctx.async_callback != NULL) + ctx->ctx.async_callback(ctx->ctx.async_context); +} + static bool sql_dict_iterate_next_query(struct sql_dict_iterate_context *ctx) { struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; @@ -413,14 +487,17 @@ ret = sql_dict_iterate_build_next_query(ctx, query); if (!ret) { /* failed */ + } else if ((ctx->flags & DICT_ITERATE_FLAG_ASYNC) == 0) { + ctx->result = sql_query_s(dict->db, str_c(query)); } else { - ctx->result = sql_query_s(dict->db, str_c(query)); + i_assert(ctx->result == NULL); + sql_query(dict->db, str_c(query), + sql_dict_iterate_callback, ctx); } } T_END; return ret; } - static struct dict_iterate_context * sql_dict_iterate_init(struct dict *_dict, const char *const *paths, enum dict_iterate_flags flags) @@ -445,6 +522,7 @@ i_error("sql dict iterate: Invalid/unmapped path: %s", paths[0]); ctx->result = NULL; + ctx->failed = TRUE; return &ctx->ctx; } return &ctx->ctx; @@ -459,12 +537,21 @@ unsigned int i, count; int ret; - if (ctx->result == NULL) { - ctx->failed = TRUE; + _ctx->has_more = FALSE; + if (ctx->failed) return FALSE; - } - while ((ret = sql_result_next_row(ctx->result)) == 0) { + for (;;) { + if (ctx->result == NULL) { + /* wait for async lookup to finish */ + i_assert((ctx->flags & DICT_ITERATE_FLAG_ASYNC) != 0); + _ctx->has_more = TRUE; + return FALSE; + } + + ret = sql_result_next_row(ctx->result); + if (ret != 0) + break; /* see if there are more results in the next map. don't do it if we're looking for an exact match, since we already should have handled it. */ @@ -550,9 +637,26 @@ return FALSE; } +static void +sql_dict_transaction_commit_callback(const char *error, + struct sql_dict_transaction_context *ctx) +{ + int ret; + + if (error == NULL) + ret = sql_dict_transaction_has_nonexistent(ctx) ? 0 : 1; + else { + i_error("sql dict: commit failed: %s", error); + ret = -1; + } + + if (ctx->async_callback != NULL) + ctx->async_callback(ret, ctx->async_context); + sql_dict_transaction_free(ctx); +} + static int -sql_dict_transaction_commit(struct dict_transaction_context *_ctx, - bool async ATTR_UNUSED, +sql_dict_transaction_commit(struct dict_transaction_context *_ctx, bool async, dict_transaction_commit_callback_t *callback, void *context) { @@ -570,6 +674,12 @@ } else if (!_ctx->changed) { /* nothing changed, no need to commit */ sql_transaction_rollback(&ctx->sql_ctx); + } else if (async) { + ctx->async_callback = callback; + ctx->async_context = context; + sql_transaction_commit(&ctx->sql_ctx, + sql_dict_transaction_commit_callback, ctx); + return 1; } else { if (sql_transaction_commit_s(&ctx->sql_ctx, &error) < 0) { i_error("sql dict: commit failed: %s", error); @@ -935,7 +1045,7 @@ { sql_dict_init, sql_dict_deinit, - NULL, + sql_dict_wait, sql_dict_lookup, sql_dict_iterate_init, sql_dict_iterate, @@ -947,7 +1057,7 @@ sql_dict_unset, sql_dict_append, sql_dict_atomic_inc, - NULL + sql_dict_lookup_async } }; From dovecot at dovecot.org Wed Sep 2 14:52:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 14:52:29 +0000 Subject: dovecot-2.2: dict: Added dict-async service. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7378ab8e3b4d changeset: 19067:7378ab8e3b4d user: Timo Sirainen date: Wed Sep 02 17:51:23 2015 +0300 description: dict: Added dict-async service. This allows running separate dict processes with separate settings for async and non-async backends. diffstat: src/dict/dict-settings.c | 36 ++++++++++++++++++++++++++++++++++++ 1 files changed, 36 insertions(+), 0 deletions(-) diffs (53 lines): diff -r 0d04ac4d43ca -r 7378ab8e3b4d src/dict/dict-settings.c --- a/src/dict/dict-settings.c Wed Sep 02 17:37:16 2015 +0300 +++ b/src/dict/dict-settings.c Wed Sep 02 17:51:23 2015 +0300 @@ -16,6 +16,16 @@ static buffer_t dict_unix_listeners_buf = { dict_unix_listeners, sizeof(dict_unix_listeners), { NULL, } }; + +static struct file_listener_settings dict_async_unix_listeners_array[] = { + { "dict-async", 0600, "", "" } +}; +static struct file_listener_settings *dict_async_unix_listeners[] = { + &dict_async_unix_listeners_array[0] +}; +static buffer_t dict_async_unix_listeners_buf = { + dict_async_unix_listeners, sizeof(dict_async_unix_listeners), { NULL, } +}; /* */ struct service_settings dict_service_settings = { @@ -44,6 +54,32 @@ .inet_listeners = ARRAY_INIT }; +struct service_settings dict_async_service_settings = { + .name = "dict-async", + .protocol = "", + .type = "", + .executable = "dict", + .user = "$default_internal_user", + .group = "", + .privileged_group = "", + .extra_groups = "", + .chroot = "", + + .drop_priv_before_exec = FALSE, + + .process_min_avail = 0, + .process_limit = 0, + .client_limit = 0, + .service_count = 0, + .idle_kill = 0, + .vsz_limit = (uoff_t)-1, + + .unix_listeners = { { &dict_async_unix_listeners_buf, + sizeof(dict_async_unix_listeners[0]) } }, + .fifo_listeners = ARRAY_INIT, + .inet_listeners = ARRAY_INIT +}; + #undef DEF #define DEF(type, name) \ { type, #name, offsetof(struct dict_server_settings, name), NULL } From dovecot at dovecot.org Wed Sep 2 15:43:02 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 15:43:02 +0000 Subject: dovecot-2.2: dict-client: Prefix relative socket paths with base... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/54c4e1ed387a changeset: 19068:54c4e1ed387a user: Timo Sirainen date: Wed Sep 02 18:42:01 2015 +0300 description: dict-client: Prefix relative socket paths with base_dir. diffstat: src/lib-dict/dict-client.c | 13 +++++++++---- 1 files changed, 9 insertions(+), 4 deletions(-) diffs (25 lines): diff -r 7378ab8e3b4d -r 54c4e1ed387a src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Wed Sep 02 17:51:23 2015 +0300 +++ b/src/lib-dict/dict-client.c Wed Sep 02 18:42:01 2015 +0300 @@ -502,12 +502,17 @@ dict->fd = -1; - if (*uri != ':') { - /* path given */ + if (uri[0] == ':') { + /* default path */ + dict->path = p_strconcat(pool, set->base_dir, + "/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL); + } else if (uri[0] == '/') { + /* absolute path */ dict->path = p_strdup_until(pool, uri, dest_uri); } else { - dict->path = p_strconcat(pool, set->base_dir, - "/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL); + /* relative path to base_dir */ + dict->path = p_strconcat(pool, set->base_dir, "/", + p_strdup_until(pool, uri, dest_uri), NULL); } dict->uri = p_strdup(pool, dest_uri + 1); *dict_r = &dict->dict; From dovecot at dovecot.org Wed Sep 2 16:39:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 16:39:34 +0000 Subject: dovecot-2.2: dict-sql: Fixed iteration with DICT_ITERATE_FLAG_NO... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ccff6d5dd1b changeset: 19069:7ccff6d5dd1b user: Timo Sirainen date: Wed Sep 02 19:38:35 2015 +0300 description: dict-sql: Fixed iteration with DICT_ITERATE_FLAG_NO_VALUE diffstat: src/lib-dict/dict-sql.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 54c4e1ed387a -r 7ccff6d5dd1b src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 18:42:01 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 19:38:35 2015 +0300 @@ -573,7 +573,7 @@ str_append_c(ctx->key, '/'); count = sql_result_get_fields_count(ctx->result); - i = 1; + i = (ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0 ? 0 : 1; for (p = ctx->map->pattern + ctx->pattern_prefix_len; *p != '\0'; p++) { if (*p != '$') str_append_c(ctx->key, *p); @@ -587,7 +587,10 @@ } *key_r = str_c(ctx->key); - *value_r = sql_result_get_field_value(ctx->result, 0); + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0) + *value_r = ""; + else + *value_r = sql_result_get_field_value(ctx->result, 0); return TRUE; } From dovecot at dovecot.org Wed Sep 2 16:43:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 16:43:10 +0000 Subject: dovecot-2.2: doveadm dict iter: Added -V parameter to return onl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d6d3bd868bd4 changeset: 19070:d6d3bd868bd4 user: Timo Sirainen date: Wed Sep 02 19:41:23 2015 +0300 description: doveadm dict iter: Added -V parameter to return only keys. diffstat: src/doveadm/doveadm-dict.c | 14 ++++++++++---- 1 files changed, 10 insertions(+), 4 deletions(-) diffs (49 lines): diff -r 7ccff6d5dd1b -r d6d3bd868bd4 src/doveadm/doveadm-dict.c --- a/src/doveadm/doveadm-dict.c Wed Sep 02 19:38:35 2015 +0300 +++ b/src/doveadm/doveadm-dict.c Wed Sep 02 19:41:23 2015 +0300 @@ -14,7 +14,7 @@ cmd_dict_init_full(int *argc, char **argv[], int own_arg_count, int key_arg_idx, doveadm_command_t *cmd, enum dict_iterate_flags *iter_flags) { - const char *getopt_args = iter_flags == NULL ? "u:" : "1Ru:"; + const char *getopt_args = iter_flags == NULL ? "u:" : "1Ru:V"; struct dict *dict; const char *error, *username = ""; int c; @@ -29,6 +29,10 @@ i_assert(iter_flags != NULL); *iter_flags |= DICT_ITERATE_FLAG_RECURSE; break; + case 'V': + i_assert(iter_flags != NULL); + *iter_flags |= DICT_ITERATE_FLAG_NO_VALUE; + break; case 'u': username = optarg; break; @@ -163,12 +167,14 @@ doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); doveadm_print_header_simple("key"); - doveadm_print_header_simple("value"); + if ((iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + doveadm_print_header_simple("value"); iter = dict_iterate_init(dict, argv[0], iter_flags); while (dict_iterate(iter, &key, &value)) { doveadm_print(key); - doveadm_print(value); + if ((iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + doveadm_print(value); } if (dict_iterate_deinit(&iter) < 0) { i_error("dict_iterate_deinit(%s) failed", argv[0]); @@ -182,7 +188,7 @@ { cmd_dict_set, "dict set", "[-u ] " }, { cmd_dict_unset, "dict unset", "[-u ] " }, { cmd_dict_inc, "dict inc", "[-u ] " }, - { cmd_dict_iter, "dict iter", "[-u ] [-R] " } + { cmd_dict_iter, "dict iter", "[-u ] [-1RV] " } }; static void dict_cmd_help(doveadm_command_t *cmd) From dovecot at dovecot.org Wed Sep 2 16:43:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 16:43:48 +0000 Subject: dovecot-2.2: cassandra: Implemented support for binary values. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5119a85795f7 changeset: 19071:5119a85795f7 user: Timo Sirainen date: Wed Sep 02 19:42:45 2015 +0300 description: cassandra: Implemented support for binary values. diffstat: src/lib-sql/driver-cassandra.c | 28 ++++++++++++++++++++++------ 1 files changed, 22 insertions(+), 6 deletions(-) diffs (86 lines): diff -r d6d3bd868bd4 -r 5119a85795f7 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Wed Sep 02 19:41:23 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Wed Sep 02 19:42:45 2015 +0300 @@ -57,6 +57,7 @@ pool_t row_pool; ARRAY_TYPE(const_string) fields; + ARRAY(size_t) field_sizes; sql_query_callback_t *callback; void *context; @@ -640,9 +641,11 @@ static int driver_cassandra_get_value(struct cassandra_result *result, - const CassValue *value, const char **str_r) + const CassValue *value, const char **str_r, + size_t *len_r) { - const char *output; + const unsigned char *output; + void *output_dup; size_t output_size; CassError rc; @@ -651,13 +654,16 @@ return 0; } - rc = cass_value_get_string(value, &output, &output_size); + rc = cass_value_get_bytes(value, &output, &output_size); if (rc != CASS_OK) { i_free(result->error); result->error = i_strdup_printf("Couldn't get value as string (code=%d)", rc); return -1; } - *str_r = p_strndup(result->row_pool, output, output_size); + output_dup = p_malloc(result->row_pool, output_size + 1); + memcpy(output_dup, output, output_size); + *str_r = output_dup; + *len_r = output_size; return 0; } @@ -667,6 +673,7 @@ const CassRow *row; const CassValue *value; const char *str; + size_t size; unsigned int i; int ret = 1; @@ -678,14 +685,16 @@ p_clear(result->row_pool); p_array_init(&result->fields, result->row_pool, 8); + p_array_init(&result->field_sizes, result->row_pool, 8); row = cass_iterator_get_row(result->iterator); for (i = 0; (value = cass_row_get_column(row, i)) != NULL; i++) { - if (driver_cassandra_get_value(result, value, &str) < 0) { + if (driver_cassandra_get_value(result, value, &str, &size) < 0) { ret = -1; break; } array_append(&result->fields, &str, 1); + array_append(&result->field_sizes, &size, 1); } return ret; } @@ -728,7 +737,14 @@ unsigned int idx ATTR_UNUSED, size_t *size_r ATTR_UNUSED) { - i_unreached(); + struct cassandra_result *result = (struct cassandra_result *)_result; + const char *const *strp; + const size_t *sizep; + + strp = array_idx(&result->fields, idx); + sizep = array_idx(&result->field_sizes, idx); + *size_r = *sizep; + return (const void *)*strp; } static const char * From dovecot at dovecot.org Wed Sep 2 16:47:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 16:47:43 +0000 Subject: dovecot-2.2: dict-sql: Implemented support for binary fields. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0453c466c601 changeset: 19073:0453c466c601 user: Timo Sirainen date: Wed Sep 02 19:46:36 2015 +0300 description: dict-sql: Implemented support for binary fields. Example: map { pattern = shared/blobtest/$key table = blobtest value_field = value value_hexblob = yes fields { key = ${hexblob:key} } } Now you can access both field and value as hex data. For example: doveadm dict set proxy::sqldict shared/blobtest/746573746b6579 7465737476616c7565 This adds "testkey" and "testvalue" to key and value fields blobs. diffstat: src/lib-dict/dict-sql-settings.c | 18 +++- src/lib-dict/dict-sql-settings.h | 8 +- src/lib-dict/dict-sql.c | 161 ++++++++++++++++++++++++++++++-------- 3 files changed, 146 insertions(+), 41 deletions(-) diffs (truncated from 427 to 300 lines): diff -r 113f4e32cb19 -r 0453c466c601 src/lib-dict/dict-sql-settings.c --- a/src/lib-dict/dict-sql-settings.c Wed Sep 02 19:43:26 2015 +0300 +++ b/src/lib-dict/dict-sql-settings.c Wed Sep 02 19:46:36 2015 +0300 @@ -15,7 +15,8 @@ }; struct dict_sql_map_field { - const char *sql_field, *variable; + struct dict_sql_field sql_field; + const char *variable; }; struct setting_parser_ctx { @@ -28,12 +29,14 @@ }; #define DEF_STR(name) DEF_STRUCT_STR(name, dict_sql_map) +#define DEF_BOOL(name) DEF_STRUCT_BOOL(name, dict_sql_map) static const struct setting_def dict_sql_map_setting_defs[] = { DEF_STR(pattern), DEF_STR(table), DEF_STR(username_field), DEF_STR(value_field), + DEF_BOOL(value_hexblob), { 0, NULL, 0 } }; @@ -148,6 +151,7 @@ struct setting_parser_ctx *ctx) { struct dict_sql_map_field *field; + unsigned int value_len; switch (ctx->type) { case SECTION_ROOT: @@ -166,8 +170,16 @@ key, NULL); } field = array_append_space(&ctx->cur_fields); - field->sql_field = p_strdup(ctx->pool, key); - field->variable = p_strdup(ctx->pool, value + 1); + field->sql_field.name = p_strdup(ctx->pool, key); + value_len = strlen(value); + if (strncmp(value, "${hexblob:", 10) == 0 && + value[value_len-1] == '}') { + field->variable = p_strndup(ctx->pool, value + 10, + value_len-10-1); + field->sql_field.value_is_hexblob = TRUE; + } else { + field->variable = p_strdup(ctx->pool, value + 1); + } return NULL; } return t_strconcat("Unknown setting: ", key, NULL); diff -r 113f4e32cb19 -r 0453c466c601 src/lib-dict/dict-sql-settings.h --- a/src/lib-dict/dict-sql-settings.h Wed Sep 02 19:43:26 2015 +0300 +++ b/src/lib-dict/dict-sql-settings.h Wed Sep 02 19:46:36 2015 +0300 @@ -1,6 +1,11 @@ #ifndef DICT_SQL_SETTINGS_H #define DICT_SQL_SETTINGS_H +struct dict_sql_field { + const char *name; + bool value_is_hexblob; +}; + struct dict_sql_map { /* pattern is in simplified form: all variables are stored as simple '$' character. fields array is sorted by the variable index. */ @@ -8,8 +13,9 @@ const char *table; const char *username_field; const char *value_field; + bool value_hexblob; - ARRAY_TYPE(const_string) sql_fields; + ARRAY(struct dict_sql_field) sql_fields; }; struct dict_sql_settings { diff -r 113f4e32cb19 -r 0453c466c601 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 19:43:26 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 19:46:36 2015 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "istream.h" +#include "hex-binary.h" #include "str.h" #include "sql-api-private.h" #include "sql-db-cache.h" @@ -211,12 +212,46 @@ } static void +sql_dict_value_escape(string_t *str, struct sql_dict *dict, + bool value_is_hexblob, const char *field_name, + const char *value, const char *value_suffix) +{ + buffer_t *buf; + + if (!value_is_hexblob) { + str_printfa(str, "'%s%s'", sql_escape_string(dict->db, value), + value_suffix); + return; + } + + buf = buffer_create_dynamic(pool_datastack_create(), strlen(value)/2); + if (hex_to_binary(value, buf) < 0) { + /* we shouldn't get untrusted input here. it's also a bit + annoying to handle this error. */ + i_fatal("dict-sql: field %s value isn't hexblob: %s", + field_name, value); + } + str_append(buf, value_suffix); + str_append(str, sql_escape_blob(dict->db, buf->data, buf->used)); +} + +static void +sql_dict_field_escape_value(string_t *str, struct sql_dict *dict, + const struct dict_sql_field *field, + const char *value, const char *value_suffix) +{ + sql_dict_value_escape(str, dict, field->value_is_hexblob, + field->name, value, value_suffix); +} + +static void sql_dict_where_build(struct sql_dict *dict, const struct dict_sql_map *map, const ARRAY_TYPE(const_string) *values_arr, char key1, enum sql_recurse_type recurse_type, string_t *query) { - const char *const *sql_fields, *const *values; + const struct dict_sql_field *sql_fields; + const char *const *values; unsigned int i, count, count2, exact_count; bool priv = key1 == DICT_PATH_PRIVATE[0]; @@ -236,8 +271,8 @@ for (i = 0; i < exact_count; i++) { if (i > 0) str_append(query, " AND"); - str_printfa(query, " %s = '%s'", sql_fields[i], - sql_escape_string(dict->db, values[i])); + str_printfa(query, " %s = ", sql_fields[i].name); + sql_dict_field_escape_value(query, dict, &sql_fields[i], values[i], ""); } switch (recurse_type) { case SQL_DICT_RECURSE_NONE: @@ -246,24 +281,26 @@ if (i > 0) str_append(query, " AND"); if (i < count2) { - str_printfa(query, " %s LIKE '%s/%%' AND " - "%s NOT LIKE '%s/%%/%%'", - sql_fields[i], - sql_escape_string(dict->db, values[i]), - sql_fields[i], - sql_escape_string(dict->db, values[i])); + str_printfa(query, " %s LIKE ", sql_fields[i].name); + sql_dict_field_escape_value(query, dict, &sql_fields[i], + values[i], "/%"); + str_printfa(query, " AND %s NOT LIKE ", sql_fields[i].name); + sql_dict_field_escape_value(query, dict, &sql_fields[i], + values[i], "/%/%"); } else { str_printfa(query, " %s LIKE '%%' AND " "%s NOT LIKE '%%/%%'", - sql_fields[i], sql_fields[i]); + sql_fields[i].name, sql_fields[i].name); } break; case SQL_DICT_RECURSE_FULL: if (i < count2) { if (i > 0) str_append(query, " AND"); - str_printfa(query, " %s LIKE '%s/%%'", sql_fields[i], - sql_escape_string(dict->db, values[i])); + str_printfa(query, " %s LIKE ", + sql_fields[i].name); + sql_dict_field_escape_value(query, dict, &sql_fields[i], + values[i], "/%"); } break; } @@ -275,13 +312,14 @@ } } -static int sql_lookup_get_query(struct sql_dict *dict, const char *key, - string_t *query) +static int +sql_lookup_get_query(struct sql_dict *dict, const char *key, + string_t *query, const struct dict_sql_map **map_r) { const struct dict_sql_map *map; ARRAY_TYPE(const_string) values; - map = sql_dict_find_map(dict, key, &values); + map = *map_r = sql_dict_find_map(dict, key, &values); if (map == NULL) { i_error("sql dict lookup: Invalid/unmapped key: %s", key); return -1; @@ -293,17 +331,54 @@ return 0; } +static const char * +sql_dict_result_unescape(bool hexblob, pool_t pool, struct sql_result *result, + unsigned int result_idx) +{ + const unsigned char *data; + size_t size; + string_t *str; + + if (!hexblob) + return p_strdup(pool, sql_result_get_field_value(result, result_idx)); + + data = sql_result_get_field_value_binary(result, result_idx, &size); + str = str_new(pool, size*2 + 1); + binary_to_hex_append(str, data, size); + return str_c(str); +} + +static const char * +sql_dict_result_unescape_value(const struct dict_sql_map *map, pool_t pool, + struct sql_result *result) +{ + return sql_dict_result_unescape(map->value_hexblob, pool, result, 0); +} + +static const char * +sql_dict_result_unescape_field(const struct dict_sql_map *map, pool_t pool, + struct sql_result *result, unsigned int result_idx, + unsigned int field_idx) +{ + const struct dict_sql_field *sql_field; + + sql_field = array_idx(&map->sql_fields, field_idx); + return sql_dict_result_unescape(sql_field->value_is_hexblob, pool, + result, result_idx); +} + static int sql_dict_lookup(struct dict *_dict, pool_t pool, const char *key, const char **value_r) { struct sql_dict *dict = (struct sql_dict *)_dict; + const struct dict_sql_map *map; struct sql_result *result = NULL; int ret; T_BEGIN { string_t *query = t_str_new(256); - ret = sql_lookup_get_query(dict, key, query); + ret = sql_lookup_get_query(dict, key, query, &map); if (ret == 0) result = sql_query_s(dict->db, str_c(query)); } T_END; @@ -321,8 +396,7 @@ } *value_r = NULL; } else { - *value_r = - p_strdup(pool, sql_result_get_field_value(result, 0)); + *value_r = sql_dict_result_unescape_value(map, pool, result); } sql_result_unref(result); @@ -330,6 +404,7 @@ } struct sql_dict_lookup_context { + const struct dict_sql_map *map; dict_lookup_callback_t *callback; void *context; }; @@ -344,8 +419,10 @@ result.ret = sql_result_next_row(sql_result); if (result.ret < 0) result.error = sql_result_get_error(sql_result); - else if (result.ret > 0) - result.value = sql_result_get_field_value(sql_result, 0); + else if (result.ret > 0) { + result.value = sql_dict_result_unescape_value(ctx->map, + pool_datastack_create(), sql_result); + } ctx->callback(&result, ctx->context); i_free(ctx); @@ -356,15 +433,17 @@ dict_lookup_callback_t *callback, void *context) { struct sql_dict *dict = (struct sql_dict *)_dict; + const struct dict_sql_map *map; struct sql_dict_lookup_context *ctx; T_BEGIN { From dovecot at dovecot.org Wed Sep 2 16:47:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 16:47:42 +0000 Subject: dovecot-2.2: lib-sql: Implemented sql_escape_blob() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/113f4e32cb19 changeset: 19072:113f4e32cb19 user: Timo Sirainen date: Wed Sep 02 19:43:26 2015 +0300 description: lib-sql: Implemented sql_escape_blob() diffstat: src/lib-sql/driver-cassandra.c | 16 +++++++++++++++- src/lib-sql/driver-mysql.c | 17 ++++++++++++++++- src/lib-sql/driver-pgsql.c | 18 +++++++++++++++++- src/lib-sql/driver-sqlite.c | 17 ++++++++++++++++- src/lib-sql/driver-sqlpool.c | 23 ++++++++++++++++++++++- src/lib-sql/sql-api-private.h | 2 ++ src/lib-sql/sql-api.c | 6 ++++++ src/lib-sql/sql-api.h | 3 +++ 8 files changed, 97 insertions(+), 5 deletions(-) diffs (244 lines): diff -r 5119a85795f7 -r 113f4e32cb19 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Wed Sep 02 19:42:45 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Wed Sep 02 19:43:26 2015 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "hex-binary.h" #include "str.h" #include "ioloop.h" #include "write-full.h" @@ -957,6 +958,17 @@ sql_transaction_add_query(_ctx, ctx->query_pool, query, affected_rows); } +static const char * +driver_cassandra_escape_blob(struct sql_db *_db ATTR_UNUSED, + const unsigned char *data, size_t size) +{ + string_t *str = t_str_new(128); + + str_append(str, "0x"); + binary_to_hex_append(str, data, size); + return str_c(str); +} + const struct sql_db driver_cassandra_db = { .name = "cassandra", .flags = SQL_DB_FLAG_POOLED, @@ -976,7 +988,9 @@ driver_cassandra_transaction_commit_s, driver_cassandra_transaction_rollback, - driver_cassandra_update + driver_cassandra_update, + + driver_cassandra_escape_blob } }; diff -r 5119a85795f7 -r 113f4e32cb19 src/lib-sql/driver-mysql.c --- a/src/lib-sql/driver-mysql.c Wed Sep 02 19:42:45 2015 +0300 +++ b/src/lib-sql/driver-mysql.c Wed Sep 02 19:43:26 2015 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "ioloop.h" #include "array.h" +#include "hex-binary.h" #include "str.h" #include "net.h" #include "sql-api-private.h" @@ -619,6 +620,18 @@ query, affected_rows); } +static const char * +driver_mysql_escape_blob(struct sql_db *_db ATTR_UNUSED, + const unsigned char *data, size_t size) +{ + string_t *str = t_str_new(128); + + str_append(str, "HEX('"); + binary_to_hex_append(str, data, size); + str_append_c(str, ')'); + return str_c(str); +} + const struct sql_db driver_mysql_db = { .name = "mysql", .flags = SQL_DB_FLAG_BLOCKING | SQL_DB_FLAG_POOLED, @@ -638,7 +651,9 @@ driver_mysql_transaction_commit_s, driver_mysql_transaction_rollback, - driver_mysql_update + driver_mysql_update, + + driver_mysql_escape_blob } }; diff -r 5119a85795f7 -r 113f4e32cb19 src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Wed Sep 02 19:42:45 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Wed Sep 02 19:43:26 2015 +0300 @@ -3,6 +3,8 @@ #include "lib.h" #include "array.h" #include "ioloop.h" +#include "hex-binary.h" +#include "str.h" #include "time-util.h" #include "sql-api-private.h" @@ -1066,6 +1068,18 @@ sql_transaction_add_query(_ctx, ctx->query_pool, query, affected_rows); } +static const char * +driver_pgsql_escape_blob(struct sql_db *_db ATTR_UNUSED, + const unsigned char *data, size_t size) +{ + string_t *str = t_str_new(128); + + str_append(str, "E'\\x"); + binary_to_hex_append(str, data, size); + str_append_c(str, '\''); + return str_c(str); +} + const struct sql_db driver_pgsql_db = { .name = "pgsql", .flags = SQL_DB_FLAG_POOLED, @@ -1085,7 +1099,9 @@ driver_pgsql_transaction_commit_s, driver_pgsql_transaction_rollback, - driver_pgsql_update + driver_pgsql_update, + + driver_pgsql_escape_blob } }; diff -r 5119a85795f7 -r 113f4e32cb19 src/lib-sql/driver-sqlite.c --- a/src/lib-sql/driver-sqlite.c Wed Sep 02 19:42:45 2015 +0300 +++ b/src/lib-sql/driver-sqlite.c Wed Sep 02 19:43:26 2015 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "str.h" +#include "hex-binary.h" #include "sql-api-private.h" #ifdef BUILD_SQLITE @@ -392,6 +393,18 @@ *affected_rows = sqlite3_changes(db->sqlite); } +static const char * +driver_sqlite_escape_blob(struct sql_db *_db ATTR_UNUSED, + const unsigned char *data, size_t size) +{ + string_t *str = t_str_new(128); + + str_append(str, "x'"); + binary_to_hex_append(str, data, size); + str_append_c(str, '\''); + return str_c(str); +} + const struct sql_db driver_sqlite_db = { .name = "sqlite", .flags = SQL_DB_FLAG_BLOCKING, @@ -410,7 +423,9 @@ driver_sqlite_transaction_commit, driver_sqlite_transaction_commit_s, driver_sqlite_transaction_rollback, - driver_sqlite_update + driver_sqlite_update, + + driver_sqlite_escape_blob } }; diff -r 5119a85795f7 -r 113f4e32cb19 src/lib-sql/driver-sqlpool.c --- a/src/lib-sql/driver-sqlpool.c Wed Sep 02 19:42:45 2015 +0300 +++ b/src/lib-sql/driver-sqlpool.c Wed Sep 02 19:43:26 2015 +0300 @@ -790,6 +790,25 @@ query, affected_rows); } +static const char * +driver_sqlpool_escape_blob(struct sql_db *_db, + const unsigned char *data, size_t size) +{ + struct sqlpool_db *db = (struct sqlpool_db *)_db; + const struct sqlpool_connection *conns; + unsigned int i, count; + + /* use the first ready connection */ + conns = array_get(&db->all_connections, &count); + for (i = 0; i < count; i++) { + if (SQL_DB_IS_READY(conns[i].db)) + return sql_escape_blob(conns[i].db, data, size); + } + /* no ready connections. just use the first one (we're guaranteed + to always have one) */ + return sql_escape_blob(conns[0].db, data, size); +} + struct sql_db driver_sqlpool_db = { "", @@ -808,6 +827,8 @@ driver_sqlpool_transaction_commit_s, driver_sqlpool_transaction_rollback, - driver_sqlpool_update + driver_sqlpool_update, + + driver_sqlpool_escape_blob } }; diff -r 5119a85795f7 -r 113f4e32cb19 src/lib-sql/sql-api-private.h --- a/src/lib-sql/sql-api-private.h Wed Sep 02 19:42:45 2015 +0300 +++ b/src/lib-sql/sql-api-private.h Wed Sep 02 19:43:26 2015 +0300 @@ -76,6 +76,8 @@ void (*update)(struct sql_transaction_context *ctx, const char *query, unsigned int *affected_rows); + const char *(*escape_blob)(struct sql_db *db, + const unsigned char *data, size_t size); }; struct sql_db { diff -r 5119a85795f7 -r 113f4e32cb19 src/lib-sql/sql-api.c --- a/src/lib-sql/sql-api.c Wed Sep 02 19:42:45 2015 +0300 +++ b/src/lib-sql/sql-api.c Wed Sep 02 19:43:26 2015 +0300 @@ -126,6 +126,12 @@ return db->v.escape_string(db, string); } +const char *sql_escape_blob(struct sql_db *db, + const unsigned char *data, size_t size) +{ + return db->v.escape_blob(db, data, size); +} + void sql_exec(struct sql_db *db, const char *query) { db->v.exec(db, query); diff -r 5119a85795f7 -r 113f4e32cb19 src/lib-sql/sql-api.h --- a/src/lib-sql/sql-api.h Wed Sep 02 19:42:45 2015 +0300 +++ b/src/lib-sql/sql-api.h Wed Sep 02 19:43:26 2015 +0300 @@ -70,6 +70,9 @@ /* Escape the given string if needed and return it. */ const char *sql_escape_string(struct sql_db *db, const char *string); +/* Escape the given data as a string. */ +const char *sql_escape_blob(struct sql_db *db, + const unsigned char *data, size_t size); /* Execute SQL query without waiting for results. */ void sql_exec(struct sql_db *db, const char *query); From dovecot at dovecot.org Wed Sep 2 17:31:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 17:31:05 +0000 Subject: dovecot-2.2: dict-sql: Fixed iteration with blob fields. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/da570231062f changeset: 19074:da570231062f user: Timo Sirainen date: Wed Sep 02 20:30:07 2015 +0300 description: dict-sql: Fixed iteration with blob fields. diffstat: src/lib-dict/dict-sql.c | 15 ++++++++------- 1 files changed, 8 insertions(+), 7 deletions(-) diffs (64 lines): diff -r 0453c466c601 -r da570231062f src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 02 19:46:36 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 02 20:30:07 2015 +0300 @@ -44,7 +44,7 @@ string_t *key; const struct dict_sql_map *map; unsigned int key_prefix_len, pattern_prefix_len, next_map_idx; - unsigned int path_idx; + unsigned int path_idx, sql_fields_start_idx; bool failed; }; @@ -358,11 +358,11 @@ static const char * sql_dict_result_unescape_field(const struct dict_sql_map *map, pool_t pool, struct sql_result *result, unsigned int result_idx, - unsigned int field_idx) + unsigned int sql_field_idx) { const struct dict_sql_field *sql_field; - sql_field = array_idx(&map->sql_fields, field_idx); + sql_field = array_idx(&map->sql_fields, sql_field_idx); return sql_dict_result_unescape(sql_field->value_is_hexblob, pool, result, result_idx); } @@ -516,6 +516,7 @@ i_assert(i > 0); i--; } + ctx->sql_fields_start_idx = i; for (; i < count; i++) str_printfa(query, "%s,", sql_fields[i].name); str_truncate(query, str_len(query)-1); @@ -613,7 +614,7 @@ struct sql_dict_iterate_context *ctx = (struct sql_dict_iterate_context *)_ctx; const char *p, *value; - unsigned int i, field_i, count; + unsigned int i, sql_field_i, count; int ret; _ctx->has_more = FALSE; @@ -653,17 +654,17 @@ count = sql_result_get_fields_count(ctx->result); i = (ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0 ? 0 : 1; - field_i = 0; + sql_field_i = ctx->sql_fields_start_idx; for (p = ctx->map->pattern + ctx->pattern_prefix_len; *p != '\0'; p++) { if (*p != '$') str_append_c(ctx->key, *p); else { i_assert(i < count); value = sql_dict_result_unescape_field(ctx->map, - pool_datastack_create(), ctx->result, i, field_i); + pool_datastack_create(), ctx->result, i, sql_field_i); if (value != NULL) str_append(ctx->key, value); - i++; field_i++; + i++; sql_field_i++; } } From dovecot at dovecot.org Wed Sep 2 20:30:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Sep 2015 20:30:27 +0000 Subject: dovecot-2.2: man: doveadm-sync.1: Added description for option `... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f313cda93533 changeset: 19075:f313cda93533 user: Pascal Volk date: Mon Aug 31 22:31:19 2015 +0000 description: man: doveadm-sync.1: Added description for option `-T secs'. diffstat: doc/man/doveadm-sync.1.in | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diffs (40 lines): diff -r da570231062f -r f313cda93533 doc/man/doveadm-sync.1.in --- a/doc/man/doveadm-sync.1.in Wed Sep 02 20:30:07 2015 +0300 +++ b/doc/man/doveadm-sync.1.in Mon Aug 31 22:31:19 2015 +0000 @@ -1,5 +1,5 @@ .\" Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-SYNC 1 "2015-05-09" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-SYNC 1 "2015-08-31" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-sync \- Dovecot\(aqs two\-way mailbox synchronization utility .br @@ -17,6 +17,7 @@ [\fB\-n\fP \fInamespace\fP|\fB\-N\fP] [\fB\-x\fP \fIexclude\fP] [\fB\-s\fP \fIstate\fP] +[\fB\-T\fP \fIsecs\fP] \fB\-d\fP|\fIdestination\fP .\"------------------------------------- .PP @@ -31,6 +32,7 @@ [\fB\-n\fP \fInamespace\fP|\fB\-N\fP] [\fB\-x\fP \fIexclude\fP] [\fB\-s\fP \fIstate\fP] +[\fB\-T\fP \fIsecs\fP] \fB\-d\fP|\fIdestination\fP .\"------------------------------------------------------------------------ .SH DESCRIPTION @@ -167,6 +169,13 @@ @INCLUDE:option-S-socket@ .\"------------------------------------- .TP +.BI \-T \ secs +Specify the time in seconds, how long +.BR doveadm (1) +should wait for stalled I/O operations. +The default timeout is 600 seconds. +.\"------------------------------------- +.TP .B \-U This is used internally by replicator to have dsync notify it when the synchronization is finished. From dovecot at dovecot.org Thu Sep 3 11:03:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 11:03:08 +0000 Subject: dovecot-2.2: lib: strnum - simplify hex and oct overflow code Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/29e0a3ea7497 changeset: 19076:29e0a3ea7497 user: Phil Carmody date: Thu Sep 03 14:01:57 2015 +0300 description: lib: strnum - simplify hex and oct overflow code uintmax_t is defined to have modulo-2^n semantics, and therefore the bottom bits of (uintmax_t) are guaranteed to be all set. Therefore the checking of the next character read is unnecessary, as it's already done in the loop control statement itself. (This is not true about the bottom digit base 10, which is why the check remains in the decimal case) Signed-off-by: Phil Carmody diffstat: src/lib/strnum.c | 16 ++++------------ 1 files changed, 4 insertions(+), 12 deletions(-) diffs (33 lines): diff -r f313cda93533 -r 29e0a3ea7497 src/lib/strnum.c --- a/src/lib/strnum.c Mon Aug 31 22:31:19 2015 +0000 +++ b/src/lib/strnum.c Thu Sep 03 14:01:57 2015 +0300 @@ -160,12 +160,8 @@ return -1; do { - if (n >= ((uintmax_t)-1 >> 4)) { - if (n > (uintmax_t)-1 >> 4) - return -1; - if ((uintmax_t)hex > ((uintmax_t)-1 & 0x0f)) - return -1; - } + if (n > (uintmax_t)-1 >> 4) + return -1; n = (n << 4) + hex; str++; } while (_str_parse_hex(*str, &hex) >= 0); @@ -230,12 +226,8 @@ return -1; for (; *str >= '0' && *str <= '7'; str++) { - if (n >= ((uintmax_t)-1 >> 3)) { - if (n > (uintmax_t)-1 >> 3) - return -1; - if ((uintmax_t)(*str - '0') > ((uintmax_t)-1 & 0x03)) - return -1; - } + if (n > (uintmax_t)-1 >> 3) + return -1; n = (n << 3) + (*str - '0'); } if (endp_r != NULL) From dovecot at dovecot.org Thu Sep 3 11:11:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 11:11:37 +0000 Subject: dovecot-2.2: Removed SET_IN_PORT_ZERO - SET_IN_PORT now always a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f17065f4f9f2 changeset: 19077:f17065f4f9f2 user: Timo Sirainen date: Thu Sep 03 14:10:36 2015 +0300 description: Removed SET_IN_PORT_ZERO - SET_IN_PORT now always allows zeros as well. A zero in all the port settings means that the port is disabled, which is also the default. So it shouldn't be an error to explicitly set it to zero in the config file. diffstat: src/config/config-request.c | 4 +--- src/lib-settings/settings-parser.c | 22 +--------------------- src/lib-settings/settings-parser.h | 1 - src/master/master-settings.c | 2 +- src/replication/aggregator/aggregator-settings.c | 2 +- 5 files changed, 4 insertions(+), 27 deletions(-) diffs (123 lines): diff -r 29e0a3ea7497 -r f17065f4f9f2 src/config/config-request.c --- a/src/config/config-request.c Thu Sep 03 14:01:57 2015 +0300 +++ b/src/config/config-request.c Thu Sep 03 14:10:36 2015 +0300 @@ -116,8 +116,7 @@ } break; } - case SET_IN_PORT: - case SET_IN_PORT_ZERO: { + case SET_IN_PORT: { const in_port_t *val = value, *dval = default_value; if (dump_default || dval == NULL || *val != *dval) @@ -254,7 +253,6 @@ case SET_UINT_OCT: case SET_TIME: case SET_IN_PORT: - case SET_IN_PORT_ZERO: case SET_STR_VARS: case SET_STR: case SET_ENUM: diff -r 29e0a3ea7497 -r f17065f4f9f2 src/lib-settings/settings-parser.c --- a/src/lib-settings/settings-parser.c Thu Sep 03 14:01:57 2015 +0300 +++ b/src/lib-settings/settings-parser.c Thu Sep 03 14:10:36 2015 +0300 @@ -583,18 +583,6 @@ } static int -get_in_port(struct setting_parser_context *ctx, const char *value, - in_port_t *result_r) -{ - if (net_str2port(value, result_r) < 0) { - ctx->error = p_strdup_printf(ctx->parser_pool, - "Invalid port number %s", value); - return -1; - } - return 0; -} - -static int get_in_port_zero(struct setting_parser_context *ctx, const char *value, in_port_t *result_r) { @@ -655,10 +643,6 @@ } break; case SET_IN_PORT: - if (get_in_port(ctx, value, (in_port_t *)ptr) < 0) - return -1; - break; - case SET_IN_PORT_ZERO: if (get_in_port_zero(ctx, value, (in_port_t *)ptr) < 0) return -1; break; @@ -1264,7 +1248,6 @@ case SET_TIME: case SET_SIZE: case SET_IN_PORT: - case SET_IN_PORT_ZERO: case SET_STR: case SET_ENUM: case SET_STRLIST: @@ -1361,7 +1344,6 @@ case SET_TIME: case SET_SIZE: case SET_IN_PORT: - case SET_IN_PORT_ZERO: case SET_STR: case SET_ENUM: case SET_STRLIST: @@ -1446,8 +1428,7 @@ *dest_size = *src_size; break; } - case SET_IN_PORT: - case SET_IN_PORT_ZERO: { + case SET_IN_PORT: { const in_port_t *src_size = src; in_port_t *dest_size = dest; @@ -1568,7 +1549,6 @@ case SET_TIME: case SET_SIZE: case SET_IN_PORT: - case SET_IN_PORT_ZERO: case SET_STR_VARS: case SET_STR: case SET_ENUM: diff -r 29e0a3ea7497 -r f17065f4f9f2 src/lib-settings/settings-parser.h --- a/src/lib-settings/settings-parser.h Thu Sep 03 14:01:57 2015 +0300 +++ b/src/lib-settings/settings-parser.h Thu Sep 03 14:10:36 2015 +0300 @@ -24,7 +24,6 @@ SET_TIME, SET_SIZE, SET_IN_PORT, /* internet port */ - SET_IN_PORT_ZERO, /* internet port */ SET_STR, SET_STR_VARS, /* string with %variables */ SET_ENUM, diff -r 29e0a3ea7497 -r f17065f4f9f2 src/master/master-settings.c --- a/src/master/master-settings.c Thu Sep 03 14:01:57 2015 +0300 +++ b/src/master/master-settings.c Thu Sep 03 14:10:36 2015 +0300 @@ -62,7 +62,7 @@ static const struct setting_define inet_listener_setting_defines[] = { DEF(SET_STR, name), DEF(SET_STR, address), - DEF(SET_IN_PORT_ZERO, port), + DEF(SET_IN_PORT, port), DEF(SET_BOOL, ssl), DEF(SET_BOOL, reuse_port), DEF(SET_BOOL, haproxy), diff -r 29e0a3ea7497 -r f17065f4f9f2 src/replication/aggregator/aggregator-settings.c --- a/src/replication/aggregator/aggregator-settings.c Thu Sep 03 14:01:57 2015 +0300 +++ b/src/replication/aggregator/aggregator-settings.c Thu Sep 03 14:10:36 2015 +0300 @@ -61,7 +61,7 @@ static const struct setting_define aggregator_setting_defines[] = { DEF(SET_STR, replicator_host), - DEF(SET_IN_PORT_ZERO, replicator_port), + DEF(SET_IN_PORT, replicator_port), SETTING_DEFINE_LIST_END }; From dovecot at dovecot.org Thu Sep 3 13:05:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 13:05:43 +0000 Subject: dovecot-2.2: doveadm director update command added. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ac9ec913a39 changeset: 19078:7ac9ec913a39 user: Timo Sirainen date: Thu Sep 03 16:04:26 2015 +0300 description: doveadm director update command added. The difference to "doveadm director add" is that if the IP doesn't already exist, it's not added. diffstat: src/director/doveadm-connection.c | 32 +++++++++++++++++++++++++++----- src/doveadm/doveadm-director.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 13 deletions(-) diffs (159 lines): diff -r f17065f4f9f2 -r 7ac9ec913a39 src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Thu Sep 03 14:10:36 2015 +0300 +++ b/src/director/doveadm-connection.c Thu Sep 03 16:04:26 2015 +0300 @@ -246,7 +246,8 @@ } static bool -doveadm_cmd_host_set(struct doveadm_connection *conn, const char *line) +doveadm_cmd_host_set_or_update(struct doveadm_connection *conn, const char *line, + bool update) { struct director *dir = conn->dir; const char *const *args, *ip_str, *tag = ""; @@ -264,8 +265,10 @@ ip_str = t_strdup_until(ip_str, tag++); } if (ip_str == NULL || net_addr2ip(ip_str, &ip) < 0 || - (args[1] != NULL && str_to_uint(args[1], &vhost_count) < 0)) { - i_error("doveadm sent invalid HOST-SET parameters: %s", line); + (args[1] != NULL && str_to_uint(args[1], &vhost_count) < 0) || + (args[1] == NULL && update)) { + i_error("doveadm sent invalid %s parameters: %s", + update ? "HOST-UPDATE" : "HOST-SET", line); return FALSE; } if (vhost_count > MAX_VALID_VHOST_COUNT && vhost_count != UINT_MAX) { @@ -273,11 +276,16 @@ return TRUE; } host = mail_host_lookup(dir->mail_hosts, &ip); - if (host == NULL) + if (host == NULL) { + if (update) { + o_stream_nsend_str(conn->output, "NOTFOUND\n"); + return TRUE; + } host = mail_host_add_ip(dir->mail_hosts, &ip, tag); + } if (vhost_count != UINT_MAX) mail_host_set_vhost_count(dir->mail_hosts, host, vhost_count); - /* NOTE: we don't supporting changing a tag for an existing host. + /* NOTE: we don't support changing a tag for an existing host. it needs to be removed first. otherwise it would be a bit ugly to handle. */ director_update_host(dir, dir->self_host, NULL, host); @@ -287,6 +295,18 @@ } static bool +doveadm_cmd_host_set(struct doveadm_connection *conn, const char *line) +{ + return doveadm_cmd_host_set_or_update(conn, line, FALSE); +} + +static bool +doveadm_cmd_host_update(struct doveadm_connection *conn, const char *line) +{ + return doveadm_cmd_host_set_or_update(conn, line, TRUE); +} + +static bool doveadm_cmd_host_updown(struct doveadm_connection *conn, bool down, const char *line) { @@ -616,6 +636,8 @@ ret = doveadm_cmd_director_remove(conn, args); else if (strcmp(cmd, "HOST-SET") == 0) ret = doveadm_cmd_host_set(conn, args); + else if (strcmp(cmd, "HOST-UPDATE") == 0) + ret = doveadm_cmd_host_update(conn, args); else if (strcmp(cmd, "HOST-UP") == 0) ret = doveadm_cmd_host_updown(conn, FALSE, args); else if (strcmp(cmd, "HOST-DOWN") == 0) diff -r f17065f4f9f2 -r 7ac9ec913a39 src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Thu Sep 03 14:10:36 2015 +0300 +++ b/src/doveadm/doveadm-director.c Thu Sep 03 16:04:26 2015 +0300 @@ -407,7 +407,9 @@ pool_unref(&pool); } -static void cmd_director_add(int argc, char *argv[]) +static void +cmd_director_add_or_update(int argc, char *argv[], doveadm_command_t *cmd_func, + const char *director_cmd) { struct director_context *ctx; struct ip_addr *ips; @@ -415,18 +417,20 @@ const char *host, *line; string_t *cmd; - ctx = cmd_director_init(argc, argv, "a:t:", cmd_director_add); + ctx = cmd_director_init(argc, argv, "a:t:", cmd_func); if (ctx->tag != NULL && ctx->tag[0] == '\0') ctx->tag = NULL; host = argv[optind++]; if (host == NULL) - director_cmd_help(cmd_director_add); + director_cmd_help(cmd_func); if (argv[optind] != NULL) { if (str_to_uint(argv[optind++], &vhost_count) < 0) - director_cmd_help(cmd_director_add); - } + director_cmd_help(cmd_func); + } else if (strcmp(director_cmd, "HOST-UPDATE") == 0) + director_cmd_help(cmd_func); + if (argv[optind] != NULL) - director_cmd_help(cmd_director_add); + director_cmd_help(cmd_func); if (ctx->tag == NULL) { ctx->tag = strchr(host, '@'); @@ -437,7 +441,7 @@ cmd = t_str_new(128); for (i = 0; i < ips_count; i++) { str_truncate(cmd, 0); - str_printfa(cmd, "HOST-SET\t%s", net_ip2addr(&ips[i])); + str_printfa(cmd, "%s\t%s", director_cmd, net_ip2addr(&ips[i])); if (ctx->tag != NULL) str_printfa(cmd, "@%s", ctx->tag); if (vhost_count != UINT_MAX) @@ -449,7 +453,9 @@ line = i_stream_read_next_line(ctx->input); if (line == NULL || strcmp(line, "OK") != 0) { fprintf(stderr, "%s: %s\n", net_ip2addr(&ips[i]), - line == NULL ? "failed" : line); + line == NULL ? "failed" : + strcmp(line, "NOTFOUND") == 0 ? + "doesn't exist" : line); doveadm_exit_code = EX_TEMPFAIL; } else if (doveadm_verbose) { printf("%s: OK\n", net_ip2addr(&ips[i])); @@ -458,6 +464,16 @@ director_disconnect(ctx); } +static void cmd_director_add(int argc, char *argv[]) +{ + cmd_director_add_or_update(argc, argv, cmd_director_add, "HOST-SET"); +} + +static void cmd_director_update(int argc, char *argv[]) +{ + cmd_director_add_or_update(argc, argv, cmd_director_update, "HOST-UPDATE"); +} + static void cmd_director_ipcmd(const char *cmd_name, doveadm_command_t *cmd, const char *success_result, int argc, char *argv[]) @@ -808,6 +824,8 @@ "[-a ] [-f ] [-h | -u] []" }, { cmd_director_add, "director add", "[-a ] [-t ] []" }, + { cmd_director_update, "director update", + "[-a ] [-t ] " }, { cmd_director_up, "director up", "[-a ] " }, { cmd_director_down, "director down", From dovecot at dovecot.org Thu Sep 3 17:00:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 17:00:17 +0000 Subject: dovecot-2.2: lib-storage: Allow index_storage_mailbox_close() to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/83fe8ab89c0e changeset: 19079:83fe8ab89c0e user: Timo Sirainen date: Thu Sep 03 19:58:39 2015 +0300 description: lib-storage: Allow index_storage_mailbox_close() to be called without an opened view. Some failing mailbox_open() call may leave it in such a state. diffstat: src/lib-storage/index/index-storage.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 7ac9ec913a39 -r 83fe8ab89c0e src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Thu Sep 03 16:04:26 2015 +0300 +++ b/src/lib-storage/index/index-storage.c Thu Sep 03 19:58:39 2015 +0300 @@ -362,7 +362,8 @@ mail_index_view_close(&box->view_pvt); if (box->index_pvt != NULL) mail_index_close(box->index_pvt); - mail_index_view_close(&box->view); + if (box->view != NULL) + mail_index_view_close(&box->view); mail_index_close(box->index); box->cache = NULL; From dovecot at dovecot.org Thu Sep 3 17:00:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 17:00:22 +0000 Subject: dovecot-2.2: dsync: Crashfix in certain situation. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f116e41d868d changeset: 19080:f116e41d868d user: Timo Sirainen date: Thu Sep 03 19:59:16 2015 +0300 description: dsync: Crashfix in certain situation. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (19 lines): diff -r 83fe8ab89c0e -r f116e41d868d src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Sep 03 19:58:39 2015 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Sep 03 19:59:16 2015 +0300 @@ -1628,8 +1628,13 @@ } else { (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r); } - *result_r = t_strdup_printf("%s (next local mail UID=%u)", - *result_r, importer->cur_mail->uid); + if (importer->cur_mail != NULL) { + *result_r = t_strdup_printf("%s (next local mail UID=%u)", + *result_r, importer->cur_mail->uid); + } else { + *result_r = t_strdup_printf("%s (no more local mails)", + *result_r); + } } int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, From dovecot at dovecot.org Thu Sep 3 17:56:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 17:56:19 +0000 Subject: dovecot-2.2: dict: Various reference counting and other fixes re... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bed46aaaa769 changeset: 19081:bed46aaaa769 user: Timo Sirainen date: Thu Sep 03 20:54:27 2015 +0300 description: dict: Various reference counting and other fixes related to using freed memory. diffstat: src/dict/dict-commands.c | 52 +++++++++++++++++----------------- src/dict/dict-commands.h | 1 - src/dict/dict-connection.c | 67 ++++++++++++++++++++++++++++++++++----------- src/dict/dict-connection.h | 6 ++++ 4 files changed, 82 insertions(+), 44 deletions(-) diffs (296 lines): diff -r f116e41d868d -r bed46aaaa769 src/dict/dict-commands.c --- a/src/dict/dict-commands.c Thu Sep 03 19:59:16 2015 +0300 +++ b/src/dict/dict-commands.c Thu Sep 03 20:54:27 2015 +0300 @@ -27,7 +27,7 @@ struct dict_iterate_context *iter; enum dict_iterate_flags iter_flags; - struct dict_connection_transaction *trans; + unsigned int trans_id; }; static void dict_connection_cmd_output_more(struct dict_connection_cmd *cmd); @@ -36,10 +36,10 @@ { if (cmd->iter != NULL) (void)dict_iterate_deinit(&cmd->iter); - array_delete(&cmd->conn->cmds, 0, 1); i_free(cmd->reply); - dict_connection_continue_input(cmd->conn); + if (dict_connection_unref(cmd->conn)) + dict_connection_continue_input(cmd->conn); i_free(cmd); } @@ -51,6 +51,7 @@ cmds = array_get(&cmd->conn->cmds, &count); for (i = 0; i < count; i++) { if (cmds[i] == cmd) { + array_delete(&cmd->conn->cmds, i, 1); dict_connection_cmd_free(cmd); return; } @@ -62,6 +63,7 @@ { struct dict_connection_cmd *cmd, *const *first_cmdp; + dict_connection_ref(conn); while (array_count(&conn->cmds) > 0) { first_cmdp = array_idx(&conn->cmds, 0); cmd = *first_cmdp; @@ -74,16 +76,7 @@ o_stream_nsend_str(conn->output, cmd->reply); dict_connection_cmd_remove(cmd); } -} - -void dict_connection_cmds_free(struct dict_connection *conn) -{ - struct dict_connection_cmd *const *first_cmdp; - - while (array_count(&conn->cmds) > 0) { - first_cmdp = array_idx(&conn->cmds, 0); - dict_connection_cmd_remove(*first_cmdp); - } + dict_connection_unref(conn); } static void @@ -201,20 +194,20 @@ static void dict_connection_transaction_array_remove(struct dict_connection *conn, - struct dict_connection_transaction *trans) + unsigned int id) { const struct dict_connection_transaction *transactions; unsigned int i, count; - i_assert(trans->ctx == NULL); - transactions = array_get(&conn->transactions, &count); for (i = 0; i < count; i++) { - if (&transactions[i] == trans) { + if (transactions[i].id == id) { + i_assert(transactions[i].ctx == NULL); array_delete(&conn->transactions, i, 1); - break; + return; } } + i_unreached(); } static int cmd_begin(struct dict_connection_cmd *cmd, const char *line) @@ -279,11 +272,11 @@ } if (async) { cmd->reply = i_strdup_printf("%c%c%u\n", - DICT_PROTOCOL_REPLY_ASYNC_COMMIT, chr, cmd->trans->id); + DICT_PROTOCOL_REPLY_ASYNC_COMMIT, chr, cmd->trans_id); } else { - cmd->reply = i_strdup_printf("%c%u\n", chr, cmd->trans->id); + cmd->reply = i_strdup_printf("%c%u\n", chr, cmd->trans_id); } - dict_connection_transaction_array_remove(cmd->conn, cmd->trans); + dict_connection_transaction_array_remove(cmd->conn, cmd->trans_id); dict_connection_cmds_flush(cmd->conn); } @@ -304,20 +297,26 @@ static int cmd_commit(struct dict_connection_cmd *cmd, const char *line) { - if (dict_connection_transaction_lookup_parse(cmd->conn, line, &cmd->trans) < 0) + struct dict_connection_transaction *trans; + + if (dict_connection_transaction_lookup_parse(cmd->conn, line, &trans) < 0) return -1; + cmd->trans_id = trans->id; - dict_transaction_commit_async(&cmd->trans->ctx, cmd_commit_callback, cmd); + dict_transaction_commit_async(&trans->ctx, cmd_commit_callback, cmd); return 1; } static int cmd_commit_async(struct dict_connection_cmd *cmd, const char *line) { - if (dict_connection_transaction_lookup_parse(cmd->conn, line, &cmd->trans) < 0) + struct dict_connection_transaction *trans; + + if (dict_connection_transaction_lookup_parse(cmd->conn, line, &trans) < 0) return -1; + cmd->trans_id = trans->id; - dict_transaction_commit_async(&cmd->trans->ctx, cmd_commit_callback_async, cmd); + dict_transaction_commit_async(&trans->ctx, cmd_commit_callback_async, cmd); return 1; } @@ -329,7 +328,7 @@ return -1; dict_transaction_rollback(&trans->ctx); - dict_connection_transaction_array_remove(cmd->conn, trans); + dict_connection_transaction_array_remove(cmd->conn, trans->id); return 0; } @@ -451,6 +450,7 @@ cmd->conn = conn; cmd->cmd = cmd_func; array_append(&conn->cmds, &cmd, 1); + dict_connection_ref(conn); if ((ret = cmd_func->func(cmd, line + 1)) <= 0) { dict_connection_cmd_remove(cmd); return ret; diff -r f116e41d868d -r bed46aaaa769 src/dict/dict-commands.h --- a/src/dict/dict-commands.h Thu Sep 03 19:59:16 2015 +0300 +++ b/src/dict/dict-commands.h Thu Sep 03 20:54:27 2015 +0300 @@ -6,6 +6,5 @@ int dict_command_input(struct dict_connection *conn, const char *line); void dict_connection_cmds_output_more(struct dict_connection *conn); -void dict_connection_cmds_free(struct dict_connection *conn); #endif diff -r f116e41d868d -r bed46aaaa769 src/dict/dict-connection.c --- a/src/dict/dict-connection.c Thu Sep 03 19:59:16 2015 +0300 +++ b/src/dict/dict-connection.c Thu Sep 03 20:54:27 2015 +0300 @@ -157,7 +157,7 @@ void dict_connection_continue_input(struct dict_connection *conn) { - if (conn->io != NULL) + if (conn->io != NULL || conn->destroyed) return; conn->io = io_add(conn->fd, IO_READ, dict_connection_input, conn); @@ -183,6 +183,7 @@ struct dict_connection *conn; conn = i_new(struct dict_connection, 1); + conn->refcount = 1; conn->fd = fd; conn->input = i_stream_create_fd(fd, DICT_CLIENT_MAX_LINE_LENGTH, FALSE); @@ -195,41 +196,73 @@ return conn; } -void dict_connection_destroy(struct dict_connection *conn) +void dict_connection_ref(struct dict_connection *conn) +{ + i_assert(conn->refcount > 0); + conn->refcount++; +} + +bool dict_connection_unref(struct dict_connection *conn) { struct dict_connection_transaction *transaction; - DLLIST_REMOVE(&dict_connections, conn); + i_assert(conn->refcount > 0); + if (--conn->refcount > 0) + return TRUE; - /* deinit dict before anything else, so any pending dict operations - are aborted and their callbacks called. */ + i_assert(array_count(&conn->cmds) == 0); + + /* we should have only transactions that haven't been committed or + rollbacked yet. close those before dict is deinitialized. */ + if (array_is_created(&conn->transactions)) { + array_foreach_modifiable(&conn->transactions, transaction) { + if (transaction->ctx != NULL) + dict_transaction_rollback(&transaction->ctx); + } + } + if (conn->dict != NULL) dict_deinit(&conn->dict); - if (array_is_created(&conn->transactions)) { - array_foreach_modifiable(&conn->transactions, transaction) - dict_transaction_rollback(&transaction->ctx); + if (array_is_created(&conn->transactions)) array_free(&conn->transactions); - } - /* this may end up adding conn->io back, so keep this early */ - dict_connection_cmds_free(conn); + i_stream_destroy(&conn->input); + o_stream_destroy(&conn->output); + array_free(&conn->cmds); + i_free(conn->name); + i_free(conn->username); + i_free(conn); + + master_service_client_connection_destroyed(master_service); + return FALSE; +} + +void dict_connection_destroy(struct dict_connection *conn) +{ + conn->destroyed = TRUE; + DLLIST_REMOVE(&dict_connections, conn); if (conn->to_input != NULL) timeout_remove(&conn->to_input); if (conn->io != NULL) io_remove(&conn->io); - i_stream_destroy(&conn->input); - o_stream_destroy(&conn->output); + i_stream_close(conn->input); + o_stream_close(conn->output); if (close(conn->fd) < 0) i_error("close(dict client) failed: %m"); + conn->fd = -1; - i_free(conn->name); - i_free(conn->username); - i_free(conn); + /* the connection is closed, but there may still be commands left + running. finish them, even if the calling client can't be notified + about whether they succeeded (clients may not even care). - master_service_client_connection_destroyed(master_service); + flush the command output here in case we were waiting on iteration + output. */ + dict_connection_cmds_output_more(conn); + + dict_connection_unref(conn); } void dict_connections_destroy_all(void) diff -r f116e41d868d -r bed46aaaa769 src/dict/dict-connection.h --- a/src/dict/dict-connection.h Thu Sep 03 19:59:16 2015 +0300 +++ b/src/dict/dict-connection.h Thu Sep 03 20:54:27 2015 +0300 @@ -12,6 +12,7 @@ struct dict_connection { struct dict_connection *prev, *next; struct dict_server *server; + int refcount; char *username; char *name; @@ -28,11 +29,16 @@ array is fast enough */ ARRAY(struct dict_connection_transaction) transactions; ARRAY(struct dict_connection_cmd *) cmds; + + unsigned int destroyed:1; }; struct dict_connection *dict_connection_create(int fd); void dict_connection_destroy(struct dict_connection *conn); +void dict_connection_ref(struct dict_connection *conn); +bool dict_connection_unref(struct dict_connection *conn); + void dict_connection_continue_input(struct dict_connection *conn); void dict_connections_destroy_all(void); From dovecot at dovecot.org Thu Sep 3 17:56:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 17:56:20 +0000 Subject: dovecot-2.2: lib-storage: Another fix to using index_storage_mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/edc402c28fe7 changeset: 19082:edc402c28fe7 user: Timo Sirainen date: Thu Sep 03 20:55:18 2015 +0300 description: lib-storage: Another fix to using index_storage_mailbox_close() without an opened view. Calling mail_index_close() without mail_index_open() caused it to assert-crash. diffstat: src/lib-storage/index/index-storage.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (16 lines): diff -r bed46aaaa769 -r edc402c28fe7 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Thu Sep 03 20:54:27 2015 +0300 +++ b/src/lib-storage/index/index-storage.c Thu Sep 03 20:55:18 2015 +0300 @@ -362,9 +362,10 @@ mail_index_view_close(&box->view_pvt); if (box->index_pvt != NULL) mail_index_close(box->index_pvt); - if (box->view != NULL) + if (box->view != NULL) { mail_index_view_close(&box->view); - mail_index_close(box->index); + mail_index_close(box->index); + } box->cache = NULL; ibox->keyword_names = NULL; From dovecot at dovecot.org Thu Sep 3 18:38:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 18:38:10 +0000 Subject: dovecot-2.2: cassandra: Handle async queries internally - don't ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5d0bcb628c88 changeset: 19083:5d0bcb628c88 user: Timo Sirainen date: Thu Sep 03 21:37:09 2015 +0300 description: cassandra: Handle async queries internally - don't use sql pooling code. There's no need to create multiple Cassandra instances, since the single instance is capable of doing multiple asynchronous requests in parallel. diffstat: src/lib-sql/driver-cassandra.c | 105 ++++++++++++++++++++++++++-------------- 1 files changed, 69 insertions(+), 36 deletions(-) diffs (269 lines): diff -r edc402c28fe7 -r 5d0bcb628c88 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Sep 03 20:55:18 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Thu Sep 03 21:37:09 2015 +0300 @@ -39,9 +39,10 @@ int fd_pipe[2]; struct io *io_pipe; ARRAY(struct cassandra_callback *) callbacks; + ARRAY(struct cassandra_result *) results; unsigned int callback_ids; - struct cassandra_result *cur_result; + /* for synchronous queries: */ struct ioloop *ioloop, *orig_ioloop; struct sql_result *sync_result; @@ -63,6 +64,7 @@ sql_query_callback_t *callback; void *context; + unsigned int query_sent:1; unsigned int finished:1; unsigned int write_query:1; }; @@ -116,6 +118,7 @@ { CASS_LOG_TRACE, "trace" } }; +static void driver_cassandra_send_queries(struct cassandra_db *db); static void result_finish(struct cassandra_result *result); static int consistency_parse(const char *str, CassConsistency *consistency_r) @@ -146,8 +149,6 @@ static void driver_cassandra_set_state(struct cassandra_db *db, enum sql_db_state state) { - i_assert(state == SQL_DB_STATE_BUSY || db->cur_result == NULL); - /* switch back to original ioloop in case the caller wants to add/remove timeouts */ if (db->ioloop != NULL) @@ -157,8 +158,10 @@ io_loop_set_current(db->ioloop); } -static void driver_cassandra_close(struct cassandra_db *db) +static void driver_cassandra_close(struct cassandra_db *db, const char *error) { + struct cassandra_result *const *resultp; + if (db->io_pipe != NULL) io_remove(&db->io_pipe); if (db->fd_pipe[0] != -1) { @@ -167,6 +170,13 @@ } driver_cassandra_set_state(db, SQL_DB_STATE_DISCONNECTED); + while (array_count(&db->results) > 0) { + resultp = array_idx(&db->results, 0); + if ((*resultp)->error == NULL) + (*resultp)->error = i_strdup(error); + result_finish(*resultp); + } + if (db->ioloop != NULL) { /* running a sync query, stop it */ io_loop_stop(db->ioloop); @@ -238,7 +248,7 @@ driver_cassandra_input_id(db, ids[i]); return; } - driver_cassandra_close(db); + driver_cassandra_close(db, "IPC pipe closed"); } static void @@ -267,7 +277,7 @@ if ((rc = cass_future_error_code(future)) != CASS_OK) { driver_cassandra_log_error(future, "Couldn't connect to Cassandra"); - driver_cassandra_close(db); + driver_cassandra_close(db, "Couldn't connect to Cassandra"); return; } driver_cassandra_set_state(db, SQL_DB_STATE_IDLE); @@ -276,6 +286,7 @@ finish */ io_loop_stop(db->ioloop); } + driver_cassandra_send_queries(db); } static int driver_cassandra_connect(struct sql_db *_db) @@ -302,12 +313,7 @@ { struct cassandra_db *db = (struct cassandra_db *)_db; - if (db->cur_result != NULL && !db->cur_result->finished) { - if (db->cur_result->error == NULL) - db->cur_result->error = i_strdup("disconnecting"); - result_finish(db->cur_result); - } - driver_cassandra_close(db); + driver_cassandra_close(db, "Disconnected"); } static const char * @@ -394,6 +400,7 @@ cass_cluster_set_request_timeout(db->cluster, SQL_QUERY_TIMEOUT_SECS * 1000); cass_cluster_set_contact_points(db->cluster, db->hosts); db->session = cass_session_new(); + i_array_init(&db->results, 16); i_array_init(&db->callbacks, 16); return &db->api; } @@ -402,15 +409,12 @@ { struct cassandra_db *db = (struct cassandra_db *)_db; - if (db->cur_result != NULL && !db->cur_result->finished) { - if (db->cur_result->error == NULL) - db->cur_result->error = i_strdup("deinitializing"); - result_finish(db->cur_result); - } - driver_cassandra_close(db); + driver_cassandra_close(db, "Deinitialized"); i_assert(array_count(&db->callbacks) == 0); array_free(&db->callbacks); + i_assert(array_count(&db->results) == 0); + array_free(&db->results); cass_session_free(db->session); cass_cluster_free(db->cluster); @@ -421,11 +425,20 @@ i_free(db); } -static void driver_cassandra_set_idle(struct cassandra_db *db) +static void driver_cassandra_result_unlink(struct cassandra_db *db, + struct cassandra_result *result) { - i_assert(db->api.state == SQL_DB_STATE_BUSY); + struct cassandra_result *const *results; + unsigned int i, count; - driver_cassandra_set_state(db, SQL_DB_STATE_IDLE); + results = array_get(&db->results, &count); + for (i = 0; i < count; i++) { + if (results[i] == result) { + array_delete(&db->results, i, 1); + return; + } + } + i_unreached(); } static void driver_cassandra_result_free(struct sql_result *_result) @@ -434,14 +447,11 @@ struct cassandra_result *result = (struct cassandra_result *)_result; i_assert(!result->api.callback); - i_assert(db->cur_result == result); i_assert(result->callback == NULL); if (_result == db->sync_result) db->sync_result = NULL; - db->cur_result = NULL; - driver_cassandra_set_idle(db); if (result->result != NULL) cass_result_free(result->result); if (result->iterator != NULL) @@ -459,6 +469,7 @@ bool free_result = TRUE; result->finished = TRUE; + driver_cassandra_result_unlink(db, result); if (db->log_level >= CASS_LOG_DEBUG) { i_debug("cassandra: Finished query '%s': %s", result->query, @@ -504,26 +515,44 @@ result_finish(result); } -static void do_query(struct cassandra_result *result, const char *query) +static int driver_cassandra_send_query(struct cassandra_result *result) { struct cassandra_db *db = (struct cassandra_db *)result->api.db; CassFuture *future; + int ret; - i_assert(SQL_DB_IS_READY(&db->api)); - i_assert(db->cur_result == NULL); + if (!SQL_DB_IS_READY(&db->api)) { + if ((ret = sql_connect(&db->api)) <= 0) { + if (ret < 0) + driver_cassandra_close(db, "Couldn't connect to Cassandra"); + return ret; + } + } - driver_cassandra_set_state(db, SQL_DB_STATE_BUSY); - db->cur_result = result; - - result->query = i_strdup(query); result->row_pool = pool_alloconly_create("cassandra result", 512); - result->statement = cass_statement_new(query, 0); + result->statement = cass_statement_new(result->query, 0); if (result->write_query) cass_statement_set_consistency(result->statement, db->write_consistency); else cass_statement_set_consistency(result->statement, db->read_consistency); future = cass_session_execute(db->session, result->statement); driver_cassandra_set_callback(future, db, query_callback, result); + result->query_sent = TRUE; + return 1; +} + +static void driver_cassandra_send_queries(struct cassandra_db *db) +{ + struct cassandra_result *const *results; + unsigned int i, count; + + results = array_get(&db->results, &count); + for (i = 0; i < count; i++) { + if (!results[i]->query_sent) { + if (driver_cassandra_send_query(results[i]) <= 0) + break; + } + } } static void exec_callback(struct sql_result *_result ATTR_UNUSED, @@ -532,19 +561,23 @@ } static void -driver_cassandra_query_full(struct sql_db *db, const char *query, bool write_query, +driver_cassandra_query_full(struct sql_db *_db, const char *query, bool write_query, sql_query_callback_t *callback, void *context) { + struct cassandra_db *db = (struct cassandra_db *)_db; struct cassandra_result *result; result = i_new(struct cassandra_result, 1); result->api = driver_cassandra_result; - result->api.db = db; + result->api.db = _db; result->api.refcount = 1; result->callback = callback; result->context = context; result->write_query = write_query; - do_query(result, query); + result->query = i_strdup(query); + array_append(&db->results, &result, 1); + + (void)driver_cassandra_send_query(result); } static void driver_cassandra_exec(struct sql_db *db, const char *query) @@ -971,7 +1004,7 @@ const struct sql_db driver_cassandra_db = { .name = "cassandra", - .flags = SQL_DB_FLAG_POOLED, + .flags = 0, .v = { driver_cassandra_init_v, From dovecot at dovecot.org Thu Sep 3 19:13:18 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 19:13:18 +0000 Subject: dovecot-2.2: dsync: Assert-crashfix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c8be83ada6c changeset: 19084:5c8be83ada6c user: Timo Sirainen date: Thu Sep 03 22:12:14 2015 +0300 description: dsync: Assert-crashfix Fixes: Panic: file mail-storage.c: line 1897 (mailbox_save_alloc): assertion failed: (!ctx->unfinished) diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 5d0bcb628c88 -r 5c8be83ada6c src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Sep 03 21:37:09 2015 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Sep 03 22:12:14 2015 +0300 @@ -2271,6 +2271,7 @@ if (!remote_mail) { /* the mail isn't remote yet. we were just trying to copy a local mail to avoid downloading the remote mail. */ + mailbox_save_cancel(&save_ctx); return FALSE; } if (mail->minimal_fields) { From dovecot at dovecot.org Thu Sep 3 22:48:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 22:48:23 +0000 Subject: dovecot-2.2: login proxy: Added asserts to track that num_waitin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c98fbc17fa4e changeset: 19086:c98fbc17fa4e user: Timo Sirainen date: Fri Sep 04 01:18:04 2015 +0300 description: login proxy: Added asserts to track that num_waiting_connections are correct. diffstat: src/login-common/login-proxy-state.c | 8 ++++++++ src/login-common/login-proxy.c | 6 +++++- 2 files changed, 13 insertions(+), 1 deletions(-) diffs (52 lines): diff -r 6b901abc1819 -r c98fbc17fa4e src/login-common/login-proxy-state.c --- a/src/login-common/login-proxy-state.c Fri Sep 04 01:10:29 2015 +0300 +++ b/src/login-common/login-proxy-state.c Fri Sep 04 01:18:04 2015 +0300 @@ -66,9 +66,17 @@ void login_proxy_state_deinit(struct login_proxy_state **_state) { struct login_proxy_state *state = *_state; + struct hash_iterate_context *iter; + struct login_proxy_record *rec; *_state = NULL; + /* sanity check: */ + iter = hash_table_iterate_init(state->hash); + while (hash_table_iterate(iter, state->hash, &rec, &rec)) + i_assert(rec->num_waiting_connections == 0); + hash_table_iterate_deinit(&iter); + if (state->to_reopen != NULL) timeout_remove(&state->to_reopen); login_proxy_state_close(state); diff -r 6b901abc1819 -r c98fbc17fa4e src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Fri Sep 04 01:10:29 2015 +0300 +++ b/src/login-common/login-proxy.c Fri Sep 04 01:18:04 2015 +0300 @@ -202,6 +202,7 @@ } else { proxy->state_rec->last_failure = ioloop_timeval; } + i_assert(proxy->state_rec->num_waiting_connections > 0); proxy->state_rec->num_waiting_connections--; proxy->state_rec = NULL; } @@ -278,6 +279,7 @@ } proxy->connected = TRUE; proxy->state_rec->last_success = ioloop_timeval; + i_assert(proxy->state_rec->num_waiting_connections > 0); proxy->state_rec->num_waiting_connections--; proxy->state_rec = NULL; @@ -399,8 +401,10 @@ if (proxy->to_notify != NULL) timeout_remove(&proxy->to_notify); - if (proxy->state_rec != NULL) + if (proxy->state_rec != NULL) { + i_assert(proxy->state_rec->num_waiting_connections > 0); proxy->state_rec->num_waiting_connections--; + } if (proxy->server_io != NULL) io_remove(&proxy->server_io); From dovecot at dovecot.org Thu Sep 3 22:48:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 22:48:22 +0000 Subject: dovecot-2.2: login proxy: Small code cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6b901abc1819 changeset: 19085:6b901abc1819 user: Timo Sirainen date: Fri Sep 04 01:10:29 2015 +0300 description: login proxy: Small code cleanup diffstat: src/login-common/login-proxy.c | 20 ++++++++------------ 1 files changed, 8 insertions(+), 12 deletions(-) diffs (73 lines): diff -r 5c8be83ada6c -r 6b901abc1819 src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Thu Sep 03 22:12:14 2015 +0300 +++ b/src/login-common/login-proxy.c Fri Sep 04 01:10:29 2015 +0300 @@ -69,9 +69,9 @@ ATTR_NULL(2); static void login_proxy_free_errno(struct login_proxy **proxy, - int err, const char *who) + int err, bool server) { - const char *reason; + const char *reason, *who = server ? "server" : "client"; reason = err == 0 || err == EPIPE ? t_strdup_printf("Disconnected by %s", who) : @@ -95,7 +95,7 @@ ret = net_receive(proxy->server_fd, buf, sizeof(buf)); if (ret < 0) { - login_proxy_free_errno(&proxy, errno, "server"); + login_proxy_free_errno(&proxy, errno, TRUE); return; } o_stream_cork(proxy->client_output); @@ -103,8 +103,7 @@ o_stream_uncork(proxy->client_output); if (ret2 != ret) { login_proxy_free_errno(&proxy, - proxy->client_output->stream_errno, - "client"); + proxy->client_output->stream_errno, FALSE); } } @@ -124,7 +123,7 @@ ret = net_receive(proxy->client_fd, buf, sizeof(buf)); if (ret < 0) { - login_proxy_free_errno(&proxy, errno, "client"); + login_proxy_free_errno(&proxy, errno, FALSE); return; } o_stream_cork(proxy->server_output); @@ -132,8 +131,7 @@ o_stream_uncork(proxy->server_output); if (ret2 != ret) { login_proxy_free_errno(&proxy, - proxy->server_output->stream_errno, - "server"); + proxy->server_output->stream_errno, TRUE); } } @@ -142,8 +140,7 @@ proxy->last_io = ioloop_time; if (o_stream_flush(proxy->server_output) < 0) { login_proxy_free_errno(&proxy, - proxy->server_output->stream_errno, - "server"); + proxy->server_output->stream_errno, TRUE); return 1; } @@ -163,8 +160,7 @@ proxy->last_io = ioloop_time; if (o_stream_flush(proxy->client_output) < 0) { login_proxy_free_errno(&proxy, - proxy->client_output->stream_errno, - "client"); + proxy->client_output->stream_errno, FALSE); return 1; } From dovecot at dovecot.org Thu Sep 3 22:48:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 22:48:23 +0000 Subject: dovecot-2.2: login proxy: Always keep proxy->state_rec available. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1a4445b58279 changeset: 19087:1a4445b58279 user: Timo Sirainen date: Fri Sep 04 01:21:28 2015 +0300 description: login proxy: Always keep proxy->state_rec available. diffstat: src/login-common/login-proxy.c | 13 ++++++++----- 1 files changed, 8 insertions(+), 5 deletions(-) diffs (69 lines): diff -r c98fbc17fa4e -r 1a4445b58279 src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Fri Sep 04 01:18:04 2015 +0300 +++ b/src/login-common/login-proxy.c Fri Sep 04 01:21:28 2015 +0300 @@ -53,6 +53,7 @@ unsigned int connected:1; unsigned int destroying:1; unsigned int disconnecting:1; + unsigned int num_waiting_connections_updated:1; }; static struct login_proxy_state *proxy_state; @@ -204,7 +205,7 @@ } i_assert(proxy->state_rec->num_waiting_connections > 0); proxy->state_rec->num_waiting_connections--; - proxy->state_rec = NULL; + proxy->num_waiting_connections_updated = TRUE; } static void @@ -278,10 +279,10 @@ return; } proxy->connected = TRUE; + proxy->num_waiting_connections_updated = TRUE; proxy->state_rec->last_success = ioloop_timeval; i_assert(proxy->state_rec->num_waiting_connections > 0); proxy->state_rec->num_waiting_connections--; - proxy->state_rec = NULL; if ((proxy->ssl_flags & PROXY_SSL_FLAG_YES) != 0 && (proxy->ssl_flags & PROXY_SSL_FLAG_STARTTLS) == 0) { @@ -306,9 +307,8 @@ static int login_proxy_connect(struct login_proxy *proxy) { - struct login_proxy_record *rec; + struct login_proxy_record *rec = proxy->state_rec; - rec = login_proxy_state_get(proxy_state, &proxy->ip, proxy->port); if (rec->last_success.tv_sec == 0) { /* first connect to this IP. don't start immediately failing the check below. */ @@ -339,6 +339,7 @@ proxy_connect_timeout, proxy); } + proxy->num_waiting_connections_updated = FALSE; proxy->state_rec = rec; proxy->state_rec->num_waiting_connections++; return 0; @@ -375,6 +376,8 @@ proxy->connect_timeout_msecs = set->connect_timeout_msecs; proxy->notify_refresh_secs = set->notify_refresh_secs; proxy->ssl_flags = set->ssl_flags; + proxy->state_rec = login_proxy_state_get(proxy_state, &proxy->ip, + proxy->port); client_ref(client); if (set->ip.family == 0 && @@ -401,7 +404,7 @@ if (proxy->to_notify != NULL) timeout_remove(&proxy->to_notify); - if (proxy->state_rec != NULL) { + if (!proxy->num_waiting_connections_updated) { i_assert(proxy->state_rec->num_waiting_connections > 0); proxy->state_rec->num_waiting_connections--; } From dovecot at dovecot.org Thu Sep 3 22:48:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Sep 2015 22:48:33 +0000 Subject: dovecot-2.2: login proxy: Code cleanup - separate login_proxy_fr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/574ba1f25346 changeset: 19088:574ba1f25346 user: Timo Sirainen date: Fri Sep 04 01:23:28 2015 +0300 description: login proxy: Code cleanup - separate login_proxy_free_final() diffstat: src/login-common/login-proxy.c | 22 ++++++++++++++-------- 1 files changed, 14 insertions(+), 8 deletions(-) diffs (48 lines): diff -r 1a4445b58279 -r 574ba1f25346 src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Fri Sep 04 01:21:28 2015 +0300 +++ b/src/login-common/login-proxy.c Fri Sep 04 01:23:28 2015 +0300 @@ -419,6 +419,18 @@ net_disconnect(proxy->server_fd); } +static void login_proxy_free_final(struct login_proxy *proxy) +{ + if (proxy->client_output != NULL) + o_stream_destroy(&proxy->client_output); + if (proxy->client_fd != -1) + net_disconnect(proxy->client_fd); + if (proxy->ssl_server_proxy != NULL) + ssl_proxy_free(&proxy->ssl_server_proxy); + i_free(proxy->host); + i_free(proxy); +} + static void ATTR_NULL(2) login_proxy_free_reason(struct login_proxy **_proxy, const char *reason) { @@ -447,23 +459,17 @@ if (proxy->client_io != NULL) io_remove(&proxy->client_io); - if (proxy->client_output != NULL) - o_stream_destroy(&proxy->client_output); - net_disconnect(proxy->client_fd); } else { i_assert(proxy->client_io == NULL); i_assert(proxy->client_output == NULL); + i_assert(proxy->client_fd == -1); DLLIST_REMOVE(&login_proxies_pending, proxy); if (proxy->callback != NULL) proxy->callback(proxy->client); } - - if (proxy->ssl_server_proxy != NULL) - ssl_proxy_free(&proxy->ssl_server_proxy); - i_free(proxy->host); - i_free(proxy); + login_proxy_free_final(proxy); client->login_proxy = NULL; client_unref(&client); From dovecot at dovecot.org Fri Sep 4 08:22:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Sep 2015 08:22:45 +0000 Subject: dovecot-2.2: dsync: If we disconnect with I/O timeout, log the l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3ab749f1ed71 changeset: 19089:3ab749f1ed71 user: Timo Sirainen date: Fri Sep 04 11:17:30 2015 +0300 description: dsync: If we disconnect with I/O timeout, log the last sent/recv state. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 47 ++++++++++++++++++++++++++--------- 1 files changed, 35 insertions(+), 12 deletions(-) diffs (164 lines): diff -r 574ba1f25346 -r 3ab749f1ed71 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Sep 04 01:23:28 2015 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Fri Sep 04 11:17:30 2015 +0300 @@ -159,6 +159,10 @@ struct dsync_mailbox_attribute *cur_attr; char value_output_last; + enum item_type last_recv_item, last_sent_item; + unsigned int last_recv_item_eol:1; + unsigned int last_sent_item_eol:1; + unsigned int version_received:1; unsigned int handshake_received:1; unsigned int has_pending_data:1; @@ -290,8 +294,13 @@ static void dsync_ibc_stream_timeout(struct dsync_ibc_stream *ibc) { - i_error("dsync(%s): I/O has stalled, no activity for %u seconds", - ibc->name, ibc->timeout_secs); + i_error("dsync(%s): I/O has stalled, no activity for %u seconds " + "(last sent=%s%s, last recv=%s%s)", + ibc->name, ibc->timeout_secs, + items[ibc->last_sent_item].name, + ibc->last_sent_item_eol ? " (EOL)" : "", + items[ibc->last_recv_item].name, + ibc->last_recv_item_eol ? " (EOL)" : ""); ibc->ibc.timeout = TRUE; dsync_ibc_stream_stop(ibc); } @@ -521,6 +530,7 @@ if (dsync_ibc_check_missing_deserializers(ibc) < 0) return FALSE; ibc->handshake_received = TRUE; + ibc->last_recv_item = ITEM_HANDSHAKE; return FALSE; } @@ -564,8 +574,12 @@ return DSYNC_IBC_RECV_RET_TRYAGAIN; } while (!dsync_ibc_stream_handshake(ibc, line)); + ibc->last_recv_item = item; + ibc->last_recv_item_eol = FALSE; + if (strcmp(line, END_OF_LIST_LINE) == 0) { /* end of this list */ + ibc->last_recv_item_eol = TRUE; return DSYNC_IBC_RECV_RET_FINISHED; } if (line[0] == items[ITEM_DONE].chr) { @@ -602,6 +616,14 @@ return DSYNC_IBC_RECV_RET_OK; } +static struct dsync_serializer_encoder * +dsync_ibc_send_encode_begin(struct dsync_ibc_stream *ibc, enum item_type item) +{ + ibc->last_sent_item = item; + ibc->last_sent_item_eol = FALSE; + return dsync_serializer_encode_begin(ibc->serializers[item]); +} + static void dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc, const struct dsync_ibc_settings *set) @@ -612,7 +634,7 @@ char sync_type[2]; str_append_c(str, items[ITEM_HANDSHAKE].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_HANDSHAKE]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_HANDSHAKE); dsync_serializer_encode_add(encoder, "hostname", set->hostname); if (set->sync_ns_prefixes != NULL) { dsync_serializer_encode_add(encoder, "sync_ns_prefix", @@ -815,6 +837,7 @@ break; } + ibc->last_sent_item_eol = TRUE; o_stream_nsend_str(ibc->output, END_OF_LIST_LINE"\n"); } @@ -827,7 +850,7 @@ string_t *str = t_str_new(128); str_append_c(str, items[ITEM_MAILBOX_STATE].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX_STATE]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_STATE); dsync_serializer_encode_add(encoder, "mailbox_guid", guid_128_to_string(state->mailbox_guid)); dsync_serializer_encode_add(encoder, "last_uidvalidity", @@ -921,7 +944,7 @@ } str_truncate(namestr, str_len(namestr)-1); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX_TREE_NODE]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_TREE_NODE); dsync_serializer_encode_add(encoder, "name", str_c(namestr)); switch (node->existence) { case DSYNC_MAILBOX_NODE_NONEXISTENT: @@ -1066,7 +1089,7 @@ str = t_str_new(128); str_append_c(str, items[ITEM_MAILBOX_DELETE].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX_DELETE]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_DELETE); sep[0] = hierarchy_sep; sep[1] = '\0'; dsync_serializer_encode_add(encoder, "hierarchy_sep", sep); @@ -1213,7 +1236,7 @@ const char *value; str_append_c(str, items[ITEM_MAILBOX].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX); dsync_serializer_encode_add(encoder, "mailbox_guid", guid_128_to_string(dsync_box->mailbox_guid)); @@ -1387,7 +1410,7 @@ return; str_append_c(str, items[ITEM_MAILBOX_ATTRIBUTE].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX_ATTRIBUTE]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_ATTRIBUTE); type[0] = type[1] = '\0'; switch (attr->type) { @@ -1514,7 +1537,7 @@ char type[2]; str_append_c(str, items[ITEM_MAIL_CHANGE].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAIL_CHANGE]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_CHANGE); type[0] = type[1] = '\0'; switch (change->type) { @@ -1700,7 +1723,7 @@ string_t *str = t_str_new(128); str_append_c(str, items[ITEM_MAIL_REQUEST].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAIL_REQUEST]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_REQUEST); if (request->guid != NULL) dsync_serializer_encode_add(encoder, "guid", request->guid); if (request->uid != 0) { @@ -1752,7 +1775,7 @@ i_assert(ibc->value_output == NULL); str_append_c(str, items[ITEM_MAIL].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAIL]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL); if (mail->guid != NULL) dsync_serializer_encode_add(encoder, "guid", mail->guid); if (mail->uid != 0) @@ -1862,7 +1885,7 @@ string_t *str = t_str_new(128); str_append_c(str, items[ITEM_FINISH].chr); - encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_FINISH]); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_FINISH); if (error != NULL) dsync_serializer_encode_add(encoder, "error", error); if (mail_error != 0) { From dovecot at dovecot.org Fri Sep 4 08:22:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Sep 2015 08:22:46 +0000 Subject: dovecot-2.2: dsync: If remote disconnects, log the last sent/rec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3abdcbde7f39 changeset: 19090:3abdcbde7f39 user: Timo Sirainen date: Fri Sep 04 11:21:39 2015 +0300 description: dsync: If remote disconnects, log the last sent/recv state. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 39 ++++++++++++++++++++++------------- 1 files changed, 24 insertions(+), 15 deletions(-) diffs (78 lines): diff -r 3ab749f1ed71 -r 3abdcbde7f39 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Sep 04 11:17:30 2015 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Fri Sep 04 11:21:39 2015 +0300 @@ -169,6 +169,20 @@ unsigned int stopped:1; }; +static const char *dsync_ibc_stream_get_state(struct dsync_ibc_stream *ibc) +{ + if (!ibc->version_received) + return "version not received"; + else if (!ibc->handshake_received) + return "handshake not received"; + + return t_strdup_printf("last sent=%s%s, last recv=%s%s", + items[ibc->last_sent_item].name, + ibc->last_sent_item_eol ? " (EOL)" : "", + items[ibc->last_recv_item].name, + ibc->last_recv_item_eol ? " (EOL)" : ""); +} + static void dsync_ibc_stream_stop(struct dsync_ibc_stream *ibc) { ibc->stopped = TRUE; @@ -185,8 +199,10 @@ } while (i_stream_read(ibc->value_input) > 0); if (ibc->value_input->eof) { if (ibc->value_input->stream_errno != 0) { - i_error("dsync(%s): read() failed: %s", ibc->name, - i_stream_get_error(ibc->value_input)); + i_error("dsync(%s): read(%s) failed: %s (%s)", ibc->name, + i_stream_get_name(ibc->value_input), + i_stream_get_error(ibc->value_input), + dsync_ibc_stream_get_state(ibc)); dsync_ibc_stream_stop(ibc); return -1; } @@ -258,9 +274,10 @@ i_assert(ret == -1); if (ibc->value_output->stream_errno != 0) { - i_error("dsync(%s): read(%s) failed: %s", + i_error("dsync(%s): read(%s) failed: %s (%s)", ibc->name, i_stream_get_name(ibc->value_output), - i_stream_get_error(ibc->value_output)); + i_stream_get_error(ibc->value_output), + dsync_ibc_stream_get_state(ibc)); dsync_ibc_stream_stop(ibc); return -1; } @@ -294,13 +311,8 @@ static void dsync_ibc_stream_timeout(struct dsync_ibc_stream *ibc) { - i_error("dsync(%s): I/O has stalled, no activity for %u seconds " - "(last sent=%s%s, last recv=%s%s)", - ibc->name, ibc->timeout_secs, - items[ibc->last_sent_item].name, - ibc->last_sent_item_eol ? " (EOL)" : "", - items[ibc->last_recv_item].name, - ibc->last_recv_item_eol ? " (EOL)" : ""); + i_error("dsync(%s): I/O has stalled, no activity for %u seconds (%s)", + ibc->name, ibc->timeout_secs, dsync_ibc_stream_get_state(ibc)); ibc->ibc.timeout = TRUE; dsync_ibc_stream_stop(ibc); } @@ -396,10 +408,7 @@ i_assert(ibc->input->eof); str_printfa(error, "read(%s) failed: EOF", ibc->name); } - if (!ibc->version_received) - str_append(error, " (version not received)"); - else if (!ibc->handshake_received) - str_append(error, " (handshake not received)"); + str_printfa(error, " (%s)", dsync_ibc_stream_get_state(ibc)); i_error("%s", str_c(error)); dsync_ibc_stream_stop(ibc); return -1; From dovecot at dovecot.org Fri Sep 4 10:01:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Sep 2015 10:01:31 +0000 Subject: dovecot-2.2: acl: acl_object_list_next() should return -1 if acl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/48684e4af247 changeset: 19091:48684e4af247 user: Timo Sirainen date: Fri Sep 04 13:00:14 2015 +0300 description: acl: acl_object_list_next() should return -1 if acl refreshing failed earlier. diffstat: src/plugins/acl/acl-api.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 3abdcbde7f39 -r 48684e4af247 src/plugins/acl/acl-api.c --- a/src/plugins/acl/acl-api.c Fri Sep 04 11:21:39 2015 +0300 +++ b/src/plugins/acl/acl-api.c Fri Sep 04 13:00:14 2015 +0300 @@ -210,6 +210,9 @@ { const struct acl_rights *rights; + if (iter->failed) + return -1; + if (iter->idx == array_count(&iter->aclobj->rights)) return 0; From dovecot at dovecot.org Fri Sep 4 10:01:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Sep 2015 10:01:33 +0000 Subject: dovecot-2.2: acl: Install most of the header files. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ef68e77f7e71 changeset: 19092:ef68e77f7e71 user: Timo Sirainen date: Fri Sep 04 13:00:25 2015 +0300 description: acl: Install most of the header files. diffstat: src/plugins/acl/Makefile.am | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (23 lines): diff -r 48684e4af247 -r ef68e77f7e71 src/plugins/acl/Makefile.am --- a/src/plugins/acl/Makefile.am Fri Sep 04 13:00:14 2015 +0300 +++ b/src/plugins/acl/Makefile.am Fri Sep 04 13:00:25 2015 +0300 @@ -33,14 +33,17 @@ acl-storage.c noinst_HEADERS = \ + acl-backend-vfile.h \ + acl-shared-storage.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = \ acl-api.h \ acl-api-private.h \ - acl-backend-vfile.h \ acl-cache.h \ acl-global-file.h \ acl-lookup-dict.h \ acl-plugin.h \ - acl-shared-storage.h \ acl-storage.h doveadm_module_LTLIBRARIES = \ From dovecot at dovecot.org Sun Sep 6 19:29:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 06 Sep 2015 19:29:28 +0000 Subject: dovecot-2.2: lib-storage: Store pointer to mail_storage_service_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e5527c5bee2f changeset: 19094:e5527c5bee2f user: Timo Sirainen date: Sun Sep 06 22:28:07 2015 +0300 description: lib-storage: Store pointer to mail_storage_service_user to mail_user if it exists. diffstat: src/lib-storage/index/shared/shared-storage.c | 1 + src/lib-storage/mail-storage-service.c | 1 + src/lib-storage/mail-user.c | 1 + src/lib-storage/mail-user.h | 2 ++ 4 files changed, 5 insertions(+), 0 deletions(-) diffs (45 lines): diff -r 5d583816b7c9 -r e5527c5bee2f src/lib-storage/index/shared/shared-storage.c --- a/src/lib-storage/index/shared/shared-storage.c Sun Sep 06 22:27:29 2015 +0300 +++ b/src/lib-storage/index/shared/shared-storage.c Sun Sep 06 22:28:07 2015 +0300 @@ -255,6 +255,7 @@ owner = mail_user_alloc(userdomain, user->set_info, user->unexpanded_set); + owner->_service_user = user->_service_user; owner->creator = user; owner->autocreated = TRUE; owner->session_id = p_strdup(owner->pool, user->session_id); diff -r 5d583816b7c9 -r e5527c5bee2f src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Sun Sep 06 22:27:29 2015 +0300 +++ b/src/lib-storage/mail-storage-service.c Sun Sep 06 22:28:07 2015 +0300 @@ -642,6 +642,7 @@ mail_user_dup() */ mail_user = mail_user_alloc(user->input.username, user->user_info, user->user_set); + mail_user->_service_user = user; mail_user_set_home(mail_user, *home == '\0' ? NULL : home); mail_user_set_vars(mail_user, ctx->service->name, &user->input.local_ip, &user->input.remote_ip); diff -r 5d583816b7c9 -r e5527c5bee2f src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Sun Sep 06 22:27:29 2015 +0300 +++ b/src/lib-storage/mail-user.c Sun Sep 06 22:28:07 2015 +0300 @@ -521,6 +521,7 @@ user2 = mail_user_alloc(user->username, user->set_info, user->unexpanded_set); + user2->_service_user = user->_service_user; if (user->_home != NULL) mail_user_set_home(user2, user->_home); mail_user_set_vars(user2, user->service, diff -r 5d583816b7c9 -r e5527c5bee2f src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Sun Sep 06 22:27:29 2015 +0300 +++ b/src/lib-storage/mail-user.h Sun Sep 06 22:28:07 2015 +0300 @@ -23,6 +23,8 @@ /* User's creator if such exists. For example for autocreated shared mailbox users their creator is the logged in user. */ struct mail_user *creator; + /* Set if user was created via mail_storage_service. */ + struct mail_storage_service_user *_service_user; const char *username; /* don't access the home directly. It may be set lazily. */ From dovecot at dovecot.org Sun Sep 6 19:29:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 06 Sep 2015 19:29:17 +0000 Subject: dovecot-2.2: lib-storage: Added mail_storage_service_user_get_se... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5d583816b7c9 changeset: 19093:5d583816b7c9 user: Timo Sirainen date: Sun Sep 06 22:27:29 2015 +0300 description: lib-storage: Added mail_storage_service_user_get_service_ctx() diffstat: src/lib-storage/mail-storage-service.c | 6 ++++++ src/lib-storage/mail-storage-service.h | 2 ++ 2 files changed, 8 insertions(+), 0 deletions(-) diffs (28 lines): diff -r ef68e77f7e71 -r 5d583816b7c9 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Fri Sep 04 13:00:25 2015 +0300 +++ b/src/lib-storage/mail-storage-service.c Sun Sep 06 22:27:29 2015 +0300 @@ -1547,6 +1547,12 @@ return user->set_parser; } +struct mail_storage_service_ctx * +mail_storage_service_user_get_service_ctx(struct mail_storage_service_user *user) +{ + return user->service_ctx; +} + void *mail_storage_service_get_settings(struct master_service *service) { void **sets, *set; diff -r ef68e77f7e71 -r 5d583816b7c9 src/lib-storage/mail-storage-service.h --- a/src/lib-storage/mail-storage-service.h Fri Sep 04 13:00:25 2015 +0300 +++ b/src/lib-storage/mail-storage-service.h Sun Sep 06 22:27:29 2015 +0300 @@ -127,6 +127,8 @@ mail_storage_service_user_get_input(struct mail_storage_service_user *user); struct setting_parser_context * mail_storage_service_user_get_settings_parser(struct mail_storage_service_user *user); +struct mail_storage_service_ctx * +mail_storage_service_user_get_service_ctx(struct mail_storage_service_user *user); const struct var_expand_table * mail_storage_service_get_var_expand_table(struct mail_storage_service_ctx *ctx, From dovecot at dovecot.org Mon Sep 7 08:41:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 08:41:14 +0000 Subject: dovecot-2.2: auth: Aborting auth request didn't abort a pending ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a4c13ab4647b changeset: 19095:a4c13ab4647b user: Timo Sirainen date: Mon Sep 07 11:40:08 2015 +0300 description: auth: Aborting auth request didn't abort a pending proxy DNS lookup. diffstat: src/auth/auth-request.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r e5527c5bee2f -r a4c13ab4647b src/auth/auth-request.c --- a/src/auth/auth-request.c Sun Sep 06 22:28:07 2015 +0300 +++ b/src/auth/auth-request.c Mon Sep 07 11:40:08 2015 +0300 @@ -1834,6 +1834,7 @@ ctx = p_new(request->pool, struct auth_request_proxy_dns_lookup_ctx, 1); ctx->request = request; auth_request_ref(request); + request->dns_lookup_ctx = ctx; if (dns_lookup(host, &dns_set, auth_request_proxy_dns_callback, ctx, &ctx->dns_lookup) < 0) { From dovecot at dovecot.org Mon Sep 7 10:41:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 10:41:46 +0000 Subject: dovecot-2.2: imap: Don't allow IMAP METADATA to access Dovecot's... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ff2440bb1aa changeset: 19096:0ff2440bb1aa user: Timo Sirainen date: Mon Sep 07 13:40:41 2015 +0300 description: imap: Don't allow IMAP METADATA to access Dovecot's private server attributes. diffstat: src/lib-imap-storage/imap-metadata.c | 15 +++++++++------ 1 files changed, 9 insertions(+), 6 deletions(-) diffs (27 lines): diff -r a4c13ab4647b -r 0ff2440bb1aa src/lib-imap-storage/imap-metadata.c --- a/src/lib-imap-storage/imap-metadata.c Mon Sep 07 11:40:08 2015 +0300 +++ b/src/lib-imap-storage/imap-metadata.c Mon Sep 07 13:40:41 2015 +0300 @@ -100,14 +100,17 @@ i_assert((*key_r)[0] == '/'); *key_r += 1; } + if (strncmp(*key_r, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) == 0) { + /* Dovecot's internal attribute (mailbox or server). + don't allow accessing this. */ + return FALSE; + } + /* Add the server-prefix (after checking for the above internal + attribute). */ if (key_prefix != NULL) *key_r = t_strconcat(key_prefix, *key_r, NULL); - - /* skip over dovecot's internal attributes. (server metadata is handled - inside the private metadata.) */ - return (imtrans->server || - strncmp(*key_r, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, - strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) != 0); + return TRUE; } static int From dovecot at dovecot.org Mon Sep 7 11:10:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 11:10:04 +0000 Subject: dovecot-2.2: auth: Fixed passdb skip_password_check / result_suc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/43723e4bf750 changeset: 19097:43723e4bf750 user: Timo Sirainen date: Mon Sep 07 14:08:52 2015 +0300 description: auth: Fixed passdb skip_password_check / result_success=continue-fail handling If passdb returned success, but result_success=continue-fail, it means that the authentication didn't succeed. So we still want to check the password again and in general treat the request as unauthenticated (especially for the passdb { skip } setting). So the current logic means that there are 2 ways for the request to be treated as authenticated and skipping any password checking: 1) passdb lookup succeeding, with result_success=continue, continue-ok, return or return-ok 2) passdb lookup not succeeding, with result_failure=continue-ok or return-ok It's a bit questionable though if 2) should be allowed. diffstat: src/auth/auth-request.c | 14 ++++++++------ 1 files changed, 8 insertions(+), 6 deletions(-) diffs (35 lines): diff -r 0ff2440bb1aa -r 43723e4bf750 src/auth/auth-request.c --- a/src/auth/auth-request.c Mon Sep 07 13:40:41 2015 +0300 +++ b/src/auth/auth-request.c Mon Sep 07 14:08:52 2015 +0300 @@ -593,10 +593,18 @@ break; case AUTH_DB_RULE_CONTINUE: passdb_continue = TRUE; + if (*result == PASSDB_RESULT_OK) { + /* password was successfully verified. don't bother + checking it again. */ + request->skip_password_check = TRUE; + } break; case AUTH_DB_RULE_CONTINUE_OK: passdb_continue = TRUE; request->passdb_success = TRUE; + /* password was successfully verified. don't bother + checking it again. */ + request->skip_password_check = TRUE; break; case AUTH_DB_RULE_CONTINUE_FAIL: passdb_continue = TRUE; @@ -604,12 +612,6 @@ break; } - if (*result == PASSDB_RESULT_OK && passdb_continue) { - /* password was successfully verified. don't bother - checking it again. */ - request->skip_password_check = TRUE; - } - if (request->requested_login_user != NULL && *result == PASSDB_RESULT_OK) { auth_request_master_lookup_finish(request); From dovecot at dovecot.org Mon Sep 7 11:17:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 11:17:45 +0000 Subject: dovecot-2.2: dovecot.m4: Added LIBDOVECOT_ACL_INCLUDE Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/74fb129251e2 changeset: 19098:74fb129251e2 user: Timo Sirainen date: Mon Sep 07 14:16:39 2015 +0300 description: dovecot.m4: Added LIBDOVECOT_ACL_INCLUDE diffstat: dovecot-config.in.in | 1 + dovecot.m4 | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diffs (32 lines): diff -r 43723e4bf750 -r 74fb129251e2 dovecot-config.in.in --- a/dovecot-config.in.in Mon Sep 07 14:08:52 2015 +0300 +++ b/dovecot-config.in.in Mon Sep 07 14:16:39 2015 +0300 @@ -31,6 +31,7 @@ LIBDOVECOT_IMAPC_INCLUDE="-I$(incdir)/src/lib-imap-client -I$(incdir)/src/lib-storage/index/imapc" LIBDOVECOT_FTS_INCLUDE="-I$(incdir)/src/plugins/fts" LIBDOVECOT_NOTIFY_INCLUDE="-I$(incdir)/src/plugins/notify" +LIBDOVECOT_ACL_INCLUDE="-I$(incdir)/src/plugins/acl" dovecot_pkgincludedir= dovecot_pkglibdir= diff -r 43723e4bf750 -r 74fb129251e2 dovecot.m4 --- a/dovecot.m4 Mon Sep 07 14:08:52 2015 +0300 +++ b/dovecot.m4 Mon Sep 07 14:16:39 2015 +0300 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 16 +# serial 17 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -128,7 +128,7 @@ AX_SUBST_L([DOVECOT_INSTALLED], [DOVECOT_CFLAGS], [DOVECOT_LIBS], [DOVECOT_SSL_LIBS], [DOVECOT_SQL_LIBS], [DOVECOT_COMPRESS_LIBS]) AX_SUBST_L([LIBDOVECOT], [LIBDOVECOT_LOGIN], [LIBDOVECOT_SQL], [LIBDOVECOT_SSL], [LIBDOVECOT_COMPRESS], [LIBDOVECOT_LDA], [LIBDOVECOT_STORAGE], [LIBDOVECOT_DSYNC]) AX_SUBST_L([LIBDOVECOT_DEPS], [LIBDOVECOT_LOGIN_DEPS], [LIBDOVECOT_SQL_DEPS], [LIBDOVECOT_SSL_DEPS], [LIBDOVECOT_COMPRESS_DEPS], [LIBDOVECOT_LDA_DEPS], [LIBDOVECOT_STORAGE_DEPS], [LIBDOVECOT_DSYNC_DEPS]) - AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_FTS_INCLUDE], [LIBDOVECOT_NOTIFY_INCLUDE]) + AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_FTS_INCLUDE], [LIBDOVECOT_NOTIFY_INCLUDE], [LIBDOVECOT_ACL_INCLUDE]) AM_CONDITIONAL(DOVECOT_INSTALLED, test "$DOVECOT_INSTALLED" = "yes") From dovecot at dovecot.org Mon Sep 7 12:09:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 12:09:12 +0000 Subject: dovecot-2.2: lib-storage: mail_user_init() wasn't always expandi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ae159e69cb6f changeset: 19100:ae159e69cb6f user: Timo Sirainen date: Mon Sep 07 15:07:55 2015 +0300 description: lib-storage: mail_user_init() wasn't always expanding %h correctly to settings. This mainly affected users autocreated for accessing shared mailboxes. diffstat: src/lib-storage/mail-user.c | 42 +++++++++++++++++++++++++++++++----------- src/lib-storage/mail-user.h | 4 ++++ 2 files changed, 35 insertions(+), 11 deletions(-) diffs (91 lines): diff -r 1a49a8d12fe0 -r ae159e69cb6f src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Mon Sep 07 15:06:40 2015 +0300 +++ b/src/lib-storage/mail-user.c Mon Sep 07 15:07:55 2015 +0300 @@ -115,17 +115,17 @@ need_home_dir = user->_home == NULL && settings_vars_have_key(user->set_info, user->set, 'h', "home", &key, &value); - - /* expand mail_home setting before calling mail_user_get_home() */ - settings_var_expand_with_funcs(user->set_info, user->set, - user->pool, mail_user_var_expand_table(user), - mail_user_var_expand_func_table, user); - if (need_home_dir && mail_user_get_home(user, &home) <= 0) { user->error = p_strdup_printf(user->pool, "userdb didn't return a home directory, " "but %s used it (%%h): %s", key, value); } + + /* expand settings after we can expand %h */ + settings_var_expand_with_funcs(user->set_info, user->set, + user->pool, mail_user_var_expand_table(user), + mail_user_var_expand_func_table, user); + user->settings_expanded = TRUE; mail_user_expand_plugins_envs(user); /* autocreated users for shared mailboxes need to be fully initialized @@ -362,6 +362,26 @@ return ret; } +static void mail_user_get_mail_home(struct mail_user *user) +{ + const char *home = user->set->mail_home; + string_t *str; + + if (user->settings_expanded) { + user->_home = home[0] != '\0' ? home : NULL; + return; + } + /* we're still initializing user. need to do the expansion ourself. */ + i_assert(home[0] == SETTING_STRVAR_UNEXPANDED[0]); + home++; + if (home[0] == '\0') + return; + + str = t_str_new(128); + var_expand(str, home, mail_user_var_expand_table(user)); + user->_home = p_strdup(user->pool, str_c(str)); +} + int mail_user_get_home(struct mail_user *user, const char **home_r) { int ret; @@ -373,17 +393,17 @@ if (mail_user_auth_master_conn == NULL) { /* no userdb connection. we can only use mail_home setting. */ - user->_home = user->set->mail_home; + mail_user_get_mail_home(user); } else if ((ret = mail_user_userdb_lookup_home(user)) < 0) { /* userdb lookup failed */ return -1; } else if (ret == 0) { /* user doesn't exist */ user->nonexistent = TRUE; - } else if (user->_home == NULL && *user->set->mail_home != '\0') { - /* no home returned by userdb lookup, fallback to mail_home - setting. */ - user->_home = user->set->mail_home; + } else if (user->_home == NULL) { + /* no home returned by userdb lookup, fallback to + mail_home setting. */ + mail_user_get_mail_home(user); } user->home_looked_up = TRUE; diff -r 1a49a8d12fe0 -r ae159e69cb6f src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Mon Sep 07 15:06:40 2015 +0300 +++ b/src/lib-storage/mail-user.h Mon Sep 07 15:07:55 2015 +0300 @@ -69,6 +69,10 @@ unsigned int autocreated:1; /* mail_user_init() has been called */ unsigned int initialized:1; + /* SET_STR_VARS in user's all settings have been expanded. + This happens near the beginning of the user initialization, + so this is rarely needed to be checked. */ + unsigned int settings_expanded:1; /* Shortcut to mail_storage_settings.mail_debug */ unsigned int mail_debug:1; /* If INBOX can't be opened, log an error, but only once. */ From dovecot at dovecot.org Mon Sep 7 12:09:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 12:09:10 +0000 Subject: dovecot-2.2: lib-storage: Updated MAILBOX_ATTRIBUTE_KEY_IS_USER_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1a49a8d12fe0 changeset: 19099:1a49a8d12fe0 user: Timo Sirainen date: Mon Sep 07 15:06:40 2015 +0300 description: lib-storage: Updated MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE() to not allow private server attributes. diffstat: src/lib-storage/mailbox-attribute.h | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (22 lines): diff -r 74fb129251e2 -r 1a49a8d12fe0 src/lib-storage/mailbox-attribute.h --- a/src/lib-storage/mailbox-attribute.h Mon Sep 07 14:16:39 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.h Mon Sep 07 15:06:40 2015 +0300 @@ -20,12 +20,15 @@ #define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER \ MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT"server/" -/* User can get/set all non-pvt/ attributes and also pvt/server/ attributes. */ +/* User can get/set all non-pvt/ attributes and also pvt/server/ + (but not pvt/server/pvt/) attributes. */ #define MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE(key) \ (strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, \ strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) != 0 || \ - strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER, \ - strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER)) == 0) + (strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER, \ + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER)) == 0 && \ + strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, \ + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) != 0)) enum mail_attribute_type { MAIL_ATTRIBUTE_TYPE_PRIVATE, From dovecot at dovecot.org Mon Sep 7 13:40:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 13:40:38 +0000 Subject: dovecot-2.2: mail-storage: Added registration API for builtin ma... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a2e28ee1a673 changeset: 19101:a2e28ee1a673 user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: mail-storage: Added registration API for builtin mailbox attributes. diffstat: src/lib-storage/mail-storage.c | 2 + src/lib-storage/mailbox-attribute-private.h | 3 + src/lib-storage/mailbox-attribute.c | 312 +++++++++++++++++++++++++++- src/lib-storage/mailbox-attribute.h | 44 +++ 4 files changed, 350 insertions(+), 11 deletions(-) diffs (truncated from 463 to 300 lines): diff -r ae159e69cb6f -r a2e28ee1a673 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Mon Sep 07 15:07:55 2015 +0300 +++ b/src/lib-storage/mail-storage.c Sat Apr 25 11:42:06 2015 +0200 @@ -45,6 +45,7 @@ void mail_storage_init(void) { dsasl_clients_init(); + mailbox_attributes_init(); mailbox_lists_init(); mail_storage_hooks_init(); i_array_init(&mail_storage_classes, 8); @@ -60,6 +61,7 @@ array_free(&mail_storage_classes); mail_storage_hooks_deinit(); mailbox_lists_deinit(); + mailbox_attributes_deinit(); dsasl_clients_deinit(); } diff -r ae159e69cb6f -r a2e28ee1a673 src/lib-storage/mailbox-attribute-private.h --- a/src/lib-storage/mailbox-attribute-private.h Mon Sep 07 15:07:55 2015 +0300 +++ b/src/lib-storage/mailbox-attribute-private.h Sat Apr 25 11:42:06 2015 +0200 @@ -5,6 +5,9 @@ struct mailbox *box; }; +void mailbox_attributes_init(void); +void mailbox_attributes_deinit(void); + int mailbox_attribute_value_to_string(struct mail_storage *storage, const struct mail_attribute_value *value, const char **str_r); diff -r ae159e69cb6f -r a2e28ee1a673 src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Mon Sep 07 15:07:55 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.c Sat Apr 25 11:42:06 2015 +0200 @@ -1,19 +1,168 @@ /* Copyright (c) 2003-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "str.h" #include "istream.h" #include "mail-storage-private.h" +#include "bsearch-insert-pos.h" + +static ARRAY(struct mailbox_attribute_internal) mailbox_internal_attributes; +static pool_t mailbox_attribute_pool; + +void mailbox_attributes_init(void) +{ + mailbox_attribute_pool = + pool_alloconly_create("mailbox attributes", 2048); + i_array_init(&mailbox_internal_attributes, 32); +} + +void mailbox_attributes_deinit(void) +{ + pool_unref(&mailbox_attribute_pool); + array_free(&mailbox_internal_attributes); +} + +/* + * Internal attributes + */ + +static int +mailbox_attribute_internal_cmp( + const struct mailbox_attribute_internal *reg1, + const struct mailbox_attribute_internal *reg2) +{ + if (reg1->type != reg2->type) + return (int)reg1->type - (int)reg2->type; + return strcmp(reg1->key, reg2->key); +} + +void mailbox_attribute_register_internal( + const struct mailbox_attribute_internal *iattr) +{ + struct mailbox_attribute_internal ireg; + unsigned int insert_idx; + + (void)array_bsearch_insert_pos(&mailbox_internal_attributes, + iattr, mailbox_attribute_internal_cmp, &insert_idx); + + ireg = *iattr; + ireg.key = p_strdup(mailbox_attribute_pool, iattr->key); + array_insert(&mailbox_internal_attributes, insert_idx, &ireg, 1); +} + +static const struct mailbox_attribute_internal * +mailbox_internal_attribute_get(enum mail_attribute_type type, + const char *key) +{ + struct mailbox_attribute_internal dreg; + unsigned int insert_idx; + + memset(&dreg, 0, sizeof(dreg)); + dreg.type = type; + dreg.key = key; + + if (!array_bsearch_insert_pos(&mailbox_internal_attributes, + &dreg, mailbox_attribute_internal_cmp, + &insert_idx)) + return NULL; + + return array_idx(&mailbox_internal_attributes, insert_idx); +} + +static void +mailbox_internal_attributes_get(enum mail_attribute_type type, + const char *prefix, ARRAY_TYPE(const_string) *attrs) +{ + const struct mailbox_attribute_internal *regs; + struct mailbox_attribute_internal dreg; + char *bare_prefix; + size_t plen; + unsigned int count, i; + + bare_prefix = t_strdup_noconst(prefix); + plen = strlen(bare_prefix); + if (plen > 0 && bare_prefix[plen-1] == '/') { + bare_prefix[plen-1] = '\0'; + plen--; + } + + memset(&dreg, 0, sizeof(dreg)); + dreg.type = type; + dreg.key = bare_prefix; + + (void)array_bsearch_insert_pos(&mailbox_internal_attributes, + &dreg, mailbox_attribute_internal_cmp, &i); + + regs = array_get(&mailbox_internal_attributes, &count); + for (; i < count; i++) { + const char *key = regs[i].key; + + if (regs[i].type != type) + return; + if (plen > 0) { + if (strncmp(key, bare_prefix, plen) != 0) + return; + if (key[plen] != '/' && strlen(key) != plen) + return; + /* remove prefix */ + key += plen + 1; + } + + array_append(attrs, &key, 1); + } +} /* * Attribute API */ +static int +mailbox_attribute_set_common(struct mailbox_transaction_context *t, + enum mail_attribute_type type, const char *key, + const struct mail_attribute_value *value) +{ + const struct mailbox_attribute_internal *iattr; + + iattr = mailbox_internal_attribute_get(type, key); + + /* allow internal server attribute only for inbox */ + if (iattr != NULL && !t->box->inbox_any && + strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER, + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER)) == 0) + iattr = NULL; + + /* handle internal attribute */ + if (iattr != NULL) { + switch (iattr->rank) { + case MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT: + case MAIL_ATTRIBUTE_INTERNAL_RANK_OVERRIDE: + /* notify about assignment */ + if (iattr->set != NULL && iattr->set(t, value) < 0) + return -1; + break; + case MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY: + if (iattr->set == NULL) { + mail_storage_set_error(t->box->storage, MAIL_ERROR_PARAMS, t_strdup_printf( + "The /%s/%s attribute cannot be changed", + (type == MAIL_ATTRIBUTE_TYPE_SHARED ? "shared" : "private"), key)); + return -1; + } + /* assign internal attribute */ + return iattr->set(t, value); + default: + i_unreached(); + } + } + + return t->box->v.attribute_set(t, type, key, value); +} + int mailbox_attribute_set(struct mailbox_transaction_context *t, enum mail_attribute_type type, const char *key, const struct mail_attribute_value *value) { - return t->box->v.attribute_set(t, type, key, value); + return mailbox_attribute_set_common(t, type, key, value); } int mailbox_attribute_unset(struct mailbox_transaction_context *t, @@ -22,7 +171,7 @@ struct mail_attribute_value value; memset(&value, 0, sizeof(value)); - return t->box->v.attribute_set(t, type, key, &value); + return mailbox_attribute_set_common(t, type, key, &value); } int mailbox_attribute_value_to_string(struct mail_storage *storage, @@ -59,14 +208,74 @@ return 0; } +static int +mailbox_attribute_get_common(struct mailbox_transaction_context *t, + enum mail_attribute_type type, const char *key, + struct mail_attribute_value *value_r) +{ + const struct mailbox_attribute_internal *iattr; + int ret; + + iattr = mailbox_internal_attribute_get(type, key); + + /* allow internal server attributes only for the inbox */ + if (iattr != NULL && !t->box->inbox_user && + strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER, + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER)) == 0) + iattr = NULL; + + /* internal attribute */ + if (iattr != NULL) { + switch (iattr->rank) { + case MAIL_ATTRIBUTE_INTERNAL_RANK_OVERRIDE: + if ((ret = iattr->get(t, value_r)) != 0) { + if (ret < 0) + return -1; + value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_READONLY; + return 1; + } + case MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT: + break; + case MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY: + if ((ret = iattr->get(t, value_r)) <= 0) + return ret; + value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_READONLY; + return 1; + default: + i_unreached(); + } + } + + /* user entries */ + if ((ret = t->box->v.attribute_get(t, type, key, value_r)) != 0) + return ret; + + /* default entries */ + if (iattr != NULL) { + switch (iattr->rank) { + case MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT: + if ((ret = iattr->get(t, value_r)) < 0) + return ret; + if (ret > 0) { + value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_READONLY; + return 1; + } + case MAIL_ATTRIBUTE_INTERNAL_RANK_OVERRIDE: + break; + default: + i_unreached(); + } + } + return 0; +} + int mailbox_attribute_get(struct mailbox_transaction_context *t, enum mail_attribute_type type, const char *key, struct mail_attribute_value *value_r) { int ret; - memset(value_r, 0, sizeof(*value_r)); - if ((ret = t->box->v.attribute_get(t, type, key, value_r)) <= 0) + if ((ret = mailbox_attribute_get_common(t, type, key, value_r)) <= 0) return ret; i_assert(value_r->value != NULL); return 1; @@ -80,29 +289,110 @@ memset(value_r, 0, sizeof(*value_r)); value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_INT_STREAMS; - if ((ret = t->box->v.attribute_get(t, type, key, value_r)) <= 0) + if ((ret = mailbox_attribute_get_common(t, type, key, value_r)) <= 0) return ret; i_assert(value_r->value != NULL || value_r->value_stream != NULL); return 1; From dovecot at dovecot.org Mon Sep 7 13:40:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 13:40:50 +0000 Subject: dovecot-2.2: lib-storage: Added MAIL_ATTRIBUTE_INTERNAL_FLAG_CHI... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/31f8e377b632 changeset: 19102:31f8e377b632 user: Timo Sirainen date: Mon Sep 07 16:38:24 2015 +0300 description: lib-storage: Added MAIL_ATTRIBUTE_INTERNAL_FLAG_CHILDREN This also adds the key to get() and set() functions, so they can know exactly what key is wanted to be accessed. diffstat: src/lib-storage/mailbox-attribute.c | 35 +++++++++++++++++++++++++---------- src/lib-storage/mailbox-attribute.h | 10 ++++++++-- 2 files changed, 33 insertions(+), 12 deletions(-) diffs (116 lines): diff -r a2e28ee1a673 -r 31f8e377b632 src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-storage/mailbox-attribute.c Mon Sep 07 16:38:24 2015 +0300 @@ -55,6 +55,7 @@ mailbox_internal_attribute_get(enum mail_attribute_type type, const char *key) { + const struct mailbox_attribute_internal *iattr; struct mailbox_attribute_internal dreg; unsigned int insert_idx; @@ -62,12 +63,26 @@ dreg.type = type; dreg.key = key; - if (!array_bsearch_insert_pos(&mailbox_internal_attributes, - &dreg, mailbox_attribute_internal_cmp, - &insert_idx)) + if (array_bsearch_insert_pos(&mailbox_internal_attributes, + &dreg, mailbox_attribute_internal_cmp, + &insert_idx)) { + /* exact match */ + return array_idx(&mailbox_internal_attributes, insert_idx); + } + if (insert_idx == 0) { + /* not found at all */ return NULL; - - return array_idx(&mailbox_internal_attributes, insert_idx); + } + iattr = array_idx(&mailbox_internal_attributes, insert_idx-1); + if (strncmp(iattr->key, key, strlen(iattr->key)) != 0) { + /* iattr isn't a prefix of key */ + return NULL; + } else if ((iattr->flags & MAIL_ATTRIBUTE_INTERNAL_FLAG_CHILDREN) != 0) { + /* iattr is a prefix of key and it wants to handle the key */ + return iattr; + } else { + return NULL; + } } static void @@ -138,7 +153,7 @@ case MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT: case MAIL_ATTRIBUTE_INTERNAL_RANK_OVERRIDE: /* notify about assignment */ - if (iattr->set != NULL && iattr->set(t, value) < 0) + if (iattr->set != NULL && iattr->set(t, key, value) < 0) return -1; break; case MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY: @@ -149,7 +164,7 @@ return -1; } /* assign internal attribute */ - return iattr->set(t, value); + return iattr->set(t, key, value); default: i_unreached(); } @@ -228,7 +243,7 @@ if (iattr != NULL) { switch (iattr->rank) { case MAIL_ATTRIBUTE_INTERNAL_RANK_OVERRIDE: - if ((ret = iattr->get(t, value_r)) != 0) { + if ((ret = iattr->get(t, key, value_r)) != 0) { if (ret < 0) return -1; value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_READONLY; @@ -237,7 +252,7 @@ case MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT: break; case MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY: - if ((ret = iattr->get(t, value_r)) <= 0) + if ((ret = iattr->get(t, key, value_r)) <= 0) return ret; value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_READONLY; return 1; @@ -254,7 +269,7 @@ if (iattr != NULL) { switch (iattr->rank) { case MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT: - if ((ret = iattr->get(t, value_r)) < 0) + if ((ret = iattr->get(t, key, value_r)) < 0) return ret; if (ret > 0) { value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_READONLY; diff -r a2e28ee1a673 -r 31f8e377b632 src/lib-storage/mailbox-attribute.h --- a/src/lib-storage/mailbox-attribute.h Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-storage/mailbox-attribute.h Mon Sep 07 16:38:24 2015 +0300 @@ -82,16 +82,22 @@ MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY }; +enum mail_attribute_internal_flags { + /* Apply this attribute to the given key and its children. */ + MAIL_ATTRIBUTE_INTERNAL_FLAG_CHILDREN = 0x01 +}; + struct mailbox_attribute_internal { enum mail_attribute_type type; const char *key; enum mail_attribute_internal_rank rank; + enum mail_attribute_internal_flags flags; /* Get the value of this internal attribute */ - int (*get)(struct mailbox_transaction_context *t, + int (*get)(struct mailbox_transaction_context *t, const char *key, struct mail_attribute_value *value_r); /* Set the value of this internal attribute */ - int (*set)(struct mailbox_transaction_context *t, + int (*set)(struct mailbox_transaction_context *t, const char *key, const struct mail_attribute_value *value); }; From dovecot at dovecot.org Mon Sep 7 13:41:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 13:41:07 +0000 Subject: dovecot-2.2: lib-storage: Allow MAIL_ATTRIBUTE_INTERNAL_RANK_DEF... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/04ed3c9c330f changeset: 19104:04ed3c9c330f user: Timo Sirainen date: Mon Sep 07 16:23:40 2015 +0300 description: lib-storage: Allow MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT to use get=NULL This is useful for registering internal attributes whose only purpose is to be able to set/get them via dict. (Because normally the dict access would be denied completely.) diffstat: src/lib-storage/mailbox-attribute.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 5d3c7d882bfb -r 04ed3c9c330f src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Mon Sep 07 16:22:11 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.c Mon Sep 07 16:23:40 2015 +0300 @@ -282,8 +282,12 @@ if (iattr != NULL) { switch (iattr->rank) { case MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT: - if ((ret = iattr->get(t, key, value_r)) < 0) - return ret; + if (iattr->get == NULL) + ret = 0; + else { + if ((ret = iattr->get(t, key, value_r)) < 0) + return ret; + } if (ret > 0) { value_r->flags |= MAIL_ATTRIBUTE_VALUE_FLAG_READONLY; return 1; From dovecot at dovecot.org Mon Sep 7 13:40:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 13:40:56 +0000 Subject: dovecot-2.2: lib-storage: Allow set/get for Dovecot-private attr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5d3c7d882bfb changeset: 19103:5d3c7d882bfb user: Timo Sirainen date: Mon Sep 07 16:22:11 2015 +0300 description: lib-storage: Allow set/get for Dovecot-private attributes via internal attributes. This allows registering attributes with MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT prefix and having them be get/set via dict, instead of failing them. diffstat: src/lib-storage/index/index-attribute.c | 6 ++++-- src/lib-storage/mail-storage-private.h | 2 ++ src/lib-storage/mailbox-attribute.c | 21 +++++++++++++++++---- 3 files changed, 23 insertions(+), 6 deletions(-) diffs (80 lines): diff -r 31f8e377b632 -r 5d3c7d882bfb src/lib-storage/index/index-attribute.c --- a/src/lib-storage/index/index-attribute.c Mon Sep 07 16:38:24 2015 +0300 +++ b/src/lib-storage/index/index-attribute.c Mon Sep 07 16:22:11 2015 +0300 @@ -197,7 +197,8 @@ time_t ts = value->last_change != 0 ? value->last_change : ioloop_time; int ret = 0; - if (!MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE(key)) { + if (!t->internal_attribute && + !MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE(key)) { mail_storage_set_error(t->box->storage, MAIL_ERROR_PARAMS, "Internal attributes cannot be changed directly"); return -1; @@ -237,7 +238,8 @@ memset(value_r, 0, sizeof(*value_r)); - if (!MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE(key)) + if (!t->internal_attribute && + !MAILBOX_ATTRIBUTE_KEY_IS_USER_ACCESSIBLE(key)) return 0; if (index_storage_get_dict(t->box, type, &dict, &mailbox_prefix) < 0) diff -r 31f8e377b632 -r 5d3c7d882bfb src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Mon Sep 07 16:38:24 2015 +0300 +++ b/src/lib-storage/mail-storage-private.h Mon Sep 07 16:22:11 2015 +0300 @@ -529,6 +529,8 @@ unsigned int stats_track:1; /* We've done some non-transactional (e.g. dovecot-uidlist updates) */ unsigned int nontransactional_changes:1; + /* FIXME: v2.3: this should be in attribute_get/set() parameters */ + unsigned int internal_attribute:1; }; union mail_search_module_context { diff -r 31f8e377b632 -r 5d3c7d882bfb src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Mon Sep 07 16:38:24 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.c Mon Sep 07 16:22:11 2015 +0300 @@ -138,7 +138,8 @@ const struct mail_attribute_value *value) { const struct mailbox_attribute_internal *iattr; - + int ret; + iattr = mailbox_internal_attribute_get(type, key); /* allow internal server attribute only for inbox */ @@ -170,7 +171,13 @@ } } - return t->box->v.attribute_set(t, type, key, value); + /* FIXME: v2.3 should move the internal_attribute to attribute_set() + parameter (as flag). not done yet for API backwards compatibility */ + t->internal_attribute = iattr != NULL && + iattr->rank != MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY; + ret = t->box->v.attribute_set(t, type, key, value); + t->internal_attribute = FALSE; + return ret; } int mailbox_attribute_set(struct mailbox_transaction_context *t, @@ -261,8 +268,14 @@ } } - /* user entries */ - if ((ret = t->box->v.attribute_get(t, type, key, value_r)) != 0) + /* user entries - FIXME: v2.3 should move the internal_attribute to + attribute_get() parameter (as flag). not done yet for API backwards + compatibility */ + t->internal_attribute = iattr != NULL && + iattr->rank != MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY; + ret = t->box->v.attribute_get(t, type, key, value_r); + t->internal_attribute = FALSE; + if (ret != 0) return ret; /* default entries */ From dovecot at dovecot.org Mon Sep 7 13:41:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 13:41:07 +0000 Subject: dovecot-2.2: lib-storage: Added mailbox_attribute_register_inter... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2e420ceaee1d changeset: 19105:2e420ceaee1d user: Timo Sirainen date: Mon Sep 07 16:39:28 2015 +0300 description: lib-storage: Added mailbox_attribute_register_internals() Just for making it easier to register multiple attributes. diffstat: src/lib-storage/mailbox-attribute.c | 9 +++++++++ src/lib-storage/mailbox-attribute.h | 2 ++ 2 files changed, 11 insertions(+), 0 deletions(-) diffs (31 lines): diff -r 04ed3c9c330f -r 2e420ceaee1d src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Mon Sep 07 16:23:40 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.c Mon Sep 07 16:39:28 2015 +0300 @@ -51,6 +51,15 @@ array_insert(&mailbox_internal_attributes, insert_idx, &ireg, 1); } +void mailbox_attribute_register_internals( + const struct mailbox_attribute_internal *iattrs, unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) + mailbox_attribute_register_internal(&iattrs[i]); +} + static const struct mailbox_attribute_internal * mailbox_internal_attribute_get(enum mail_attribute_type type, const char *key) diff -r 04ed3c9c330f -r 2e420ceaee1d src/lib-storage/mailbox-attribute.h --- a/src/lib-storage/mailbox-attribute.h Mon Sep 07 16:23:40 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.h Mon Sep 07 16:39:28 2015 +0300 @@ -103,6 +103,8 @@ void mailbox_attribute_register_internal( const struct mailbox_attribute_internal *iattr); +void mailbox_attribute_register_internals( + const struct mailbox_attribute_internal *iattrs, unsigned int count); /* * Attribute API From dovecot at dovecot.org Mon Sep 7 14:11:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 14:11:22 +0000 Subject: dovecot-2.2: auth: Ignore first passdbs that contain skip=unauth... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/757a566a0d1c changeset: 19106:757a566a0d1c user: Timo Sirainen date: Mon Sep 07 17:10:19 2015 +0300 description: auth: Ignore first passdbs that contain skip=unauthenticated. They can never match anything. diffstat: src/auth/auth.c | 12 ++++++++++++ 1 files changed, 12 insertions(+), 0 deletions(-) diffs (29 lines): diff -r 2e420ceaee1d -r 757a566a0d1c src/auth/auth.c --- a/src/auth/auth.c Mon Sep 07 16:39:28 2015 +0300 +++ b/src/auth/auth.c Mon Sep 07 17:10:19 2015 +0300 @@ -227,6 +227,13 @@ if (passdbs[i]->master) continue; + /* passdb { skip=unauthenticated } as the first passdb doesn't + make sense, since user is never authenticated at that point. + skip over them silently. */ + if (auth->passdbs == NULL && + auth_passdb_skip_parse(passdbs[i]->skip) == AUTH_PASSDB_SKIP_UNAUTHENTICATED) + continue; + auth_passdb_preinit(auth, passdbs[i], &auth->passdbs); passdb_count++; last_passdb = i; @@ -238,6 +245,11 @@ if (!passdbs[i]->master) continue; + /* skip skip=unauthenticated, as explained above */ + if (auth->masterdbs == NULL && + auth_passdb_skip_parse(passdbs[i]->skip) == AUTH_PASSDB_SKIP_UNAUTHENTICATED) + continue; + if (passdbs[i]->deny) i_fatal("Master passdb can't have deny=yes"); if (passdbs[i]->pass && passdb_count == 0) { From dovecot at dovecot.org Mon Sep 7 15:51:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 15:51:35 +0000 Subject: dovecot-2.2: login proxy: Added delayed disconnection of clients... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2eeef1619161 changeset: 19107:2eeef1619161 user: Timo Sirainen date: Mon Sep 07 18:50:24 2015 +0300 description: login proxy: Added delayed disconnection of clients on server mass-disconnect. login_proxy_max_disconnect_delay setting (default 0 = disabled) controls for how long time period the disconnections are spread to. The idea behind this is to avoid load spikes due to client reconnections when a backend server dies or is restarted. diffstat: src/login-common/login-proxy-state.h | 14 +++ src/login-common/login-proxy.c | 134 +++++++++++++++++++++++++++++++++- src/login-common/login-settings.c | 2 + src/login-common/login-settings.h | 1 + 4 files changed, 144 insertions(+), 7 deletions(-) diffs (truncated from 326 to 300 lines): diff -r 757a566a0d1c -r 2eeef1619161 src/login-common/login-proxy-state.h --- a/src/login-common/login-proxy-state.h Mon Sep 07 17:10:19 2015 +0300 +++ b/src/login-common/login-proxy-state.h Mon Sep 07 18:50:24 2015 +0300 @@ -7,8 +7,22 @@ struct ip_addr ip; in_port_t port; + /* These are used to spread client-visible disconnects over longer + periods of time to avoid reconnect spikes when a server dies. + + If num_disconnects_since_ts=0 when server disconnects us, it's + increased and disconnect_timestamp is updated. Afterwards it's + increased for each new disconnection. num_disconnects_since_ts gets + reset back to zero whenever a) last_success gets updated or b) + num_delayed_client_disconnects drops to 0. */ + struct timeval disconnect_timestamp; + unsigned int num_disconnects_since_ts; + unsigned int num_delayed_client_disconnects; + /* these are tracking connect()s, not necessarily logins: */ unsigned int num_waiting_connections; + /* number of connections we're proxying now (post-login) */ + unsigned int num_proxying_connections; struct timeval last_failure; struct timeval last_success; }; diff -r 757a566a0d1c -r 2eeef1619161 src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Mon Sep 07 17:10:19 2015 +0300 +++ b/src/login-common/login-proxy.c Mon Sep 07 18:50:24 2015 +0300 @@ -16,6 +16,8 @@ #include "login-proxy-state.h" #include "login-proxy.h" +#include + #define MAX_PROXY_INPUT_SIZE 4096 #define OUTBUF_THRESHOLD 1024 #define LOGIN_PROXY_DIE_IDLE_SECS 2 @@ -24,6 +26,7 @@ #define KILLED_BY_ADMIN_REASON "Killed by admin" #define PROXY_IMMEDIATE_FAILURE_SECS 30 #define PROXY_CONNECT_RETRY_MSECS 1000 +#define PROXY_DISCONNECT_INTERVAL_MSECS 100 struct login_proxy { struct login_proxy *prev, *next; @@ -53,21 +56,27 @@ unsigned int connected:1; unsigned int destroying:1; unsigned int disconnecting:1; + unsigned int delayed_disconnect:1; unsigned int num_waiting_connections_updated:1; }; static struct login_proxy_state *proxy_state; static struct login_proxy *login_proxies = NULL; static struct login_proxy *login_proxies_pending = NULL; +static struct login_proxy *login_proxies_disconnecting = NULL; static struct ipc_server *login_proxy_ipc_server; static int login_proxy_connect(struct login_proxy *proxy); static void login_proxy_disconnect(struct login_proxy *proxy); static void login_proxy_ipc_cmd(struct ipc_cmd *cmd, const char *line); +static void login_proxy_free_final(struct login_proxy *proxy); static void login_proxy_free_reason(struct login_proxy **_proxy, const char *reason) ATTR_NULL(2); +static void +login_proxy_free_delayed(struct login_proxy **_proxy, const char *reason) + ATTR_NULL(2); static void login_proxy_free_errno(struct login_proxy **proxy, int err, bool server) @@ -77,7 +86,10 @@ reason = err == 0 || err == EPIPE ? t_strdup_printf("Disconnected by %s", who) : t_strdup_printf("Disconnected by %s: %s", who, strerror(errno)); - login_proxy_free_reason(proxy, reason); + if (server) + login_proxy_free_delayed(proxy, reason); + else + login_proxy_free_reason(proxy, reason); } static void server_input(struct login_proxy *proxy) @@ -136,6 +148,16 @@ } } +static void proxy_client_disconnected_input(struct login_proxy *proxy) +{ + unsigned char buf[OUTBUF_THRESHOLD]; + + /* we're already disconnected from server. either wait for + disconnection timeout or for client to disconnect itself. */ + if (net_receive(proxy->client_fd, buf, sizeof(buf)) < 0) + login_proxy_free_final(proxy); +} + static int server_output(struct login_proxy *proxy) { proxy->last_io = ioloop_time; @@ -283,6 +305,8 @@ proxy->state_rec->last_success = ioloop_timeval; i_assert(proxy->state_rec->num_waiting_connections > 0); proxy->state_rec->num_waiting_connections--; + proxy->state_rec->num_proxying_connections++; + proxy->state_rec->num_disconnects_since_ts = 0; if ((proxy->ssl_flags & PROXY_SSL_FLAG_YES) != 0 && (proxy->ssl_flags & PROXY_SSL_FLAG_STARTTLS) == 0) { @@ -408,6 +432,10 @@ i_assert(proxy->state_rec->num_waiting_connections > 0); proxy->state_rec->num_waiting_connections--; } + if (proxy->connected) { + i_assert(proxy->state_rec->num_proxying_connections > 0); + proxy->state_rec->num_proxying_connections--; + } if (proxy->server_io != NULL) io_remove(&proxy->server_io); @@ -421,6 +449,17 @@ static void login_proxy_free_final(struct login_proxy *proxy) { + if (proxy->delayed_disconnect) { + DLLIST_REMOVE(&login_proxies_disconnecting, proxy); + + i_assert(proxy->state_rec->num_delayed_client_disconnects > 0); + if (--proxy->state_rec->num_delayed_client_disconnects == 0) + proxy->state_rec->num_disconnects_since_ts = 0; + timeout_remove(&proxy->to); + } + + if (proxy->client_io != NULL) + io_remove(&proxy->client_io); if (proxy->client_output != NULL) o_stream_destroy(&proxy->client_output); if (proxy->client_fd != -1) @@ -431,12 +470,69 @@ i_free(proxy); } +static unsigned int login_proxy_delay_disconnect(struct login_proxy *proxy) +{ + struct login_proxy_record *rec = proxy->state_rec; + const unsigned int max_delay = + proxy->client->set->login_proxy_max_disconnect_delay; + struct timeval disconnect_time_offset; + unsigned int max_disconnects_per_sec, delay_msecs_since_ts, max_conns; + int delay_msecs; + + if (rec->num_disconnects_since_ts == 0) { + rec->disconnect_timestamp = ioloop_timeval; + /* start from a slightly random timestamp. this way all proxy + processes will disconnect at slightly different times to + spread the load. */ + timeval_add_msecs(&rec->disconnect_timestamp, + rand() % PROXY_DISCONNECT_INTERVAL_MSECS); + } + rec->num_disconnects_since_ts++; + if (proxy->to != NULL) { + /* we were already lazily disconnecting this */ + return 0; + } + if (max_delay == 0) { + /* delaying is disabled */ + return 0; + } + max_conns = rec->num_proxying_connections + rec->num_disconnects_since_ts; + max_disconnects_per_sec = (max_conns + max_delay-1) / max_delay; + if (rec->num_disconnects_since_ts <= max_disconnects_per_sec && + rec->num_delayed_client_disconnects == 0) { + /* wait delaying until we have 1 second's worth of clients + disconnected */ + return 0; + } + + /* see at which time we should be disconnecting the client. + do it in 100ms intervals so the timeouts are triggered together. */ + disconnect_time_offset = rec->disconnect_timestamp; + delay_msecs_since_ts = PROXY_DISCONNECT_INTERVAL_MSECS * + (max_delay * rec->num_disconnects_since_ts * + (1000/PROXY_DISCONNECT_INTERVAL_MSECS) / max_conns); + timeval_add_msecs(&disconnect_time_offset, delay_msecs_since_ts); + delay_msecs = timeval_diff_msecs(&disconnect_time_offset, &ioloop_timeval); + if (delay_msecs <= 0) { + /* we already reached the time */ + return 0; + } + + rec->num_delayed_client_disconnects++; + proxy->delayed_disconnect = TRUE; + proxy->to = timeout_add(delay_msecs, login_proxy_free_final, proxy); + DLLIST_PREPEND(&login_proxies_disconnecting, proxy); + return delay_msecs; +} + static void ATTR_NULL(2) -login_proxy_free_reason(struct login_proxy **_proxy, const char *reason) +login_proxy_free_full(struct login_proxy **_proxy, const char *reason, + bool delayed) { struct login_proxy *proxy = *_proxy; struct client *client = proxy->client; const char *ipstr; + unsigned int delay_ms = 0; *_proxy = NULL; @@ -444,18 +540,23 @@ return; proxy->destroying = TRUE; + /* we'll disconnect server side in any case. */ login_proxy_disconnect(proxy); if (proxy->client_fd != -1) { /* detached proxy */ DLLIST_REMOVE(&login_proxies, proxy); + if (delayed) + delay_ms = login_proxy_delay_disconnect(proxy); + ipstr = net_ip2addr(&proxy->client->ip); client_log(proxy->client, t_strdup_printf( - "proxy(%s): disconnecting %s%s", + "proxy(%s): disconnecting %s%s%s", proxy->client->virtual_user, ipstr != NULL ? ipstr : "", - reason == NULL ? "" : t_strdup_printf(" (%s)", reason))); + reason == NULL ? "" : t_strdup_printf(" (%s)", reason), + delay_ms == 0 ? "" : t_strdup_printf(" - disconnecting client in %ums", delay_ms))); if (proxy->client_io != NULL) io_remove(&proxy->client_io); @@ -469,12 +570,29 @@ if (proxy->callback != NULL) proxy->callback(proxy->client); } - login_proxy_free_final(proxy); + if (delay_ms == 0) + login_proxy_free_final(proxy); + else { + proxy->client_io = io_add(proxy->client_fd, IO_READ, + proxy_client_disconnected_input, proxy); + } client->login_proxy = NULL; client_unref(&client); } +static void ATTR_NULL(2) +login_proxy_free_reason(struct login_proxy **_proxy, const char *reason) +{ + login_proxy_free_full(_proxy, reason, FALSE); +} + +static void ATTR_NULL(2) +login_proxy_free_delayed(struct login_proxy **_proxy, const char *reason) +{ + login_proxy_free_full(_proxy, reason, TRUE); +} + void login_proxy_free(struct login_proxy **_proxy) { login_proxy_free_reason(_proxy, NULL); @@ -680,7 +798,7 @@ next = proxy->next; if (strcmp(proxy->client->virtual_user, args[0]) == 0) { - login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON); + login_proxy_free_delayed(&proxy, KILLED_BY_ADMIN_REASON); count++; } } @@ -726,7 +844,7 @@ if (director_username_hash(proxy->client) == hash && !net_ip_compare(&proxy->ip, &except_ip)) { - login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON); + login_proxy_free_delayed(&proxy, KILLED_BY_ADMIN_REASON); count++; } } @@ -799,6 +917,8 @@ proxy = login_proxies; login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON); } + while (login_proxies_disconnecting != NULL) + login_proxy_free_final(login_proxies_disconnecting); if (login_proxy_ipc_server != NULL) ipc_server_deinit(&login_proxy_ipc_server); login_proxy_state_deinit(&proxy_state); diff -r 757a566a0d1c -r 2eeef1619161 src/login-common/login-settings.c --- a/src/login-common/login-settings.c Mon Sep 07 17:10:19 2015 +0300 +++ b/src/login-common/login-settings.c Mon Sep 07 18:50:24 2015 +0300 @@ -26,6 +26,7 @@ From dovecot at dovecot.org Mon Sep 7 17:25:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 17:25:41 +0000 Subject: dovecot-2.2: director: Added director_user_kick_delay setting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3ebba506c2c7 changeset: 19108:3ebba506c2c7 user: Timo Sirainen date: Mon Sep 07 20:24:25 2015 +0300 description: director: Added director_user_kick_delay setting. This replaces the hardcoded 2 seconds delay. This setting specifies how long to wait for after user has been kicked from all directors before letting the user login to the new server. This timeout should be large enough that the user's existing processes in the old backend should be finished. diffstat: src/director/director-settings.c | 2 ++ src/director/director-settings.h | 1 + src/director/director.c | 6 ++++-- 3 files changed, 7 insertions(+), 2 deletions(-) diffs (53 lines): diff -r 2eeef1619161 -r 3ebba506c2c7 src/director/director-settings.c --- a/src/director/director-settings.c Mon Sep 07 18:50:24 2015 +0300 +++ b/src/director/director-settings.c Mon Sep 07 20:24:25 2015 +0300 @@ -72,6 +72,7 @@ DEF(SET_STR, director_mail_servers), DEF(SET_STR, director_username_hash), DEF(SET_TIME, director_user_expire), + DEF(SET_TIME, director_user_kick_delay), DEF(SET_IN_PORT, director_doveadm_port), DEF(SET_BOOL, director_consistent_hashing), @@ -85,6 +86,7 @@ .director_mail_servers = "", .director_username_hash = "%Lu", .director_user_expire = 60*15, + .director_user_kick_delay = 2, .director_doveadm_port = 0 }; diff -r 2eeef1619161 -r 3ebba506c2c7 src/director/director-settings.h --- a/src/director/director-settings.h Mon Sep 07 18:50:24 2015 +0300 +++ b/src/director/director-settings.h Mon Sep 07 20:24:25 2015 +0300 @@ -10,6 +10,7 @@ const char *director_mail_servers; const char *director_username_hash; unsigned int director_user_expire; + unsigned int director_user_kick_delay; in_port_t director_doveadm_port; bool director_consistent_hashing; }; diff -r 2eeef1619161 -r 3ebba506c2c7 src/director/director.c --- a/src/director/director.c Mon Sep 07 18:50:24 2015 +0300 +++ b/src/director/director.c Mon Sep 07 20:24:25 2015 +0300 @@ -17,7 +17,6 @@ #define DIRECTOR_RECONNECT_RETRY_SECS 60 #define DIRECTOR_RECONNECT_TIMEOUT_MSECS (30*1000) #define DIRECTOR_USER_MOVE_TIMEOUT_MSECS (30*1000) -#define DIRECTOR_USER_MOVE_FINISH_DELAY_MSECS (2*1000) #define DIRECTOR_SYNC_TIMEOUT_MSECS (5*1000) #define DIRECTOR_RING_MIN_WAIT_SECS 20 #define DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS 1000 @@ -667,7 +666,10 @@ user->kill_state = USER_KILL_STATE_DELAY; timeout_remove(&user->to_move); - user->to_move = timeout_add(DIRECTOR_USER_MOVE_FINISH_DELAY_MSECS, + /* wait for a while for the kills to finish in the backend server, + so there are no longer any processes running for the user before we + start letting new in connections to the new server. */ + user->to_move = timeout_add(dir->set->director_user_kick_delay * 1000, director_user_kill_finish_delayed_to, ctx); } From dovecot at dovecot.org Mon Sep 7 18:03:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 18:03:55 +0000 Subject: dovecot-2.2: auth: If multiple userdbs are used, default_fields ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f3ed414944b8 changeset: 19109:f3ed414944b8 user: Timo Sirainen date: Mon Sep 07 21:02:51 2015 +0300 description: auth: If multiple userdbs are used, default_fields was ignored for all but the first one. diffstat: src/auth/auth-request.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 3ebba506c2c7 -r f3ed414944b8 src/auth/auth-request.c --- a/src/auth/auth-request.c Mon Sep 07 20:24:25 2015 +0300 +++ b/src/auth/auth-request.c Mon Sep 07 21:02:51 2015 +0300 @@ -1148,6 +1148,12 @@ request->userdb_lookup = TRUE; if (request->userdb_reply == NULL) auth_request_init_userdb_reply(request); + else { + /* we still want to set default_fields. these override any + existing fields set by previous userdbs (because if that is + unwanted, ":protected" can be used). */ + userdb_template_export(userdb->default_fields_tmpl, request); + } /* (for now) auth_cache is shared between passdb and userdb */ cache_key = passdb_cache == NULL ? NULL : userdb->cache_key; From dovecot at dovecot.org Mon Sep 7 18:25:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 18:25:07 +0000 Subject: dovecot-2.2: auth: If userdb lookup was found from auth cache, d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7f7ad2bc4d0 changeset: 19110:b7f7ad2bc4d0 user: Timo Sirainen date: Mon Sep 07 21:24:01 2015 +0300 description: auth: If userdb lookup was found from auth cache, don't clear the earlier userdb fields. diffstat: src/auth/auth-request.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (18 lines): diff -r f3ed414944b8 -r b7f7ad2bc4d0 src/auth/auth-request.c --- a/src/auth/auth-request.c Mon Sep 07 21:02:51 2015 +0300 +++ b/src/auth/auth-request.c Mon Sep 07 21:24:01 2015 +0300 @@ -1024,7 +1024,13 @@ return TRUE; } - request->userdb_reply = auth_fields_init(request->pool); + /* We want to preserve any userdb fields set by the earlier passdb + lookup, so initialize userdb_reply only if it doesn't exist. + Don't use auth_request_init_userdb_reply(), because the entire + userdb part of the result comes from the cache so we don't want to + initialize it with default_fields. */ + if (request->userdb_reply == NULL) + request->userdb_reply = auth_fields_init(request->pool); auth_request_userdb_import(request, value); *result_r = USERDB_RESULT_OK; return TRUE; From dovecot at dovecot.org Mon Sep 7 18:55:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 18:55:34 +0000 Subject: dovecot-2.2: doveadm: Added new "auth login" command to simulate... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/caaaa7c12edd changeset: 19111:caaaa7c12edd user: Timo Sirainen date: Mon Sep 07 21:54:20 2015 +0300 description: doveadm: Added new "auth login" command to simulate an actual client login. This performs both passdb and userdb lookups and prints their results. diffstat: src/doveadm/doveadm-auth.c | 127 ++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 125 insertions(+), 2 deletions(-) diffs (183 lines): diff -r b7f7ad2bc4d0 -r caaaa7c12edd src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Mon Sep 07 21:24:01 2015 +0300 +++ b/src/doveadm/doveadm-auth.c Mon Sep 07 21:54:20 2015 +0300 @@ -5,6 +5,7 @@ #include "array.h" #include "askpass.h" #include "base64.h" +#include "hex-binary.h" #include "str.h" #include "wildcard-match.h" #include "settings-parser.h" @@ -13,6 +14,8 @@ #include "auth-client.h" #include "auth-master.h" #include "auth-server-connection.h" +#include "master-auth.h" +#include "master-login-auth.h" #include "mail-storage-service.h" #include "mail-user.h" #include "doveadm.h" @@ -23,10 +26,19 @@ #include struct authtest_input { + pool_t pool; const char *username; const char *password; struct auth_user_info info; bool success; + + struct auth_client_request *request; + struct master_auth_request master_auth_req; + + unsigned int auth_id; + unsigned int auth_pid; + const char *auth_cookie; + }; static void auth_cmd_help(doveadm_command_t *cmd); @@ -106,6 +118,12 @@ { struct authtest_input *input = context; + input->request = NULL; + input->auth_id = auth_client_request_get_id(request); + input->auth_pid = auth_client_request_get_server_pid(request); + input->auth_cookie = input->pool == NULL ? NULL : + p_strdup(input->pool, auth_client_request_get_cookie(request)); + if (!io_loop_is_running(current_ioloop)) return; @@ -155,8 +173,8 @@ info.remote_port = input->info.remote_port; info.initial_resp_base64 = str_c(base64_resp); - (void)auth_client_request_new(client, &info, - auth_callback, input); + input->request = auth_client_request_new(client, &info, + auth_callback, input); } static void @@ -293,6 +311,109 @@ doveadm_exit_code = EX_NOPERM; } +static void +master_auth_callback(const char *const *auth_args, + const char *errormsg, void *context) +{ + struct authtest_input *input = context; + unsigned int i; + + io_loop_stop(current_ioloop); + if (errormsg != NULL) { + i_error("userdb lookup failed: %s", errormsg); + return; + } + printf("userdb extra fields:\n"); + for (i = 0; auth_args[i] != NULL; i++) + printf(" %s\n", auth_args[i]); + input->success = TRUE; +} + +static void +cmd_auth_master_input(const char *auth_master_socket_path, + struct authtest_input *input) +{ + struct master_login_auth *master_auth; + struct master_auth_request master_auth_req; + buffer_t buf; + + memset(&master_auth_req, 0, sizeof(master_auth_req)); + master_auth_req.tag = 1; + master_auth_req.auth_pid = input->auth_pid; + master_auth_req.auth_id = input->auth_id; + master_auth_req.client_pid = getpid(); + master_auth_req.local_ip = input->info.local_ip; + master_auth_req.remote_ip = input->info.remote_ip; + + buffer_create_from_data(&buf, master_auth_req.cookie, + sizeof(master_auth_req.cookie)); + if (strlen(input->auth_cookie) == MASTER_AUTH_COOKIE_SIZE*2) + (void)hex_to_binary(input->auth_cookie, &buf); + + input->success = FALSE; + master_auth = master_login_auth_init(auth_master_socket_path, FALSE); + io_loop_set_running(current_ioloop); + master_login_auth_request(master_auth, &master_auth_req, + master_auth_callback, input); + if (io_loop_is_running(current_ioloop)) + io_loop_run(current_ioloop); + master_login_auth_deinit(&master_auth); +} + +static void cmd_auth_login(int argc, char *argv[]) +{ + const char *auth_login_socket_path, *auth_master_socket_path; + struct auth_client *auth_client; + struct authtest_input input; + int c; + + memset(&input, 0, sizeof(input)); + input.info.service = "doveadm"; + + auth_login_socket_path = t_strconcat(doveadm_settings->base_dir, + "/auth-login", NULL); + auth_master_socket_path = t_strconcat(doveadm_settings->base_dir, + "/auth-master", NULL); + while ((c = getopt(argc, argv, "a:m:x:")) > 0) { + switch (c) { + case 'a': + auth_login_socket_path = optarg; + break; + case 'm': + auth_master_socket_path = optarg; + break; + case 'x': + auth_user_info_parse(&input.info, optarg); + break; + default: + auth_cmd_help(cmd_auth_login); + } + } + + if (optind == argc) + auth_cmd_help(cmd_auth_login); + + input.pool = pool_alloconly_create("auth login", 256); + input.username = argv[optind++]; + input.password = argv[optind] != NULL ? argv[optind++] : + t_askpass("Password: "); + if (argv[optind] != NULL) + i_fatal("Unexpected parameter: %s", argv[optind]); + /* authenticate */ + auth_client = auth_client_init(auth_login_socket_path, getpid(), FALSE); + auth_client_set_connect_notify(auth_client, auth_connected, &input); + if (!auth_client_is_disconnected(auth_client)) + io_loop_run(current_ioloop); + auth_client_set_connect_notify(auth_client, NULL, NULL); + /* finish login with userdb lookup */ + if (input.success) + cmd_auth_master_input(auth_master_socket_path, &input); + if (!input.success) + doveadm_exit_code = EX_NOPERM; + auth_client_deinit(&auth_client); + pool_unref(&input.pool); +} + static void cmd_auth_lookup(int argc, char *argv[]) { const char *auth_socket_path = doveadm_settings->auth_socket_path; @@ -522,6 +643,8 @@ struct doveadm_cmd doveadm_cmd_auth[] = { { cmd_auth_test, "auth test", "[-a ] [-x ] []" }, + { cmd_auth_login, "auth login", + "[-a ] [-m ] [-x ] []" }, { cmd_auth_lookup, "auth lookup", "[-a ] [-x ] [-f field] [...]" }, { cmd_auth_cache_flush, "auth cache flush", From dovecot at dovecot.org Mon Sep 7 19:07:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 19:07:21 +0000 Subject: dovecot-2.2: lib-storage: Fixed assert-crash when reading binary... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/865405fce42e changeset: 19112:865405fce42e user: Timo Sirainen date: Mon Sep 07 22:06:16 2015 +0300 description: lib-storage: Fixed assert-crash when reading binary streams. diffstat: src/lib-storage/index/index-mail-binary.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r caaaa7c12edd -r 865405fce42e src/lib-storage/index/index-mail-binary.c --- a/src/lib-storage/index/index-mail-binary.c Mon Sep 07 21:54:20 2015 +0300 +++ b/src/lib-storage/index/index-mail-binary.c Mon Sep 07 22:06:16 2015 +0300 @@ -345,6 +345,7 @@ /* go to the next block */ if (++block_idx == block_count) { i_assert(i_stream_is_eof(full_input)); + ret = -1; break; } cur_block++; From dovecot at dovecot.org Mon Sep 7 19:15:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 19:15:28 +0000 Subject: dovecot-2.2: imap: Fixed assert-crash in NOTIFY when using multi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fae5feef70af changeset: 19113:fae5feef70af user: Timo Sirainen date: Mon Sep 07 22:14:18 2015 +0300 description: imap: Fixed assert-crash in NOTIFY when using multiple namespaces. I'm not sure why the original code was trying to add it to multiple namespaces. A single mailbox name should be matching only a single namespace (visible one at least). In any case we can't use mail_namespace_find() with only partial namespaces-list, because it'll assert-crash if it can't find a namespace for the mailbox. diffstat: src/imap/cmd-notify.c | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) diffs (17 lines): diff -r 865405fce42e -r fae5feef70af src/imap/cmd-notify.c --- a/src/imap/cmd-notify.c Mon Sep 07 22:06:16 2015 +0300 +++ b/src/imap/cmd-notify.c Mon Sep 07 22:14:18 2015 +0300 @@ -225,11 +225,8 @@ { struct mail_namespace *ns; - /* add to all matching namespaces */ - for (ns = ctx->client->user->namespaces; ns != NULL; ns = ns->next) { - if (mail_namespace_find(ns, name) == ns) - cmd_notify_add_mailbox(ctx, ns, name, type, events); - } + ns = mail_namespace_find(ctx->client->user->namespaces, name); + cmd_notify_add_mailbox(ctx, ns, name, type, events); } static int From dovecot at dovecot.org Mon Sep 7 19:23:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 19:23:21 +0000 Subject: dovecot-2.2: lib: Fix hang in safe_sendfile on SmartOS Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c6554cb1961d changeset: 19114:c6554cb1961d user: Sebastian Wiedenroth date: Thu Jul 16 14:41:24 2015 +0200 description: lib: Fix hang in safe_sendfile on SmartOS The call to sendfile on SmartOS can fail with EOPNOTSUPP. This is a valid error code and documented in the man page. This error code needs to be handled or else dovecot will retry the sendfile call endlessly and hang. diffstat: src/lib/sendfile-util.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r fae5feef70af -r c6554cb1961d src/lib/sendfile-util.c --- a/src/lib/sendfile-util.c Mon Sep 07 22:14:18 2015 +0300 +++ b/src/lib/sendfile-util.c Thu Jul 16 14:41:24 2015 +0200 @@ -116,7 +116,7 @@ if (errno == EINVAL) { /* most likely trying to read past EOF */ ret = 0; - } else if (errno == EAFNOSUPPORT) { + } else if (errno == EAFNOSUPPORT || errno == EOPNOTSUPP) { /* not supported, return Linux-like EINVAL so caller sees only consistent errnos. */ errno = EINVAL; From dovecot at dovecot.org Mon Sep 7 19:42:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 19:42:38 +0000 Subject: dovecot-2.2: Various passthrough istreams didn't preserve readab... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bb80b907a116 changeset: 19115:bb80b907a116 user: Timo Sirainen date: Mon Sep 07 22:35:30 2015 +0300 description: Various passthrough istreams didn't preserve readable_fd. diffstat: src/lib-storage/index/istream-mail.c | 1 + src/lib/istream-failure-at.c | 1 + src/lib/istream-hash.c | 1 + src/lib/istream-timeout.c | 1 + 4 files changed, 4 insertions(+), 0 deletions(-) diffs (44 lines): diff -r c6554cb1961d -r bb80b907a116 src/lib-storage/index/istream-mail.c --- a/src/lib-storage/index/istream-mail.c Thu Jul 16 14:41:24 2015 +0200 +++ b/src/lib-storage/index/istream-mail.c Mon Sep 07 22:35:30 2015 +0300 @@ -146,6 +146,7 @@ mstream->istream.read = i_stream_mail_read; + mstream->istream.istream.readable_fd = input->readable_fd; mstream->istream.istream.blocking = input->blocking; mstream->istream.istream.seekable = input->seekable; return i_stream_create(&mstream->istream, input, diff -r c6554cb1961d -r bb80b907a116 src/lib/istream-failure-at.c --- a/src/lib/istream-failure-at.c Thu Jul 16 14:41:24 2015 +0200 +++ b/src/lib/istream-failure-at.c Mon Sep 07 22:35:30 2015 +0300 @@ -72,6 +72,7 @@ fstream->istream.read = i_stream_failure_at_read; fstream->istream.iostream.destroy = i_stream_failure_at_destroy; + fstream->istream.istream.readable_fd = input->readable_fd; fstream->istream.istream.blocking = input->blocking; fstream->istream.istream.seekable = input->seekable; diff -r c6554cb1961d -r bb80b907a116 src/lib/istream-hash.c --- a/src/lib/istream-hash.c Thu Jul 16 14:41:24 2015 +0200 +++ b/src/lib/istream-hash.c Mon Sep 07 22:35:30 2015 +0300 @@ -74,6 +74,7 @@ hstream->istream.read = i_stream_hash_read; hstream->istream.seek = i_stream_hash_seek; + hstream->istream.istream.readable_fd = input->readable_fd; hstream->istream.istream.blocking = input->blocking; hstream->istream.istream.seekable = input->seekable; diff -r c6554cb1961d -r bb80b907a116 src/lib/istream-timeout.c --- a/src/lib/istream-timeout.c Thu Jul 16 14:41:24 2015 +0200 +++ b/src/lib/istream-timeout.c Mon Sep 07 22:35:30 2015 +0300 @@ -135,6 +135,7 @@ tstream->istream.switch_ioloop = i_stream_timeout_switch_ioloop; tstream->istream.iostream.close = i_stream_timeout_close; + tstream->istream.istream.readable_fd = input->readable_fd; tstream->istream.istream.blocking = input->blocking; tstream->istream.istream.seekable = input->seekable; return i_stream_create(&tstream->istream, input, From dovecot at dovecot.org Mon Sep 7 20:09:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 20:09:50 +0000 Subject: dovecot-2.2: imap: NOTIFY (SUBSCRIPTIONS) assert-crashed when su... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/73acc7075146 changeset: 19116:73acc7075146 user: Timo Sirainen date: Mon Sep 07 23:08:44 2015 +0300 description: imap: NOTIFY (SUBSCRIPTIONS) assert-crashed when subscriptions hadn't been refreshed. diffstat: src/imap/cmd-notify.c | 27 +++++++++++++++++++++++++++ src/imap/imap-notify.h | 1 + 2 files changed, 28 insertions(+), 0 deletions(-) diffs (60 lines): diff -r bb80b907a116 -r 73acc7075146 src/imap/cmd-notify.c --- a/src/imap/cmd-notify.c Mon Sep 07 22:35:30 2015 +0300 +++ b/src/imap/cmd-notify.c Mon Sep 07 23:08:44 2015 +0300 @@ -206,11 +206,34 @@ } } +static int +imap_notify_refresh_subscriptions(struct client_command_context *cmd, + struct imap_notify_context *ctx) +{ + struct mailbox_list_iterate_context *iter; + struct mail_namespace *ns; + + if (!ctx->have_subscriptions) + return 0; + + /* make sure subscriptions are refreshed at least once */ + for (ns = ctx->client->user->namespaces; ns != NULL; ns = ns->next) { + iter = mailbox_list_iter_init(ns->list, "*", MAILBOX_LIST_ITER_SELECT_SUBSCRIBED); + (void)mailbox_list_iter_next(iter); + if (mailbox_list_iter_deinit(&iter) < 0) { + client_send_list_error(cmd, ns->list); + return -1; + } + } + return 0; +} + static void cmd_notify_add_subscribed(struct imap_notify_context *ctx, enum imap_notify_event events) { struct mail_namespace *ns; + ctx->have_subscriptions = TRUE; for (ns = ctx->client->user->namespaces; ns != NULL; ns = ns->next) { cmd_notify_add_mailbox(ctx, ns, "", IMAP_NOTIFY_TYPE_SUBSCRIBED, events); @@ -523,6 +546,10 @@ "NO [NOTIFICATIONOVERFLOW] Too many mailbox names"); pool_unref(&pool); return TRUE; + } else if (imap_notify_refresh_subscriptions(cmd, ctx) < 0) { + /* tagline already sent */ + pool_unref(&pool); + return TRUE; } else if (imap_notify_begin(ctx) < 0) { client_send_tagline(cmd, "NO [NOTIFICATIONOVERFLOW] NOTIFY not supported for these mailboxes."); diff -r bb80b907a116 -r 73acc7075146 src/imap/imap-notify.h --- a/src/imap/imap-notify.h Mon Sep 07 22:35:30 2015 +0300 +++ b/src/imap/imap-notify.h Mon Sep 07 23:08:44 2015 +0300 @@ -49,6 +49,7 @@ struct imap_fetch_context *fetch_ctx; struct timeout *to_watch; + unsigned int have_subscriptions:1; unsigned int selected_set:1; unsigned int selected_immediate_expunges:1; unsigned int send_immediate_status:1; From dovecot at dovecot.org Mon Sep 7 20:56:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 20:56:38 +0000 Subject: dovecot-2.2: Mailbox list notify API changed to return multiple ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa979ccfa34c changeset: 19117:fa979ccfa34c user: Timo Sirainen date: Mon Sep 07 23:55:31 2015 +0300 description: Mailbox list notify API changed to return multiple events at once. This fixes some issues where a single event could actually trigger multiple different kinds of events. diffstat: src/imap/imap-notify.c | 156 ++++++++++------------ src/lib-storage/list/mailbox-list-index-notify.c | 35 ++-- src/lib-storage/mailbox-list-notify.h | 3 +- 3 files changed, 93 insertions(+), 101 deletions(-) diffs (truncated from 303 to 300 lines): diff -r 73acc7075146 -r fa979ccfa34c src/imap/imap-notify.c --- a/src/imap/imap-notify.c Mon Sep 07 23:08:44 2015 +0300 +++ b/src/imap/imap-notify.c Mon Sep 07 23:55:31 2015 +0300 @@ -57,27 +57,17 @@ items.status |= STATUS_HIGHESTMODSEQ; box = mailbox_alloc(notify_ns->ns->list, rec->vname, 0); - switch (rec->event) { - case MAILBOX_LIST_NOTIFY_UIDVALIDITY: + if ((rec->events & MAILBOX_LIST_NOTIFY_UIDVALIDITY) != 0) { items.status |= STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES | STATUS_UNSEEN; - break; - case MAILBOX_LIST_NOTIFY_APPENDS: - case MAILBOX_LIST_NOTIFY_EXPUNGES: + } + if ((rec->events & (MAILBOX_LIST_NOTIFY_APPENDS | + MAILBOX_LIST_NOTIFY_EXPUNGES)) != 0) items.status |= STATUS_UIDNEXT | STATUS_MESSAGES | STATUS_UNSEEN; - break; - case MAILBOX_LIST_NOTIFY_SEEN_CHANGES: + if ((rec->events & MAILBOX_LIST_NOTIFY_SEEN_CHANGES) != 0) items.status |= STATUS_UNSEEN; - break; - case MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES: + if ((rec->events & MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES) != 0) { /* if HIGHESTMODSEQ isn't being sent, don't send anything */ - break; - case MAILBOX_LIST_NOTIFY_CREATE: - case MAILBOX_LIST_NOTIFY_DELETE: - case MAILBOX_LIST_NOTIFY_RENAME: - case MAILBOX_LIST_NOTIFY_SUBSCRIBE: - case MAILBOX_LIST_NOTIFY_UNSUBSCRIBE: - i_unreached(); } if (items.status == 0) { /* don't send anything */ @@ -100,46 +90,50 @@ const struct mailbox_list_notify_rec *rec) { enum mailbox_info_flags mailbox_flags; - int ret = 1; + int ret; - switch (rec->event) { - case MAILBOX_LIST_NOTIFY_CREATE: + if ((rec->events & MAILBOX_LIST_NOTIFY_CREATE) != 0) { if (mailbox_list_mailbox(notify_ns->ns->list, rec->storage_name, &mailbox_flags) < 0) mailbox_flags = 0; - ret = imap_notify_list(notify_ns, rec, mailbox_flags); - break; - case MAILBOX_LIST_NOTIFY_DELETE: - ret = imap_notify_list(notify_ns, rec, MAILBOX_NONEXISTENT); - break; - case MAILBOX_LIST_NOTIFY_RENAME: + if ((ret = imap_notify_list(notify_ns, rec, mailbox_flags)) <= 0) + return ret; + } + if ((rec->events & MAILBOX_LIST_NOTIFY_DELETE) != 0) { + if ((ret = imap_notify_list(notify_ns, rec, MAILBOX_NONEXISTENT)) < 0) + return ret; + } + if ((rec->events & MAILBOX_LIST_NOTIFY_RENAME) != 0) { if (mailbox_list_mailbox(notify_ns->ns->list, rec->storage_name, &mailbox_flags) < 0) mailbox_flags = 0; - ret = imap_notify_list(notify_ns, rec, mailbox_flags); - break; - case MAILBOX_LIST_NOTIFY_SUBSCRIBE: + if ((ret = imap_notify_list(notify_ns, rec, mailbox_flags)) < 0) + return ret; + } + if ((rec->events & MAILBOX_LIST_NOTIFY_SUBSCRIBE) != 0) { if (mailbox_list_mailbox(notify_ns->ns->list, rec->storage_name, &mailbox_flags) < 0) mailbox_flags = 0; - ret = imap_notify_list(notify_ns, rec, - mailbox_flags | MAILBOX_SUBSCRIBED); - break; - case MAILBOX_LIST_NOTIFY_UNSUBSCRIBE: + if ((ret = imap_notify_list(notify_ns, rec, + mailbox_flags | MAILBOX_SUBSCRIBED)) < 0) + return ret; + } + if ((rec->events & MAILBOX_LIST_NOTIFY_UNSUBSCRIBE) != 0) { if (mailbox_list_mailbox(notify_ns->ns->list, rec->storage_name, &mailbox_flags) < 0) mailbox_flags = 0; - ret = imap_notify_list(notify_ns, rec, mailbox_flags); - break; - case MAILBOX_LIST_NOTIFY_UIDVALIDITY: - case MAILBOX_LIST_NOTIFY_APPENDS: - case MAILBOX_LIST_NOTIFY_EXPUNGES: - case MAILBOX_LIST_NOTIFY_SEEN_CHANGES: - case MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES: - ret = imap_notify_status(notify_ns, rec); - break; + if ((ret = imap_notify_list(notify_ns, rec, mailbox_flags)) < 0) + return ret; } - return ret; + if ((rec->events & (MAILBOX_LIST_NOTIFY_UIDVALIDITY | + MAILBOX_LIST_NOTIFY_APPENDS | + MAILBOX_LIST_NOTIFY_EXPUNGES | + MAILBOX_LIST_NOTIFY_SEEN_CHANGES | + MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES)) != 0) { + if ((ret = imap_notify_status(notify_ns, rec)) < 0) + return ret; + } + return 1; } static bool @@ -149,53 +143,45 @@ { enum imap_notify_event wanted_events = notify_boxes->events; struct mailbox *box; - bool mailbox_event = FALSE; - switch (rec->event) { - case MAILBOX_LIST_NOTIFY_CREATE: - case MAILBOX_LIST_NOTIFY_DELETE: - case MAILBOX_LIST_NOTIFY_RENAME: - if ((wanted_events & IMAP_NOTIFY_EVENT_MAILBOX_NAME) == 0) - return FALSE; - break; - case MAILBOX_LIST_NOTIFY_SUBSCRIBE: - case MAILBOX_LIST_NOTIFY_UNSUBSCRIBE: - if ((wanted_events & IMAP_NOTIFY_EVENT_SUBSCRIPTION_CHANGE) == 0) - return FALSE; - break; - case MAILBOX_LIST_NOTIFY_UIDVALIDITY: - if ((wanted_events & (IMAP_NOTIFY_EVENT_MESSAGE_NEW | - IMAP_NOTIFY_EVENT_MESSAGE_EXPUNGE | - IMAP_NOTIFY_EVENT_FLAG_CHANGE)) == 0) - return FALSE; - mailbox_event = TRUE; - break; - case MAILBOX_LIST_NOTIFY_APPENDS: - if ((wanted_events & IMAP_NOTIFY_EVENT_MESSAGE_NEW) == 0) - return FALSE; - mailbox_event = TRUE; - break; - case MAILBOX_LIST_NOTIFY_EXPUNGES: - if ((wanted_events & IMAP_NOTIFY_EVENT_MESSAGE_EXPUNGE) == 0) - return FALSE; - mailbox_event = TRUE; - break; - case MAILBOX_LIST_NOTIFY_SEEN_CHANGES: - case MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES: - if ((wanted_events & IMAP_NOTIFY_EVENT_FLAG_CHANGE) == 0) - return FALSE; - mailbox_event = TRUE; - break; + /* check for mailbox list events first */ + if ((wanted_events & IMAP_NOTIFY_EVENT_MAILBOX_NAME) != 0) { + if ((rec->events & (MAILBOX_LIST_NOTIFY_CREATE | + MAILBOX_LIST_NOTIFY_DELETE | + MAILBOX_LIST_NOTIFY_RENAME)) != 0) + return TRUE; + } + if ((wanted_events & IMAP_NOTIFY_EVENT_SUBSCRIPTION_CHANGE) != 0) { + if ((rec->events & (MAILBOX_LIST_NOTIFY_SUBSCRIBE | + MAILBOX_LIST_NOTIFY_UNSUBSCRIBE)) != 0) + return TRUE; } - if (mailbox_event) { - /* if this is an even for selected mailbox, ignore it */ - box = notify_ns->ctx->client->mailbox; - if (box != NULL && - mailbox_equals(box, notify_ns->ns, rec->vname)) - return FALSE; + /* if this is an event for the selected mailbox, ignore it */ + box = notify_ns->ctx->client->mailbox; + if (box != NULL && mailbox_equals(box, notify_ns->ns, rec->vname)) + return FALSE; + + if ((wanted_events & (IMAP_NOTIFY_EVENT_MESSAGE_NEW | + IMAP_NOTIFY_EVENT_MESSAGE_EXPUNGE | + IMAP_NOTIFY_EVENT_FLAG_CHANGE)) != 0) { + if ((rec->events & MAILBOX_LIST_NOTIFY_UIDVALIDITY) != 0) + return TRUE; } - return TRUE; + if ((wanted_events & IMAP_NOTIFY_EVENT_MESSAGE_NEW) != 0) { + if ((rec->events & MAILBOX_LIST_NOTIFY_APPENDS) != 0) + return TRUE; + } + if ((wanted_events & IMAP_NOTIFY_EVENT_MESSAGE_EXPUNGE) != 0) { + if ((rec->events & MAILBOX_LIST_NOTIFY_EXPUNGES) != 0) + return TRUE; + } + if ((wanted_events & IMAP_NOTIFY_EVENT_FLAG_CHANGE) != 0) { + if ((rec->events & (MAILBOX_LIST_NOTIFY_SEEN_CHANGES | + MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES)) != 0) + return TRUE; + } + return FALSE; } bool imap_notify_match_mailbox(struct imap_notify_namespace *notify_ns, diff -r 73acc7075146 -r fa979ccfa34c src/lib-storage/list/mailbox-list-index-notify.c --- a/src/lib-storage/list/mailbox-list-index-notify.c Mon Sep 07 23:08:44 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-notify.c Mon Sep 07 23:55:31 2015 +0300 @@ -591,7 +591,7 @@ return FALSE; rec->old_vname = old_vname; - rec->event = MAILBOX_LIST_NOTIFY_RENAME; + rec->events = MAILBOX_LIST_NOTIFY_RENAME; return TRUE; } @@ -607,7 +607,7 @@ rec->vname = *vnamep; rec->storage_name = mailbox_list_get_storage_name(inotify->notify.list, rec->vname); - rec->event = MAILBOX_LIST_NOTIFY_SUBSCRIBE; + rec->events = MAILBOX_LIST_NOTIFY_SUBSCRIBE; return TRUE; } @@ -623,7 +623,7 @@ rec->vname = *vnamep; rec->storage_name = mailbox_list_get_storage_name(inotify->notify.list, rec->vname); - rec->event = MAILBOX_LIST_NOTIFY_UNSUBSCRIBE; + rec->events = MAILBOX_LIST_NOTIFY_UNSUBSCRIBE; return TRUE; } @@ -637,7 +637,7 @@ if (!mailbox_list_index_notify_lookup(inotify, inotify->old_view, uid, 0, &status, &rec)) return FALSE; - rec->event = MAILBOX_LIST_NOTIFY_DELETE; + rec->events = MAILBOX_LIST_NOTIFY_DELETE; return TRUE; } @@ -651,7 +651,7 @@ if (!mailbox_list_index_notify_lookup(inotify, inotify->view, uid, 0, &status, &rec)) i_unreached(); - rec->event = MAILBOX_LIST_NOTIFY_CREATE; + rec->events = MAILBOX_LIST_NOTIFY_CREATE; return TRUE; } @@ -674,15 +674,20 @@ nnode = mailbox_list_notify_tree_lookup(inotify->tree, rec->storage_name); if (nnode == NULL || nnode->uidvalidity != status.uidvalidity) - rec->event = MAILBOX_LIST_NOTIFY_UIDVALIDITY; - else if (nnode->uidnext != status.uidnext) - rec->event = MAILBOX_LIST_NOTIFY_APPENDS; - else if (nnode->messages > status.messages) - rec->event = MAILBOX_LIST_NOTIFY_EXPUNGES; - else if (nnode->unseen != status.unseen) - rec->event = MAILBOX_LIST_NOTIFY_SEEN_CHANGES; - else if (nnode->highest_modseq < status.highest_modseq) - rec->event = MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES; + rec->events |= MAILBOX_LIST_NOTIFY_UIDVALIDITY; + if (nnode->uidnext != status.uidnext) + rec->events |= MAILBOX_LIST_NOTIFY_APPENDS; + if (nnode->messages > status.messages) { + /* NOTE: not entirely reliable, since there could be both + expunges and appends.. but it shouldn't make any difference + in practise, since anybody interested in expunges is most + likely also interested in appends. */ + rec->events |= MAILBOX_LIST_NOTIFY_EXPUNGES; + } + if (nnode->unseen != status.unseen) + rec->events |= MAILBOX_LIST_NOTIFY_SEEN_CHANGES; + if (nnode->highest_modseq < status.highest_modseq) + rec->events |= MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES; else { /* nothing changed */ return FALSE; @@ -748,7 +753,7 @@ if (!inotify->initialized) mailbox_list_index_notify_read_init(inotify); while (mailbox_list_index_notify_try_next(inotify)) { - if ((inotify->notify_rec.event & inotify->notify.mask) != 0) { + if ((inotify->notify_rec.events & inotify->notify.mask) != 0) { *rec_r = &inotify->notify_rec; return 1; } else { diff -r 73acc7075146 -r fa979ccfa34c src/lib-storage/mailbox-list-notify.h --- a/src/lib-storage/mailbox-list-notify.h Mon Sep 07 23:08:44 2015 +0300 +++ b/src/lib-storage/mailbox-list-notify.h Mon Sep 07 23:55:31 2015 +0300 @@ -30,7 +30,8 @@ }; struct mailbox_list_notify_rec { - enum mailbox_list_notify_event event; + /* Each record can contain multiple events */ + enum mailbox_list_notify_event events; From dovecot at dovecot.org Mon Sep 7 21:09:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 21:09:03 +0000 Subject: dovecot-2.2: lib-storage: Mailbox list notifications didn't work... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f600285c3df2 changeset: 19118:f600285c3df2 user: Timo Sirainen date: Tue Sep 08 00:07:55 2015 +0300 description: lib-storage: Mailbox list notifications didn't work for INBOX. Because of some earlier optimizations that changes to INBOX aren't written to dovecot.list.index.log file. diffstat: src/lib-storage/list/mailbox-list-index-notify.c | 88 ++++++++++++++++++----- 1 files changed, 69 insertions(+), 19 deletions(-) diffs (186 lines): diff -r fa979ccfa34c -r f600285c3df2 src/lib-storage/list/mailbox-list-index-notify.c --- a/src/lib-storage/list/mailbox-list-index-notify.c Mon Sep 07 23:55:31 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-notify.c Tue Sep 08 00:07:55 2015 +0300 @@ -5,7 +5,7 @@ #include "str.h" #include "mail-index-private.h" #include "mail-transaction-log-private.h" -#include "mail-storage.h" +#include "mail-storage-private.h" #include "mailbox-list-notify.h" #include "mailbox-list-notify-tree.h" #include "mailbox-list-index.h" @@ -44,7 +44,7 @@ void (*wait_callback)(void *context); void *wait_context; - struct io *io_wait; + struct io *io_wait, *io_wait_inbox; struct timeout *to_wait, *to_notify; ARRAY_TYPE(seq_range) new_uids, expunged_uids, changed_uids; @@ -58,10 +58,12 @@ struct mailbox_list_notify_rec notify_rec; string_t *rec_name; - struct stat last_st; + char *list_log_path, *inbox_log_path; + struct stat list_last_st, inbox_last_st; unsigned int initialized:1; unsigned int read_failed:1; + unsigned int inbox_event_pending:1; }; int mailbox_list_index_notify_init(struct mailbox_list *list, @@ -70,6 +72,7 @@ { struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(list); struct mailbox_list_notify_index *inotify; + const char *index_dir; if (ilist == NULL) { /* can't do this without mailbox list indexes */ @@ -97,6 +100,14 @@ mailbox_tree_sort(list->subscriptions); inotify->subscriptions = mailbox_tree_dup(list->subscriptions); } + inotify->list_log_path = i_strdup(ilist->index->log->filepath); + if ((list->ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0 && + mailbox_list_get_path(list, "INBOX", MAILBOX_LIST_PATH_TYPE_INDEX, + &index_dir) > 0) { + /* FIXME: annoyingly hardcoded filename. */ + inotify->inbox_log_path = i_strdup_printf( + "%s/"MAIL_INDEX_PREFIX".log", index_dir); + } *notify_r = &inotify->notify; return 1; @@ -112,6 +123,8 @@ mailbox_tree_deinit(&inotify->subscriptions); if (inotify->io_wait != NULL) io_remove(&inotify->io_wait); + if (inotify->io_wait_inbox != NULL) + io_remove(&inotify->io_wait_inbox); if (inotify->to_wait != NULL) timeout_remove(&inotify->to_wait); if (inotify->to_notify != NULL) @@ -128,6 +141,8 @@ array_free(&inotify->changed_uids); array_free(&inotify->renames); str_free(&inotify->rec_name); + i_free(inotify->list_log_path); + i_free(inotify->inbox_log_path); i_free(inotify); } @@ -166,14 +181,22 @@ static void notify_update_stat(struct mailbox_list_notify_index *inotify) { - struct mailbox_list_index *ilist = - INDEX_LIST_CONTEXT(inotify->notify.list); - const char *path = ilist->index->log->filepath; + bool call = FALSE; - if (stat(path, &inotify->last_st) < 0 && errno != ENOENT) { - i_error("stat(%s) failed: %m", path); + if (stat(inotify->list_log_path, &inotify->list_last_st) < 0 && + errno != ENOENT) { + i_error("stat(%s) failed: %m", inotify->list_log_path); + call = TRUE; + } + if (inotify->inbox_log_path != NULL) { + if (stat(inotify->inbox_log_path, &inotify->inbox_last_st) < 0 && + errno != ENOENT) { + i_error("stat(%s) failed: %m", inotify->inbox_log_path); + call = TRUE; + } + } + if (call) mailbox_list_index_notify_wait(&inotify->notify, NULL, NULL); - } } static void @@ -760,6 +783,20 @@ /* caller doesn't care about this change */ } } + if (inotify->inbox_event_pending) { + inotify->inbox_event_pending = FALSE; + memset(&inotify->notify_rec, 0, sizeof(inotify->notify_rec)); + inotify->notify_rec.vname = "INBOX"; + inotify->notify_rec.storage_name = "INBOX"; + /* Don't bother trying to figure out which event exactly this + is. Just send them all and let the caller handle it. */ + inotify->notify_rec.events = MAILBOX_LIST_NOTIFY_APPENDS | + MAILBOX_LIST_NOTIFY_EXPUNGES | + MAILBOX_LIST_NOTIFY_SEEN_CHANGES | + MAILBOX_LIST_NOTIFY_MODSEQ_CHANGES; + *rec_r = &inotify->notify_rec; + return 1; + } mailbox_list_index_notify_read_deinit(inotify); return inotify->read_failed ? -1 : 0; @@ -773,13 +810,19 @@ static void notify_callback(struct mailbox_list_notify_index *inotify) { - struct stat prev_st = inotify->last_st; - +#define INOTIFY_ST_CHANGED(last_st, prev_st) \ + ((last_st).st_mtime != (prev_st).st_mtime || \ + ST_MTIME_NSEC(last_st) != ST_MTIME_NSEC(prev_st) || \ + (last_st).st_size != (prev_st).st_size || \ + (last_st).st_ino != (prev_st).st_ino) + struct stat list_prev_st = inotify->list_last_st; + struct stat inbox_prev_st = inotify->inbox_last_st; + notify_update_stat(inotify); - if (inotify->last_st.st_mtime != prev_st.st_mtime || - ST_MTIME_NSEC(inotify->last_st) != ST_MTIME_NSEC(prev_st) || - inotify->last_st.st_size != prev_st.st_size || - inotify->last_st.st_ino != prev_st.st_ino) { + if (INOTIFY_ST_CHANGED(inotify->inbox_last_st, inbox_prev_st)) + inotify->inbox_event_pending = TRUE; + if (inotify->inbox_event_pending || + INOTIFY_ST_CHANGED(inotify->list_last_st, list_prev_st)) { /* log has changed. call the callback with a small delay to allow bundling multiple changes together */ if (inotify->to_notify != NULL) { @@ -798,7 +841,6 @@ { struct mailbox_list_notify_index *inotify = (struct mailbox_list_notify_index *)notify; - const char *path; unsigned int check_interval; inotify->wait_callback = callback; @@ -807,14 +849,22 @@ if (callback == NULL) { if (inotify->io_wait != NULL) io_remove(&inotify->io_wait); + if (inotify->io_wait_inbox != NULL) + io_remove(&inotify->io_wait_inbox); if (inotify->to_wait != NULL) timeout_remove(&inotify->to_wait); if (inotify->to_notify != NULL) timeout_remove(&inotify->to_notify); } else if (inotify->to_wait == NULL) { - path = inotify->view->index->log->filepath; - (void)io_add_notify(path, notify_callback, inotify, - &inotify->io_wait); + (void)io_add_notify(inotify->list_log_path, notify_callback, + inotify, &inotify->io_wait); + /* we need to check for INBOX explicitly, because INBOX changes + don't get added to mailbox.list.index.log */ + if (inotify->inbox_log_path != NULL) { + (void)io_add_notify(inotify->inbox_log_path, + notify_callback, inotify, + &inotify->io_wait_inbox); + } /* check with timeout as well, in case io_add_notify() doesn't work (e.g. NFS) */ check_interval = notify->list->mail_set->mailbox_idle_check_interval; From dovecot at dovecot.org Mon Sep 7 21:35:18 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 21:35:18 +0000 Subject: dovecot-2.2: lib-charset: Updated test-charset unit test to chec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/93991734ae75 changeset: 19119:93991734ae75 user: Timo Sirainen date: Tue Sep 08 00:33:21 2015 +0300 description: lib-charset: Updated test-charset unit test to check for iconv() E2BIG result diffstat: src/lib-charset/test-charset.c | 14 +++++++++++++- 1 files changed, 13 insertions(+), 1 deletions(-) diffs (31 lines): diff -r f600285c3df2 -r 93991734ae75 src/lib-charset/test-charset.c --- a/src/lib-charset/test-charset.c Tue Sep 08 00:07:55 2015 +0300 +++ b/src/lib-charset/test-charset.c Tue Sep 08 00:33:21 2015 +0300 @@ -25,7 +25,7 @@ { "p\xC3\xA4\xC3", "p\xC3\xA4", CHARSET_RET_INCOMPLETE_INPUT }, { "p\xC3\xA4\xC3""a", "p\xC3\xA4"UNICODE_REPLACEMENT_CHAR_UTF8"a", CHARSET_RET_INVALID_INPUT } }; - string_t *str = t_str_new(128); + string_t *src, *str = t_str_new(256); enum charset_result result; unsigned int i; @@ -36,6 +36,18 @@ test_assert_idx(strcmp(tests[i].output, str_c(str)) == 0, i); test_assert_idx(result == tests[i].result, i); } + /* check that E2BIG handling works. We assume that iconv() is called + with 8192 byte buffer (tmpbuf[8192]) */ + src = str_new(default_pool, 16384); + for (i = 0; i < 8190; i++) + str_append_c(src, 'a' + i % ('z'-'a'+1)); + for (i = 0; i < 256; i++) { + str_truncate(str, 0); + str_append_c(src, 'A' + i % ('Z'-'A'+1)); + test_assert_idx(charset_to_utf8_str(input_charset, NULL, + str_c(src), str, &result) == 0, i); + } + str_free(&src); } static void test_charset_utf8(void) From dovecot at dovecot.org Mon Sep 7 21:35:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 21:35:19 +0000 Subject: dovecot-2.2: lib-charset: Added more asserts to checking iconv()... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9fdbb3b220ec changeset: 19120:9fdbb3b220ec user: Timo Sirainen date: Tue Sep 08 00:34:14 2015 +0300 description: lib-charset: Added more asserts to checking iconv() results. diffstat: src/lib-charset/charset-iconv.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diffs (25 lines): diff -r 93991734ae75 -r 9fdbb3b220ec src/lib-charset/charset-iconv.c --- a/src/lib-charset/charset-iconv.c Tue Sep 08 00:33:21 2015 +0300 +++ b/src/lib-charset/charset-iconv.c Tue Sep 08 00:34:14 2015 +0300 @@ -74,15 +74,17 @@ ic_srcbuf = (ICONV_CONST char *) src; if (iconv(t->cd, &ic_srcbuf, &srcleft, - &ic_destbuf, &destleft) != (size_t)-1) + &ic_destbuf, &destleft) != (size_t)-1) { + i_assert(srcleft == 0); *result = CHARSET_RET_OK; - else if (errno == E2BIG) { + } else if (errno == E2BIG) { /* set result just to avoid compiler warning */ *result = CHARSET_RET_INCOMPLETE_INPUT; ret = FALSE; - } else if (errno == EINVAL) + } else if (errno == EINVAL) { + i_assert(srcleft <= CHARSET_MAX_PENDING_BUF_SIZE); *result = CHARSET_RET_INCOMPLETE_INPUT; - else { + } else { /* should be EILSEQ */ *result = CHARSET_RET_INVALID_INPUT; ret = FALSE; From dovecot at dovecot.org Mon Sep 7 21:47:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 21:47:15 +0000 Subject: dovecot-2.2: Make static analyzer happier. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/06b884831f25 changeset: 19121:06b884831f25 user: Timo Sirainen date: Tue Sep 08 00:46:11 2015 +0300 description: Make static analyzer happier. diffstat: src/config/config-request.c | 5 +++-- src/lib-storage/mail-search-args-simplify.c | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diffs (40 lines): diff -r 9fdbb3b220ec -r 06b884831f25 src/config/config-request.c --- a/src/config/config-request.c Tue Sep 08 00:34:14 2015 +0300 +++ b/src/config/config-request.c Tue Sep 08 00:46:11 2015 +0300 @@ -206,7 +206,7 @@ { const struct setting_define *def; const void *value, *default_value, *change_value; - void *const *children = NULL, *const *change_children = NULL; + void *const *children, *const *change_children = NULL; unsigned int i, count, count2, prefix_len; const char *str; char *key; @@ -244,7 +244,7 @@ } dump = FALSE; - count = 0; + count = 0; children = NULL; str_truncate(ctx->value, 0); switch (def->type) { case SET_BOOL: @@ -333,6 +333,7 @@ } } + i_assert(count == 0 || children != NULL); prefix_len = str_len(ctx->prefix); for (i = 0; i < count; i++) { str_append(ctx->prefix, def->key); diff -r 9fdbb3b220ec -r 06b884831f25 src/lib-storage/mail-search-args-simplify.c --- a/src/lib-storage/mail-search-args-simplify.c Tue Sep 08 00:34:14 2015 +0300 +++ b/src/lib-storage/mail-search-args-simplify.c Tue Sep 08 00:46:11 2015 +0300 @@ -346,6 +346,7 @@ break; } if (merged) { + i_assert(prev_arg != NULL); prev_arg->next = args->next; args = args->next; ctx.removals = TRUE; From dovecot at dovecot.org Mon Sep 7 21:52:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 21:52:35 +0000 Subject: dovecot-2.2: doveadm: Fixed hiding titles' tabs with tab formatt... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b8f09586ab33 changeset: 19122:b8f09586ab33 user: Timo Sirainen date: Tue Sep 08 00:51:30 2015 +0300 description: doveadm: Fixed hiding titles' tabs with tab formatter (-h -f tab) diffstat: src/doveadm/doveadm-print-tab.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (19 lines): diff -r 06b884831f25 -r b8f09586ab33 src/doveadm/doveadm-print-tab.c --- a/src/doveadm/doveadm-print-tab.c Tue Sep 08 00:46:11 2015 +0300 +++ b/src/doveadm/doveadm-print-tab.c Tue Sep 08 00:51:30 2015 +0300 @@ -26,10 +26,12 @@ static void doveadm_print_tab_header(const struct doveadm_print_header *hdr) { - if (ctx.header_count++ > 0) - printf("\t"); - if (!doveadm_print_hide_titles) + ctx.header_count++; + if (!doveadm_print_hide_titles) { + if (ctx.header_count > 1) + printf("\t"); printf("%s", hdr->title); + } } static void doveadm_print_tab_print(const char *value) From dovecot at dovecot.org Mon Sep 7 22:21:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 22:21:55 +0000 Subject: dovecot-2.2: lib: Added assert to Solaris sendfile() return value. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f6dd24658fb1 changeset: 19123:f6dd24658fb1 user: Timo Sirainen date: Tue Sep 08 01:20:51 2015 +0300 description: lib: Added assert to Solaris sendfile() return value. diffstat: src/lib/sendfile-util.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b8f09586ab33 -r f6dd24658fb1 src/lib/sendfile-util.c --- a/src/lib/sendfile-util.c Tue Sep 08 00:51:30 2015 +0300 +++ b/src/lib/sendfile-util.c Tue Sep 08 01:20:51 2015 +0300 @@ -127,6 +127,7 @@ } } *offset = (uoff_t)s_offset; + i_assert(ret < 0 || (size_t)ret <= count); return ret; } From dovecot at dovecot.org Mon Sep 7 22:31:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 22:31:29 +0000 Subject: dovecot-2.2: imap: Implemented /private/specialuse METADATA entr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c4932f3f4acd changeset: 19125:c4932f3f4acd user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: imap: Implemented /private/specialuse METADATA entry for SPECIAL-USE capability. diffstat: src/lib-storage/Makefile.am | 2 + src/lib-storage/mailbox-attribute-internal.c | 48 ++++++++++++++++++++++++++++ src/lib-storage/mailbox-attribute-internal.h | 9 +++++ src/lib-storage/mailbox-attribute.c | 4 ++ 4 files changed, 63 insertions(+), 0 deletions(-) diffs (105 lines): diff -r d85bf97ab8b7 -r c4932f3f4acd src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Tue Sep 08 01:27:18 2015 +0300 +++ b/src/lib-storage/Makefile.am Sat Apr 25 11:42:06 2015 +0200 @@ -45,6 +45,7 @@ mail-thread.c \ mail-user.c \ mailbox-attribute.c \ + mailbox-attribute-internal.c \ mailbox-get.c \ mailbox-guid-cache.c \ mailbox-header.c \ @@ -75,6 +76,7 @@ mail-storage-settings.h \ mail-user.h \ mailbox-attribute.h \ + mailbox-attribute-internal.h \ mailbox-attribute-private.h \ mailbox-guid-cache.h \ mailbox-list.h \ diff -r d85bf97ab8b7 -r c4932f3f4acd src/lib-storage/mailbox-attribute-internal.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mailbox-attribute-internal.c Sat Apr 25 11:42:06 2015 +0200 @@ -0,0 +1,48 @@ +/* Copyright (c) 2003-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-storage-private.h" +#include "mailbox-attribute-internal.h" + +/* + * Internal mailbox attributes + */ + + /* /private/specialuse (RFC 6154) */ + +static int +mailbox_attribute_specialuse_get(struct mailbox_transaction_context *t, + const char *key ATTR_UNUSED, + struct mail_attribute_value *value_r) +{ + const struct mailbox_settings *set = t->box->set; + + if (set == NULL || *set->special_use == '\0') + return 0; + + value_r->value = set->special_use; + return 1; +} + +static struct mailbox_attribute_internal +iattr_mbox_prv_special_use = { + .type = MAIL_ATTRIBUTE_TYPE_PRIVATE, + .key = MAILBOX_ATTRIBUTE_SPECIALUSE, + .rank = MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY, + + .get = mailbox_attribute_specialuse_get +}; + +/* + * Registry + */ + +void mailbox_attributes_internal_init(void) +{ + /* + * Internal mailbox attributes + */ + + /* /private/specialuse (RFC 6154) */ + mailbox_attribute_register_internal(&iattr_mbox_prv_special_use); +} diff -r d85bf97ab8b7 -r c4932f3f4acd src/lib-storage/mailbox-attribute-internal.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mailbox-attribute-internal.h Sat Apr 25 11:42:06 2015 +0200 @@ -0,0 +1,9 @@ +#ifndef MAILBOX_ATTRIBUTE_INTERNAL_H +#define MAILBOX_ATTRIBUTE_INTERNAL_H + +/* RFC 6154, Section 4: IMAP METADATA Entry for Special-Use Attributes */ +#define MAILBOX_ATTRIBUTE_SPECIALUSE "specialuse" + +void mailbox_attributes_internal_init(void); + +#endif diff -r d85bf97ab8b7 -r c4932f3f4acd src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Tue Sep 08 01:27:18 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.c Sat Apr 25 11:42:06 2015 +0200 @@ -6,6 +6,7 @@ #include "istream.h" #include "mail-storage-private.h" #include "bsearch-insert-pos.h" +#include "mailbox-attribute-internal.h" static ARRAY(struct mailbox_attribute_internal) mailbox_internal_attributes; static pool_t mailbox_attribute_pool; @@ -15,6 +16,9 @@ mailbox_attribute_pool = pool_alloconly_create("mailbox attributes", 2048); i_array_init(&mailbox_internal_attributes, 32); + + /* internal mailbox attributes */ + mailbox_attributes_internal_init(); } void mailbox_attributes_deinit(void) From dovecot at dovecot.org Mon Sep 7 22:31:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 22:31:28 +0000 Subject: dovecot-2.2: lib: Added some tests for the new strnum _hex() and... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d85bf97ab8b7 changeset: 19124:d85bf97ab8b7 user: Stephan Bosch date: Tue Sep 08 01:27:18 2015 +0300 description: lib: Added some tests for the new strnum _hex() and _oct() function variants. diffstat: src/lib/test-strnum.c | 149 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 148 insertions(+), 1 deletions(-) diffs (179 lines): diff -r f6dd24658fb1 -r d85bf97ab8b7 src/lib/test-strnum.c --- a/src/lib/test-strnum.c Tue Sep 08 01:20:51 2015 +0300 +++ b/src/lib/test-strnum.c Tue Sep 08 01:27:18 2015 +0300 @@ -23,7 +23,6 @@ #undef STRINGIFY #undef BIGBASE } - static void test_str_to_uintmax(void) { unsigned int i=0; @@ -73,12 +72,158 @@ while (buff[--j] == '9') buff[j] = '0'; buff[j]++; + ret = str_to_uintmax(buff, &value); test_assert_idx(ret < 0 && value == UINTMAX_MAX/9-1, i); } test_end(); } +/* always pads with leading zeros to a size of 9 digits */ +static int crappy_uintmax_to_str_hex(char *into, uintmax_t val) +{ +#define BIGBASE 0x1000000000 +#define STRINGIFY(s) #s +#define STRINGIFY2(s) STRINGIFY(s) + int len = 0; + if(val >= BIGBASE) { + len = crappy_uintmax_to_str_hex(into, val/BIGBASE); + } + i_snprintf(into + len, 10, "%09lx", (unsigned long)(val % BIGBASE)); + return len + strlen(STRINGIFY2(BIGBASE))-3; +#undef STRINGIFY2 +#undef STRINGIFY +#undef BIGBASE +} +static void test_str_to_uintmax_hex(void) +{ + unsigned int i=0; + int randrange = rand()%15+1; /* when 1, will max out on 1s */ + uintmax_t value = 0; + int len, ret; + char buff[52]; /* totally assumes < 200 bits */ + + test_begin("str_to_uintmax_hex in range"); + while (i < sizeof(uintmax_t)*CHAR_BIT) { + uintmax_t value_back; + const char *endp; + + value = (value << 1) + 1; + if (value >= 64) + value -= rand()%randrange; /* don't always test the same numbers */ + len = crappy_uintmax_to_str_hex(buff, value); + ret = str_to_uintmax_hex(buff, &value_back); + test_assert_idx(ret == 0, i); + test_assert_idx(value == value_back, i); + + /* test with trailing noise */ + buff[len] = 'x'; /* don't even null-terminate, let's be evil */ + value_back = 0x1234567890123456; + ret = str_to_uintmax_hex(buff, &value_back); + test_assert_idx(ret < 0, i); + test_assert_idx(value_back == 0x1234567890123456, i); + ret = str_parse_uintmax_hex(buff, &value_back, &endp); + test_assert_idx(ret == 0, i); + test_assert_idx(value_back == value, i); + test_assert_idx(endp == &buff[len], i); + i++; + } + test_end(); + + /* not knowing exactly how large a uintmax_t is, we have to construct + the troublesome near-0x10/0x0F*MAX strings manually by appending digits + to a MAX/0x0f string which we can easily create. Do a wider range + of 0x30 rather than the obvious 0x10, just in case - all are too large.*/ + test_begin("str_to_uintmax_hex overflow corner case"); + value = (UINTMAX_MAX/0x0f)-1; + len = crappy_uintmax_to_str_hex(buff, value); + buff[len] = '0'; + buff[len+1] = '\0'; + for(i = 0; i <= 0x30; ++i) { + int j = len + 1; + while (buff[--j] == 'f') + buff[j] = '0'; + if (buff[j] == '9') + buff[j] = 'a'; + else + buff[j]++; + ret = str_to_uintmax_hex(buff, &value); + test_assert_idx(ret < 0 && value == UINTMAX_MAX/0x0f-1, i); + } + test_end(); +} + +/* always pads with leading zeros to a size of 9 digits */ +static int crappy_uintmax_to_str_oct(char *into, uintmax_t val) +{ +#define BIGBASE 01000000000 +#define STRINGIFY(s) #s +#define STRINGIFY2(s) STRINGIFY(s) + int len = 0; + if(val >= BIGBASE) { + len = crappy_uintmax_to_str_oct(into, val/BIGBASE); + } + i_snprintf(into + len, 10, "%09lo", (unsigned long)(val % BIGBASE)); + return len + strlen(STRINGIFY2(BIGBASE))-2; +#undef STRINGIFY2 +#undef STRINGIFY +#undef BIGBASE +} +static void test_str_to_uintmax_oct(void) +{ + unsigned int i=0; + int randrange = rand()%15+1; /* when 1, will max out on 1s */ + uintmax_t value = 0; + int len, ret; + char buff[69]; /* totally assumes < 200 bits */ + + test_begin("str_to_uintmax_oct in range"); + while (i < sizeof(uintmax_t)*CHAR_BIT) { + uintmax_t value_back; + const char *endp; + + value = (value << 1) + 1; + if (value >= 64) + value -= rand()%randrange; /* don't always test the same numbers */ + len = crappy_uintmax_to_str_oct(buff, value); + ret = str_to_uintmax_oct(buff, &value_back); + test_assert_idx(ret == 0, i); + test_assert_idx(value == value_back, i); + + /* test with trailing noise */ + buff[len] = 'x'; /* don't even null-terminate, let's be evil */ + value_back = 0x1234567890123456; + ret = str_to_uintmax_oct(buff, &value_back); + test_assert_idx(ret < 0, i); + test_assert_idx(value_back == 0x1234567890123456, i); + ret = str_parse_uintmax_oct(buff, &value_back, &endp); + test_assert_idx(ret == 0, i); + test_assert_idx(value_back == value, i); + test_assert_idx(endp == &buff[len], i); + i++; + } + test_end(); + + /* not knowing exactly how large a uintmax_t is, we have to construct + the troublesome near-010/007*MAX strings manually by appending digits + to a MAX/007 string which we can easily create. Do a wider range + of 030 rather than the obvious 010, just in case - all are too large.*/ + test_begin("str_to_uintmax_oct overflow corner case"); + value = (UINTMAX_MAX/007)-1; + len = crappy_uintmax_to_str_oct(buff, value); + buff[len] = '0'; + buff[len+1] = '\0'; + for(i = 0; i <= 030; ++i) { + int j = len + 1; + while (buff[--j] == '7') + buff[j] = '0'; + buff[j]++; + ret = str_to_uintmax_oct(buff, &value); + test_assert_idx(ret < 0 && value == UINTMAX_MAX/007-1, i); + } + test_end(); +} + static void test_str_to_u64(void) { unsigned int i; @@ -213,6 +358,8 @@ { /* If the above isn't true, then we do expect some failures possibly */ test_str_to_uintmax(); + test_str_to_uintmax_hex(); + test_str_to_uintmax_oct(); test_str_to_u64(); test_str_to_u32(); test_str_to_llong(); From dovecot at dovecot.org Mon Sep 7 22:31:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Sep 2015 22:31:30 +0000 Subject: dovecot-2.2: imap: METADATA: Implemented mailbox and server comm... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/793bf4ab439f changeset: 19126:793bf4ab439f user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: imap: METADATA: Implemented mailbox and server comment entries and the server admin entry. diffstat: src/lib-storage/mail-storage-settings.c | 11 ++- src/lib-storage/mail-storage-settings.h | 3 + src/lib-storage/mailbox-attribute-internal.c | 100 +++++++++++++++++++++++++++ src/lib-storage/mailbox-attribute-internal.h | 5 + 4 files changed, 118 insertions(+), 1 deletions(-) diffs (201 lines): diff -r c4932f3f4acd -r 793bf4ab439f src/lib-storage/mail-storage-settings.c --- a/src/lib-storage/mail-storage-settings.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-storage/mail-storage-settings.c Sat Apr 25 11:42:06 2015 +0200 @@ -35,6 +35,8 @@ DEF(SET_STR, mail_cache_fields), DEF(SET_STR, mail_always_cache_fields), DEF(SET_STR, mail_never_cache_fields), + DEF(SET_STR, mail_server_comment), + DEF(SET_STR, mail_server_admin), DEF(SET_UINT, mail_cache_min_mail_count), DEF(SET_TIME, mailbox_idle_check_interval), DEF(SET_UINT, mail_max_keyword_length), @@ -73,6 +75,8 @@ .mail_cache_fields = "flags", .mail_always_cache_fields = "", .mail_never_cache_fields = "imap.envelope", + .mail_server_comment = "", + .mail_server_admin = "", .mail_cache_min_mail_count = 0, .mailbox_idle_check_interval = 30, .mail_max_keyword_length = 50, @@ -121,6 +125,7 @@ { SET_ENUM, "auto", offsetof(struct mailbox_settings, autocreate), NULL } , DEF(SET_STR, special_use), DEF(SET_STR, driver), + DEF(SET_STR, comment), SETTING_DEFINE_LIST_END }; @@ -131,7 +136,8 @@ MAILBOX_SET_AUTO_CREATE":" MAILBOX_SET_AUTO_SUBSCRIBE, .special_use = "", - .driver = "" + .driver = "", + .comment = "" }; const struct setting_parser_info mailbox_setting_parser_info = { @@ -444,6 +450,9 @@ return FALSE; } #endif + + // FIXME: check set->mail_server_admin syntax (RFC 5464, Section 6.2.2) + return TRUE; } diff -r c4932f3f4acd -r 793bf4ab439f src/lib-storage/mail-storage-settings.h --- a/src/lib-storage/mail-storage-settings.h Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-storage/mail-storage-settings.h Sat Apr 25 11:42:06 2015 +0200 @@ -20,6 +20,8 @@ const char *mail_cache_fields; const char *mail_always_cache_fields; const char *mail_never_cache_fields; + const char *mail_server_comment; + const char *mail_server_admin; unsigned int mail_cache_min_mail_count; unsigned int mailbox_idle_check_interval; unsigned int mail_max_keyword_length; @@ -78,6 +80,7 @@ const char *autocreate; const char *special_use; const char *driver; + const char *comment; }; struct mail_user_settings { diff -r c4932f3f4acd -r 793bf4ab439f src/lib-storage/mailbox-attribute-internal.c --- a/src/lib-storage/mailbox-attribute-internal.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-storage/mailbox-attribute-internal.c Sat Apr 25 11:42:06 2015 +0200 @@ -33,6 +33,93 @@ .get = mailbox_attribute_specialuse_get }; +/* /private/comment, /shared/comment (RFC 5464) */ + +static int +mailbox_attribute_comment_get(struct mailbox_transaction_context *t, + const char *key ATTR_UNUSED, + struct mail_attribute_value *value_r) +{ + const struct mailbox_settings *set = t->box->set; + + if (set == NULL || *set->comment == '\0') + return 0; + value_r->value = set->comment; + return 1; +} + +static struct mailbox_attribute_internal +iattr_mbox_prv_comment = { + .type = MAIL_ATTRIBUTE_TYPE_PRIVATE, + .key = MAILBOX_ATTRIBUTE_COMMENT, + .rank = MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT, + + .get = mailbox_attribute_comment_get +}; + +static struct mailbox_attribute_internal +iattr_mbox_shd_comment = { + .type = MAIL_ATTRIBUTE_TYPE_SHARED, + .key = MAILBOX_ATTRIBUTE_COMMENT, + .rank = MAIL_ATTRIBUTE_INTERNAL_RANK_DEFAULT, + + .get = mailbox_attribute_comment_get +}; + +/* + * Internal server attributes + */ + +/* /shared/comment (RFC 5464) */ + +static int +server_attribute_comment_get(struct mailbox_transaction_context *t, + const char *key ATTR_UNUSED, + struct mail_attribute_value *value_r) +{ + const struct mail_storage_settings *set = t->box->storage->set; + + if (*set->mail_server_comment == '\0') + return 0; + value_r->value = set->mail_server_comment; + return 1; +} + +static struct mailbox_attribute_internal +iattr_serv_shd_comment = { + .type = MAIL_ATTRIBUTE_TYPE_SHARED, + .key = MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER + MAIL_SERVER_ATTRIBUTE_COMMENT, + .rank = MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY, + + .get = server_attribute_comment_get +}; + +/* /shared/admin (RFC 5464) */ + +static int +server_attribute_admin_get(struct mailbox_transaction_context *t, + const char *key ATTR_UNUSED, + struct mail_attribute_value *value_r) +{ + const struct mail_storage_settings *set = t->box->storage->set; + + if (*set->mail_server_admin == '\0') + return 0; + value_r->value = set->mail_server_admin; + return 1; +} + +static struct mailbox_attribute_internal +iattr_serv_shd_admin = { + .type = MAIL_ATTRIBUTE_TYPE_SHARED, + .key = MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER + MAIL_SERVER_ATTRIBUTE_ADMIN, + .rank = MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY, + + .get = server_attribute_admin_get +}; + /* * Registry */ @@ -45,4 +132,17 @@ /* /private/specialuse (RFC 6154) */ mailbox_attribute_register_internal(&iattr_mbox_prv_special_use); + /* /private/comment (RFC 5464) */ + mailbox_attribute_register_internal(&iattr_mbox_prv_comment); + /* /shared/comment (RFC 5464) */ + mailbox_attribute_register_internal(&iattr_mbox_shd_comment); + + /* + * internal server attributes + */ + + /* /shared/comment (RFC 5464) */ + mailbox_attribute_register_internal(&iattr_serv_shd_comment); + /* /shared/admin (RFC 5464) */ + mailbox_attribute_register_internal(&iattr_serv_shd_admin); } diff -r c4932f3f4acd -r 793bf4ab439f src/lib-storage/mailbox-attribute-internal.h --- a/src/lib-storage/mailbox-attribute-internal.h Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-storage/mailbox-attribute-internal.h Sat Apr 25 11:42:06 2015 +0200 @@ -1,8 +1,13 @@ #ifndef MAILBOX_ATTRIBUTE_INTERNAL_H #define MAILBOX_ATTRIBUTE_INTERNAL_H +/* RFC 5464, Section 3.2.1.2: Mailbox entries */ +#define MAILBOX_ATTRIBUTE_COMMENT "comment" /* RFC 6154, Section 4: IMAP METADATA Entry for Special-Use Attributes */ #define MAILBOX_ATTRIBUTE_SPECIALUSE "specialuse" +/* RFC 5464, Section 3.2.1.1: Server entries */ +#define MAIL_SERVER_ATTRIBUTE_COMMENT "comment" +#define MAIL_SERVER_ATTRIBUTE_ADMIN "admin" void mailbox_attributes_internal_init(void); From pigeonhole at rename-it.nl Mon Sep 7 23:08:40 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 08 Sep 2015 01:08:40 +0200 Subject: dovecot-2.2-pigeonhole: managesieve: Fixed assert failure occuri... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/5df1b6d72ec2 changeset: 2096:5df1b6d72ec2 user: Stephan Bosch date: Tue Sep 08 01:08:34 2015 +0200 description: managesieve: Fixed assert failure occuring when client disconnects during GETSCRIPT command. Passed NULL as reason to client_disconnect(), which is not allowed. diffstat: src/managesieve/cmd-getscript.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 633bf3c89a39 -r 5df1b6d72ec2 src/managesieve/cmd-getscript.c --- a/src/managesieve/cmd-getscript.c Tue Sep 08 01:07:43 2015 +0200 +++ b/src/managesieve/cmd-getscript.c Tue Sep 08 01:08:34 2015 +0200 @@ -4,6 +4,7 @@ #include "lib.h" #include "ostream.h" #include "istream.h" +#include "iostream.h" #include "sieve-script.h" #include "sieve-storage.h" @@ -64,7 +65,9 @@ sieve_script_location(ctx->script), i_stream_get_error(ctx->script_stream)); } else { - client_disconnect(ctx->client, NULL); + client_disconnect(ctx->client, + io_stream_get_disconnect_reason + (client->input, client->output)); } ctx->failed = TRUE; return cmd_getscript_finish(ctx); From pigeonhole at rename-it.nl Mon Sep 7 23:08:40 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 08 Sep 2015 01:08:40 +0200 Subject: dovecot-2.2-pigeonhole: managesieve: Started using io_stream_get... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/633bf3c89a39 changeset: 2095:633bf3c89a39 user: Stephan Bosch date: Tue Sep 08 01:07:43 2015 +0200 description: managesieve: Started using io_stream_get_disconnect_reason() for default client_destroy() reason. diffstat: src/managesieve/managesieve-client.c | 20 +++++--------------- 1 files changed, 5 insertions(+), 15 deletions(-) diffs (44 lines): diff -r 45854bc3a431 -r 633bf3c89a39 src/managesieve/managesieve-client.c --- a/src/managesieve/managesieve-client.c Sat Aug 29 13:12:58 2015 +0200 +++ b/src/managesieve/managesieve-client.c Tue Sep 08 01:07:43 2015 +0200 @@ -9,6 +9,7 @@ #include "net.h" #include "istream.h" #include "ostream.h" +#include "iostream.h" #include "var-expand.h" #include "master-service.h" #include "mail-storage-service.h" @@ -211,19 +212,6 @@ return str_c(str); } -static const char *client_get_disconnect_reason(struct client *client) -{ - errno = client->input->stream_errno != 0 ? - client->input->stream_errno : - client->output->stream_errno; - if (errno == 0 || errno == EPIPE) - return "Connection closed"; - return t_strdup_printf("Connection closed: %s", - client->input->stream_errno != 0 ? - i_stream_get_error(client->input) : - o_stream_get_error(client->output)); -} - void client_destroy(struct client *client, const char *reason) { int ret; @@ -234,8 +222,10 @@ if (!client->disconnected) { client->disconnected = TRUE; - if (reason == NULL) - reason = client_get_disconnect_reason(client); + if (reason == NULL) { + reason = io_stream_get_disconnect_reason(client->input, + client->output); + } i_info("%s %s", reason, client_stats(client)); } From dovecot at dovecot.org Tue Sep 8 10:09:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 10:09:10 +0000 Subject: dovecot-2.2: lib-charset: Fixed assert-crash with some iconv() v... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cb4fcdc716e2 changeset: 19127:cb4fcdc716e2 user: Timo Sirainen date: Tue Sep 08 13:07:59 2015 +0300 description: lib-charset: Fixed assert-crash with some iconv() versions. Older glibc iconv() versions seem to skip over invalid characters, at least with some charsets, while newer versions don't. We were assuming that the skipping never happened, so if the invalid character was at the end of the string we could have wrapped size to (size_t)-1 and caused a crash later on. diffstat: src/lib-charset/charset-iconv.c | 4 +++- src/lib-charset/test-charset.c | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletions(-) diffs (61 lines): diff -r 793bf4ab439f -r cb4fcdc716e2 src/lib-charset/charset-iconv.c --- a/src/lib-charset/charset-iconv.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-charset/charset-iconv.c Tue Sep 08 13:07:59 2015 +0300 @@ -111,6 +111,7 @@ bool ret; for (pos = 0;;) { + i_assert(pos <= *src_size); size = *src_size - pos; ret = charset_to_utf8_try(t, src + pos, &size, dest, &result); pos += size; @@ -124,7 +125,8 @@ strlen(UNICODE_REPLACEMENT_CHAR_UTF8)); prev_invalid_pos = dest->used; } - pos++; + if (pos < *src_size) + pos++; } } diff -r 793bf4ab439f -r cb4fcdc716e2 src/lib-charset/test-charset.c --- a/src/lib-charset/test-charset.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-charset/test-charset.c Tue Sep 08 13:07:59 2015 +0300 @@ -85,6 +85,28 @@ test_charset_utf8_common("UTF-8//IGNORE"); test_end(); } +static void test_charset_iconv_crashes(void) +{ + struct { + const char *charset; + const char *input; + } tests[] = { + { "CP932", "\203\334" } + }; + string_t *str = t_str_new(128); + enum charset_result result; + unsigned int i; + + test_begin("charset iconv crashes"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + str_truncate(str, 0); + /* we don't care about checking the result. we only want to + verify that there's no crash. */ + (void)charset_to_utf8_str(tests[i].charset, NULL, + tests[i].input, str, &result); + } + test_end(); +} #endif int main(void) @@ -94,6 +116,7 @@ test_charset_utf8, #ifdef HAVE_ICONV test_charset_iconv, + test_charset_iconv_crashes, #endif NULL }; From dovecot at dovecot.org Tue Sep 8 10:20:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 10:20:16 +0000 Subject: dovecot-2.2: acl: Log which ACL groups are enabled with mail_deb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/500e8dd7a389 changeset: 19128:500e8dd7a389 user: Timo Sirainen date: Tue Sep 08 13:18:58 2015 +0300 description: acl: Log which ACL groups are enabled with mail_debug=yes diffstat: src/plugins/acl/acl-backend.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (16 lines): diff -r cb4fcdc716e2 -r 500e8dd7a389 src/plugins/acl/acl-backend.c --- a/src/plugins/acl/acl-backend.c Tue Sep 08 13:07:59 2015 +0300 +++ b/src/plugins/acl/acl-backend.c Tue Sep 08 13:18:58 2015 +0300 @@ -66,8 +66,11 @@ backend->group_count = group_count; backend->groups = p_new(backend->pool, const char *, group_count); - for (i = 0; i < group_count; i++) + for (i = 0; i < group_count; i++) { backend->groups[i] = p_strdup(backend->pool, groups[i]); + if (user->mail_debug) + i_debug("acl: group added: %s", groups[i]); + } i_qsort(backend->groups, group_count, sizeof(const char *), i_strcmp_p); } From dovecot at dovecot.org Tue Sep 8 10:32:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 10:32:38 +0000 Subject: dovecot-2.2: example-config: Added more detailed documentation f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/14964e7e91b9 changeset: 19130:14964e7e91b9 user: Stephan Bosch date: Tue Sep 08 13:27:41 2015 +0300 description: example-config: Added more detailed documentation for mailbox settings. diffstat: doc/example-config/conf.d/10-mail.conf | 2 + doc/example-config/conf.d/15-mailboxes.conf | 44 +++++++++++++++++++++------- 2 files changed, 35 insertions(+), 11 deletions(-) diffs (68 lines): diff -r e4438e342208 -r 14964e7e91b9 doc/example-config/conf.d/10-mail.conf --- a/doc/example-config/conf.d/10-mail.conf Tue Sep 08 13:26:38 2015 +0300 +++ b/doc/example-config/conf.d/10-mail.conf Tue Sep 08 13:27:41 2015 +0300 @@ -75,6 +75,8 @@ # Namespace handles its own subscriptions. If set to "no", the parent # namespace handles them (empty prefix should always have this as "yes") #subscriptions = yes + + # See 15-mailboxes.conf for definitions of special mailboxes. } # Example shared namespace configuration diff -r e4438e342208 -r 14964e7e91b9 doc/example-config/conf.d/15-mailboxes.conf --- a/doc/example-config/conf.d/15-mailboxes.conf Tue Sep 08 13:26:38 2015 +0300 +++ b/doc/example-config/conf.d/15-mailboxes.conf Tue Sep 08 13:27:41 2015 +0300 @@ -2,19 +2,41 @@ ## Mailbox definitions ## +# Each mailbox is specified in a separate mailbox section. The section name +# specifies the mailbox name. If it has spaces, you can put the name +# "in quotes". These sections can contain the following mailbox settings: +# +# auto: +# Indicates whether the mailbox with this name is automatically created +# implicitly when it is first accessed. The user can also be automatically +# subscribed to the mailbox after creation. The following values are +# defined for this setting: +# +# no - Never created automatically. +# create - Automatically created, but no automatic subscription. +# subscribe - Automatically created and subscribed. +# +# special_use: +# A space-separated list of SPECIAL-USE flags (RFC 6154) to use for the +# mailbox. There are no validity checks, so you could specify anything +# you want in here, but it's not a good idea to use flags other than the +# standard ones specified in the RFC: +# +# \All - This (virtual) mailbox presents all messages in the +# user's message store. +# \Archive - This mailbox is used to archive messages. +# \Drafts - This mailbox is used to hold draft messages. +# \Flagged - This (virtual) mailbox presents all messages in the +# user's message store marked with the IMAP \Flagged flag. +# \Junk - This mailbox is where messages deemed to be junk mail +# are held. +# \Sent - This mailbox is used to hold copies of messages that +# have been sent. +# \Trash - This mailbox is used to hold messages that have been +# deleted. + # NOTE: Assumes "namespace inbox" has been defined in 10-mail.conf. namespace inbox { - - #mailbox name { - # auto=create will automatically create this mailbox. - # auto=subscribe will both create and subscribe to the mailbox. - #auto = no - - # Space separated list of IMAP SPECIAL-USE attributes as specified by - # RFC 6154: \All \Archive \Drafts \Flagged \Junk \Sent \Trash - #special_use = - #} - # These mailboxes are widely used and could perhaps be created automatically: mailbox Drafts { special_use = \Drafts From dovecot at dovecot.org Tue Sep 8 10:32:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 10:32:37 +0000 Subject: dovecot-2.2: lib: strnum: Fixed test suite failure on 32 bit sys... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4438e342208 changeset: 19129:e4438e342208 user: Stephan Bosch date: Tue Sep 08 13:26:38 2015 +0300 description: lib: strnum: Fixed test suite failure on 32 bit systems. diffstat: src/lib/test-strnum.c | 21 ++++++++++++--------- 1 files changed, 12 insertions(+), 9 deletions(-) diffs (63 lines): diff -r 500e8dd7a389 -r e4438e342208 src/lib/test-strnum.c --- a/src/lib/test-strnum.c Tue Sep 08 13:18:58 2015 +0300 +++ b/src/lib/test-strnum.c Tue Sep 08 13:26:38 2015 +0300 @@ -10,15 +10,16 @@ /* always pads with leading zeros to a size of 9 digits */ static int crappy_uintmax_to_str(char *into, uintmax_t val) { -#define BIGBASE 1000000000u +#define BIGBASE 1000000000ull #define STRINGIFY(s) #s #define STRINGIFY2(s) STRINGIFY(s) int len = 0; if(val >= BIGBASE) { len = crappy_uintmax_to_str(into, val/BIGBASE); } - i_snprintf(into + len, 10, "%09lu", (unsigned long)(val % BIGBASE)); - return len + strlen(STRINGIFY2(BIGBASE))-2; + i_snprintf(into + len, 10, "%09llu", + (unsigned long long)(val % BIGBASE)); + return len + strlen(STRINGIFY2(BIGBASE))-4; #undef STRINGIFY2 #undef STRINGIFY #undef BIGBASE @@ -82,15 +83,16 @@ /* always pads with leading zeros to a size of 9 digits */ static int crappy_uintmax_to_str_hex(char *into, uintmax_t val) { -#define BIGBASE 0x1000000000 +#define BIGBASE 0x1000000000ull #define STRINGIFY(s) #s #define STRINGIFY2(s) STRINGIFY(s) int len = 0; if(val >= BIGBASE) { len = crappy_uintmax_to_str_hex(into, val/BIGBASE); } - i_snprintf(into + len, 10, "%09lx", (unsigned long)(val % BIGBASE)); - return len + strlen(STRINGIFY2(BIGBASE))-3; + i_snprintf(into + len, 10, "%09llx", + (unsigned long long)(val % BIGBASE)); + return len + strlen(STRINGIFY2(BIGBASE))-6; #undef STRINGIFY2 #undef STRINGIFY #undef BIGBASE @@ -156,15 +158,16 @@ /* always pads with leading zeros to a size of 9 digits */ static int crappy_uintmax_to_str_oct(char *into, uintmax_t val) { -#define BIGBASE 01000000000 +#define BIGBASE 01000000000ull #define STRINGIFY(s) #s #define STRINGIFY2(s) STRINGIFY(s) int len = 0; if(val >= BIGBASE) { len = crappy_uintmax_to_str_oct(into, val/BIGBASE); } - i_snprintf(into + len, 10, "%09lo", (unsigned long)(val % BIGBASE)); - return len + strlen(STRINGIFY2(BIGBASE))-2; + i_snprintf(into + len, 10, "%09llo", + (unsigned long long)(val % BIGBASE)); + return len + strlen(STRINGIFY2(BIGBASE))-5; #undef STRINGIFY2 #undef STRINGIFY #undef BIGBASE From dovecot at dovecot.org Tue Sep 8 10:32:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 10:32:39 +0000 Subject: dovecot-2.2: example-config: 10-mail.conf: Added example configu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4b581d735292 changeset: 19131:4b581d735292 user: Stephan Bosch date: Tue Sep 08 13:30:21 2015 +0300 description: example-config: 10-mail.conf: Added example configuration for the new settings relating to IMAP METADATA. diffstat: doc/example-config/conf.d/10-mail.conf | 12 ++++++++++++ doc/example-config/conf.d/15-mailboxes.conf | 9 +++++++++ 2 files changed, 21 insertions(+), 0 deletions(-) diffs (52 lines): diff -r 14964e7e91b9 -r 4b581d735292 doc/example-config/conf.d/10-mail.conf --- a/doc/example-config/conf.d/10-mail.conf Tue Sep 08 13:27:41 2015 +0300 +++ b/doc/example-config/conf.d/10-mail.conf Tue Sep 08 13:30:21 2015 +0300 @@ -130,6 +130,18 @@ # URLAUTH and METADATA extensions. #mail_attribute_dict = +# A comment or note that is associated with the server. This value is +# accessible for authenticated users through the IMAP METADATA server +# entry "/shared/comment". +#mail_server_comment = "" + +# Indicates a method for contacting the server administrator. According to +# RFC 5464, this value MUST be a URI (e.g., a mailto: or tel: URL), but that +# is currently not enforced. Use for example mailto:admin at example.com. This +# value is accessible for authenticated users through the IMAP METADATA server +# entry "/shared/admin". +#mail_server_admin = + ## ## Mail processes ## diff -r 14964e7e91b9 -r 4b581d735292 doc/example-config/conf.d/15-mailboxes.conf --- a/doc/example-config/conf.d/15-mailboxes.conf Tue Sep 08 13:27:41 2015 +0300 +++ b/doc/example-config/conf.d/15-mailboxes.conf Tue Sep 08 13:30:21 2015 +0300 @@ -34,6 +34,13 @@ # have been sent. # \Trash - This mailbox is used to hold messages that have been # deleted. +# +# comment: +# Defines a default comment or note associated with the mailbox. This +# value is accessible through the IMAP METADATA mailbox entries +# "/shared/comment" and "/private/comment". Users with sufficient +# privileges can override the default value for entries with a custom +# value. # NOTE: Assumes "namespace inbox" has been defined in 10-mail.conf. namespace inbox { @@ -60,10 +67,12 @@ # If you have a virtual "All messages" mailbox: #mailbox virtual/All { # special_use = \All + # comment = All my messages #} # If you have a virtual "Flagged" mailbox: #mailbox virtual/Flagged { # special_use = \Flagged + # comment = All my flagged messages #} } From dovecot at dovecot.org Tue Sep 8 10:42:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 10:42:26 +0000 Subject: dovecot-2.2: lib-storage: If mail_attribute_dict is set, ignore ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e027fa256c55 changeset: 19132:e027fa256c55 user: Timo Sirainen date: Tue Sep 08 13:41:21 2015 +0300 description: lib-storage: If mail_attribute_dict is set, ignore non-authority rank internal attributes. This way dsync can still sync things like Sieve scripts, but it doesn't attempt to sync anything that actually requires dict. diffstat: src/lib-storage/mailbox-attribute.c | 12 +++++++----- 1 files changed, 7 insertions(+), 5 deletions(-) diffs (41 lines): diff -r 4b581d735292 -r e027fa256c55 src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Tue Sep 08 13:30:21 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.c Tue Sep 08 13:41:21 2015 +0300 @@ -100,7 +100,7 @@ static void mailbox_internal_attributes_get(enum mail_attribute_type type, - const char *prefix, ARRAY_TYPE(const_string) *attrs) + const char *prefix, bool have_dict, ARRAY_TYPE(const_string) *attrs) { const struct mailbox_attribute_internal *regs; struct mailbox_attribute_internal dreg; @@ -136,8 +136,8 @@ /* remove prefix */ key += plen + 1; } - - array_append(attrs, &key, 1); + if (have_dict || regs[i].rank == MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY) + array_append(attrs, &key, 1); } } @@ -357,13 +357,15 @@ struct mailbox_attribute_iter *iter; ARRAY_TYPE(const_string) extra_attrs; const char *const *attr; - + bool have_dict; + iter = box->v.attribute_iter_init(box, type, prefix); i_assert(iter->box != NULL); /* check which internal attributes may apply */ t_array_init(&extra_attrs, 4); - mailbox_internal_attributes_get(type, prefix, &extra_attrs); + have_dict = box->storage->set->mail_attribute_dict[0] != '\0'; + mailbox_internal_attributes_get(type, prefix, have_dict, &extra_attrs); /* any extra internal attributes to add? */ if (array_count(&extra_attrs) == 0) { From dovecot at dovecot.org Tue Sep 8 15:47:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 15:47:27 +0000 Subject: dovecot-2.2: lib-storage: Use MAIL_ERROR_NOTPOSSIBLE if mailbox_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/72d6f1f235d2 changeset: 19133:72d6f1f235d2 user: Timo Sirainen date: Tue Sep 08 18:45:28 2015 +0300 description: lib-storage: Use MAIL_ERROR_NOTPOSSIBLE if mailbox_attribute_*set() fails for read-only attribute. MAIL_ERROR_PARAMS indicates that there was something wrong with the parameters itself. But the caller can't know (at least with current APIs) if it's a read-only attribute it's trying to change before actually doing it. diffstat: src/lib-storage/mailbox-attribute.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e027fa256c55 -r 72d6f1f235d2 src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Tue Sep 08 13:41:21 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.c Tue Sep 08 18:45:28 2015 +0300 @@ -172,7 +172,7 @@ break; case MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY: if (iattr->set == NULL) { - mail_storage_set_error(t->box->storage, MAIL_ERROR_PARAMS, t_strdup_printf( + mail_storage_set_error(t->box->storage, MAIL_ERROR_NOTPOSSIBLE, t_strdup_printf( "The /%s/%s attribute cannot be changed", (type == MAIL_ATTRIBUTE_TYPE_SHARED ? "shared" : "private"), key)); return -1; From dovecot at dovecot.org Tue Sep 8 15:47:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 15:47:28 +0000 Subject: dovecot-2.2: lib-storage: Mailbox deletion shouldn't fail when t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/962ff5ebd807 changeset: 19134:962ff5ebd807 user: Timo Sirainen date: Tue Sep 08 18:45:57 2015 +0300 description: lib-storage: Mailbox deletion shouldn't fail when trying to delete read-only attributes. diffstat: src/lib-storage/index/index-storage.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 72d6f1f235d2 -r 962ff5ebd807 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Tue Sep 08 18:45:28 2015 +0300 +++ b/src/lib-storage/index/index-storage.c Tue Sep 08 18:45:57 2015 +0300 @@ -632,8 +632,12 @@ strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER)) == 0) continue; - if (mailbox_attribute_unset(t, type, key) < 0) - ret = -1; + if (mailbox_attribute_unset(t, type, key) < 0) { + if (mailbox_get_last_mail_error(t->box) != MAIL_ERROR_NOTPOSSIBLE) { + ret = -1; + break; + } + } } if (mailbox_attribute_iter_deinit(&iter) < 0) ret = -1; From dovecot at dovecot.org Tue Sep 8 16:07:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 16:07:23 +0000 Subject: dovecot-2.2: lib: Added i_unlink() and i_unlink_if_exists() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2ec2b0a51fec changeset: 19135:2ec2b0a51fec user: Timo Sirainen date: Tue Sep 08 18:49:00 2015 +0300 description: lib: Added i_unlink() and i_unlink_if_exists() These log the error message on a failed unlink(). They also include the source code file and line number to make it easier to find which unlink() actually failed if the path itself doesn't already clearly identify it. This can be especially useful if the path is (null), "" or contains some corrupted garbage. diffstat: src/lib/lib.c | 27 +++++++++++++++++++++++++++ src/lib/lib.h | 10 ++++++++++ 2 files changed, 37 insertions(+), 0 deletions(-) diffs (57 lines): diff -r 962ff5ebd807 -r 2ec2b0a51fec src/lib/lib.c --- a/src/lib/lib.c Tue Sep 08 18:45:57 2015 +0300 +++ b/src/lib/lib.c Tue Sep 08 18:49:00 2015 +0300 @@ -30,6 +30,33 @@ return ret; } +#undef i_unlink +int i_unlink(const char *path, const char *source_fname, + unsigned int source_linenum) +{ + if (unlink(path) < 0) { + i_error("unlink(%s) failed: %m (in %s:%u)", + path, source_fname, source_linenum); + return -1; + } + return 0; +} + +#undef i_unlink_if_exists +int i_unlink_if_exists(const char *path, const char *source_fname, + unsigned int source_linenum) +{ + if (unlink(path) == 0) + return 1; + else if (errno == ENOENT) + return 0; + else { + i_error("unlink(%s) failed: %m (in %s:%u)", + path, source_fname, source_linenum); + return -1; + } +} + void lib_atexit(lib_atexit_callback_t *callback) { lib_atexit_priority(callback, 0); diff -r 962ff5ebd807 -r 2ec2b0a51fec src/lib/lib.h --- a/src/lib/lib.h Tue Sep 08 18:45:57 2015 +0300 +++ b/src/lib/lib.h Tue Sep 08 18:49:00 2015 +0300 @@ -49,6 +49,16 @@ #define LIB_ATEXIT_PRIORITY_LOW 10 int close_keep_errno(int *fd); +/* Call unlink(). If it fails, log an error including the source filename + and line number. */ +int i_unlink(const char *path, const char *source_fname, + unsigned int source_linenum); +#define i_unlink(path) i_unlink(path, __FILE__, __LINE__) +/* Same as i_unlink(), but don't log an error if errno=ENOENT. Returns 1 on + unlink() success, 0 if errno=ENOENT, -1 on other errors. */ +int i_unlink_if_exists(const char *path, const char *source_fname, + unsigned int source_linenum); +#define i_unlink_if_exists(path) i_unlink_if_exists(path, __FILE__, __LINE__) /* Call the given callback at the beginning of lib_deinit(). The main difference to atexit() is that liblib's memory allocation and logging From dovecot at dovecot.org Tue Sep 8 16:08:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 16:08:34 +0000 Subject: dovecot-2.2: Replaced unlink() calls with i_unlink*() wherever p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fefaa6d09a81 changeset: 19136:fefaa6d09a81 user: Timo Sirainen date: Tue Sep 08 19:07:02 2015 +0300 description: Replaced unlink() calls with i_unlink*() wherever possible. diffstat: src/auth/auth-token.c | 12 ++---- src/doveadm/doveadm-sis.c | 12 ++---- src/doveadm/dsync/dsync-brain.c | 3 +- src/doveadm/dsync/dsync-ibc-stream.c | 3 +- src/lda/main.c | 3 +- src/lib-compression/test-compression.c | 3 +- src/lib-fs/fs-posix.c | 3 +- src/lib-imap-client/imapc-client.c | 3 +- src/lib-imap-urlauth/imap-urlauth-connection.c | 3 +- src/lib-index/mail-cache-compress.c | 3 +- src/lib-index/mail-cache.c | 2 +- src/lib-index/mail-index-strmap.c | 6 +- src/lib-index/mail-index-transaction.c | 3 +- src/lib-index/mail-index.c | 4 +- src/lib-index/mailbox-log.c | 3 +- src/lib-index/test-mail-transaction-log-append.c | 2 +- src/lib-lda/duplicate.c | 6 +-- src/lib-lda/smtp-client.c | 3 +- src/lib-mail/test-istream-attachment.c | 2 +- src/lib-master/mountpoint-list.c | 2 +- src/lib-storage/index/dbox-common/dbox-file-fix.c | 6 +-- src/lib-storage/index/dbox-common/dbox-storage.c | 3 +- src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 3 +- src/lib-storage/index/dbox-single/sdbox-file.c | 8 ++-- src/lib-storage/index/index-mail-binary.c | 3 +- src/lib-storage/index/maildir/maildir-keywords.c | 2 +- src/lib-storage/index/maildir/maildir-save.c | 2 +- src/lib-storage/index/maildir/maildir-uidlist.c | 2 +- src/lib-storage/index/pop3c/pop3c-client.c | 3 +- src/lib-storage/mailbox-uidvalidity.c | 3 +- src/lib/file-copy.c | 6 +-- src/lib/file-create-locked.c | 6 +-- src/lib/file-dotlock.c | 31 ++++----------- src/lib/iostream-rawlog.c | 2 +- src/lib/iostream-temp.c | 3 +- src/lib/istream-seekable.c | 3 +- src/lib/net.c | 3 +- src/lib/safe-mkstemp.c | 2 +- src/lib/test-istream-seekable.c | 2 +- src/lib/test-ostream-file.c | 3 +- src/lib/unlink-old-files.c | 3 +- src/lmtp/commands.c | 3 +- src/login-common/ssl-proxy-gnutls.c | 8 ++-- src/master/main.c | 9 +--- src/master/master-settings.c | 3 +- src/master/service-listen.c | 5 +-- src/plugins/acl/acl-backend-vfile-acllist.c | 9 +--- src/plugins/fts-squat/squat-test.c | 4 +- src/plugins/fts-squat/squat-trie.c | 6 +-- src/plugins/fts-squat/squat-uidlist.c | 9 +--- src/plugins/fts/fts-expunge-log.c | 10 ++--- src/plugins/mail-filter/mail-filter-plugin.c | 3 +- src/plugins/quota/quota-maildir.c | 6 +-- src/ssl-params/ssl-params.c | 2 +- 54 files changed, 91 insertions(+), 166 deletions(-) diffs (truncated from 989 to 300 lines): diff -r 2ec2b0a51fec -r fefaa6d09a81 src/auth/auth-token.c --- a/src/auth/auth-token.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/auth/auth-token.c Tue Sep 08 19:07:02 2015 +0300 @@ -57,8 +57,7 @@ if (st.st_size != AUTH_TOKEN_SECRET_LEN || !S_ISREG(st.st_mode)) { i_error("Corrupted token secret file: %s", path); i_close_fd(&fd); - if (unlink(path) < 0) - i_error("unlink(%s) failed: %m", path); + i_unlink(path); return -1; } @@ -76,8 +75,7 @@ !CMP_DEV_T(st.st_dev, lst.st_dev)) { i_error("Compromised token secret file: %s", path); i_close_fd(&fd); - if (unlink(path) < 0) - i_error("unlink(%s) failed: %m", path); + i_unlink(path); return -1; } @@ -126,15 +124,13 @@ } if (ret < 0) { - if (unlink(temp_path) < 0) - i_error("unlink(%s) failed: %m", temp_path); + i_unlink(temp_path); return -1; } if (rename(temp_path, path) < 0) { i_error("rename(%s, %s) failed: %m", temp_path, path); - if (unlink(temp_path) < 0) - i_error("unlink(%s) failed: %m", temp_path); + i_unlink(temp_path); return -1; } diff -r 2ec2b0a51fec -r fefaa6d09a81 src/doveadm/doveadm-sis.c --- a/src/doveadm/doveadm-sis.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/doveadm/doveadm-sis.c Tue Sep 08 19:07:02 2015 +0300 @@ -118,14 +118,12 @@ return -1; } if (st.st_ino != src_inode) { - if (unlink(tmppath) < 0) - i_error("unlink(%s) failed: %m", tmppath); + i_unlink(tmppath); return 0; } if (rename(tmppath, dest) < 0) { i_error("rename(%s, %s) failed: %m", src, tmppath); - if (unlink(tmppath) < 0) - i_error("unlink(%s) failed: %m", tmppath); + i_unlink(tmppath); return -1; } return 1; @@ -258,10 +256,8 @@ T_BEGIN { ret = sis_try_deduplicate(rootdir, d->d_name); } T_END; - if (ret == 0) { - if (unlink(str_c(path)) < 0) - i_error("unlink(%s) failed: %m", str_c(path)); - } + if (ret == 0) + i_unlink(str_c(path)); } if (closedir(dir) < 0) i_error("closedir(%s) failed: %m", queuedir); diff -r 2ec2b0a51fec -r fefaa6d09a81 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Tue Sep 08 19:07:02 2015 +0300 @@ -332,8 +332,7 @@ if (brain->lock_fd != -1) { /* unlink the lock file before it gets unlocked */ - if (unlink(brain->lock_path) < 0) - i_error("unlink(%s) failed: %m", brain->lock_path); + i_unlink(brain->lock_path); file_lock_free(&brain->lock); i_close_fd(&brain->lock_fd); } diff -r 2ec2b0a51fec -r fefaa6d09a81 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Tue Sep 08 19:07:02 2015 +0300 @@ -467,9 +467,8 @@ } /* we just want the fd, unlink it */ - if (unlink(str_c(path)) < 0) { + if (i_unlink(str_c(path)) < 0) { /* shouldn't happen.. */ - i_error("unlink(%s) failed: %m", str_c(path)); i_close_fd(&fd); return -1; } diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lda/main.c --- a/src/lda/main.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lda/main.c Tue Sep 08 19:07:02 2015 +0300 @@ -98,9 +98,8 @@ } /* we just want the fd, unlink it */ - if (unlink(str_c(path)) < 0) { + if (i_unlink(str_c(path)) < 0) { /* shouldn't happen.. */ - i_error("unlink(%s) failed: %m", str_c(path)); i_close_fd(&fd); return -1; } diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-compression/test-compression.c --- a/src/lib-compression/test-compression.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-compression/test-compression.c Tue Sep 08 19:07:02 2015 +0300 @@ -80,8 +80,7 @@ sha1_result(&sha1, input_sha1); test_assert(memcmp(input_sha1, output_sha1, sizeof(input_sha1)) == 0); - if (unlink(path) < 0) - i_error("unlink(%s) failed: %m", path); + i_unlink(path); test_end(); } diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-fs/fs-posix.c Tue Sep 08 19:07:02 2015 +0300 @@ -630,8 +630,7 @@ ret = link(src->full_path, dest->full_path); if (errno == EEXIST && dest->open_mode == FS_OPEN_MODE_REPLACE) { /* destination file already exists - replace it */ - if (unlink(dest->full_path) < 0 && errno != ENOENT) - i_error("unlink(%s) failed: %m", dest->full_path); + i_unlink_if_exists(dest->full_path); ret = link(src->full_path, dest->full_path); } while (ret < 0 && errno == ENOENT && diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-imap-client/imapc-client.c --- a/src/lib-imap-client/imapc-client.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-imap-client/imapc-client.c Tue Sep 08 19:07:02 2015 +0300 @@ -460,9 +460,8 @@ } /* we just want the fd, unlink it */ - if (unlink(str_c(path)) < 0) { + if (i_unlink(str_c(path)) < 0) { /* shouldn't happen.. */ - i_error("unlink(%s) failed: %m", str_c(path)); i_close_fd(&fd); return -1; } diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-imap-urlauth/imap-urlauth-connection.c --- a/src/lib-imap-urlauth/imap-urlauth-connection.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth-connection.c Tue Sep 08 19:07:02 2015 +0300 @@ -493,9 +493,8 @@ } /* we just want the fd, unlink it */ - if (unlink(str_c(path)) < 0) { + if (i_unlink(str_c(path)) < 0) { /* shouldn't happen.. */ - i_error("unlink(%s) failed: %m", str_c(path)); i_close_fd(&fd); return -1; } diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-index/mail-cache-compress.c Tue Sep 08 19:07:02 2015 +0300 @@ -458,8 +458,7 @@ return -1; if (mail_cache_compress_write(cache, trans, fd, temp_path, unlock) < 0) { i_close_fd(&fd); - if (unlink(temp_path) < 0) - i_error("unlink(%s) failed: %m", temp_path); + i_unlink(temp_path); return -1; } if (cache->file_cache != NULL) diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-index/mail-cache.c Tue Sep 08 19:07:02 2015 +0300 @@ -25,7 +25,7 @@ static void mail_cache_unlink(struct mail_cache *cache) { if (!cache->index->readonly) - (void)unlink(cache->filepath); + i_unlink(cache->filepath); } void mail_cache_reset(struct mail_cache *cache) diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-index/mail-index-strmap.c --- a/src/lib-index/mail-index-strmap.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-index/mail-index-strmap.c Tue Sep 08 19:07:02 2015 +0300 @@ -243,7 +243,7 @@ mail_index_set_error(view->strmap->index, "Corrupted strmap index file: %s", view->strmap->path); - (void)unlink(view->strmap->path); + i_unlink(view->strmap->path); mail_index_strmap_close(view->strmap); mail_index_strmap_view_reset(view); } @@ -285,7 +285,7 @@ hdr.uid_validity != idx_hdr->uid_validity) { /* need to rebuild. if we already had something in the strmap, we can keep it. */ - (void)unlink(strmap->path); + i_unlink(strmap->path); mail_index_strmap_close(strmap); return 0; } @@ -1034,7 +1034,7 @@ ret = -1; } if (ret < 0) - (void)unlink(temp_path); + i_unlink(temp_path); return ret; } diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-index/mail-index-transaction.c --- a/src/lib-index/mail-index-transaction.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-index/mail-index-transaction.c Tue Sep 08 19:07:02 2015 +0300 @@ -176,8 +176,7 @@ if (t->reset) { /* get rid of the old index. it might just confuse readers, especially if it's broken. */ - if (unlink(log->index->filepath) < 0 && errno != ENOENT) - i_error("unlink(%s) failed: %m", log->index->filepath); + i_unlink_if_exists(log->index->filepath); } *commit_size_r = log_offset2 - log_offset1; diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-index/mail-index.c --- a/src/lib-index/mail-index.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-index/mail-index.c Tue Sep 08 19:07:02 2015 +0300 @@ -468,10 +468,8 @@ if (fd == -1 && errno == EEXIST) { /* stale temp file. unlink and recreate rather than overwriting, just to make sure locking problems won't cause corruption */ - if (unlink(path) < 0) { - i_error("unlink(%s) failed: %m", path); + if (i_unlink(path) < 0) return -1; - } old_mask = umask(0); fd = open(path, O_RDWR|O_CREAT|O_EXCL, index->mode); umask(old_mask); diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-index/mailbox-log.c --- a/src/lib-index/mailbox-log.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-index/mailbox-log.c Tue Sep 08 19:07:02 2015 +0300 @@ -276,8 +276,7 @@ (iter->count - iter->idx) * sizeof(iter->buf[0]); i_error("Corrupted mailbox log %s at offset %"PRIuUOFF_T": " "type=%d", iter->filepath, offset, rec->type); - if (unlink(iter->filepath) < 0) - i_error("unlink(%s) failed: %m", iter->filepath); + i_unlink(iter->filepath); return NULL; } return rec; diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-index/test-mail-transaction-log-append.c --- a/src/lib-index/test-mail-transaction-log-append.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-index/test-mail-transaction-log-append.c Tue Sep 08 19:07:02 2015 +0300 @@ -162,7 +162,7 @@ i_free(log->head); i_free(log->index); i_free(log); - unlink(tmp_path); + i_unlink(tmp_path); } int main(void) diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-lda/duplicate.c --- a/src/lib-lda/duplicate.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-lda/duplicate.c Tue Sep 08 19:07:02 2015 +0300 @@ -184,10 +184,8 @@ } if (record_size == 0 || - duplicate_read_records(file, input, record_size) < 0) { - if (unlink(file->path) < 0 && errno != ENOENT) - i_error("unlink(%s) failed: %m", file->path); - } + duplicate_read_records(file, input, record_size) < 0) + i_unlink_if_exists(file->path); i_stream_unref(&input); if (close(fd) < 0) diff -r 2ec2b0a51fec -r fefaa6d09a81 src/lib-lda/smtp-client.c --- a/src/lib-lda/smtp-client.c Tue Sep 08 18:49:00 2015 +0300 +++ b/src/lib-lda/smtp-client.c Tue Sep 08 19:07:02 2015 +0300 @@ -87,9 +87,8 @@ } /* we just want the fd, unlink it */ From dovecot at dovecot.org Tue Sep 8 16:14:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 16:14:15 +0000 Subject: dovecot-2.2: lib-index: Replaced some unlink()s with i_unlink*()... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/27a629e444b2 changeset: 19137:27a629e444b2 user: Timo Sirainen date: Tue Sep 08 19:11:45 2015 +0300 description: lib-index: Replaced some unlink()s with i_unlink*()s where we could. This changes some mail_index_set_error() calls to i_error()s, but because these unlink() failures don't actually fail the operation it doesn't matter. In fact it may be even better that it doesn't overwrite the existing index->error if it exists. diffstat: src/lib-index/mail-index-write.c | 8 ++------ src/lib-index/mail-transaction-log-file.c | 11 +++-------- src/lib-index/mail-transaction-log.c | 8 ++------ 3 files changed, 7 insertions(+), 20 deletions(-) diffs (64 lines): diff -r fefaa6d09a81 -r 27a629e444b2 src/lib-index/mail-index-write.c --- a/src/lib-index/mail-index-write.c Tue Sep 08 19:07:02 2015 +0300 +++ b/src/lib-index/mail-index-write.c Tue Sep 08 19:11:45 2015 +0300 @@ -110,12 +110,8 @@ ret = -1; } - if (ret < 0) { - if (unlink(path) < 0) { - mail_index_set_error(index, "unlink(%s) failed: %m", - path); - } - } + if (ret < 0) + i_unlink(path); return ret; } diff -r fefaa6d09a81 -r 27a629e444b2 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Tue Sep 08 19:07:02 2015 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Tue Sep 08 19:11:45 2015 +0300 @@ -794,9 +794,7 @@ file_dotlock_replace(). during that time the log file doesn't exist, which could cause problems. */ path2 = t_strconcat(file->filepath, ".2", NULL); - if (unlink(path2) < 0 && errno != ENOENT) { - mail_index_set_error(index, "unlink(%s) failed: %m", - path2); + if (i_unlink_if_exists(path2) < 0) { /* try to link() anyway */ } if (nfs_safe_link(file->filepath, path2, FALSE) < 0 && @@ -916,11 +914,8 @@ /* corrupted */ if (index->readonly) { /* don't delete */ - } else if (unlink(file->filepath) < 0 && - errno != ENOENT) { - mail_index_set_error(index, - "unlink(%s) failed: %m", - file->filepath); + } else { + i_unlink_if_exists(file->filepath); } return 0; } diff -r fefaa6d09a81 -r 27a629e444b2 src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Tue Sep 08 19:07:02 2015 +0300 +++ b/src/lib-index/mail-transaction-log.c Tue Sep 08 19:11:45 2015 +0300 @@ -52,12 +52,8 @@ } if (st.st_mtime + MAIL_TRANSACTION_LOG2_STALE_SECS <= ioloop_time && - !log->index->readonly) { - if (unlink(log->filepath2) < 0 && errno != ENOENT) { - mail_index_set_error(log->index, - "unlink(%s) failed: %m", log->filepath2); - } - } + !log->index->readonly) + i_unlink_if_exists(log->filepath2); } int mail_transaction_log_open(struct mail_transaction_log *log) From dovecot at dovecot.org Tue Sep 8 16:14:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 16:14:16 +0000 Subject: dovecot-2.2: lib-storage: Replaced some unlink()s with i_unlink*... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/11e13b7df9f6 changeset: 19138:11e13b7df9f6 user: Timo Sirainen date: Tue Sep 08 19:12:26 2015 +0300 description: lib-storage: Replaced some unlink()s with i_unlink*()s where we could. This changes some mail_storage_set_critical() calls to i_error()s, but because these unlink() failures don't actually fail the operation it doesn't matter. diffstat: src/lib-storage/index/cydir/cydir-save.c | 8 ++------ src/lib-storage/index/cydir/cydir-sync.c | 8 +++----- src/lib-storage/index/maildir/maildir-save.c | 5 +---- src/lib-storage/index/maildir/maildir-sync.c | 7 +------ src/lib-storage/index/maildir/maildir-uidlist.c | 9 +++------ 5 files changed, 10 insertions(+), 27 deletions(-) diffs (93 lines): diff -r 27a629e444b2 -r 11e13b7df9f6 src/lib-storage/index/cydir/cydir-save.c --- a/src/lib-storage/index/cydir/cydir-save.c Tue Sep 08 19:11:45 2015 +0300 +++ b/src/lib-storage/index/cydir/cydir-save.c Tue Sep 08 19:12:26 2015 +0300 @@ -218,12 +218,8 @@ if (!ctx->failed) ctx->mail_count++; - else { - if (unlink(path) < 0) { - mail_storage_set_critical(&ctx->mbox->storage->storage, - "unlink(%s) failed: %m", path); - } - } + else + i_unlink(path); index_mail_cache_parse_deinit(_ctx->dest_mail, _ctx->data.received_date, !ctx->failed); diff -r 27a629e444b2 -r 11e13b7df9f6 src/lib-storage/index/cydir/cydir-sync.c --- a/src/lib-storage/index/cydir/cydir-sync.c Tue Sep 08 19:11:45 2015 +0300 +++ b/src/lib-storage/index/cydir/cydir-sync.c Tue Sep 08 19:12:26 2015 +0300 @@ -41,16 +41,14 @@ str_truncate(ctx->path, ctx->path_dir_prefix_len); str_printfa(ctx->path, "%u.", uid); - if (unlink(str_c(ctx->path)) == 0) { + if (i_unlink_if_exists(str_c(ctx->path)) < 0) { + /* continue anyway */ + } else { if (box->v.sync_notify != NULL) { box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE); } mail_index_expunge(ctx->trans, seq1); - } else if (errno != ENOENT) { - mail_storage_set_critical(&ctx->mbox->storage->storage, - "unlink(%s) failed: %m", str_c(ctx->path)); - /* continue anyway */ } } } diff -r 27a629e444b2 -r 11e13b7df9f6 src/lib-storage/index/maildir/maildir-save.c --- a/src/lib-storage/index/maildir/maildir-save.c Tue Sep 08 19:11:45 2015 +0300 +++ b/src/lib-storage/index/maildir/maildir-save.c Tue Sep 08 19:12:26 2015 +0300 @@ -624,10 +624,7 @@ if (ctx->failed) { /* delete the tmp file */ - if (unlink(path) < 0 && errno != ENOENT) { - mail_storage_set_critical(storage, - "unlink(%s) failed: %m", path); - } + i_unlink_if_exists(path); errno = output_errno; if (ENOQUOTA(errno)) { diff -r 27a629e444b2 -r 11e13b7df9f6 src/lib-storage/index/maildir/maildir-sync.c --- a/src/lib-storage/index/maildir/maildir-sync.c Tue Sep 08 19:11:45 2015 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.c Tue Sep 08 19:12:26 2015 +0300 @@ -325,13 +325,8 @@ and hope that another process didn't just decide to unlink() the other (uidlist lock prevents this from happening) */ - if (unlink(path2) == 0) + if (i_unlink(path2) == 0) i_warning("Unlinked a duplicate: %s", path2); - else { - mail_storage_set_critical( - &ctx->mbox->storage->storage, - "unlink(%s) failed: %m", path2); - } } return 0; } diff -r 27a629e444b2 -r 11e13b7df9f6 src/lib-storage/index/maildir/maildir-uidlist.c --- a/src/lib-storage/index/maildir/maildir-uidlist.c Tue Sep 08 19:11:45 2015 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Tue Sep 08 19:12:26 2015 +0300 @@ -1453,12 +1453,9 @@ } } - if (ret < 0) { - if (unlink(temp_path) < 0) { - mail_storage_set_critical(box->storage, - "unlink(%s) failed: %m", temp_path); - } - } else if (fstat(fd, &st) < 0) { + if (ret < 0) + i_unlink(temp_path); + else if (fstat(fd, &st) < 0) { mail_storage_set_critical(box->storage, "fstat(%s) failed: %m", temp_path); ret = -1; From dovecot at dovecot.org Tue Sep 8 16:30:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Sep 2015 16:30:15 +0000 Subject: dovecot-2.2: lib: test-strnum - do not invite undetectable errors Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8687e24d9019 changeset: 19139:8687e24d9019 user: Phil Carmody date: Tue Sep 08 19:28:31 2015 +0300 description: lib: test-strnum - do not invite undetectable errors The very numbers which could cause a broken parser to over-run, the ones we are testing, are the ones which will be parsed as having a value similar to 'value', so check that no value was returned by using a number completely dissimilar to that. Otherwise, there might be an accidental mis-parse that overwrote value, but left its value the same. Signed-off-by: Phil Carmody diffstat: src/lib/test-strnum.c | 16 +++++++++------- 1 files changed, 9 insertions(+), 7 deletions(-) diffs (64 lines): diff -r 11e13b7df9f6 -r 8687e24d9019 src/lib/test-strnum.c --- a/src/lib/test-strnum.c Tue Sep 08 19:12:26 2015 +0300 +++ b/src/lib/test-strnum.c Tue Sep 08 19:28:31 2015 +0300 @@ -28,7 +28,7 @@ { unsigned int i=0; int randrange = rand()%15+1; /* when 1, will max out on 1s */ - uintmax_t value = 0; + uintmax_t value = 0, valbase = rand() * 1000ull; int len, ret; char buff[50]; /* totally assumes < 159 bits */ @@ -73,9 +73,9 @@ while (buff[--j] == '9') buff[j] = '0'; buff[j]++; - + value = valbase + i; ret = str_to_uintmax(buff, &value); - test_assert_idx(ret < 0 && value == UINTMAX_MAX/9-1, i); + test_assert_idx(ret < 0 && value == valbase + i, i); } test_end(); } @@ -101,7 +101,7 @@ { unsigned int i=0; int randrange = rand()%15+1; /* when 1, will max out on 1s */ - uintmax_t value = 0; + uintmax_t value = 0, valbase = rand() * 1000ull; int len, ret; char buff[52]; /* totally assumes < 200 bits */ @@ -149,8 +149,9 @@ buff[j] = 'a'; else buff[j]++; + value = valbase + i; ret = str_to_uintmax_hex(buff, &value); - test_assert_idx(ret < 0 && value == UINTMAX_MAX/0x0f-1, i); + test_assert_idx(ret < 0 && value == valbase + i, i); } test_end(); } @@ -176,7 +177,7 @@ { unsigned int i=0; int randrange = rand()%15+1; /* when 1, will max out on 1s */ - uintmax_t value = 0; + uintmax_t value = 0, valbase = rand() * 1000ull; int len, ret; char buff[69]; /* totally assumes < 200 bits */ @@ -221,8 +222,9 @@ while (buff[--j] == '7') buff[j] = '0'; buff[j]++; + value = valbase + i; ret = str_to_uintmax_oct(buff, &value); - test_assert_idx(ret < 0 && value == UINTMAX_MAX/007-1, i); + test_assert_idx(ret < 0 && value == valbase + i, i); } test_end(); } From dovecot at dovecot.org Wed Sep 9 08:32:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Sep 2015 08:32:48 +0000 Subject: dovecot-2.2: lib-storage: Fixed crash in mailbox list index noti... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/390d55d627a8 changeset: 19140:390d55d627a8 user: Timo Sirainen date: Wed Sep 09 11:31:41 2015 +0300 description: lib-storage: Fixed crash in mailbox list index notifying caused by fa979ccfa34c diffstat: src/lib-storage/list/mailbox-list-index-notify.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (26 lines): diff -r 8687e24d9019 -r 390d55d627a8 src/lib-storage/list/mailbox-list-index-notify.c --- a/src/lib-storage/list/mailbox-list-index-notify.c Tue Sep 08 19:28:31 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-notify.c Wed Sep 09 11:31:41 2015 +0300 @@ -686,7 +686,7 @@ STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES | STATUS_UNSEEN | STATUS_HIGHESTMODSEQ; struct mailbox_list_notify_rec *rec; - struct mailbox_notify_node *nnode; + struct mailbox_notify_node *nnode, empty_node; struct mailbox_status status; if (!mailbox_list_index_notify_lookup(inotify, inotify->view, @@ -696,7 +696,12 @@ /* get the old status */ nnode = mailbox_list_notify_tree_lookup(inotify->tree, rec->storage_name); - if (nnode == NULL || nnode->uidvalidity != status.uidvalidity) + if (nnode == NULL) { + /* mailbox didn't exist earlier - report all events as new */ + memset(&empty_node, 0, sizeof(empty_node)); + nnode = &empty_node; + } + if (nnode->uidvalidity != status.uidvalidity) rec->events |= MAILBOX_LIST_NOTIFY_UIDVALIDITY; if (nnode->uidnext != status.uidnext) rec->events |= MAILBOX_LIST_NOTIFY_APPENDS; From dovecot at dovecot.org Thu Sep 10 22:21:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Sep 2015 22:21:37 +0000 Subject: dovecot-2.2: master: systemd's sd_listen_fds() error handling wa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/179477c59392 changeset: 19141:179477c59392 user: Timo Sirainen date: Fri Sep 11 01:20:28 2015 +0300 description: master: systemd's sd_listen_fds() error handling was wrong. diffstat: src/master/service-listen.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 390d55d627a8 -r 179477c59392 src/master/service-listen.c --- a/src/master/service-listen.c Wed Sep 09 11:31:41 2015 +0300 +++ b/src/master/service-listen.c Fri Sep 11 01:20:28 2015 +0300 @@ -166,8 +166,8 @@ if (sd_fds < 0) { sd_fds = sd_listen_fds(0); - if (sd_fds == -1) { - i_error("sd_listen_fds() failed: %m"); + if (sd_fds < 0) { + i_error("sd_listen_fds() failed: %s", strerror(-sd_fds)); return -1; } } From dovecot at dovecot.org Fri Sep 11 11:03:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Sep 2015 11:03:31 +0000 Subject: dovecot-2.2: lib-storage: Fixed iterating attribute prefix that ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d190ee84158b changeset: 19142:d190ee84158b user: Timo Sirainen date: Fri Sep 11 14:02:01 2015 +0300 description: lib-storage: Fixed iterating attribute prefix that matched the attribute itself. So if attribute key was "foo", iterating "foo" returned garbage because it skipped over the trailing \0. diffstat: src/lib-storage/mailbox-attribute.c | 13 ++++++++++--- 1 files changed, 10 insertions(+), 3 deletions(-) diffs (24 lines): diff -r 179477c59392 -r d190ee84158b src/lib-storage/mailbox-attribute.c --- a/src/lib-storage/mailbox-attribute.c Fri Sep 11 01:20:28 2015 +0300 +++ b/src/lib-storage/mailbox-attribute.c Fri Sep 11 14:02:01 2015 +0300 @@ -131,10 +131,17 @@ if (plen > 0) { if (strncmp(key, bare_prefix, plen) != 0) return; - if (key[plen] != '/' && strlen(key) != plen) + if (key[plen] == '/') { + /* remove prefix */ + key += plen + 1; + } else if (key[plen] == '\0') { + /* list the key itself, so this becomes an + empty key string. it's the same as how the + dict backend works too. */ + key += plen; + } else { return; - /* remove prefix */ - key += plen + 1; + } } if (have_dict || regs[i].rank == MAIL_ATTRIBUTE_INTERNAL_RANK_AUTHORITY) array_append(attrs, &key, 1); From dovecot at dovecot.org Fri Sep 11 11:03:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Sep 2015 11:03:32 +0000 Subject: dovecot-2.2: doveadm mailbox attribute list: Prefix parameter wa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/80a70ccadb85 changeset: 19143:80a70ccadb85 user: Timo Sirainen date: Fri Sep 11 14:02:21 2015 +0300 description: doveadm mailbox attribute list: Prefix parameter was ignored. diffstat: src/doveadm/doveadm-mail-mailbox-metadata.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r d190ee84158b -r 80a70ccadb85 src/doveadm/doveadm-mail-mailbox-metadata.c --- a/src/doveadm/doveadm-mail-mailbox-metadata.c Fri Sep 11 14:02:01 2015 +0300 +++ b/src/doveadm/doveadm-mail-mailbox-metadata.c Fri Sep 11 14:02:21 2015 +0300 @@ -270,6 +270,7 @@ if (args[1] != NULL) cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key); ctx->mailbox = p_strdup(_ctx->pool, args[0]); + ctx->key = key == NULL ? "" : p_strdup(_ctx->pool, key); doveadm_print_header("key", "key", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); } From dovecot at dovecot.org Sat Sep 12 10:02:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Sep 2015 10:02:10 +0000 Subject: dovecot-2.2: doveadm mailbox metadata list: Fixed not giving the... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/29249b209deb changeset: 19144:29249b209deb user: Timo Sirainen date: Sat Sep 12 13:00:58 2015 +0300 description: doveadm mailbox metadata list: Fixed not giving the prefix parameter. diffstat: src/doveadm/doveadm-mail-mailbox-metadata.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 80a70ccadb85 -r 29249b209deb src/doveadm/doveadm-mail-mailbox-metadata.c --- a/src/doveadm/doveadm-mail-mailbox-metadata.c Fri Sep 11 14:02:21 2015 +0300 +++ b/src/doveadm/doveadm-mail-mailbox-metadata.c Sat Sep 12 13:00:58 2015 +0300 @@ -263,7 +263,7 @@ const char *const args[]) { struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx; - const char *key; + const char *key = NULL; if (args[0] == NULL) doveadm_mail_help_name("mailbox metadata list"); From dovecot at dovecot.org Sun Sep 13 13:43:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 13 Sep 2015 13:43:43 +0000 Subject: dovecot-2.2: lib: file_create_locked() may have leaked memory on... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9ce9bd7fba83 changeset: 19145:9ce9bd7fba83 user: Timo Sirainen date: Sun Sep 13 16:42:32 2015 +0300 description: lib: file_create_locked() may have leaked memory on some race conditions. diffstat: src/lib/file-create-locked.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 29249b209deb -r 9ce9bd7fba83 src/lib/file-create-locked.c --- a/src/lib/file-create-locked.c Sat Sep 12 13:00:58 2015 +0300 +++ b/src/lib/file-create-locked.c Sun Sep 13 16:42:32 2015 +0300 @@ -76,6 +76,7 @@ *error_r = t_strdup_printf("link(%s, %s) failed: %m", str_c(temp_path), path); } + file_lock_free(lock_r); } else { i_unlink_if_exists(str_c(temp_path)); *fd_r = fd; From dovecot at dovecot.org Tue Sep 15 06:21:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Sep 2015 06:21:24 +0000 Subject: dovecot-2.2: cassandra: Added delete_consistency parameter. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/602efd349b5c changeset: 19146:602efd349b5c user: Timo Sirainen date: Tue Sep 15 15:20:08 2015 +0900 description: cassandra: Added delete_consistency parameter. diffstat: src/lib-sql/driver-cassandra.c | 43 ++++++++++++++++++++++++++++++++--------- 1 files changed, 33 insertions(+), 10 deletions(-) diffs (136 lines): diff -r 9ce9bd7fba83 -r 602efd349b5c src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Sun Sep 13 16:42:32 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Tue Sep 15 15:20:08 2015 +0900 @@ -18,6 +18,12 @@ typedef void driver_cassandra_callback_t(CassFuture *future, void *context); +enum cassandra_query_type { + CASSANDRA_QUERY_TYPE_READ, + CASSANDRA_QUERY_TYPE_WRITE, + CASSANDRA_QUERY_TYPE_DELETE +}; + struct cassandra_callback { unsigned int id; CassFuture *future; @@ -30,7 +36,7 @@ struct sql_db api; char *hosts, *keyspace; - CassConsistency read_consistency, write_consistency; + CassConsistency read_consistency, write_consistency, delete_consistency; CassLogLevel log_level; CassCluster *cluster; @@ -56,6 +62,7 @@ CassIterator *iterator; char *query; char *error; + enum cassandra_query_type query_type; pool_t row_pool; ARRAY_TYPE(const_string) fields; @@ -66,7 +73,6 @@ unsigned int query_sent:1; unsigned int finished:1; - unsigned int write_query:1; }; struct cassandra_transaction_context { @@ -343,6 +349,7 @@ db->log_level = CASS_LOG_WARN; db->read_consistency = CASS_CONSISTENCY_LOCAL_QUORUM; db->write_consistency = CASS_CONSISTENCY_LOCAL_QUORUM; + db->delete_consistency = CASS_CONSISTENCY_LOCAL_QUORUM; args = t_strsplit_spaces(connect_string, " "); for (; *args != NULL; args++) { @@ -367,6 +374,9 @@ } else if (strcmp(key, "write_consistency") == 0) { if (consistency_parse(value, &db->write_consistency) < 0) i_fatal("cassandra: Unknown write_consistency: %s", value); + } else if (strcmp(key, "delete_consistency") == 0) { + if (consistency_parse(value, &db->delete_consistency) < 0) + i_fatal("cassandra: Unknown delete_consistency: %s", value); } else if (strcmp(key, "log_level") == 0) { if (log_level_parse(value, &db->log_level) < 0) i_fatal("cassandra: Unknown log_level: %s", value); @@ -531,10 +541,17 @@ result->row_pool = pool_alloconly_create("cassandra result", 512); result->statement = cass_statement_new(result->query, 0); - if (result->write_query) + switch (result->query_type) { + case CASSANDRA_QUERY_TYPE_READ: + cass_statement_set_consistency(result->statement, db->read_consistency); + break; + case CASSANDRA_QUERY_TYPE_WRITE: cass_statement_set_consistency(result->statement, db->write_consistency); - else - cass_statement_set_consistency(result->statement, db->read_consistency); + break; + case CASSANDRA_QUERY_TYPE_DELETE: + cass_statement_set_consistency(result->statement, db->delete_consistency); + break; + } future = cass_session_execute(db->session, result->statement); driver_cassandra_set_callback(future, db, query_callback, result); result->query_sent = TRUE; @@ -561,7 +578,8 @@ } static void -driver_cassandra_query_full(struct sql_db *_db, const char *query, bool write_query, +driver_cassandra_query_full(struct sql_db *_db, const char *query, + enum cassandra_query_type query_type, sql_query_callback_t *callback, void *context) { struct cassandra_db *db = (struct cassandra_db *)_db; @@ -573,7 +591,7 @@ result->api.refcount = 1; result->callback = callback; result->context = context; - result->write_query = write_query; + result->query_type = query_type; result->query = i_strdup(query); array_append(&db->results, &result, 1); @@ -582,13 +600,13 @@ static void driver_cassandra_exec(struct sql_db *db, const char *query) { - driver_cassandra_query_full(db, query, TRUE, exec_callback, NULL); + driver_cassandra_query_full(db, query, CASSANDRA_QUERY_TYPE_WRITE, exec_callback, NULL); } static void driver_cassandra_query(struct sql_db *db, const char *query, sql_query_callback_t *callback, void *context) { - driver_cassandra_query_full(db, query, FALSE, callback, context); + driver_cassandra_query_full(db, query, CASSANDRA_QUERY_TYPE_READ, callback, context); } static void cassandra_query_s_callback(struct sql_result *result, void *context) @@ -865,6 +883,7 @@ { struct cassandra_transaction_context *ctx = (struct cassandra_transaction_context *)_ctx; + enum cassandra_query_type query_type; ctx->callback = callback; ctx->context = context; @@ -874,7 +893,11 @@ driver_cassandra_transaction_unref(&ctx); } else if (_ctx->head->next == NULL) { /* just a single query, send it */ - driver_cassandra_query_full(_ctx->db, _ctx->head->query, TRUE, + if (strncasecmp(_ctx->head->query, "DELETE ", 7) == 0) + query_type = CASSANDRA_QUERY_TYPE_DELETE; + else + query_type = CASSANDRA_QUERY_TYPE_WRITE; + driver_cassandra_query_full(_ctx->db, _ctx->head->query, query_type, transaction_commit_callback, ctx); } else { /* multiple queries - we don't actually have a transaction though */ From dovecot at dovecot.org Tue Sep 15 19:46:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Sep 2015 19:46:03 +0000 Subject: dovecot-2.2: lib: Added a simple timing.h API for tracking min/m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9add45266550 changeset: 19147:9add45266550 user: Timo Sirainen date: Wed Sep 16 04:44:37 2015 +0900 description: lib: Added a simple timing.h API for tracking min/max/avg for events. diffstat: src/lib/Makefile.am | 3 + src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + src/lib/test-timing.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/timing.c | 59 +++++++++++++++++++++++++++++++++++++++ src/lib/timing.h | 20 +++++++++++++ 6 files changed, 161 insertions(+), 0 deletions(-) diffs (217 lines): diff -r 602efd349b5c -r 9add45266550 src/lib/Makefile.am --- a/src/lib/Makefile.am Tue Sep 15 15:20:08 2015 +0900 +++ b/src/lib/Makefile.am Wed Sep 16 04:44:37 2015 +0900 @@ -136,6 +136,7 @@ strfuncs.c \ strnum.c \ time-util.c \ + timing.c \ unix-socket-create.c \ unlink-directory.c \ unlink-old-files.c \ @@ -266,6 +267,7 @@ strfuncs.h \ strnum.h \ time-util.h \ + timing.h \ unix-socket-create.h \ unlink-directory.h \ unlink-old-files.h \ @@ -335,6 +337,7 @@ test-str-sanitize.c \ test-str-table.c \ test-time-util.c \ + test-timing.c \ test-unichar.c \ test-utc-mktime.c \ test-var-expand.c \ diff -r 602efd349b5c -r 9add45266550 src/lib/test-lib.c --- a/src/lib/test-lib.c Tue Sep 15 15:20:08 2015 +0900 +++ b/src/lib/test-lib.c Wed Sep 16 04:44:37 2015 +0900 @@ -51,6 +51,7 @@ test_str_sanitize, test_str_table, test_time_util, + test_timing, test_unichar, test_utc_mktime, test_var_expand, diff -r 602efd349b5c -r 9add45266550 src/lib/test-lib.h --- a/src/lib/test-lib.h Tue Sep 15 15:20:08 2015 +0900 +++ b/src/lib/test-lib.h Wed Sep 16 04:44:37 2015 +0900 @@ -54,6 +54,7 @@ void test_str_sanitize(void); void test_str_table(void); void test_time_util(void); +void test_timing(void); void test_unichar(void); void test_utc_mktime(void); void test_var_expand(void); diff -r 602efd349b5c -r 9add45266550 src/lib/test-timing.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-timing.c Wed Sep 16 04:44:37 2015 +0900 @@ -0,0 +1,77 @@ +/* Copyright (c) 2007-2015 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "timing.h" + +#include + +static int uint64_cmp(const uint64_t *i1, const uint64_t *i2) +{ + if (*i1 < *i2) + return -1; + else if (*i1 > *i2) + return 1; + else + return 0; +} + +static void +test_timing_verify(const struct timing *t, const int64_t *input, + unsigned int input_size) +{ + uint64_t min = INT_MAX, max = 0, sum = 0; + uint64_t *copy; + unsigned int i; + + copy = i_new(uint64_t, input_size); + for (i = 0; i < input_size; i++) { + uint64_t value = input[i]; + + if (min > value) + min = value; + if (max < value) + max = value; + sum += value; + copy[i] = value; + } + i_qsort(copy, input_size, sizeof(*copy), uint64_cmp); + + test_assert_idx(timing_get_count(t) == input_size, input_size); + test_assert_idx(timing_get_min(t) == min, input_size); + test_assert_idx(timing_get_max(t) == max, input_size); + test_assert_idx(timing_get_avg(t) == sum/input_size, input_size); + + i_free(copy); +} + +void test_timing(void) +{ + static int64_t test_input1[] = { + 20, 19, 18, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 17, -1 + }; + static int64_t test_input2[] = { + 20, 21, 19, 18, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 17, -1 + }; + static int64_t test_input3[] = { + 20, 21, 19, 18, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 17, 22, -1 + }; + static int64_t *test_inputs[] = { + test_input1, test_input2, test_input3 + }; + struct timing *t; + unsigned int i, j; + + for (i = 0; i < N_ELEMENTS(test_inputs); i++) { + test_begin(t_strdup_printf("timings %u", i)); + t = timing_init(); + for (j = 0; test_inputs[i][j] >= 0; j++) { + timing_add_usecs(t, test_inputs[i][j]); + test_timing_verify(t, test_inputs[i], j+1); + } + timing_deinit(&t); + test_end(); + } +} diff -r 602efd349b5c -r 9add45266550 src/lib/timing.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/timing.c Wed Sep 16 04:44:37 2015 +0900 @@ -0,0 +1,59 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "bsearch-insert-pos.h" +#include "timing.h" + +#define TIMING_MAX_BUCKET_COUNT 20 + +struct timing { + unsigned int count; + uint64_t min, max, sum; +}; + +struct timing *timing_init(void) +{ + return i_new(struct timing, 1); +} + +void timing_deinit(struct timing **_timing) +{ + i_free_and_null(*_timing); +} + + +void timing_add_usecs(struct timing *timing, uint64_t usecs) +{ + if (timing->count++ == 0) { + timing->min = timing->max = timing->sum = usecs; + } else { + if (timing->min > usecs) + timing->min = usecs; + if (timing->max < usecs) + timing->max = usecs; + timing->sum += usecs; + } +} + +unsigned int timing_get_count(const struct timing *timing) +{ + return timing->count; +} + +uint64_t timing_get_min(const struct timing *timing) +{ + return timing->min; +} + +uint64_t timing_get_max(const struct timing *timing) +{ + return timing->max; +} + +uint64_t timing_get_avg(const struct timing *timing) +{ + if (timing->count == 0) + return 0; + + return timing->sum / timing->count; +} diff -r 602efd349b5c -r 9add45266550 src/lib/timing.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/timing.h Wed Sep 16 04:44:37 2015 +0900 @@ -0,0 +1,20 @@ +#ifndef TIMING_H +#define TIMING_H + +struct timing *timing_init(void); +void timing_deinit(struct timing **timing); + +/* Add a new event that took the specified number of usecs. */ +void timing_add_usecs(struct timing *timing, uint64_t usecs); + +/* Returns number of events added. */ +unsigned int timing_get_count(const struct timing *timing); + +/* Returns events' minimum. */ +uint64_t timing_get_min(const struct timing *timing); +/* Returns events' maximum. */ +uint64_t timing_get_max(const struct timing *timing); +/* Returns events' average. */ +uint64_t timing_get_avg(const struct timing *timing); + +#endif From dovecot at dovecot.org Tue Sep 15 20:03:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Sep 2015 20:03:06 +0000 Subject: dovecot-2.2: lib-fs: Track operation timing statistics if enable... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d8c27e220815 changeset: 19148:d8c27e220815 user: Timo Sirainen date: Wed Sep 16 05:01:40 2015 +0900 description: lib-fs: Track operation timing statistics if enable_timing setting is set. diffstat: src/lib-fs/Makefile.am | 2 + src/lib-fs/fs-api-private.h | 22 ++------ src/lib-fs/fs-api.c | 103 ++++++++++++++++++++++++++++++++++++++++- src/lib-fs/fs-api.h | 23 +++++++++ src/lib-fs/istream-fs-stats.c | 47 +++++++++++++++++++ src/lib-fs/istream-fs-stats.h | 9 +++ 6 files changed, 186 insertions(+), 20 deletions(-) diffs (truncated from 484 to 300 lines): diff -r 9add45266550 -r d8c27e220815 src/lib-fs/Makefile.am --- a/src/lib-fs/Makefile.am Wed Sep 16 04:44:37 2015 +0900 +++ b/src/lib-fs/Makefile.am Wed Sep 16 05:01:40 2015 +0900 @@ -16,6 +16,7 @@ fs-sis-common.c \ fs-sis-queue.c \ istream-fs-file.c \ + istream-fs-stats.c \ istream-metawrap.c \ ostream-metawrap.c \ ostream-cmp.c @@ -25,6 +26,7 @@ fs-api-private.h \ fs-sis-common.h \ istream-fs-file.h \ + istream-fs-stats.h \ istream-metawrap.h \ ostream-metawrap.h \ ostream-cmp.h diff -r 9add45266550 -r d8c27e220815 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Wed Sep 16 04:44:37 2015 +0900 +++ b/src/lib-fs/fs-api-private.h Wed Sep 16 05:01:40 2015 +0900 @@ -4,23 +4,6 @@ #include "fs-api.h" #include "module-context.h" -enum fs_op { - FS_OP_WAIT, - FS_OP_METADATA, - FS_OP_PREFETCH, - FS_OP_READ, - FS_OP_WRITE, - FS_OP_LOCK, - FS_OP_EXISTS, - FS_OP_STAT, - FS_OP_COPY, - FS_OP_RENAME, - FS_OP_DELETE, - FS_OP_ITER, - - FS_OP_COUNT -}; - struct fs_api_module_register { unsigned int id; }; @@ -127,6 +110,8 @@ struct istream *copy_input; struct ostream *copy_output; + struct timeval timing_start[FS_OP_COUNT]; + unsigned int write_pending:1; unsigned int metadata_changed:1; @@ -145,6 +130,7 @@ struct fs *fs; enum fs_iter_flags flags; + struct timeval start_time; bool async_have_more; fs_file_async_callback_t *async_callback; @@ -172,4 +158,6 @@ const char *key, const char *value); int fs_default_copy(struct fs_file *src, struct fs_file *dest); +void fs_file_timing_end(struct fs_file *file, enum fs_op op); + #endif diff -r 9add45266550 -r d8c27e220815 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Wed Sep 16 04:44:37 2015 +0900 +++ b/src/lib-fs/fs-api.c Wed Sep 16 05:01:40 2015 +0900 @@ -9,6 +9,9 @@ #include "istream.h" #include "istream-seekable.h" #include "ostream.h" +#include "timing.h" +#include "time-util.h" +#include "istream-fs-stats.h" #include "fs-api-private.h" struct fs_api_module_register fs_api_module_register = { 0 }; @@ -128,6 +131,7 @@ { const struct fs *fs_class; const char *temp_file_prefix; + unsigned int i; fs_class = fs_class_find(driver); if (fs_class == NULL) { @@ -143,6 +147,9 @@ if (fs_alloc(fs_class, args, set, fs_r, error_r) < 0) return -1; + for (i = 0; i < FS_OP_COUNT; i++) + (*fs_r)->stats.timings[i] = timing_init(); + temp_file_prefix = set->temp_file_prefix != NULL ? set->temp_file_prefix : ".temp.dovecot"; (*fs_r)->temp_path_prefix = i_strconcat(set->temp_dir, "/", @@ -167,6 +174,7 @@ struct fs *fs = *_fs; string_t *last_error = fs->last_error; struct array module_contexts_arr = fs->module_contexts.arr; + unsigned int i; i_assert(fs->refcount > 0); @@ -187,6 +195,8 @@ T_BEGIN { fs->v.deinit(fs); } T_END; + for (i = 0; i < FS_OP_COUNT; i++) + timing_deinit(&fs->stats.timings[i]); array_free_i(&module_contexts_arr); str_free(&last_error); } @@ -296,6 +306,40 @@ } T_END; } +static void fs_file_timing_start(struct fs_file *file, enum fs_op op) +{ + if (!file->fs->set.enable_timing) + return; + if (file->timing_start[op].tv_sec == 0) { + if (gettimeofday(&file->timing_start[op], NULL) < 0) + i_fatal("gettimeofday() failed: %m"); + } +} + +static void +fs_timing_end(struct timing *timing, const struct timeval *start_tv) +{ + struct timeval now; + long long diff; + + if (gettimeofday(&now, NULL) < 0) + i_fatal("gettimeofday() failed: %m"); + + diff = timeval_diff_usecs(&now, start_tv); + if (diff > 0) + timing_add_usecs(timing, diff); +} + +void fs_file_timing_end(struct fs_file *file, enum fs_op op) +{ + if (!file->fs->set.enable_timing || file->timing_start[op].tv_sec == 0) + return; + + fs_timing_end(file->fs->stats.timings[op], &file->timing_start[op]); + /* don't count this again */ + file->timing_start[op].tv_sec = 0; +} + int fs_get_metadata(struct fs_file *file, const ARRAY_TYPE(fs_metadata) **metadata_r) { @@ -309,10 +353,13 @@ !file->lookup_metadata_counted) { file->lookup_metadata_counted = TRUE; file->fs->stats.lookup_metadata_count++; + fs_file_timing_start(file, FS_OP_METADATA); } T_BEGIN { ret = file->fs->v.get_metadata(file, metadata_r); } T_END; + if (!(ret < 0 && errno == EAGAIN)) + fs_file_timing_end(file, FS_OP_METADATA); return ret; } @@ -380,10 +427,12 @@ if (!file->read_or_prefetch_counted) { file->read_or_prefetch_counted = TRUE; file->fs->stats.prefetch_count++; + fs_file_timing_start(file, FS_OP_PREFETCH); } T_BEGIN { ret = file->fs->v.prefetch(file, length); } T_END; + fs_file_timing_end(file, FS_OP_PREFETCH); return ret; } @@ -422,12 +471,15 @@ if (!file->read_or_prefetch_counted) { file->read_or_prefetch_counted = TRUE; file->fs->stats.read_count++; + fs_file_timing_start(file, FS_OP_READ); } if (file->fs->v.read != NULL) { T_BEGIN { ret = file->fs->v.read(file, buf, size); } T_END; + if (!(ret < 0 && errno == EAGAIN)) + fs_file_timing_end(file, FS_OP_READ); return ret; } @@ -447,6 +499,7 @@ if (!file->read_or_prefetch_counted) { file->read_or_prefetch_counted = TRUE; file->fs->stats.read_count++; + fs_file_timing_start(file, FS_OP_READ); } if (file->seekable_input != NULL) { @@ -459,8 +512,15 @@ } T_END; if (input->stream_errno != 0) { /* read failed already */ + fs_file_timing_end(file, FS_OP_READ); return input; } + if (file->fs->set.enable_timing) { + struct istream *input2 = i_stream_create_fs_stats(input, file); + + i_stream_unref(&input); + input = input2; + } if ((file->flags & FS_OPEN_FLAG_SEEKABLE) != 0) want_seekable = TRUE; @@ -532,11 +592,14 @@ int ret; if (file->fs->v.write != NULL) { + fs_file_timing_start(file, FS_OP_WRITE); T_BEGIN { ret = file->fs->v.write(file, data, size); } T_END; - if (!(ret < 0 && errno == EAGAIN)) + if (!(ret < 0 && errno == EAGAIN)) { file->fs->stats.write_count++; + fs_file_timing_end(file, FS_OP_WRITE); + } return ret; } @@ -560,10 +623,12 @@ { int ret; + fs_file_timing_start(file, FS_OP_WRITE); T_BEGIN { ret = file->fs->v.write_stream_finish(file, success); } T_END; if (ret != 0) { + fs_file_timing_end(file, FS_OP_WRITE); file->metadata_changed = FALSE; } else { /* write didn't finish yet. this shouldn't happen if we @@ -681,9 +746,12 @@ return errno == ENOENT ? 0 : -1; } file->fs->stats.exists_count++; + fs_file_timing_start(file, FS_OP_EXISTS); T_BEGIN { ret = file->fs->v.exists(file); } T_END; + if (!(ret < 0 && errno == EAGAIN)) + fs_file_timing_end(file, FS_OP_EXISTS); return ret; } @@ -695,10 +763,13 @@ !file->lookup_metadata_counted && !file->stat_counted) { file->stat_counted = TRUE; file->fs->stats.stat_count++; + fs_file_timing_start(file, FS_OP_STAT); } T_BEGIN { ret = file->fs->v.stat(file, st_r); } T_END; + if (!(ret < 0 && errno == EAGAIN)) + fs_file_timing_end(file, FS_OP_STAT); return ret; } @@ -759,11 +830,14 @@ i_assert(src->fs == dest->fs); dest->fs->stats.copy_count++; + fs_file_timing_start(dest, FS_OP_COPY); T_BEGIN { ret = src->fs->v.copy(src, dest); } T_END; - if (!(ret < 0 && errno == EAGAIN)) + if (!(ret < 0 && errno == EAGAIN)) { + fs_file_timing_end(dest, FS_OP_COPY); dest->metadata_changed = FALSE; + } return ret; } @@ -774,8 +848,10 @@ T_BEGIN { From dovecot at dovecot.org Wed Sep 16 01:48:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Sep 2015 01:48:06 +0000 Subject: dovecot-2.2: lib: file_wait_lock_error() assert-crashes now on E... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/19576fdbf950 changeset: 19149:19576fdbf950 user: Timo Sirainen date: Wed Sep 16 10:46:44 2015 +0900 description: lib: file_wait_lock_error() assert-crashes now on EDEADLOCK It's always a bug and a crash can help debug it. diffstat: src/lib/file-lock.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (15 lines): diff -r d8c27e220815 -r 19576fdbf950 src/lib/file-lock.c --- a/src/lib/file-lock.c Wed Sep 16 05:01:40 2015 +0900 +++ b/src/lib/file-lock.c Wed Sep 16 10:46:44 2015 +0900 @@ -292,8 +292,10 @@ int ret; ret = file_lock_do(fd, path, lock_type, lock_method, timeout_secs, error_r); - if (ret <= 0) + if (ret <= 0) { + i_assert(errno != EDEADLOCK); return ret; + } lock = i_new(struct file_lock, 1); lock->fd = fd; From dovecot at dovecot.org Wed Sep 16 22:09:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Sep 2015 22:09:25 +0000 Subject: dovecot-2.2: cassandra: If log_level=debug, log also how long th... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a72a65771d8c changeset: 19150:a72a65771d8c user: Timo Sirainen date: Thu Sep 17 07:07:57 2015 +0900 description: cassandra: If log_level=debug, log also how long the queries take. diffstat: src/lib-sql/driver-cassandra.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (46 lines): diff -r 19576fdbf950 -r a72a65771d8c src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Wed Sep 16 10:46:44 2015 +0900 +++ b/src/lib-sql/driver-cassandra.c Thu Sep 17 07:07:57 2015 +0900 @@ -6,6 +6,7 @@ #include "str.h" #include "ioloop.h" #include "write-full.h" +#include "time-util.h" #include "sql-api-private.h" #ifdef BUILD_CASSANDRA @@ -63,6 +64,7 @@ char *query; char *error; enum cassandra_query_type query_type; + struct timeval start_time; pool_t row_pool; ARRAY_TYPE(const_string) fields; @@ -476,13 +478,17 @@ static void result_finish(struct cassandra_result *result) { struct cassandra_db *db = (struct cassandra_db *)result->api.db; + struct timeval now; bool free_result = TRUE; result->finished = TRUE; driver_cassandra_result_unlink(db, result); if (db->log_level >= CASS_LOG_DEBUG) { - i_debug("cassandra: Finished query '%s': %s", result->query, + if (gettimeofday(&now, NULL) < 0) + i_fatal("gettimeofday() failed: %m"); + i_debug("cassandra: Finished query '%s' (%lld us): %s", result->query, + timeval_diff_usecs(&now, &result->start_time), result->error != NULL ? result->error : "success"); } @@ -539,6 +545,7 @@ } } + result->start_time = ioloop_timeval; result->row_pool = pool_alloconly_create("cassandra result", 512); result->statement = cass_statement_new(result->query, 0); switch (result->query_type) { From dovecot at dovecot.org Wed Sep 16 22:22:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Sep 2015 22:22:05 +0000 Subject: dovecot-2.2: cassandra: Don't crash if connection to Cassandra f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ddc7d2a3036 changeset: 19151:7ddc7d2a3036 user: Timo Sirainen date: Thu Sep 17 07:20:32 2015 +0900 description: cassandra: Don't crash if connection to Cassandra failed. diffstat: src/lib-sql/driver-cassandra.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r a72a65771d8c -r 7ddc7d2a3036 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Sep 17 07:07:57 2015 +0900 +++ b/src/lib-sql/driver-cassandra.c Thu Sep 17 07:20:32 2015 +0900 @@ -468,8 +468,10 @@ cass_result_free(result->result); if (result->iterator != NULL) cass_iterator_free(result->iterator); - cass_statement_free(result->statement); - pool_unref(&result->row_pool); + if (result->statement != NULL) + cass_statement_free(result->statement); + if (result->row_pool != NULL) + pool_unref(&result->row_pool); i_free(result->query); i_free(result->error); i_free(result); From dovecot at dovecot.org Wed Sep 16 22:28:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Sep 2015 22:28:23 +0000 Subject: dovecot-2.2: cassandra: With debugging, log also how long result... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ffb377cccd22 changeset: 19152:ffb377cccd22 user: Timo Sirainen date: Thu Sep 17 07:24:28 2015 +0900 description: cassandra: With debugging, log also how long result was used for before it was freed. This includes the time spent on SELECT query's iterator. diffstat: src/lib-sql/driver-cassandra.c | 22 ++++++++++++---------- 1 files changed, 12 insertions(+), 10 deletions(-) diffs (58 lines): diff -r 7ddc7d2a3036 -r ffb377cccd22 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Sep 17 07:20:32 2015 +0900 +++ b/src/lib-sql/driver-cassandra.c Thu Sep 17 07:24:28 2015 +0900 @@ -64,7 +64,7 @@ char *query; char *error; enum cassandra_query_type query_type; - struct timeval start_time; + struct timeval start_time, finish_time; pool_t row_pool; ARRAY_TYPE(const_string) fields; @@ -457,6 +457,7 @@ { struct cassandra_db *db = (struct cassandra_db *)_result->db; struct cassandra_result *result = (struct cassandra_result *)_result; + struct timeval now; i_assert(!result->api.callback); i_assert(result->callback == NULL); @@ -464,6 +465,15 @@ if (_result == db->sync_result) db->sync_result = NULL; + if (db->log_level >= CASS_LOG_DEBUG) { + if (gettimeofday(&now, NULL) < 0) + i_fatal("gettimeofday() failed: %m"); + i_debug("cassandra: Finished query '%s' (%lld+%lld us): %s", result->query, + timeval_diff_usecs(&result->finish_time, &result->start_time), + timeval_diff_usecs(&now, &result->finish_time), + result->error != NULL ? result->error : "success"); + } + if (result->result != NULL) cass_result_free(result->result); if (result->iterator != NULL) @@ -480,20 +490,12 @@ static void result_finish(struct cassandra_result *result) { struct cassandra_db *db = (struct cassandra_db *)result->api.db; - struct timeval now; bool free_result = TRUE; result->finished = TRUE; + result->finish_time = ioloop_timeval; driver_cassandra_result_unlink(db, result); - if (db->log_level >= CASS_LOG_DEBUG) { - if (gettimeofday(&now, NULL) < 0) - i_fatal("gettimeofday() failed: %m"); - i_debug("cassandra: Finished query '%s' (%lld us): %s", result->query, - timeval_diff_usecs(&now, &result->start_time), - result->error != NULL ? result->error : "success"); - } - i_assert((result->error != NULL) == (result->iterator == NULL)); result->api.callback = TRUE; From dovecot at dovecot.org Wed Sep 16 22:28:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Sep 2015 22:28:25 +0000 Subject: dovecot-2.2: cassandra: With debugging, log also how many rows w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7fc709a7b5a5 changeset: 19153:7fc709a7b5a5 user: Timo Sirainen date: Thu Sep 17 07:26:49 2015 +0900 description: cassandra: With debugging, log also how many rows were iterated. diffstat: src/lib-sql/driver-cassandra.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (29 lines): diff -r ffb377cccd22 -r 7fc709a7b5a5 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Sep 17 07:24:28 2015 +0900 +++ b/src/lib-sql/driver-cassandra.c Thu Sep 17 07:26:49 2015 +0900 @@ -65,6 +65,7 @@ char *error; enum cassandra_query_type query_type; struct timeval start_time, finish_time; + unsigned int row_count; pool_t row_pool; ARRAY_TYPE(const_string) fields; @@ -468,7 +469,8 @@ if (db->log_level >= CASS_LOG_DEBUG) { if (gettimeofday(&now, NULL) < 0) i_fatal("gettimeofday() failed: %m"); - i_debug("cassandra: Finished query '%s' (%lld+%lld us): %s", result->query, + i_debug("cassandra: Finished query '%s' (%u rows, %lld+%lld us): %s", result->query, + result->row_count, timeval_diff_usecs(&result->finish_time, &result->start_time), timeval_diff_usecs(&now, &result->finish_time), result->error != NULL ? result->error : "success"); @@ -745,6 +747,7 @@ if (!cass_iterator_next(result->iterator)) return 0; + result->row_count++; p_clear(result->row_pool); p_array_init(&result->fields, result->row_pool, 8); From dovecot at dovecot.org Wed Sep 16 22:34:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Sep 2015 22:34:48 +0000 Subject: dovecot-2.2: lib-fs: Fixed compiling with some OSes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7cf44f5f32bd changeset: 19155:7cf44f5f32bd user: Timo Sirainen date: Thu Sep 17 07:33:22 2015 +0900 description: lib-fs: Fixed compiling with some OSes diffstat: src/lib-fs/fs-api-private.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r a20b72fd97d6 -r 7cf44f5f32bd src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Thu Sep 17 07:33:01 2015 +0900 +++ b/src/lib-fs/fs-api-private.h Thu Sep 17 07:33:22 2015 +0900 @@ -4,6 +4,8 @@ #include "fs-api.h" #include "module-context.h" +#include + struct fs_api_module_register { unsigned int id; }; From dovecot at dovecot.org Wed Sep 16 22:34:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Sep 2015 22:34:47 +0000 Subject: dovecot-2.2: lib-fs: timing_deinit()s was called too late - fs w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a20b72fd97d6 changeset: 19154:a20b72fd97d6 user: Timo Sirainen date: Thu Sep 17 07:33:01 2015 +0900 description: lib-fs: timing_deinit()s was called too late - fs was already freed. diffstat: src/lib-fs/fs-api.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (17 lines): diff -r 7fc709a7b5a5 -r a20b72fd97d6 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu Sep 17 07:26:49 2015 +0900 +++ b/src/lib-fs/fs-api.c Thu Sep 17 07:33:01 2015 +0900 @@ -192,11 +192,11 @@ i_free(fs->username); i_free(fs->session_id); i_free(fs->temp_path_prefix); + for (i = 0; i < FS_OP_COUNT; i++) + timing_deinit(&fs->stats.timings[i]); T_BEGIN { fs->v.deinit(fs); } T_END; - for (i = 0; i < FS_OP_COUNT; i++) - timing_deinit(&fs->stats.timings[i]); array_free_i(&module_contexts_arr); str_free(&last_error); } From dovecot at dovecot.org Sat Sep 19 22:06:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 19 Sep 2015 22:06:09 +0000 Subject: dovecot-2.2: lib: file_wait_lock*(): Improve the panic log messa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4f4243794ba1 changeset: 19156:4f4243794ba1 user: Timo Sirainen date: Sun Sep 20 01:04:50 2015 +0300 description: lib: file_wait_lock*(): Improve the panic log message on EDEADLOCK. diffstat: src/lib/file-lock.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (33 lines): diff -r 7cf44f5f32bd -r 4f4243794ba1 src/lib/file-lock.c --- a/src/lib/file-lock.c Thu Sep 17 07:33:22 2015 +0900 +++ b/src/lib/file-lock.c Sun Sep 20 01:04:50 2015 +0300 @@ -211,6 +211,8 @@ } *error_r = t_strdup_printf("fcntl(%s, %s, %s) locking failed: %m", path, lock_type_str, timeout_secs == 0 ? "F_SETLK" : "F_SETLKW"); + if (errno == EDEADLOCK) + i_panic("%s%s", *error_r, file_lock_find(fd, lock_method, lock_type)); return -1; #endif } @@ -257,6 +259,8 @@ } *error_r = t_strdup_printf("flock(%s, %s) failed: %m", path, lock_type_str); + if (errno == EDEADLOCK) + i_panic("%s%s", *error_r, file_lock_find(fd, lock_method, lock_type)); return -1; #endif } @@ -292,10 +296,8 @@ int ret; ret = file_lock_do(fd, path, lock_type, lock_method, timeout_secs, error_r); - if (ret <= 0) { - i_assert(errno != EDEADLOCK); + if (ret <= 0) return ret; - } lock = i_new(struct file_lock, 1); lock->fd = fd; From dovecot at dovecot.org Sun Sep 20 18:51:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 20 Sep 2015 18:51:19 +0000 Subject: dovecot-2.2: auth: The mechanisms configured using the auth_mech... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d482a8097362 changeset: 19157:d482a8097362 user: Stephan Bosch date: Sun Sep 20 21:49:51 2015 +0300 description: auth: The mechanisms configured using the auth_mechanisms setting were not enforced. The login service would check whether the mechanism is supported by auth, but auth performed no such check of its own. This means that any implemented mechanism was accessible from a login, even though was presumably disabled. diffstat: src/auth/auth-request-handler.c | 5 +++-- src/auth/mech.c | 12 ++++++++++++ src/auth/mech.h | 2 ++ 3 files changed, 17 insertions(+), 2 deletions(-) diffs (49 lines): diff -r 4f4243794ba1 -r d482a8097362 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Sun Sep 20 01:04:50 2015 +0300 +++ b/src/auth/auth-request-handler.c Sun Sep 20 21:49:51 2015 +0300 @@ -475,8 +475,9 @@ handler->client_pid, str_sanitize(list[1], MAX_MECH_NAME_LEN)); return FALSE; } - } else { - mech = mech_module_find(list[1]); + } else { + struct auth *auth_default = auth_default_service(); + mech = mech_register_find(auth_default->reg, list[1]); if (mech == NULL) { /* unsupported mechanism */ i_error("BUG: Authentication client %u requested unsupported " diff -r 4f4243794ba1 -r d482a8097362 src/auth/mech.c --- a/src/auth/mech.c Sun Sep 20 01:04:50 2015 +0300 +++ b/src/auth/mech.c Sun Sep 20 21:49:51 2015 +0300 @@ -175,6 +175,18 @@ pool_unref(®->pool); } +const struct mech_module * +mech_register_find(const struct mechanisms_register *reg, const char *name) +{ + const struct mech_module_list *list; + + for (list = reg->modules; list != NULL; list = list->next) { + if (strcasecmp(list->module.mech_name, name) == 0) + return &list->module; + } + return NULL; +} + void mech_init(const struct auth_settings *set) { mech_register_module(&mech_plain); diff -r 4f4243794ba1 -r d482a8097362 src/auth/mech.h --- a/src/auth/mech.h Sun Sep 20 01:04:50 2015 +0300 +++ b/src/auth/mech.h Sun Sep 20 21:49:51 2015 +0300 @@ -68,6 +68,8 @@ struct mechanisms_register * mech_register_init(const struct auth_settings *set); void mech_register_deinit(struct mechanisms_register **reg); +const struct mech_module * +mech_register_find(const struct mechanisms_register *reg, const char *name); void mech_init(const struct auth_settings *set); void mech_deinit(const struct auth_settings *set); From dovecot at dovecot.org Sun Sep 20 19:26:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 20 Sep 2015 19:26:29 +0000 Subject: dovecot-2.2: lib-index: Added mail_index_sync_have_any_expunges() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/91f68c1b9b36 changeset: 19158:91f68c1b9b36 user: Timo Sirainen date: Sun Sep 20 22:25:11 2015 +0300 description: lib-index: Added mail_index_sync_have_any_expunges() This can be used to quickly check before mail_index_sync_begin() if there are likely to be any expunges that will be synced. diffstat: src/lib-index/mail-index-sync.c | 25 ++++++++++++++++++++----- src/lib-index/mail-index.h | 2 ++ 2 files changed, 22 insertions(+), 5 deletions(-) diffs (71 lines): diff -r d482a8097362 -r 91f68c1b9b36 src/lib-index/mail-index-sync.c --- a/src/lib-index/mail-index-sync.c Sun Sep 20 21:49:51 2015 +0300 +++ b/src/lib-index/mail-index-sync.c Sun Sep 20 22:25:11 2015 +0300 @@ -526,7 +526,8 @@ } static bool mail_index_sync_view_have_any(struct mail_index_view *view, - enum mail_index_sync_flags flags) + enum mail_index_sync_flags flags, + bool expunges_only) { const struct mail_transaction_header *hdr; const void *data; @@ -560,19 +561,22 @@ continue; switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + case MAIL_TRANSACTION_EXPUNGE_GUID: + return TRUE; case MAIL_TRANSACTION_EXT_REC_UPDATE: case MAIL_TRANSACTION_EXT_ATOMIC_INC: /* extension record updates aren't exactly needed to be synced, but cache syncing relies on tail offsets being updated. */ - case MAIL_TRANSACTION_EXPUNGE: - case MAIL_TRANSACTION_EXPUNGE_GUID: case MAIL_TRANSACTION_FLAG_UPDATE: case MAIL_TRANSACTION_KEYWORD_UPDATE: case MAIL_TRANSACTION_KEYWORD_RESET: case MAIL_TRANSACTION_INDEX_DELETED: case MAIL_TRANSACTION_INDEX_UNDELETED: - return TRUE; + if (!expunges_only) + return TRUE; + break; default: break; } @@ -587,7 +591,18 @@ bool ret; view = mail_index_view_open(index); - ret = mail_index_sync_view_have_any(view, flags); + ret = mail_index_sync_view_have_any(view, flags, FALSE); + mail_index_view_close(&view); + return ret; +} + +bool mail_index_sync_have_any_expunges(struct mail_index *index) +{ + struct mail_index_view *view; + bool ret; + + view = mail_index_view_open(index); + ret = mail_index_sync_view_have_any(view, 0, TRUE); mail_index_view_close(&view); return ret; } diff -r d482a8097362 -r 91f68c1b9b36 src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Sun Sep 20 21:49:51 2015 +0300 +++ b/src/lib-index/mail-index.h Sun Sep 20 22:25:11 2015 +0300 @@ -363,6 +363,8 @@ /* Returns TRUE if it currently looks like syncing would return changes. */ bool mail_index_sync_have_any(struct mail_index *index, enum mail_index_sync_flags flags); +/* Returns TRUE if it currently looks like syncing would return expunges. */ +bool mail_index_sync_have_any_expunges(struct mail_index *index); /* Returns the log file seq+offsets for the area which this sync is handling. */ void mail_index_sync_get_offsets(struct mail_index_sync_ctx *ctx, uint32_t *seq1_r, uoff_t *offset1_r, From dovecot at dovecot.org Mon Sep 21 11:04:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 11:04:07 +0000 Subject: dovecot-2.2: imap: When running SEARCH on "background", run it t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c607ef95a785 changeset: 19159:c607ef95a785 user: Timo Sirainen date: Mon Sep 21 13:59:22 2015 +0300 description: imap: When running SEARCH on "background", run it through command_exec() Otherwise all the timing information isn't right and the pre/post hooks weren't being called so stats counting was also wrong. diffstat: src/imap/imap-search.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 91f68c1b9b36 -r c607ef95a785 src/imap/imap-search.c --- a/src/imap/imap-search.c Sun Sep 20 22:25:11 2015 +0300 +++ b/src/imap/imap-search.c Mon Sep 21 13:59:22 2015 +0300 @@ -516,7 +516,7 @@ bool finished; o_stream_cork(client->output); - finished = cmd_search_more(cmd); + finished = command_exec(cmd); o_stream_uncork(client->output); if (!finished) From dovecot at dovecot.org Mon Sep 21 11:04:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 11:04:08 +0000 Subject: dovecot-2.2: lib: Added io_loop_get_wait_usecs() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1076679adf9f changeset: 19160:1076679adf9f user: Timo Sirainen date: Mon Sep 21 13:59:41 2015 +0300 description: lib: Added io_loop_get_wait_usecs() diffstat: src/lib/ioloop-private.h | 1 + src/lib/ioloop.c | 31 +++++++++++++++++++++++-------- src/lib/ioloop.h | 2 ++ 3 files changed, 26 insertions(+), 8 deletions(-) diffs (81 lines): diff -r c607ef95a785 -r 1076679adf9f src/lib/ioloop-private.h --- a/src/lib/ioloop-private.h Mon Sep 21 13:59:22 2015 +0300 +++ b/src/lib/ioloop-private.h Mon Sep 21 13:59:41 2015 +0300 @@ -25,6 +25,7 @@ io_loop_time_moved_callback_t *time_moved_callback; time_t next_max_time; + uint64_t ioloop_wait_usecs; unsigned int io_pending_count; diff -r c607ef95a785 -r 1076679adf9f src/lib/ioloop.c --- a/src/lib/ioloop.c Mon Sep 21 13:59:22 2015 +0300 +++ b/src/lib/ioloop.c Mon Sep 21 13:59:41 2015 +0300 @@ -402,6 +402,12 @@ tv_now.tv_sec = 0; msecs = timeout_get_wait_time(timeout, tv_r, &tv_now); ioloop->next_max_time = (tv_now.tv_sec + msecs/1000) + 1; + + /* update ioloop_timeval - this is meant for io_loop_handle_timeouts()'s + ioloop_wait_usecs calculation. normally after this we go to the + ioloop and after that we update ioloop_timeval immediately again. */ + ioloop_timeval = tv_now; + ioloop_time = tv_now.tv_sec; return msecs; } @@ -466,7 +472,7 @@ static void io_loop_handle_timeouts_real(struct ioloop *ioloop) { struct priorityq_item *item; - struct timeval tv, tv_call; + struct timeval tv, tv_call, prev_ioloop_timeval = ioloop_timeval; unsigned int t_id; if (gettimeofday(&ioloop_timeval, NULL) < 0) @@ -482,13 +488,17 @@ /* the callback may have slept, so check the time again. */ if (gettimeofday(&ioloop_timeval, NULL) < 0) i_fatal("gettimeofday(): %m"); - } else if (unlikely(ioloop_timeval.tv_sec > - ioloop->next_max_time)) { - io_loops_timeouts_update(ioloop_timeval.tv_sec - - ioloop->next_max_time); - /* time moved forwards */ - ioloop->time_moved_callback(ioloop->next_max_time, - ioloop_timeval.tv_sec); + } else { + if (unlikely(ioloop_timeval.tv_sec > + ioloop->next_max_time)) { + io_loops_timeouts_update(ioloop_timeval.tv_sec - + ioloop->next_max_time); + /* time moved forwards */ + ioloop->time_moved_callback(ioloop->next_max_time, + ioloop_timeval.tv_sec); + } + ioloop->ioloop_wait_usecs += + timeval_diff_usecs(&ioloop_timeval, &prev_ioloop_timeval); } ioloop_time = ioloop_timeval.tv_sec; @@ -933,3 +943,8 @@ return io_loop_get_wait_time(ioloop, &tv) == 0; } + +uint64_t io_loop_get_wait_usecs(struct ioloop *ioloop) +{ + return ioloop->ioloop_wait_usecs; +} diff -r c607ef95a785 -r 1076679adf9f src/lib/ioloop.h --- a/src/lib/ioloop.h Mon Sep 21 13:59:22 2015 +0300 +++ b/src/lib/ioloop.h Mon Sep 21 13:59:41 2015 +0300 @@ -185,5 +185,7 @@ /* Returns TRUE if there is a pending timeout that is going to be run immediately. */ bool io_loop_have_immediate_timeouts(struct ioloop *ioloop); +/* Returns number of microseconds spent on the ioloop waiting itself. */ +uint64_t io_loop_get_wait_usecs(struct ioloop *ioloop); #endif From dovecot at dovecot.org Mon Sep 21 11:04:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 11:04:08 +0000 Subject: dovecot-2.2: imap: Improved command timing information reporting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/127c88e75185 changeset: 19161:127c88e75185 user: Timo Sirainen date: Mon Sep 21 14:02:47 2015 +0300 description: imap: Improved command timing information reporting. We now report also the time we spent waiting on ioloop. Also fixed reporting command timing information when multiple commands were running in parallel (e.g. SEARCH + FETCH). If multiple commands are running in parallel they all report the same ioloop wait time, because there's no easy way to know which one of them caused the wait. diffstat: src/imap/imap-client.c | 38 +++++++++++++++++++++++++------------- src/imap/imap-client.h | 9 ++++----- src/imap/imap-commands.c | 16 +++++++--------- 3 files changed, 36 insertions(+), 27 deletions(-) diffs (119 lines): diff -r 1076679adf9f -r 127c88e75185 src/imap/imap-client.c --- a/src/imap/imap-client.c Mon Sep 21 13:59:41 2015 +0300 +++ b/src/imap/imap-client.c Mon Sep 21 14:02:47 2015 +0300 @@ -396,11 +396,32 @@ return 1; } +static void +client_cmd_append_timing_stats(struct client_command_context *cmd, + string_t *str) +{ + unsigned int msecs_in_cmd, msecs_in_ioloop; + uint64_t ioloop_wait_usecs; + + if (cmd->start_time.tv_sec == 0) + return; + + ioloop_wait_usecs = io_loop_get_wait_usecs(current_ioloop); + msecs_in_cmd = (cmd->running_usecs + 999) / 1000; + msecs_in_ioloop = (ioloop_wait_usecs - + cmd->start_ioloop_wait_usecs + 999) / 1000; + + if (str_data(str)[str_len(str)-1] == '.') + str_truncate(str, str_len(str)-1); + str_printfa(str, " (%d.%03d + %d.%03d secs).", + msecs_in_cmd / 1000, msecs_in_cmd % 1000, + msecs_in_ioloop / 1000, msecs_in_ioloop % 1000); +} + void client_send_tagline(struct client_command_context *cmd, const char *data) { struct client *client = cmd->client; const char *tag = cmd->tag; - int time_msecs; if (client->output->closed || cmd->cancel) return; @@ -414,18 +435,7 @@ T_BEGIN { string_t *str = t_str_new(256); str_printfa(str, "%s %s", tag, data); - if (cmd->start_time.tv_sec != 0) { - if (str_data(str)[str_len(str)-1] == '.') - str_truncate(str, str_len(str)-1); - io_loop_time_refresh(); - time_msecs = timeval_diff_msecs(&ioloop_timeval, - &cmd->start_time); - time_msecs -= cmd->usecs_in_ioloop/1000; - if (time_msecs >= 0) { - str_printfa(str, " (%d.%03d secs).", - time_msecs/1000, time_msecs%1000); - } - } + client_cmd_append_timing_stats(cmd, str); str_append(str, "\r\n"); o_stream_nsend(client->output, str_data(str), str_len(str)); } T_END; @@ -496,6 +506,8 @@ imap_write_args(str, *args_r); cmd->args = p_strdup(cmd->pool, str_c(str)); cmd->start_time = ioloop_timeval; + cmd->start_ioloop_wait_usecs = + io_loop_get_wait_usecs(current_ioloop); cmd->client->input_lock = NULL; return TRUE; diff -r 1076679adf9f -r 127c88e75185 src/imap/imap-client.h --- a/src/imap/imap-client.h Mon Sep 21 13:59:41 2015 +0300 +++ b/src/imap/imap-client.h Mon Sep 21 14:02:47 2015 +0300 @@ -77,11 +77,10 @@ /* time when command handling was started - typically this is after reading all the parameters. */ struct timeval start_time; - /* time when an unfinished command handling entered back to ioloop. - used for calculating usecs_in_ioloop */ - struct timeval last_ioloop_time; - /* how much time was spent waiting for the client in ioloop */ - uint64_t usecs_in_ioloop; + /* io_loop_get_wait_usecs()'s value when the command was started */ + uint64_t start_ioloop_wait_usecs; + /* how many usecs this command itself has spent running */ + uint64_t running_usecs; struct client_sync_context *sync; diff -r 1076679adf9f -r 127c88e75185 src/imap/imap-commands.c --- a/src/imap/imap-commands.c Mon Sep 21 13:59:41 2015 +0300 +++ b/src/imap/imap-commands.c Mon Sep 21 14:02:47 2015 +0300 @@ -156,12 +156,10 @@ const struct command_hook *hook; long long diff; bool finished; + struct timeval cmd_start_timeval; - if (cmd->last_ioloop_time.tv_sec != 0) { - diff = timeval_diff_usecs(&ioloop_timeval, &cmd->last_ioloop_time); - if (diff > 0) - cmd->usecs_in_ioloop += diff; - } + io_loop_time_refresh(); + cmd_start_timeval = ioloop_timeval; array_foreach(&command_hooks, hook) hook->pre(cmd); @@ -170,10 +168,10 @@ hook->post(cmd); if (cmd->state == CLIENT_COMMAND_STATE_DONE) finished = TRUE; - if (!finished) { - io_loop_time_refresh(); - cmd->last_ioloop_time = ioloop_timeval; - } + + io_loop_time_refresh(); + cmd->running_usecs += + timeval_diff_usecs(&ioloop_timeval, &cmd_start_timeval); return finished; } From dovecot at dovecot.org Mon Sep 21 11:20:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 11:20:07 +0000 Subject: dovecot-2.2: Compiler warning fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e39e5867fa7b changeset: 19162:e39e5867fa7b user: Timo Sirainen date: Mon Sep 21 14:18:51 2015 +0300 description: Compiler warning fix. diffstat: src/imap/imap-commands.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 127c88e75185 -r e39e5867fa7b src/imap/imap-commands.c --- a/src/imap/imap-commands.c Mon Sep 21 14:02:47 2015 +0300 +++ b/src/imap/imap-commands.c Mon Sep 21 14:18:51 2015 +0300 @@ -154,7 +154,6 @@ bool command_exec(struct client_command_context *cmd) { const struct command_hook *hook; - long long diff; bool finished; struct timeval cmd_start_timeval; From dovecot at dovecot.org Mon Sep 21 12:39:58 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 12:39:58 +0000 Subject: dovecot-2.2: imap: If client disconnects, log how long the in-pr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/12dda0010920 changeset: 19163:12dda0010920 user: Timo Sirainen date: Mon Sep 21 15:32:02 2015 +0300 description: imap: If client disconnects, log how long the in-progress commands were running. diffstat: src/imap/imap-client.c | 37 ++++++++++++++++++++++++++++++++++++- 1 files changed, 36 insertions(+), 1 deletions(-) diffs (58 lines): diff -r e39e5867fa7b -r 12dda0010920 src/imap/imap-client.c --- a/src/imap/imap-client.c Mon Sep 21 14:18:51 2015 +0300 +++ b/src/imap/imap-client.c Mon Sep 21 15:32:02 2015 +0300 @@ -255,9 +255,43 @@ client->v.destroy(client, reason); } +static const char *client_get_commands_status(struct client *client) +{ + struct client_command_context *cmd; + unsigned int msecs_in_ioloop; + uint64_t running_usecs = 0, ioloop_wait_usecs; + string_t *str; + + if (client->command_queue == NULL) + return ""; + + str = t_str_new(128); + str_append(str, " ("); + for (cmd = client->command_queue; cmd != NULL; cmd = cmd->next) { + str_append(str, cmd->name); + if (cmd->next != NULL) + str_append_c(str, ','); + running_usecs += cmd->running_usecs; + } + + ioloop_wait_usecs = io_loop_get_wait_usecs(current_ioloop); + msecs_in_ioloop = (ioloop_wait_usecs - + client->command_queue->start_ioloop_wait_usecs + 999) / 1000; + str_printfa(str, " running for %d.%03d + waiting for %d.%03d secs", + (int)((running_usecs+999)/1000 / 1000), + (int)((running_usecs+999)/1000 % 1000), + msecs_in_ioloop / 1000, msecs_in_ioloop % 1000); + if (o_stream_get_buffer_used_size(client->output) > 0) + str_printfa(str, ", %"PRIuSIZE_T" B output buffered", + o_stream_get_buffer_used_size(client->output)); + str_append_c(str, ')'); + return str_c(str); +} + static void client_default_destroy(struct client *client, const char *reason) { struct client_command_context *cmd; + const char *cmd_status = ""; i_assert(!client->destroyed); client->destroyed = TRUE; @@ -267,8 +301,9 @@ if (reason == NULL) { reason = io_stream_get_disconnect_reason(client->input, client->output); + cmd_status = client_get_commands_status(client); } - i_info("%s %s", reason, client_stats(client)); + i_info("%s%s %s", reason, cmd_status, client_stats(client)); } i_stream_close(client->input); From dovecot at dovecot.org Mon Sep 21 12:43:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 12:43:06 +0000 Subject: dovecot-2.2: imap: If client disconnects, log the in-progress co... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e937b8e7a6a3 changeset: 19164:e937b8e7a6a3 user: Timo Sirainen date: Mon Sep 21 15:41:49 2015 +0300 description: imap: If client disconnects, log the in-progress commands' input/output bytes. diffstat: src/imap/imap-client.c | 10 ++++++---- src/imap/imap-client.h | 2 ++ src/imap/imap-commands.c | 8 ++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diffs (81 lines): diff -r 12dda0010920 -r e937b8e7a6a3 src/imap/imap-client.c --- a/src/imap/imap-client.c Mon Sep 21 15:32:02 2015 +0300 +++ b/src/imap/imap-client.c Mon Sep 21 15:41:49 2015 +0300 @@ -260,6 +260,7 @@ struct client_command_context *cmd; unsigned int msecs_in_ioloop; uint64_t running_usecs = 0, ioloop_wait_usecs; + unsigned long long bytes_in = 0, bytes_out = 0; string_t *str; if (client->command_queue == NULL) @@ -272,6 +273,8 @@ if (cmd->next != NULL) str_append_c(str, ','); running_usecs += cmd->running_usecs; + bytes_in += cmd->bytes_in; + bytes_out += cmd->bytes_out; } ioloop_wait_usecs = io_loop_get_wait_usecs(current_ioloop); @@ -281,10 +284,9 @@ (int)((running_usecs+999)/1000 / 1000), (int)((running_usecs+999)/1000 % 1000), msecs_in_ioloop / 1000, msecs_in_ioloop % 1000); - if (o_stream_get_buffer_used_size(client->output) > 0) - str_printfa(str, ", %"PRIuSIZE_T" B output buffered", - o_stream_get_buffer_used_size(client->output)); - str_append_c(str, ')'); + str_printfa(str, ", %llu B in + %llu+%"PRIuSIZE_T" B out)", + bytes_in, bytes_out, + o_stream_get_buffer_used_size(client->output)); return str_c(str); } diff -r 12dda0010920 -r e937b8e7a6a3 src/imap/imap-client.h --- a/src/imap/imap-client.h Mon Sep 21 15:32:02 2015 +0300 +++ b/src/imap/imap-client.h Mon Sep 21 15:41:49 2015 +0300 @@ -81,6 +81,8 @@ uint64_t start_ioloop_wait_usecs; /* how many usecs this command itself has spent running */ uint64_t running_usecs; + /* how many bytes of client input/output command has used */ + uint64_t bytes_in, bytes_out; struct client_sync_context *sync; diff -r 12dda0010920 -r e937b8e7a6a3 src/imap/imap-commands.c --- a/src/imap/imap-commands.c Mon Sep 21 15:32:02 2015 +0300 +++ b/src/imap/imap-commands.c Mon Sep 21 15:41:49 2015 +0300 @@ -4,6 +4,8 @@ #include "array.h" #include "buffer.h" #include "ioloop.h" +#include "istream.h" +#include "ostream.h" #include "time-util.h" #include "imap-commands.h" @@ -156,9 +158,12 @@ const struct command_hook *hook; bool finished; struct timeval cmd_start_timeval; + uint64_t cmd_start_bytes_in, cmd_start_bytes_out; io_loop_time_refresh(); cmd_start_timeval = ioloop_timeval; + cmd_start_bytes_in = i_stream_get_absolute_offset(cmd->client->input); + cmd_start_bytes_out = cmd->client->output->offset; array_foreach(&command_hooks, hook) hook->pre(cmd); @@ -171,6 +176,9 @@ io_loop_time_refresh(); cmd->running_usecs += timeval_diff_usecs(&ioloop_timeval, &cmd_start_timeval); + cmd->bytes_in += i_stream_get_absolute_offset(cmd->client->input) - + cmd_start_bytes_in; + cmd->bytes_out += cmd->client->output->offset - cmd_start_bytes_out; return finished; } From dovecot at dovecot.org Mon Sep 21 13:25:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 13:25:57 +0000 Subject: dovecot-2.2: quota: Code cleanup - extract default init() handli... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff5c34d62daf changeset: 19165:ff5c34d62daf user: Timo Sirainen date: Mon Sep 21 16:09:37 2015 +0300 description: quota: Code cleanup - extract default init() handling to quota_root_default_init() diffstat: src/plugins/quota/quota-private.h | 2 + src/plugins/quota/quota.c | 48 ++++++++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 18 deletions(-) diffs (83 lines): diff -r e937b8e7a6a3 -r ff5c34d62daf src/plugins/quota/quota-private.h --- a/src/plugins/quota/quota-private.h Mon Sep 21 15:41:49 2015 +0300 +++ b/src/plugins/quota/quota-private.h Mon Sep 21 16:09:37 2015 +0300 @@ -172,6 +172,8 @@ void quota_add_user_namespace(struct quota *quota, struct mail_namespace *ns); void quota_remove_user_namespace(struct mail_namespace *ns); +int quota_root_default_init(struct quota_root *root, const char *args, + const char **error_r); struct quota *quota_get_mail_user_quota(struct mail_user *user); bool quota_root_is_namespace_visible(struct quota_root *root, diff -r e937b8e7a6a3 -r ff5c34d62daf src/plugins/quota/quota.c --- a/src/plugins/quota/quota.c Mon Sep 21 15:41:49 2015 +0300 +++ b/src/plugins/quota/quota.c Mon Sep 21 16:09:37 2015 +0300 @@ -277,12 +277,39 @@ pool_unref(&pool); } +int quota_root_default_init(struct quota_root *root, const char *args, + const char **error_r) +{ + const char *const *tmp; + + if (args == NULL) + return 0; + + tmp = t_strsplit_spaces(args, " "); + for (; *tmp != NULL; tmp++) { + if (strcmp(*tmp, "noenforcing") == 0) + root->no_enforcing = TRUE; + else if (strcmp(*tmp, "hidden") == 0) + root->hidden = TRUE; + else if (strcmp(*tmp, "ignoreunlimited") == 0) + root->disable_unlimited_tracking = TRUE; + else + break; + } + if (*tmp != NULL) { + *error_r = t_strdup_printf( + "Unknown parameter for backend %s: %s", + root->backend.name, *tmp); + return -1; + } + return 0; +} + static int quota_root_init(struct quota_root_settings *root_set, struct quota *quota, struct quota_root **root_r, const char **error_r) { struct quota_root *root; - const char *const *tmp; root = root_set->backend->v.alloc(); root->resource_ret = -1; @@ -302,24 +329,9 @@ root->backend.name, *error_r); return -1; } - } else if (root_set->args != NULL) { - tmp = t_strsplit_spaces(root_set->args, " "); - for (; *tmp != NULL; tmp++) { - if (strcmp(*tmp, "noenforcing") == 0) - root->no_enforcing = TRUE; - else if (strcmp(*tmp, "hidden") == 0) - root->hidden = TRUE; - else if (strcmp(*tmp, "ignoreunlimited") == 0) - root->disable_unlimited_tracking = TRUE; - else - break; - } - if (*tmp != NULL) { - *error_r = t_strdup_printf( - "Unknown parameter for backend %s: %s", - root->backend.name, *tmp); + } else { + if (quota_root_default_init(root, root_set->args, error_r) < 0) return -1; - } } if (root_set->default_rule.bytes_limit == 0 && root_set->default_rule.count_limit == 0 && From dovecot at dovecot.org Mon Sep 21 13:25:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 13:25:59 +0000 Subject: dovecot-2.2: quota: Commit expunge transaction instead of rollin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ea1b001a82e changeset: 19166:7ea1b001a82e user: Timo Sirainen date: Mon Sep 21 16:12:43 2015 +0300 description: quota: Commit expunge transaction instead of rolling back. This likely won't make much of a difference, since it should have accessed only expunged mails. But it's a bit cleaner this way. diffstat: src/plugins/quota/quota-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r ff5c34d62daf -r 7ea1b001a82e src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Mon Sep 21 16:09:37 2015 +0300 +++ b/src/plugins/quota/quota-storage.c Mon Sep 21 16:12:43 2015 +0300 @@ -293,7 +293,7 @@ if (qbox->expunge_qt != NULL && qbox->expunge_qt->tmp_mail != NULL) { mail_free(&qbox->expunge_qt->tmp_mail); - mailbox_transaction_rollback(&qbox->expunge_trans); + (void)mailbox_transaction_commit(&qbox->expunge_trans); } qbox->sync_transaction_expunge = FALSE; } From dovecot at dovecot.org Mon Sep 21 13:25:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 13:25:59 +0000 Subject: dovecot-2.2: quota: Added quota_vsizes=yes setting to count quot... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7f26cfb08fcd changeset: 19167:7f26cfb08fcd user: Timo Sirainen date: Mon Sep 21 16:24:30 2015 +0300 description: quota: Added quota_vsizes=yes setting to count quotas using virtual sizes instead of physical sizes. This doesn't work with all the quota backends. diffstat: src/plugins/quota/quota-count.c | 9 ++++++--- src/plugins/quota/quota-private.h | 1 + src/plugins/quota/quota-storage.c | 21 +++++++++++++++++---- src/plugins/quota/quota.c | 1 + 4 files changed, 25 insertions(+), 7 deletions(-) diffs (105 lines): diff -r 7ea1b001a82e -r 7f26cfb08fcd src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Mon Sep 21 16:12:43 2015 +0300 +++ b/src/plugins/quota/quota-count.c Mon Sep 21 16:24:30 2015 +0300 @@ -25,12 +25,14 @@ } box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); - if (mailbox_get_metadata(box, MAILBOX_METADATA_PHYSICAL_SIZE, + if (mailbox_get_metadata(box, root->quota->set->vsizes ? + MAILBOX_METADATA_VIRTUAL_SIZE : + MAILBOX_METADATA_PHYSICAL_SIZE, &metadata) < 0 || mailbox_get_status(box, STATUS_MESSAGES, &status) < 0) { errstr = mailbox_get_last_error(box, &error); if (error == MAIL_ERROR_TEMP) { - i_error("quota: Couldn't get physical size of mailbox %s: %s", + i_error("quota: Couldn't get size of mailbox %s: %s", vname, errstr); ret = -1; } else { @@ -39,7 +41,8 @@ } } else { ret = 1; - *bytes_r = metadata.physical_size; + *bytes_r = root->quota->set->vsizes ? + metadata.virtual_size : metadata.physical_size; *count_r = status.messages; } mailbox_free(&box); diff -r 7ea1b001a82e -r 7f26cfb08fcd src/plugins/quota/quota-private.h --- a/src/plugins/quota/quota-private.h Mon Sep 21 16:12:43 2015 +0300 +++ b/src/plugins/quota/quota-private.h Mon Sep 21 16:24:30 2015 +0300 @@ -28,6 +28,7 @@ const char *quota_exceeded_msg; unsigned int debug:1; unsigned int initialized:1; + unsigned int vsizes:1; }; struct quota_rule { diff -r 7ea1b001a82e -r 7f26cfb08fcd src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Mon Sep 21 16:12:43 2015 +0300 +++ b/src/plugins/quota/quota-storage.c Mon Sep 21 16:24:30 2015 +0300 @@ -48,14 +48,20 @@ { struct mail_private *mail = (struct mail_private *)_mail; struct quota_mailbox *qbox = QUOTA_CONTEXT(_mail->box); + struct quota_user *quser = QUOTA_USER_CONTEXT(_mail->box->storage->user); union mail_module_context *qmail = QUOTA_MAIL_CONTEXT(mail); uoff_t size; + int ret; /* We need to handle the situation where multiple transactions expunged the mail at the same time. In here we'll just save the message's physical size and do the quota freeing later when the message was known to be expunged. */ - if (mail_get_physical_size(_mail, &size) == 0) { + if (quser->quota->set->vsizes) + ret = mail_get_virtual_size(_mail, &size); + else + ret = mail_get_physical_size(_mail, &size); + if (ret == 0) { if (!array_is_created(&qbox->expunge_uids)) { i_array_init(&qbox->expunge_uids, 64); i_array_init(&qbox->expunge_sizes, 64); @@ -310,6 +316,7 @@ enum mailbox_sync_type sync_type) { struct quota_mailbox *qbox = QUOTA_CONTEXT(box); + struct quota_user *quser = QUOTA_USER_CONTEXT(box->storage->user); const uint32_t *uids; const uoff_t *sizep; unsigned int i, count; @@ -369,10 +376,16 @@ mail_alloc(qbox->expunge_trans, MAIL_FETCH_PHYSICAL_SIZE, NULL); } - if (mail_set_uid(qbox->expunge_qt->tmp_mail, uid) && - mail_get_physical_size(qbox->expunge_qt->tmp_mail, &size) == 0) + if (!mail_set_uid(qbox->expunge_qt->tmp_mail, uid)) + ; + else if (!quser->quota->set->vsizes) { + if (mail_get_physical_size(qbox->expunge_qt->tmp_mail, &size) == 0) { + quota_free_bytes(qbox->expunge_qt, size); + return; + } + } else if (mail_get_virtual_size(qbox->expunge_qt->tmp_mail, &size) == 0) { quota_free_bytes(qbox->expunge_qt, size); - else { + } else { /* there's no way to get the size. recalculate the quota. */ quota_recalculate(qbox->expunge_qt); qbox->recalculate = TRUE; diff -r 7ea1b001a82e -r 7f26cfb08fcd src/plugins/quota/quota.c --- a/src/plugins/quota/quota.c Mon Sep 21 16:12:43 2015 +0300 +++ b/src/plugins/quota/quota.c Mon Sep 21 16:24:30 2015 +0300 @@ -229,6 +229,7 @@ mail_user_plugin_getenv(user, "quota_exceeded_message"); if (quota_set->quota_exceeded_msg == NULL) quota_set->quota_exceeded_msg = DEFAULT_QUOTA_EXCEEDED_MSG; + quota_set->vsizes = mail_user_plugin_getenv(user, "quota_vsizes") != NULL; p_array_init("a_set->root_sets, pool, 4); if (i_strocpy(root_name, "quota", sizeof(root_name)) < 0) From dovecot at dovecot.org Mon Sep 21 13:37:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 13:37:34 +0000 Subject: dovecot-2.2: quota: Added "count" backend, which simply sums up ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f205f3d56093 changeset: 19169:f205f3d56093 user: Timo Sirainen date: Mon Sep 21 16:36:05 2015 +0300 description: quota: Added "count" backend, which simply sums up mailboxes' vsizes. In a way this is similar to the simple "dirsize" backend, but much more efficient. As long as mailbox_list_index=yes, the quota can typically be looked up only by reading the dovecot.list.index* files. This backend enforces using quota_vsizes=yes setting to keep the performance good, because physical sizes don't have a similar optimized vsize header. There's also no especially good reason why this backend should support physical sizes - they were originally mainly used to allow quickly stat()ing Maildir files. diffstat: src/plugins/quota/quota-count.c | 75 +++++++++++++++++++++++++++++++++++++++++ src/plugins/quota/quota.c | 2 + 2 files changed, 77 insertions(+), 0 deletions(-) diffs (101 lines): diff -r e29d2f7fe53f -r f205f3d56093 src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Mon Sep 21 16:32:27 2015 +0300 +++ b/src/plugins/quota/quota-count.c Mon Sep 21 16:36:05 2015 +0300 @@ -107,3 +107,78 @@ root->recounting = FALSE; return ret; } + +static struct quota_root *count_quota_alloc(void) +{ + return i_new(struct quota_root, 1); +} + +static int count_quota_init(struct quota_root *root, const char *args, + const char **error_r) +{ + if (!root->quota->set->vsizes) { + *error_r = "quota count backend requires quota_vsizes=yes"; + return -1; + } + return quota_root_default_init(root, args, error_r); +} + +static void count_quota_deinit(struct quota_root *_root) +{ + i_free(_root); +} + +static const char *const * +count_quota_root_get_resources(struct quota_root *root ATTR_UNUSED) +{ + static const char *resources[] = { + QUOTA_NAME_STORAGE_KILOBYTES, QUOTA_NAME_MESSAGES, NULL + }; + return resources; +} + +static int +count_quota_get_resource(struct quota_root *root, + const char *name, uint64_t *value_r) +{ + uint64_t bytes, count; + + if (quota_count(root, &bytes, &count) < 0) + return -1; + + if (strcmp(name, QUOTA_NAME_STORAGE_BYTES) == 0) + *value_r = bytes; + else if (strcmp(name, QUOTA_NAME_MESSAGES) == 0) + *value_r = count; + else + return 0; + return 1; +} + +static int +count_quota_update(struct quota_root *root ATTR_UNUSED, + struct quota_transaction_context *ctx ATTR_UNUSED) +{ + if (ctx->recalculate) { + //FIXME: remove cached values from index + } + return 0; +} + +struct quota_backend quota_backend_count = { + "count", + + { + count_quota_alloc, + count_quota_init, + count_quota_deinit, + NULL, + NULL, + NULL, + count_quota_root_get_resources, + count_quota_get_resource, + count_quota_update, + NULL, + NULL + } +}; diff -r e29d2f7fe53f -r f205f3d56093 src/plugins/quota/quota.c --- a/src/plugins/quota/quota.c Mon Sep 21 16:32:27 2015 +0300 +++ b/src/plugins/quota/quota.c Mon Sep 21 16:36:05 2015 +0300 @@ -29,6 +29,7 @@ unsigned int quota_module_id = 0; +extern struct quota_backend quota_backend_count; extern struct quota_backend quota_backend_dict; extern struct quota_backend quota_backend_dirsize; extern struct quota_backend quota_backend_fs; @@ -38,6 +39,7 @@ #ifdef HAVE_FS_QUOTA "a_backend_fs, #endif + "a_backend_count, "a_backend_dict, "a_backend_dirsize, "a_backend_maildir From dovecot at dovecot.org Mon Sep 21 13:37:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 13:37:32 +0000 Subject: dovecot-2.2: lib-storage: Update mailbox vsize header on save/co... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e29d2f7fe53f changeset: 19168:e29d2f7fe53f user: Timo Sirainen date: Mon Sep 21 16:32:27 2015 +0300 description: lib-storage: Update mailbox vsize header on save/copy/expunge. This allows always efficiently looking up maiboxes' vsizes after they're initially calculated. The expunge handling is unfortunately done currently in quota handling code, so it works only if quota is enabled. Ideally this would be solved in v2.3 with some lib-storage core changes. diffstat: src/lib-storage/index/Makefile.am | 1 + src/lib-storage/index/cydir/cydir-sync.c | 10 +- src/lib-storage/index/dbox-multi/mdbox-sync.c | 19 +- src/lib-storage/index/dbox-single/sdbox-sync.c | 12 +- src/lib-storage/index/index-mailbox-size.c | 344 +++++++++++++++++--- src/lib-storage/index/index-storage.c | 69 ++++ src/lib-storage/index/index-storage.h | 9 + src/lib-storage/index/index-sync.c | 3 + src/lib-storage/index/maildir/maildir-sync-index.c | 7 +- src/lib-storage/index/mbox/mbox-sync.c | 11 +- src/plugins/quota/quota-storage.c | 9 + 11 files changed, 407 insertions(+), 87 deletions(-) diffs (truncated from 772 to 300 lines): diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/Makefile.am --- a/src/lib-storage/index/Makefile.am Mon Sep 21 16:24:30 2015 +0300 +++ b/src/lib-storage/index/Makefile.am Mon Sep 21 16:32:27 2015 +0300 @@ -40,6 +40,7 @@ istream-mail.h \ index-attachment.h \ index-mail.h \ + index-mailbox-size.h \ index-rebuild.h \ index-search-private.h \ index-search-result.h \ diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/cydir/cydir-sync.c --- a/src/lib-storage/index/cydir/cydir-sync.c Mon Sep 21 16:24:30 2015 +0300 +++ b/src/lib-storage/index/cydir/cydir-sync.c Mon Sep 21 16:32:27 2015 +0300 @@ -112,18 +112,18 @@ if (!force) sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES; - ret = mail_index_sync_begin(mbox->box.index, &ctx->index_sync_ctx, - &ctx->sync_view, &ctx->trans, - sync_flags); + ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, + sync_flags); if (ret <= 0) { - if (ret < 0) - mailbox_set_index_error(&mbox->box); i_free(ctx); *ctx_r = NULL; return ret; } cydir_sync_index(ctx); + index_storage_expunging_deinit(&mbox->box); + *ctx_r = ctx; return 0; } diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/dbox-multi/mdbox-sync.c --- a/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Sep 21 16:24:30 2015 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Sep 21 16:32:27 2015 +0300 @@ -198,24 +198,20 @@ struct mdbox_mailbox *mbox = ctx->mbox; int ret; - ret = mail_index_sync_begin(mbox->box.index, &ctx->index_sync_ctx, - &ctx->sync_view, &ctx->trans, sync_flags); + ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx, + &ctx->sync_view, &ctx->trans, sync_flags); if (mail_index_reset_fscked(mbox->box.index)) mdbox_storage_set_corrupted(mbox->storage); - if (ret < 0) { - mailbox_set_index_error(&mbox->box); - return -1; - } - if (ret == 0) { - /* nothing to do */ - return 0; - } + if (ret <= 0) + return ret; /* error / nothing to do */ if (!mdbox_map_atomic_is_locked(ctx->atomic) && mail_index_sync_has_expunges(ctx->index_sync_ctx)) { /* we have expunges, so we need to write to map. it needs to be locked before mailbox index. */ mail_index_sync_rollback(&ctx->index_sync_ctx); + index_storage_expunging_deinit(&ctx->mbox->box); + if (mdbox_map_atomic_lock(ctx->atomic) < 0) return -1; return mdbox_sync_try_begin(ctx, sync_flags); @@ -262,12 +258,14 @@ ret = mdbox_sync_try_begin(ctx, sync_flags); if (ret <= 0) { /* failed / nothing to do */ + index_storage_expunging_deinit(&mbox->box); i_free(ctx); return ret; } if ((ret = mdbox_sync_index(ctx)) <= 0) { mail_index_sync_rollback(&ctx->index_sync_ctx); + index_storage_expunging_deinit(&mbox->box); i_free_and_null(ctx); if (ret < 0) @@ -292,6 +290,7 @@ } return mdbox_sync_begin(mbox, flags, atomic, ctx_r); } + index_storage_expunging_deinit(&mbox->box); *ctx_r = ctx; return 0; diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/dbox-single/sdbox-sync.c --- a/src/lib-storage/index/dbox-single/sdbox-sync.c Mon Sep 21 16:24:30 2015 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Mon Sep 21 16:32:27 2015 +0300 @@ -216,15 +216,12 @@ sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES; for (i = 0;; i++) { - ret = mail_index_sync_begin(mbox->box.index, - &ctx->index_sync_ctx, - &ctx->sync_view, &ctx->trans, - sync_flags); + ret = index_storage_expunged_sync_begin(&mbox->box, + &ctx->index_sync_ctx, &ctx->sync_view, + &ctx->trans, sync_flags); if (mail_index_reset_fscked(mbox->box.index)) sdbox_set_mailbox_corrupted(&mbox->box); if (ret <= 0) { - if (ret < 0) - mailbox_set_index_error(&mbox->box); array_free(&ctx->expunged_uids); i_free(ctx); *ctx_r = NULL; @@ -255,6 +252,7 @@ } mail_index_sync_rollback(&ctx->index_sync_ctx); if (ret < 0) { + index_storage_expunging_deinit(&ctx->mbox->box); array_free(&ctx->expunged_uids); i_free(ctx); return -1; @@ -274,6 +272,7 @@ if (success) { mail_index_view_ref(ctx->sync_view); + if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { mailbox_set_index_error(&ctx->mbox->box); ret = -1; @@ -285,6 +284,7 @@ mail_index_sync_rollback(&ctx->index_sync_ctx); } + index_storage_expunging_deinit(&ctx->mbox->box); array_free(&ctx->expunged_uids); i_free(ctx); return ret; diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/index-mailbox-size.c --- a/src/lib-storage/index/index-mailbox-size.c Mon Sep 21 16:24:30 2015 +0300 +++ b/src/lib-storage/index/index-mailbox-size.c Mon Sep 21 16:32:27 2015 +0300 @@ -1,53 +1,283 @@ /* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "file-create-locked.h" #include "mail-search-build.h" #include "index-storage.h" +#include "index-mailbox-size.h" + +/* + Saving new mails: After transaction is committed and synced, trigger + vsize updating. Lock vsize updates. Check if the message count + + last-indexed-uid are still valid. If they are, add all the missing new + mails. Unlock. + + Fetching vsize: Lock vsize updates. Check if the message count + + last-indexed-uid are still valid. If not, set them to zero. Add all + the missing mails. Unlock. + + Expunging mails: Check if syncing would expunge any mails. If so, lock the + vsize updates before locking syncing (to avoid deadlocks). Check if the + message count + last-indexed-uid are still valid. If not, unlock vsize and + do nothing else. Otherwise, for each expunged mail whose UID <= + last-indexed-uid, decrease the message count and the vsize in memory. After + syncing is successfully committed, write the changes to header. Unlock. + + Note that the final expunge handling with some mailbox formats is done while + syncing is no longer locked. Because of this we need to have the vsize + locking. The final vsize header update requires committing a transaction, + which internally is the same as a sync lock. So to avoid deadlocks we always + need to lock vsize updates before sync. +*/ + +#define VSIZE_LOCK_SUFFIX ".vsize.lock" +#define VSIZE_UPDATE_MAX_LOCK_SECS 10 + +struct mailbox_vsize_update { + struct mailbox *box; + struct mail_index_view *view; + struct mailbox_index_vsize vsize_hdr, orig_vsize_hdr; + + char *lock_path; + int lock_fd; + struct file_lock *lock; + bool rebuild; + bool written; +}; + +static void vsize_header_refresh(struct mailbox_vsize_update *update) +{ + const void *data; + size_t size; + + if (update->view != NULL) + mail_index_view_close(&update->view); + (void)mail_index_refresh(update->box->index); + update->view = mail_index_view_open(update->box->index); + + mail_index_get_header_ext(update->view, update->box->vsize_hdr_ext_id, + &data, &size); + memcpy(&update->orig_vsize_hdr, data, + I_MIN(size, sizeof(update->orig_vsize_hdr))); + if (size == sizeof(update->vsize_hdr)) + memcpy(&update->vsize_hdr, data, sizeof(update->vsize_hdr)); + else { + if (size != 0) { + mail_storage_set_critical(update->box->storage, + "vsize-hdr has invalid size: %"PRIuSIZE_T, + size); + } + update->rebuild = TRUE; + memset(&update->vsize_hdr, 0, sizeof(update->vsize_hdr)); + } +} + +static void +index_mailbox_vsize_check_rebuild(struct mailbox_vsize_update *update) +{ + uint32_t seq1, seq2; + + if (update->vsize_hdr.highest_uid == 0) + return; + if (!mail_index_lookup_seq_range(update->view, 1, + update->vsize_hdr.highest_uid, + &seq1, &seq2)) + seq2 = 0; + + if (update->vsize_hdr.message_count != seq2) { + if (update->vsize_hdr.message_count < seq2) { + mail_storage_set_critical(update->box->storage, + "vsize-hdr has invalid message-count (%u < %u)", + update->vsize_hdr.message_count, seq2); + } else { + /* some messages have been expunged, rescan */ + } + memset(&update->vsize_hdr, 0, sizeof(update->vsize_hdr)); + update->rebuild = TRUE; + } +} + +struct mailbox_vsize_update * +index_mailbox_vsize_update_init(struct mailbox *box) +{ + struct mailbox_vsize_update *update; + + update = i_new(struct mailbox_vsize_update, 1); + update->box = box; + update->lock_fd = -1; + + vsize_header_refresh(update); + return update; +} + +static bool vsize_update_lock_full(struct mailbox_vsize_update *update, + unsigned int lock_secs) +{ + struct mailbox *box = update->box; + const struct mailbox_permissions *perm; + struct file_create_settings set; + const char *error; + bool created; + + if (update->lock_path != NULL) + return update->lock != NULL; + if (MAIL_INDEX_IS_IN_MEMORY(box->index)) + return FALSE; + + perm = mailbox_get_permissions(box); + memset(&set, 0, sizeof(set)); + set.lock_timeout_secs = + mail_storage_get_lock_timeout(box->storage, lock_secs); + set.lock_method = box->storage->set->parsed_lock_method; + set.mode = perm->file_create_mode; + set.gid = perm->file_create_gid; + set.gid_origin = perm->file_create_gid_origin; + + update->lock_path = i_strdup_printf("%s/"VSIZE_LOCK_SUFFIX, + box->index->dir); + update->lock_fd = file_create_locked(update->lock_path, &set, + &update->lock, &created, &error); + if (update->lock_fd == -1) { + if (errno != EAGAIN) { + i_error("file_create_locked(%s) failed: %m", + update->lock_path); + } + return FALSE; + } + update->rebuild = FALSE; + vsize_header_refresh(update); + index_mailbox_vsize_check_rebuild(update); + return TRUE; +} + +bool index_mailbox_vsize_update_try_lock(struct mailbox_vsize_update *update) +{ + return vsize_update_lock_full(update, 0); From dovecot at dovecot.org Mon Sep 21 14:02:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 14:02:35 +0000 Subject: dovecot-2.2: fts: Send session ID to indexer and indexer-worker ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/250d85e784d1 changeset: 19170:250d85e784d1 user: Timo Sirainen date: Mon Sep 21 17:01:05 2015 +0300 description: fts: Send session ID to indexer and indexer-worker for logging purposes. diffstat: src/indexer/indexer-client.c | 7 +++++-- src/indexer/indexer-queue.c | 10 +++++++--- src/indexer/indexer-queue.h | 4 +++- src/indexer/master-connection.c | 9 +++++---- src/indexer/worker-connection.c | 3 +++ src/plugins/fts/fts-indexer.c | 5 +++-- src/plugins/fts/fts-storage.c | 5 ++++- 7 files changed, 30 insertions(+), 13 deletions(-) diffs (176 lines): diff -r f205f3d56093 -r 250d85e784d1 src/indexer/indexer-client.c --- a/src/indexer/indexer-client.c Mon Sep 21 16:36:05 2015 +0300 +++ b/src/indexer/indexer-client.c Mon Sep 21 17:01:05 2015 +0300 @@ -67,9 +67,10 @@ const char *const *args, const char **error_r) { struct indexer_client_request *ctx = NULL; + const char *session_id = NULL; unsigned int tag, max_recent_msgs; - /* [] */ + /* [ []] */ if (str_array_length(args) < 3) { *error_r = "Wrong parameter count"; return -1; @@ -83,6 +84,8 @@ else if (str_to_uint(args[3], &max_recent_msgs) < 0) { *error_r = "Invalid max_recent_msgs"; return -1; + } else { + session_id = args[4]; } if (tag != 0) { @@ -93,7 +96,7 @@ } indexer_queue_append(client->queue, append, args[1], args[2], - max_recent_msgs, ctx); + session_id, max_recent_msgs, ctx); o_stream_nsend_str(client->output, t_strdup_printf("%u\tOK\n", tag)); return 0; } diff -r f205f3d56093 -r 250d85e784d1 src/indexer/indexer-queue.c --- a/src/indexer/indexer-queue.c Mon Sep 21 16:36:05 2015 +0300 +++ b/src/indexer/indexer-queue.c Mon Sep 21 17:01:05 2015 +0300 @@ -82,6 +82,7 @@ static struct indexer_request * indexer_queue_append_request(struct indexer_queue *queue, bool append, const char *username, const char *mailbox, + const char *session_id, unsigned int max_recent_msgs, void *context) { struct indexer_request *request; @@ -91,6 +92,7 @@ request = i_new(struct indexer_request, 1); request->username = i_strdup(username); request->mailbox = i_strdup(mailbox); + request->session_id = i_strdup(session_id); request->max_recent_msgs = max_recent_msgs; request_add_context(request, context); hash_table_insert(queue->requests, request, request); @@ -130,12 +132,14 @@ void indexer_queue_append(struct indexer_queue *queue, bool append, const char *username, const char *mailbox, - unsigned int max_recent_msgs, void *context) + const char *session_id, unsigned int max_recent_msgs, + void *context) { struct indexer_request *request; request = indexer_queue_append_request(queue, append, username, mailbox, - max_recent_msgs, context); + session_id, max_recent_msgs, + context); request->index = TRUE; indexer_queue_append_finish(queue); } @@ -147,7 +151,7 @@ struct indexer_request *request; request = indexer_queue_append_request(queue, TRUE, username, mailbox, - 0, context); + NULL, 0, context); request->optimize = TRUE; indexer_queue_append_finish(queue); } diff -r f205f3d56093 -r 250d85e784d1 src/indexer/indexer-queue.h --- a/src/indexer/indexer-queue.h Mon Sep 21 16:36:05 2015 +0300 +++ b/src/indexer/indexer-queue.h Mon Sep 21 17:01:05 2015 +0300 @@ -8,6 +8,7 @@ char *username; char *mailbox; + char *session_id; unsigned int max_recent_msgs; /* index messages in this mailbox */ @@ -38,7 +39,8 @@ void indexer_queue_append(struct indexer_queue *queue, bool append, const char *username, const char *mailbox, - unsigned int max_recent_msgs, void *context); + const char *session_id, unsigned int max_recent_msgs, + void *context); void indexer_queue_append_optimize(struct indexer_queue *queue, const char *username, const char *mailbox, void *context); diff -r f205f3d56093 -r 250d85e784d1 src/indexer/master-connection.c --- a/src/indexer/master-connection.c Mon Sep 21 16:36:05 2015 +0300 +++ b/src/indexer/master-connection.c Mon Sep 21 17:01:05 2015 +0300 @@ -201,9 +201,9 @@ unsigned int max_recent_msgs; int ret; - /* [i][o] */ - if (str_array_length(args) != 4 || - str_to_uint(args[2], &max_recent_msgs) < 0 || args[3][0] == '\0') { + /* [i][o] */ + if (str_array_length(args) != 5 || + str_to_uint(args[3], &max_recent_msgs) < 0 || args[4][0] == '\0') { i_error("Invalid input from master: %s", line); return -1; } @@ -212,6 +212,7 @@ input.module = "mail"; input.service = "indexer-worker"; input.username = args[0]; + input.session_id = args[2][0] == '\0' ? NULL : args[2]; if (mail_storage_service_lookup_next(conn->storage_service, &input, &service_user, &user, &error) <= 0) { @@ -220,7 +221,7 @@ } else { indexer_worker_refresh_proctitle(user->username, args[1], 0, 0); ret = index_mailbox(conn, user, args[1], - max_recent_msgs, args[3]); + max_recent_msgs, args[4]); indexer_worker_refresh_proctitle(NULL, NULL, 0, 0); mail_user_unref(&user); mail_storage_service_user_free(&service_user); diff -r f205f3d56093 -r 250d85e784d1 src/indexer/worker-connection.c --- a/src/indexer/worker-connection.c Mon Sep 21 16:36:05 2015 +0300 +++ b/src/indexer/worker-connection.c Mon Sep 21 17:01:05 2015 +0300 @@ -241,6 +241,9 @@ str_append_tabescaped(str, request->username); str_append_c(str, '\t'); str_append_tabescaped(str, request->mailbox); + str_append_c(str, '\t'); + if (request->session_id != NULL) + str_append_tabescaped(str, request->session_id); str_printfa(str, "\t%u\t", request->max_recent_msgs); if (request->index) str_append_c(str, 'i'); diff -r f205f3d56093 -r 250d85e784d1 src/plugins/fts/fts-indexer.c --- a/src/plugins/fts/fts-indexer.c Mon Sep 21 16:36:05 2015 +0300 +++ b/src/plugins/fts/fts-indexer.c Mon Sep 21 17:01:05 2015 +0300 @@ -113,9 +113,10 @@ return 0; } - cmd = t_strdup_printf("PREPEND\t1\t%s\t%s\n", + cmd = t_strdup_printf("PREPEND\t1\t%s\t%s\t0\t%s\n", str_tabescape(box->storage->user->username), - str_tabescape(box->vname)); + str_tabescape(box->vname), + str_tabescape(box->storage->user->session_id)); fd = fts_indexer_cmd(box->storage->user, cmd, &path); if (fd == -1) return -1; diff -r f205f3d56093 -r 250d85e784d1 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Mon Sep 21 16:36:05 2015 +0300 +++ b/src/plugins/fts/fts-storage.c Mon Sep 21 17:01:05 2015 +0300 @@ -612,7 +612,10 @@ str_append_tabescaped(str, user->username); str_append_c(str, '\t'); str_append_tabescaped(str, box->vname); - str_printfa(str, "\t%u\n", max_recent_msgs); + str_printfa(str, "\t%u", max_recent_msgs); + str_append_c(str, '\t'); + str_append_tabescaped(str, box->storage->user->session_id); + str_append_c(str, '\n'); if (write_full(fd, str_data(str), str_len(str)) < 0) i_error("write(%s) failed: %m", path); i_close_fd(&fd); From dovecot at dovecot.org Mon Sep 21 14:04:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 14:04:39 +0000 Subject: dovecot-2.2: lib-storage: Forgot to add index-mailbox-size.h to ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6377910c19e3 changeset: 19171:6377910c19e3 user: Timo Sirainen date: Mon Sep 21 17:03:19 2015 +0300 description: lib-storage: Forgot to add index-mailbox-size.h to e29d2f7fe53f diffstat: src/lib-storage/index/index-mailbox-size.h | 20 ++++++++++++++++++++ 1 files changed, 20 insertions(+), 0 deletions(-) diffs (24 lines): diff -r 250d85e784d1 -r 6377910c19e3 src/lib-storage/index/index-mailbox-size.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-mailbox-size.h Mon Sep 21 17:03:19 2015 +0300 @@ -0,0 +1,20 @@ +#ifndef INDEX_MAILBOX_SIZE_H +#define INDEX_MAILBOX_SIZE_H + +struct mailbox; + +struct mailbox_vsize_update * +index_mailbox_vsize_update_init(struct mailbox *box); +void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **update); + +void index_mailbox_vsize_hdr_expunge(struct mailbox_vsize_update *update, + uint32_t uid, uoff_t vsize); + +bool index_mailbox_vsize_update_try_lock(struct mailbox_vsize_update *update); +bool index_mailbox_vsize_update_wait_lock(struct mailbox_vsize_update *update); +/* Returns TRUE if expunges & appends should be updating the header. */ +bool index_mailbox_vsize_want_updates(struct mailbox_vsize_update *update); + +void index_mailbox_vsize_update_appends(struct mailbox *box); + +#endif From dovecot at dovecot.org Mon Sep 21 16:54:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 16:54:54 +0000 Subject: dovecot-2.2: lib: timings - added quantiles Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2f492fac75b7 changeset: 19173:2f492fac75b7 user: Phil Carmody date: Mon Sep 21 19:52:32 2015 +0300 description: lib: timings - added quantiles Just sub-sample the stream. On the assumption that the samples come from one distribution, then any randomly selected subset will share the same distribution. Therefore the quantiles should be at approximately the same value. However, that's a big assumption, as there will almost certainly be time dependency, and periodicity (24hrs, 7 days). Signed-off-by: Phil Carmody diffstat: src/lib/test-timing.c | 12 ++++++- src/lib/timing.c | 78 +++++++++++++++++++++++++++++++++++++++++++------- src/lib/timing.h | 4 ++ 3 files changed, 80 insertions(+), 14 deletions(-) diffs (151 lines): diff -r 1a5a45c80687 -r 2f492fac75b7 src/lib/test-timing.c --- a/src/lib/test-timing.c Mon Sep 21 19:51:05 2015 +0300 +++ b/src/lib/test-timing.c Mon Sep 21 19:52:32 2015 +0300 @@ -2,7 +2,7 @@ #include "test-lib.h" #include "timing.h" - +#include "sort.h" #include static void @@ -29,7 +29,15 @@ test_assert_idx(timing_get_count(t) == input_size, input_size); test_assert_idx(timing_get_min(t) == min, input_size); test_assert_idx(timing_get_max(t) == max, input_size); - test_assert_idx(timing_get_avg(t) == sum/input_size, input_size); + test_assert_idx(timing_get_avg(t) == (sum + input_size/2)/input_size, input_size); + + /* these aren't always fully accurate: */ + test_assert_idx(timing_get_median(t) >= copy[(input_size-1)/2] && + timing_get_median(t) <= copy[input_size/2], + input_size); + /* when we have 20 elements, [19] is the max, not the 95th %ile, so subtract 1 */ + test_assert_idx(timing_get_95th(t) == copy[input_size*95/100 - !(input_size%20)], + input_size); i_free(copy); } diff -r 1a5a45c80687 -r 2f492fac75b7 src/lib/timing.c --- a/src/lib/timing.c Mon Sep 21 19:51:05 2015 +0300 +++ b/src/lib/timing.c Mon Sep 21 19:52:32 2015 +0300 @@ -1,14 +1,21 @@ /* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ #include "lib.h" -#include "bsearch-insert-pos.h" #include "timing.h" +#include "sort.h" +#include -#define TIMING_MAX_BUCKET_COUNT 20 +/* In order to have a vaguely accurate 95th percentile, you need way + more than 20 in your subsample. */ +#define TIMING_SUBSAMPLING_BUFFER (20*24) /* 20*24 fits in a page */ struct timing { unsigned int count; - uint64_t min, max, sum; + bool sorted; + uint64_t min; + uint64_t samples[TIMING_SUBSAMPLING_BUFFER]; + uint64_t max; + uint64_t sum; }; struct timing *timing_init(void) @@ -21,18 +28,30 @@ i_free_and_null(*_timing); } - void timing_add_usecs(struct timing *timing, uint64_t usecs) { - if (timing->count++ == 0) { - timing->min = timing->max = timing->sum = usecs; + if (timing->count < TIMING_SUBSAMPLING_BUFFER) { + timing->samples[timing->count] = usecs; + if (timing->count == 0) + timing->min = timing->max = usecs; } else { - if (timing->min > usecs) - timing->min = usecs; - if (timing->max < usecs) - timing->max = usecs; - timing->sum += usecs; + unsigned int count = timing->count; + unsigned int idx; + if (count > RAND_MAX >> 6) + idx = (rand()*((uint64_t)RAND_MAX+1) + rand()) % count; + else + idx = rand() % count; + if (idx < TIMING_SUBSAMPLING_BUFFER) + timing->samples[idx] = usecs; } + + timing->count++; + timing->sum += usecs; + if (timing->max < usecs) + timing->max = usecs; + if (timing->min > usecs) + timing->min = usecs; + timing->sorted = FALSE; } unsigned int timing_get_count(const struct timing *timing) @@ -55,5 +74,40 @@ if (timing->count == 0) return 0; - return timing->sum / timing->count; + return (timing->sum + timing->count/2) / timing->count; } + +static void timing_ensure_sorted(struct timing *timing) +{ + if (timing->sorted) + return; + i_qsort(timing->samples, timing->count, sizeof(*timing->samples), + uint64_cmp); + timing->sorted = TRUE; +} + +uint64_t timing_get_median(const struct timing *timing) +{ + if (timing->count == 0) + return 0; + /* cast-away const - reading requires sorting */ + timing_ensure_sorted((struct timing *)timing); + unsigned int count = (timing->count < TIMING_SUBSAMPLING_BUFFER) + ? timing->count + : TIMING_SUBSAMPLING_BUFFER; + unsigned int idx1 = (count-1)/2, idx2 = count/2; + return (timing->samples[idx1] + timing->samples[idx2]) / 2; +} + +uint64_t timing_get_95th(const struct timing *timing) +{ + if (timing->count == 0) + return 0; + /* cast-away const - reading requires sorting */ + timing_ensure_sorted((struct timing *)timing); + unsigned int count = (timing->count < TIMING_SUBSAMPLING_BUFFER) + ? timing->count + : TIMING_SUBSAMPLING_BUFFER; + unsigned int idx = count - count/20 - 1; + return timing->samples[idx]; +} diff -r 1a5a45c80687 -r 2f492fac75b7 src/lib/timing.h --- a/src/lib/timing.h Mon Sep 21 19:51:05 2015 +0300 +++ b/src/lib/timing.h Mon Sep 21 19:52:32 2015 +0300 @@ -16,5 +16,9 @@ uint64_t timing_get_max(const struct timing *timing); /* Returns events' average. */ uint64_t timing_get_avg(const struct timing *timing); +/* Returns events' approximate (through random subsampling) median. */ +uint64_t timing_get_median(const struct timing *timing); +/* Returns events' approximate (through random subsampling) 95th percentile. */ +uint64_t timing_get_95th(const struct timing *timing); #endif From dovecot at dovecot.org Mon Sep 21 16:54:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 16:54:53 +0000 Subject: dovecot-2.2: lib: extract sort-helpers into separate sort.h file Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1a5a45c80687 changeset: 19172:1a5a45c80687 user: Phil Carmody date: Mon Sep 21 19:51:05 2015 +0300 description: lib: extract sort-helpers into separate sort.h file The macro definition was nothing to do with strings, and we can put trivial common-type comparators here. They didn't need to be static inline, as they can never be inlined anyway, being only used via function pointers, but that preserves the closest equivalent to the current code. Signed-off-by: Phil Carmody diffstat: src/lib-imap-client/imapc-msgmap.c | 6 ------ src/lib-storage/index/pop3c/pop3c-sync.c | 6 ------ src/lib-storage/list/mailbox-list-index-sync.c | 6 ------ src/lib/sort.h | 24 ++++++++++++++++++++++++ src/lib/strfuncs.h | 8 +++----- src/lib/test-timing.c | 10 ---------- src/plugins/fts-squat/squat-uidlist.c | 5 ----- 7 files changed, 27 insertions(+), 38 deletions(-) diffs (129 lines): diff -r 6377910c19e3 -r 1a5a45c80687 src/lib-imap-client/imapc-msgmap.c --- a/src/lib-imap-client/imapc-msgmap.c Mon Sep 21 17:03:19 2015 +0300 +++ b/src/lib-imap-client/imapc-msgmap.c Mon Sep 21 19:51:05 2015 +0300 @@ -47,12 +47,6 @@ return *uidp; } -static int uint32_cmp(const uint32_t *p1, const uint32_t *p2) -{ - return *p1 < *p2 ? -1 : - (*p1 > *p2 ? 1 : 0); -} - bool imapc_msgmap_uid_to_rseq(struct imapc_msgmap *msgmap, uint32_t uid, uint32_t *rseq_r) { diff -r 6377910c19e3 -r 1a5a45c80687 src/lib-storage/index/pop3c/pop3c-sync.c --- a/src/lib-storage/index/pop3c/pop3c-sync.c Mon Sep 21 17:03:19 2015 +0300 +++ b/src/lib-storage/index/pop3c/pop3c-sync.c Mon Sep 21 19:51:05 2015 +0300 @@ -189,12 +189,6 @@ mailbox_recent_flags_set_seqs(&mbox->box, sync_view, seq1, seq2); } -static int uint32_cmp(const uint32_t *u1, const uint32_t *u2) -{ - return *u1 < *u2 ? -1 : - (*u1 > *u2 ? 1 : 0); -} - int pop3c_sync(struct pop3c_mailbox *mbox) { struct mail_index_sync_ctx *index_sync_ctx; diff -r 6377910c19e3 -r 1a5a45c80687 src/lib-storage/list/mailbox-list-index-sync.c --- a/src/lib-storage/list/mailbox-list-index-sync.c Mon Sep 21 17:03:19 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-sync.c Mon Sep 21 19:51:05 2015 +0300 @@ -143,12 +143,6 @@ } } -static int uint32_cmp(const uint32_t *p1, const uint32_t *p2) -{ - return *p1 < *p2 ? -1 : - (*p1 > *p2 ? 1 : 0); -} - static void mailbox_list_index_sync_names(struct mailbox_list_index_sync_context *ctx) { diff -r 6377910c19e3 -r 1a5a45c80687 src/lib/sort.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/sort.h Mon Sep 21 19:51:05 2015 +0300 @@ -0,0 +1,24 @@ +#ifndef SORT_H +#define SORT_H + +#define INTEGER_CMP(name, type) \ + static inline int name(const type *i1, const type *i2) \ + { \ + if (*i1 < *i2) \ + return -1; \ + else if (*i1 > *i2) \ + return 1; \ + else \ + return 0; \ + } + +INTEGER_CMP(uint64_cmp, uint64_t) +INTEGER_CMP(uint32_cmp, uint32_t) + +#define i_qsort(base, nmemb, size, cmp) \ + qsort(base, nmemb, size + \ + CALLBACK_TYPECHECK(cmp, int (*)(typeof(const typeof(*base) *), \ + typeof(const typeof(*base) *))), \ + (int (*)(const void *, const void *))cmp) + +#endif diff -r 6377910c19e3 -r 1a5a45c80687 src/lib/strfuncs.h --- a/src/lib/strfuncs.h Mon Sep 21 17:03:19 2015 +0300 +++ b/src/lib/strfuncs.h Mon Sep 21 19:51:05 2015 +0300 @@ -94,11 +94,9 @@ const char **p_strarray_dup(pool_t pool, const char *const *arr) ATTR_MALLOC ATTR_RETURNS_NONNULL; -#define i_qsort(base, nmemb, size, cmp) \ - qsort(base, nmemb, size + \ - CALLBACK_TYPECHECK(cmp, int (*)(typeof(const typeof(*base) *), \ - typeof(const typeof(*base) *))), \ - (int (*)(const void *, const void *))cmp) +/* FIXME: v2.3 - sort and search APIs belong into their own header, not here */ +#include "sort.h" + #define i_bsearch(key, base, nmemb, size, cmp) \ bsearch(key, base, nmemb, size + \ CALLBACK_TYPECHECK(cmp, int (*)(typeof(const typeof(*key) *), \ diff -r 6377910c19e3 -r 1a5a45c80687 src/lib/test-timing.c --- a/src/lib/test-timing.c Mon Sep 21 17:03:19 2015 +0300 +++ b/src/lib/test-timing.c Mon Sep 21 19:51:05 2015 +0300 @@ -5,16 +5,6 @@ #include -static int uint64_cmp(const uint64_t *i1, const uint64_t *i2) -{ - if (*i1 < *i2) - return -1; - else if (*i1 > *i2) - return 1; - else - return 0; -} - static void test_timing_verify(const struct timing *t, const int64_t *input, unsigned int input_size) diff -r 6377910c19e3 -r 1a5a45c80687 src/plugins/fts-squat/squat-uidlist.c --- a/src/plugins/fts-squat/squat-uidlist.c Mon Sep 21 17:03:19 2015 +0300 +++ b/src/plugins/fts-squat/squat-uidlist.c Mon Sep 21 19:51:05 2015 +0300 @@ -1382,11 +1382,6 @@ return 0; } -static int uint32_cmp(const uint32_t *key, const uint32_t *data) -{ - return (int)*key - (int)*data; -} - static int squat_uidlist_get_offset(struct squat_uidlist *uidlist, uint32_t uid_list_idx, uint32_t *offset_r, uint32_t *num_r) From pigeonhole at rename-it.nl Mon Sep 21 21:35:11 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 21 Sep 2015 23:35:11 +0200 Subject: dovecot-2.2-pigeonhole: vnd.dovecot.environment extension: Forgo... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/ef333e5beccd changeset: 2097:ef333e5beccd user: Stephan Bosch date: Mon Sep 21 23:34:41 2015 +0200 description: vnd.dovecot.environment extension: Forgot a \ in Makefile.am. diffstat: src/lib-sieve/plugins/vnd.dovecot/environment/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 5df1b6d72ec2 -r ef333e5beccd src/lib-sieve/plugins/vnd.dovecot/environment/Makefile.am --- a/src/lib-sieve/plugins/vnd.dovecot/environment/Makefile.am Tue Sep 08 01:08:34 2015 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/environment/Makefile.am Mon Sep 21 23:34:41 2015 +0200 @@ -11,6 +11,6 @@ ext-vnd-environment-items.c \ ext-vnd-environment-variables.c -noinst_HEADERS = +noinst_HEADERS = \ ext-vnd-environment-common.h From dovecot at dovecot.org Mon Sep 21 21:41:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 21:41:32 +0000 Subject: dovecot-2.2: Makefile: Added missing sort.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa306ce6c02a changeset: 19174:fa306ce6c02a user: Timo Sirainen date: Tue Sep 22 00:40:14 2015 +0300 description: Makefile: Added missing sort.h diffstat: src/lib/Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 2f492fac75b7 -r fa306ce6c02a src/lib/Makefile.am --- a/src/lib/Makefile.am Mon Sep 21 19:52:32 2015 +0300 +++ b/src/lib/Makefile.am Tue Sep 22 00:40:14 2015 +0300 @@ -259,6 +259,7 @@ seq-range-array.h \ sha1.h \ sha2.h \ + sort.h \ str.h \ str-find.h \ str-sanitize.h \ From dovecot at dovecot.org Mon Sep 21 21:56:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 21:56:34 +0000 Subject: dovecot-2.2: lib-http: Avoid crashes on failing http_client_requ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/052d8d7654f6 changeset: 19175:052d8d7654f6 user: Timo Sirainen date: Tue Sep 22 00:55:15 2015 +0300 description: lib-http: Avoid crashes on failing http_client_request_send_payload() calls It HTTP server connection died, ioloop might not have anything to do anymore: Panic: file ioloop-epoll.c: line 187 (io_loop_handler_run_internal): assertion failed: (msecs >= 0) diffstat: src/lib-http/http-client-request.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (21 lines): diff -r fa306ce6c02a -r 052d8d7654f6 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Sep 22 00:40:14 2015 +0300 +++ b/src/lib-http/http-client-request.c Tue Sep 22 00:55:15 2015 +0300 @@ -981,6 +981,8 @@ if (!sending && req->payload_input != NULL) i_stream_unref(&req->payload_input); } + if (req->client->ioloop != NULL) + io_loop_stop(req->client->ioloop); } void http_client_request_error_delayed(struct http_client_request **_req) @@ -1039,6 +1041,8 @@ if (req->queue != NULL) http_client_queue_drop_request(req->queue, req); + if (req->client->ioloop != NULL) + io_loop_stop(req->client->ioloop); http_client_request_unref(_req); } From dovecot at dovecot.org Mon Sep 21 22:06:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Sep 2015 22:06:31 +0000 Subject: dovecot-2.2: lib-http: Optimized http_client_request_send_payloa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0d3a438c8928 changeset: 19176:0d3a438c8928 user: Timo Sirainen date: Tue Sep 22 01:05:09 2015 +0300 description: lib-http: Optimized http_client_request_send_payload() fix Badly copy&pasted my original fix, instead of noticing that Stephan's fix was slightly different. Probably shouldn't have made actual difference other than for performance. diffstat: src/lib-http/http-client-request.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 052d8d7654f6 -r 0d3a438c8928 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Sep 22 00:55:15 2015 +0300 +++ b/src/lib-http/http-client-request.c Tue Sep 22 01:05:09 2015 +0300 @@ -981,7 +981,7 @@ if (!sending && req->payload_input != NULL) i_stream_unref(&req->payload_input); } - if (req->client->ioloop != NULL) + if (req->payload_wait && req->client->ioloop != NULL) io_loop_stop(req->client->ioloop); } @@ -1041,7 +1041,7 @@ if (req->queue != NULL) http_client_queue_drop_request(req->queue, req); - if (req->client->ioloop != NULL) + if (req->payload_wait && req->client->ioloop != NULL) io_loop_stop(req->client->ioloop); http_client_request_unref(_req); } From dovecot at dovecot.org Tue Sep 22 10:51:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 10:51:31 +0000 Subject: dovecot-2.2: virtual plugin: Disable caching via mailbox list in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/587226389203 changeset: 19177:587226389203 user: Timo Sirainen date: Tue Sep 22 13:50:08 2015 +0300 description: virtual plugin: Disable caching via mailbox list indexes. Otherwise e.g. STATUS can give stale replies. diffstat: src/plugins/virtual/virtual-storage.c | 22 ++++++++++++++++++++-- 1 files changed, 20 insertions(+), 2 deletions(-) diffs (39 lines): diff -r 0d3a438c8928 -r 587226389203 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Tue Sep 22 01:05:09 2015 +0300 +++ b/src/plugins/virtual/virtual-storage.c Tue Sep 22 13:50:08 2015 +0300 @@ -701,6 +701,24 @@ return index_storage_is_inconsistent(box); } +static int +virtual_list_index_has_changed(struct mailbox *box ATTR_UNUSED, + struct mail_index_view *list_view ATTR_UNUSED, + uint32_t seq ATTR_UNUSED) +{ + /* we don't have any quick and easy optimizations for tracking + virtual folders. ideally we'd completely disable mailbox list + indexes for them, but this is the easiest way to do it for now. */ + return 1; +} + +static void +virtual_list_index_update_sync(struct mailbox *box ATTR_UNUSED, + struct mail_index_transaction *trans ATTR_UNUSED, + uint32_t seq ATTR_UNUSED) +{ +} + struct mail_storage virtual_storage = { .name = VIRTUAL_STORAGE_NAME, .class_flags = MAIL_STORAGE_CLASS_FLAG_NOQUOTA, @@ -738,8 +756,8 @@ index_storage_attribute_iter_init, index_storage_attribute_iter_next, index_storage_attribute_iter_deinit, - NULL, - NULL, + virtual_list_index_has_changed, + virtual_list_index_update_sync, virtual_storage_sync_init, index_mailbox_sync_next, index_mailbox_sync_deinit, From dovecot at dovecot.org Tue Sep 22 10:55:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 10:55:15 +0000 Subject: dovecot-2.2: lib-master: Slightly improved "process_limit reache... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/11c1a5cf41f8 changeset: 19178:11c1a5cf41f8 user: Timo Sirainen date: Tue Sep 22 13:53:47 2015 +0300 description: lib-master: Slightly improved "process_limit reached?" error message. diffstat: src/lib-master/master-auth.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 587226389203 -r 11c1a5cf41f8 src/lib-master/master-auth.c --- a/src/lib-master/master-auth.c Tue Sep 22 13:50:08 2015 +0300 +++ b/src/lib-master/master-auth.c Tue Sep 22 13:53:47 2015 +0300 @@ -110,7 +110,7 @@ if (ret <= 0) { if (ret == 0 || errno == ECONNRESET) { i_error("read(%s) failed: Remote closed connection " - "(service's process_limit reached?)", + "(destination service { process_limit } reached?)", conn->auth->path); } else { if (errno == EAGAIN) From dovecot at dovecot.org Tue Sep 22 11:27:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 11:27:29 +0000 Subject: dovecot-2.2: quota: Make sure the limit return value is initiali... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e6b3170648c7 changeset: 19179:e6b3170648c7 user: Timo Sirainen date: Tue Sep 22 14:26:08 2015 +0300 description: quota: Make sure the limit return value is initialized when quota_get_resource() returns 0 Later on a debug log message in quota_over_flag_check_root() was accessing the limit, which wasn't necessarily initialized. Also initialized value return value just as well to be safe. diffstat: src/plugins/quota/quota.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 11c1a5cf41f8 -r e6b3170648c7 src/plugins/quota/quota.c --- a/src/plugins/quota/quota.c Tue Sep 22 13:53:47 2015 +0300 +++ b/src/plugins/quota/quota.c Tue Sep 22 14:26:08 2015 +0300 @@ -666,6 +666,8 @@ bool kilobytes = FALSE; int ret; + *value_r = *limit_r = 0; + if (strcmp(name, QUOTA_NAME_STORAGE_KILOBYTES) == 0) { name = QUOTA_NAME_STORAGE_BYTES; kilobytes = TRUE; From dovecot at dovecot.org Tue Sep 22 11:27:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 11:27:45 +0000 Subject: dovecot-2.2: Make static analyzer happier. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/46e5f4a5282d changeset: 19180:46e5f4a5282d user: Timo Sirainen date: Tue Sep 22 14:26:25 2015 +0300 description: Make static analyzer happier. diffstat: src/lib/test-timing.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r e6b3170648c7 -r 46e5f4a5282d src/lib/test-timing.c --- a/src/lib/test-timing.c Tue Sep 22 14:26:08 2015 +0300 +++ b/src/lib/test-timing.c Tue Sep 22 14:26:25 2015 +0300 @@ -13,6 +13,8 @@ uint64_t *copy; unsigned int i; + i_assert(input_size > 0); + copy = i_new(uint64_t, input_size); for (i = 0; i < input_size; i++) { uint64_t value = input[i]; From dovecot at dovecot.org Tue Sep 22 11:34:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 11:34:19 +0000 Subject: dovecot-2.2: auth: Remove redundant if Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c3dfdf9a17a3 changeset: 19181:c3dfdf9a17a3 user: Teemu Huovila date: Tue Sep 22 14:32:51 2015 +0300 description: auth: Remove redundant if diffstat: src/auth/auth-request.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 46e5f4a5282d -r c3dfdf9a17a3 src/auth/auth-request.c --- a/src/auth/auth-request.c Tue Sep 22 14:26:25 2015 +0300 +++ b/src/auth/auth-request.c Tue Sep 22 14:32:51 2015 +0300 @@ -1355,7 +1355,7 @@ else if (remote_ip->family == 0) { auth_request_log_info(request, AUTH_SUBSYS_DB, "%s check failed: Remote IP not known and 'local' missing", name); - } else if (!found) { + } else { auth_request_log_info(request, AUTH_SUBSYS_DB, "%s check failed: IP %s not in allowed networks", name, net_ip2addr(remote_ip)); From dovecot at dovecot.org Tue Sep 22 11:34:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 11:34:59 +0000 Subject: dovecot-2.2: imap: Fix typo in error message. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/13252c581016 changeset: 19182:13252c581016 user: Teemu Huovila date: Tue Sep 22 14:33:33 2015 +0300 description: imap: Fix typo in error message. diffstat: src/imap/imap-state.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c3dfdf9a17a3 -r 13252c581016 src/imap/imap-state.c --- a/src/imap/imap-state.c Tue Sep 22 14:32:51 2015 +0300 +++ b/src/imap/imap-state.c Tue Sep 22 14:33:33 2015 +0300 @@ -264,7 +264,7 @@ if (array_is_created(&client->search_updates) && array_count(&client->search_updates) > 0) { /* these could be tricky */ - *error_r = "SEARCH=CONTEXT updates not supported currently"; + *error_r = "CONTEXT=SEARCH updates not supported currently"; return 0; } if (client->notify_ctx != NULL) { From dovecot at dovecot.org Tue Sep 22 13:11:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 13:11:14 +0000 Subject: dovecot-2.2: lib: Added timing_get_sum() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a1c9e2cc574b changeset: 19183:a1c9e2cc574b user: Timo Sirainen date: Tue Sep 22 16:09:41 2015 +0300 description: lib: Added timing_get_sum() diffstat: src/lib/test-timing.c | 1 + src/lib/timing.c | 5 +++++ src/lib/timing.h | 2 ++ 3 files changed, 8 insertions(+), 0 deletions(-) diffs (38 lines): diff -r 13252c581016 -r a1c9e2cc574b src/lib/test-timing.c --- a/src/lib/test-timing.c Tue Sep 22 14:33:33 2015 +0300 +++ b/src/lib/test-timing.c Tue Sep 22 16:09:41 2015 +0300 @@ -29,6 +29,7 @@ i_qsort(copy, input_size, sizeof(*copy), uint64_cmp); test_assert_idx(timing_get_count(t) == input_size, input_size); + test_assert_idx(timing_get_sum(t) == sum, input_size); test_assert_idx(timing_get_min(t) == min, input_size); test_assert_idx(timing_get_max(t) == max, input_size); test_assert_idx(timing_get_avg(t) == (sum + input_size/2)/input_size, input_size); diff -r 13252c581016 -r a1c9e2cc574b src/lib/timing.c --- a/src/lib/timing.c Tue Sep 22 14:33:33 2015 +0300 +++ b/src/lib/timing.c Tue Sep 22 16:09:41 2015 +0300 @@ -59,6 +59,11 @@ return timing->count; } +uint64_t timing_get_sum(const struct timing *timing) +{ + return timing->sum; +} + uint64_t timing_get_min(const struct timing *timing) { return timing->min; diff -r 13252c581016 -r a1c9e2cc574b src/lib/timing.h --- a/src/lib/timing.h Tue Sep 22 14:33:33 2015 +0300 +++ b/src/lib/timing.h Tue Sep 22 16:09:41 2015 +0300 @@ -9,6 +9,8 @@ /* Returns number of events added. */ unsigned int timing_get_count(const struct timing *timing); +/* Returns the sum of all usecs added. */ +uint64_t timing_get_sum(const struct timing *timing); /* Returns events' minimum. */ uint64_t timing_get_min(const struct timing *timing); From dovecot at dovecot.org Tue Sep 22 13:42:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 13:42:27 +0000 Subject: dovecot-2.2: lib-fs: debug and enable_timing settings weren't be... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5ceab761d8d6 changeset: 19184:5ceab761d8d6 user: Timo Sirainen date: Tue Sep 22 16:38:59 2015 +0300 description: lib-fs: debug and enable_timing settings weren't being permanently copied to fs->set diffstat: src/lib-fs/fs-api.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r a1c9e2cc574b -r 5ceab761d8d6 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Tue Sep 22 16:09:41 2015 +0300 +++ b/src/lib-fs/fs-api.c Tue Sep 22 16:38:59 2015 +0300 @@ -31,6 +31,8 @@ fs = fs_class->v.alloc(); fs->refcount = 1; fs->last_error = str_new(default_pool, 64); + fs->set.debug = set->debug; + fs->set.enable_timing = set->enable_timing; i_array_init(&fs->module_contexts, 5); T_BEGIN { From dovecot at dovecot.org Tue Sep 22 13:42:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 13:42:28 +0000 Subject: dovecot-2.2: lib-storage: If stats plugin is enabled, set fs_set... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8166cb1a32ba changeset: 19185:8166cb1a32ba user: Timo Sirainen date: Tue Sep 22 16:40:58 2015 +0300 description: lib-storage: If stats plugin is enabled, set fs_settings.enable_timings=TRUE. diffstat: src/lib-storage/mail-user.c | 1 + src/lib-storage/mail-user.h | 2 ++ src/plugins/stats/stats-plugin.c | 1 + 3 files changed, 4 insertions(+), 0 deletions(-) diffs (34 lines): diff -r 5ceab761d8d6 -r 8166cb1a32ba src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Tue Sep 22 16:38:59 2015 +0300 +++ b/src/lib-storage/mail-user.c Tue Sep 22 16:40:58 2015 +0300 @@ -568,6 +568,7 @@ fs_set->base_dir = user->set->base_dir; fs_set->temp_dir = user->set->mail_temp_dir; fs_set->debug = user->mail_debug; + fs_set->enable_timing = user->stats_enabled; fs_set->ssl_client_set = ssl_set; ssl_set->ca_dir = mail_set->ssl_client_ca_dir; diff -r 5ceab761d8d6 -r 8166cb1a32ba src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Tue Sep 22 16:38:59 2015 +0300 +++ b/src/lib-storage/mail-user.h Tue Sep 22 16:40:58 2015 +0300 @@ -87,6 +87,8 @@ unsigned int deinitializing:1; /* Enable administrator user commands for the user */ unsigned int admin:1; + /* Enable all statistics gathering */ + unsigned int stats_enabled:1; }; struct mail_user_module_register { diff -r 5ceab761d8d6 -r 8166cb1a32ba src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Tue Sep 22 16:38:59 2015 +0300 +++ b/src/plugins/stats/stats-plugin.c Tue Sep 22 16:40:58 2015 +0300 @@ -425,6 +425,7 @@ p_strdup(user->pool, guid_128_to_string(guid)); } suser->last_session_update = time(NULL); + user->stats_enabled = TRUE; suser->ioloop_ctx = ioloop_ctx; io_loop_context_add_callbacks(ioloop_ctx, From dovecot at dovecot.org Tue Sep 22 18:18:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 18:18:23 +0000 Subject: dovecot-2.2: lib-http: client: Improved hash function for peer a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b4a1e4d085e2 changeset: 19186:b4a1e4d085e2 user: Stephan Bosch date: Tue Sep 22 21:15:45 2015 +0300 description: lib-http: client: Improved hash function for peer addresses. diffstat: src/lib-http/http-client-peer.c | 24 ++++++++++++++---------- 1 files changed, 14 insertions(+), 10 deletions(-) diffs (39 lines): diff -r 8166cb1a32ba -r b4a1e4d085e2 src/lib-http/http-client-peer.c --- a/src/lib-http/http-client-peer.c Tue Sep 22 16:40:58 2015 +0300 +++ b/src/lib-http/http-client-peer.c Tue Sep 22 21:15:45 2015 +0300 @@ -43,21 +43,25 @@ unsigned int http_client_peer_addr_hash (const struct http_client_peer_addr *peer) { + unsigned int hash = (unsigned int)peer->type; + switch (peer->type) { - case HTTP_CLIENT_PEER_ADDR_RAW: - return net_ip_hash(&peer->a.tcp.ip) + peer->a.tcp.port + 1; - case HTTP_CLIENT_PEER_ADDR_HTTP: - return net_ip_hash(&peer->a.tcp.ip) + peer->a.tcp.port; case HTTP_CLIENT_PEER_ADDR_HTTPS: case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: - return net_ip_hash(&peer->a.tcp.ip) + peer->a.tcp.port + - (peer->a.tcp.https_name == NULL ? - 0 : str_hash(peer->a.tcp.https_name)); + if (peer->a.tcp.https_name != NULL) + hash += str_hash(peer->a.tcp.https_name); + /* fall through */ + case HTTP_CLIENT_PEER_ADDR_RAW: + case HTTP_CLIENT_PEER_ADDR_HTTP: + hash += net_ip_hash(&peer->a.tcp.ip); + hash += peer->a.tcp.port; + break; case HTTP_CLIENT_PEER_ADDR_UNIX: - return str_hash(peer->a.un.path); + hash += str_hash(peer->a.un.path); + break; } - i_unreached(); - return 0; + + return hash; } int http_client_peer_addr_cmp From dovecot at dovecot.org Tue Sep 22 18:18:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 18:18:25 +0000 Subject: dovecot-2.2: lib-http: client: Fixed bug that caused queues to b... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1b1dbfaf7daa changeset: 19187:1b1dbfaf7daa user: Stephan Bosch date: Tue Sep 22 21:16:34 2015 +0300 description: lib-http: client: Fixed bug that caused queues to be duplicated over time when host has multiple IPs. The peer address comparison did not account for the fact that the initial queue address has no IP assigned. diffstat: src/lib-http/http-client-peer.c | 16 ++++++++++++++-- src/lib-http/http-client-request.c | 9 +++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diffs (79 lines): diff -r b4a1e4d085e2 -r 1b1dbfaf7daa src/lib-http/http-client-peer.c --- a/src/lib-http/http-client-peer.c Tue Sep 22 21:15:45 2015 +0300 +++ b/src/lib-http/http-client-peer.c Tue Sep 22 21:16:34 2015 +0300 @@ -53,7 +53,8 @@ /* fall through */ case HTTP_CLIENT_PEER_ADDR_RAW: case HTTP_CLIENT_PEER_ADDR_HTTP: - hash += net_ip_hash(&peer->a.tcp.ip); + if (peer->a.tcp.ip.family != 0) + hash += net_ip_hash(&peer->a.tcp.ip); hash += peer->a.tcp.port; break; case HTTP_CLIENT_PEER_ADDR_UNIX: @@ -77,7 +78,13 @@ case HTTP_CLIENT_PEER_ADDR_HTTP: case HTTP_CLIENT_PEER_ADDR_HTTPS: case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: - if ((ret=net_ip_cmp(&peer1->a.tcp.ip, &peer2->a.tcp.ip)) != 0) + /* Queues are created with peer addresses that have an uninitialized + IP value, because that is assigned later when the host lookup completes. + In all other other contexts, the IP is always initialized, so we do not + compare IPs when one of them is unassigned. */ + if (peer1->a.tcp.ip.family != 0 && + peer2->a.tcp.ip.family != 0 && + (ret=net_ip_cmp(&peer1->a.tcp.ip, &peer2->a.tcp.ip)) != 0) return ret; if (peer1->a.tcp.port != peer2->a.tcp.port) return (peer1->a.tcp.port > peer2->a.tcp.port ? 1 : -1); @@ -461,8 +468,13 @@ peer->addr = *addr; switch (addr->type) { + case HTTP_CLIENT_PEER_ADDR_RAW: + case HTTP_CLIENT_PEER_ADDR_HTTP: + i_assert(peer->addr.a.tcp.ip.family != 0); + break; case HTTP_CLIENT_PEER_ADDR_HTTPS: case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + i_assert(peer->addr.a.tcp.ip.family != 0); i_assert(client->ssl_ctx != NULL); peer->addr_name = i_strdup(addr->a.tcp.https_name); peer->addr.a.tcp.https_name = peer->addr_name; diff -r b4a1e4d085e2 -r 1b1dbfaf7daa src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Sep 22 21:15:45 2015 +0300 +++ b/src/lib-http/http-client-request.c Tue Sep 22 21:16:34 2015 +0300 @@ -535,12 +535,17 @@ const char *host_socket = req->host_socket; const struct http_url *host_url = req->host_url; + /* the IP address may be unassigned in the returned peer address, since + that is only available at this stage when the target URL has an + explicit IP address. */ memset(addr, 0, sizeof(*addr)); if (host_socket != NULL) { addr->type = HTTP_CLIENT_PEER_ADDR_UNIX; addr->a.un.path = host_socket; } else if (req->connect_direct) { addr->type = HTTP_CLIENT_PEER_ADDR_RAW; + if (host_url->have_host_ip) + addr->a.tcp.ip = host_url->host_ip; addr->a.tcp.port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT); } else if (host_url->have_ssl) { @@ -548,11 +553,15 @@ addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL; else addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS; + if (host_url->have_host_ip) + addr->a.tcp.ip = host_url->host_ip; addr->a.tcp.https_name = host_url->host_name; addr->a.tcp.port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT); } else { addr->type = HTTP_CLIENT_PEER_ADDR_HTTP; + if (host_url->have_host_ip) + addr->a.tcp.ip = host_url->host_ip; addr->a.tcp.port = (host_url->have_port ? host_url->port : HTTP_DEFAULT_PORT); } From dovecot at dovecot.org Tue Sep 22 18:18:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Sep 2015 18:18:30 +0000 Subject: dovecot-2.2: lib-http: client: Tunneled https peer addresses wer... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3a942625aaa0 changeset: 19188:3a942625aaa0 user: Stephan Bosch date: Tue Sep 22 21:16:51 2015 +0300 description: lib-http: client: Tunneled https peer addresses were not compared properly. The https_name field was ignored in that case. diffstat: src/lib-http/http-client-peer.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 1b1dbfaf7daa -r 3a942625aaa0 src/lib-http/http-client-peer.c --- a/src/lib-http/http-client-peer.c Tue Sep 22 21:16:34 2015 +0300 +++ b/src/lib-http/http-client-peer.c Tue Sep 22 21:16:51 2015 +0300 @@ -88,7 +88,8 @@ return ret; if (peer1->a.tcp.port != peer2->a.tcp.port) return (peer1->a.tcp.port > peer2->a.tcp.port ? 1 : -1); - if (peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS) + if (peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS && + peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL) return 0; return null_strcmp (peer1->a.tcp.https_name, peer2->a.tcp.https_name); From dovecot at dovecot.org Wed Sep 23 11:50:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 11:50:01 +0000 Subject: dovecot-2.2: Added push-notification plugin Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1afdeb1cae62 changeset: 19189:1afdeb1cae62 user: Michael M Slusarz date: Tue Sep 22 22:33:41 2015 -0600 description: Added push-notification plugin diffstat: configure.ac | 1 + dovecot-config.in.in | 1 + src/plugins/Makefile.am | 1 + src/plugins/push-notification/Makefile.am | 60 + src/plugins/push-notification/push-notification-driver-dlog.c | 116 +++ src/plugins/push-notification/push-notification-driver-ox.c | 341 ++++++++++ src/plugins/push-notification/push-notification-drivers.c | 189 +++++ src/plugins/push-notification/push-notification-drivers.h | 121 +++ src/plugins/push-notification/push-notification-event-flagsclear.c | 179 +++++ src/plugins/push-notification/push-notification-event-flagsclear.h | 25 + src/plugins/push-notification/push-notification-event-flagsset.c | 174 +++++ src/plugins/push-notification/push-notification-event-flagsset.h | 26 + src/plugins/push-notification/push-notification-event-mailboxcreate.c | 51 + src/plugins/push-notification/push-notification-event-mailboxcreate.h | 14 + src/plugins/push-notification/push-notification-event-mailboxdelete.c | 47 + src/plugins/push-notification/push-notification-event-mailboxdelete.h | 14 + src/plugins/push-notification/push-notification-event-mailboxrename.c | 52 + src/plugins/push-notification/push-notification-event-mailboxrename.h | 13 + src/plugins/push-notification/push-notification-event-mailboxsubscribe.c | 48 + src/plugins/push-notification/push-notification-event-mailboxsubscribe.h | 14 + src/plugins/push-notification/push-notification-event-mailboxunsubscribe.c | 48 + src/plugins/push-notification/push-notification-event-mailboxunsubscribe.h | 14 + src/plugins/push-notification/push-notification-event-message-common.h | 22 + src/plugins/push-notification/push-notification-event-messageappend.c | 115 +++ src/plugins/push-notification/push-notification-event-messageappend.h | 20 + src/plugins/push-notification/push-notification-event-messageexpunge.c | 55 + src/plugins/push-notification/push-notification-event-messageexpunge.h | 14 + src/plugins/push-notification/push-notification-event-messagenew.c | 136 +++ src/plugins/push-notification/push-notification-event-messagenew.h | 30 + src/plugins/push-notification/push-notification-event-messageread.c | 58 + src/plugins/push-notification/push-notification-event-messageread.h | 13 + src/plugins/push-notification/push-notification-event-messagetrash.c | 58 + src/plugins/push-notification/push-notification-event-messagetrash.h | 14 + src/plugins/push-notification/push-notification-events-rfc5423.c | 46 + src/plugins/push-notification/push-notification-events-rfc5423.h | 11 + src/plugins/push-notification/push-notification-events.c | 105 +++ src/plugins/push-notification/push-notification-events.h | 127 +++ src/plugins/push-notification/push-notification-plugin.c | 327 +++++++++ src/plugins/push-notification/push-notification-plugin.h | 13 + src/plugins/push-notification/push-notification-triggers.c | 222 ++++++ src/plugins/push-notification/push-notification-triggers.h | 78 ++ src/plugins/push-notification/push-notification-txn-mbox.c | 90 ++ src/plugins/push-notification/push-notification-txn-mbox.h | 34 + src/plugins/push-notification/push-notification-txn-msg.c | 135 +++ src/plugins/push-notification/push-notification-txn-msg.h | 43 + 45 files changed, 3315 insertions(+), 0 deletions(-) diffs (truncated from 3521 to 300 lines): diff -r 3a942625aaa0 -r 1afdeb1cae62 configure.ac --- a/configure.ac Tue Sep 22 21:16:51 2015 +0300 +++ b/configure.ac Tue Sep 22 22:33:41 2015 -0600 @@ -2929,6 +2929,7 @@ src/plugins/mail-log/Makefile src/plugins/mailbox-alias/Makefile src/plugins/notify/Makefile +src/plugins/push-notification/Makefile src/plugins/pop3-migration/Makefile src/plugins/quota/Makefile src/plugins/quota-clone/Makefile diff -r 3a942625aaa0 -r 1afdeb1cae62 dovecot-config.in.in --- a/dovecot-config.in.in Tue Sep 22 21:16:51 2015 +0300 +++ b/dovecot-config.in.in Tue Sep 22 22:33:41 2015 -0600 @@ -31,6 +31,7 @@ LIBDOVECOT_IMAPC_INCLUDE="-I$(incdir)/src/lib-imap-client -I$(incdir)/src/lib-storage/index/imapc" LIBDOVECOT_FTS_INCLUDE="-I$(incdir)/src/plugins/fts" LIBDOVECOT_NOTIFY_INCLUDE="-I$(incdir)/src/plugins/notify" +LIBDOVECOT_PUSH_NOTIFICATION_INCLUDE="-I$(incdir)/src/plugins/push-notification" LIBDOVECOT_ACL_INCLUDE="-I$(incdir)/src/plugins/acl" dovecot_pkgincludedir= diff -r 3a942625aaa0 -r 1afdeb1cae62 src/plugins/Makefile.am --- a/src/plugins/Makefile.am Tue Sep 22 21:16:51 2015 +0300 +++ b/src/plugins/Makefile.am Tue Sep 22 22:33:41 2015 -0600 @@ -21,6 +21,7 @@ lazy-expunge \ listescape \ notify \ + push-notification \ mail-filter \ mail-log \ mailbox-alias \ diff -r 3a942625aaa0 -r 1afdeb1cae62 src/plugins/push-notification/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/push-notification/Makefile.am Tue Sep 22 22:33:41 2015 -0600 @@ -0,0 +1,60 @@ +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-http \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/plugins/notify + +NOPLUGIN_LDFLAGS = +lib20_push_notification_plugin_la_LDFLAGS = -module -avoid-version + +module_LTLIBRARIES = lib20_push_notification_plugin.la + +lib20_push_notification_plugin_la_SOURCES = \ + push-notification-driver-dlog.c \ + push-notification-driver-ox.c \ + push-notification-drivers.c \ + push-notification-event-flagsclear.c \ + push-notification-event-flagsset.c \ + push-notification-event-mailboxcreate.c \ + push-notification-event-mailboxdelete.c \ + push-notification-event-mailboxrename.c \ + push-notification-event-mailboxsubscribe.c \ + push-notification-event-mailboxunsubscribe.c \ + push-notification-event-messageappend.c \ + push-notification-event-messageexpunge.c \ + push-notification-event-messagenew.c \ + push-notification-event-messageread.c \ + push-notification-event-messagetrash.c \ + push-notification-events.c \ + push-notification-events-rfc5423.c \ + push-notification-plugin.c \ + push-notification-triggers.c \ + push-notification-txn-mbox.c \ + push-notification-txn-msg.c + +headers = \ + push-notification-drivers.h \ + push-notification-event-flagsclear.h \ + push-notification-event-flagsset.h \ + push-notification-event-mailboxcreate.h \ + push-notification-event-mailboxdelete.h \ + push-notification-event-mailboxrename.h \ + push-notification-event-mailboxsubscribe.h \ + push-notification-event-mailboxunsubscribe.h \ + push-notification-event-message-common.h \ + push-notification-event-messageappend.h \ + push-notification-event-messageexpunge.h \ + push-notification-event-messagenew.h \ + push-notification-event-messageread.h \ + push-notification-event-messagetrash.h \ + push-notification-events.h \ + push-notification-events-rfc5423.h \ + push-notification-plugin.h \ + push-notification-triggers.h \ + push-notification-txn-mbox.h \ + push-notification-txn-msg.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers) diff -r 3a942625aaa0 -r 1afdeb1cae62 src/plugins/push-notification/push-notification-driver-dlog.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/push-notification/push-notification-driver-dlog.c Tue Sep 22 22:33:41 2015 -0600 @@ -0,0 +1,116 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" + +#include "push-notification-drivers.h" +#include "push-notification-events.h" +#include "push-notification-txn-mbox.h" +#include "push-notification-txn-msg.h" + + +static int +push_notification_driver_dlog_init(struct push_notification_driver_config *config, + struct mail_user *user ATTR_UNUSED, + pool_t pool ATTR_UNUSED, + void **context ATTR_UNUSED, + const char **error_r ATTR_UNUSED) +{ + i_debug("Called init push_notification plugin hook."); + + if (config->raw_config != NULL) { + i_debug("Config string for dlog push_notification driver: %s", + config->raw_config); + } + + return 0; +} + +static bool push_notification_driver_dlog_begin_txn +(struct push_notification_driver_txn *dtxn) +{ + const struct push_notification_event *const *event; + + i_debug("Called begin_txn push_notification plugin hook."); + + array_foreach(&push_notification_events, event) { + push_notification_event_init(dtxn, (*event)->name, NULL); + } + + return TRUE; +} + +static void push_notification_driver_dlog_process_mbox +(struct push_notification_driver_txn *dtxn ATTR_UNUSED, + struct push_notification_txn_mbox *mbox) +{ + struct push_notification_txn_event *const *event; + + i_debug("Called process_mbox push_notification plugin hook."); + + i_debug("Mailbox data: Mailbox [%s]", mbox->mailbox); + + if (array_is_created(&mbox->eventdata)) { + array_foreach(&mbox->eventdata, event) { + if ((*event)->event->event->mbox.debug_mbox != NULL) { + (*event)->event->event->mbox.debug_mbox(*event); + } + } + } +} + +static void push_notification_driver_dlog_process_msg +(struct push_notification_driver_txn *dtxn ATTR_UNUSED, + struct push_notification_txn_msg *msg) +{ + struct push_notification_txn_event *const *event; + + i_debug("Called process_msg push_notification plugin hook."); + + i_debug("Message data: Mailbox [%s], UID [%u], UIDVALIDITY [%u]", + msg->mailbox, msg->uid, msg->uid_validity); + + if (array_is_created(&msg->eventdata)) { + array_foreach(&msg->eventdata, event) { + if ((*event)->event->event->msg.debug_msg != NULL) { + (*event)->event->event->msg.debug_msg(*event); + } + } + } +} + +static void push_notification_driver_dlog_end_txn +(struct push_notification_driver_txn *dtxn ATTR_UNUSED, + bool success ATTR_UNUSED) +{ + i_debug("Called end_txn push_notification plugin hook."); +} + +static void push_notification_driver_dlog_deinit +(struct push_notification_driver_user *duser ATTR_UNUSED) +{ + i_debug("Called deinit push_notification plugin hook."); +} + +static void push_notification_driver_dlog_cleanup(void) +{ + i_debug("Called cleanup push_notification plugin hook."); +} + + +/* Driver definition */ + +extern struct push_notification_driver push_notification_driver_dlog; + +struct push_notification_driver push_notification_driver_dlog = { + .name = "dlog", + .v = { + .init = push_notification_driver_dlog_init, + .begin_txn = push_notification_driver_dlog_begin_txn, + .process_mbox = push_notification_driver_dlog_process_mbox, + .process_msg = push_notification_driver_dlog_process_msg, + .end_txn = push_notification_driver_dlog_end_txn, + .deinit = push_notification_driver_dlog_deinit, + .cleanup = push_notification_driver_dlog_cleanup + } +}; \ No newline at end of file diff -r 3a942625aaa0 -r 1afdeb1cae62 src/plugins/push-notification/push-notification-driver-ox.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/push-notification/push-notification-driver-ox.c Tue Sep 22 22:33:41 2015 -0600 @@ -0,0 +1,341 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hash.h" +#include "http-client.h" +#include "http-url.h" +#include "ioloop.h" +#include "istream.h" +#include "json-parser.h" +#include "mailbox-attribute.h" +#include "mail-storage-private.h" +#include "str.h" + +#include "push-notification-drivers.h" +#include "push-notification-event-messagenew.h" +#include "push-notification-events.h" +#include "push-notification-txn-msg.h" + + +#define OX_LOG_LABEL "OX Push Notification: " + +#define OX_METADATA_KEY \ + MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER "vendor/vendor.dovecot/http-notify" + +/* Default values. */ +static const char *const default_events[] = { "MessageNew", NULL }; +static const char *const default_mboxes[] = { "INBOX", NULL }; + + +/* This is data that is shared by all plugin users. */ +struct push_notification_driver_ox_global { + struct http_client *http_client; + int refcount; +}; +static struct push_notification_driver_ox_global *ox_global = NULL; + +/* This is data specific to an OX driver. */ +struct push_notification_driver_ox_config { + struct http_url *http_url; +}; + +/* This is data specific to an OX driver transaction. */ +struct push_notification_driver_ox_txn { + const char *user; +}; + +static void +push_notification_driver_ox_init_global(struct mail_user *user) +{ + struct http_client_settings http_set; + + if (ox_global->http_client == NULL) { + memset(&http_set, 0, sizeof(http_set)); + http_set.debug = user->mail_debug; + + ox_global->http_client = http_client_init(&http_set); + } +} + +static int +push_notification_driver_ox_init(struct push_notification_driver_config *config, + struct mail_user *user, pool_t pool, + void **context, const char **error_r) +{ + struct push_notification_driver_ox_config *dconfig; + const char *error, *url; + + /* Valid config keys: url */ + url = hash_table_lookup(config->config, (const char *)"url"); + if (url == NULL) { + *error_r = OX_LOG_LABEL "Driver requires the url parameter"; + return -1; + } + + dconfig = p_new(pool, struct push_notification_driver_ox_config, 1); + + if (http_url_parse(url, NULL, HTTP_URL_ALLOW_USERINFO_PART, pool, + &dconfig->http_url, &error) < 0) { From dovecot at dovecot.org Wed Sep 23 12:01:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 12:01:21 +0000 Subject: dovecot-2.2: lib: Include stdlib.h in lib.h. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/55e6910aed18 changeset: 19190:55e6910aed18 user: Teemu Huovila date: Wed Sep 23 14:52:44 2015 +0300 description: lib: Include stdlib.h in lib.h. It's already included in so many files that we might as well include it in lib.h for everything. This is especially needed to be able to use sort.h, which currently is included in lib.h anyway. diffstat: src/lib/lib.h | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 1afdeb1cae62 -r 55e6910aed18 src/lib/lib.h --- a/src/lib/lib.h Tue Sep 22 22:33:41 2015 -0600 +++ b/src/lib/lib.h Wed Sep 23 14:52:44 2015 +0300 @@ -8,6 +8,7 @@ /* default system includes - keep these at minimum.. */ #include /* Solaris defines NULL wrong unless this is used */ +#include #include /* strcmp() etc. */ #ifdef HAVE_STRINGS_H # include /* strcasecmp() etc. */ From dovecot at dovecot.org Wed Sep 23 12:01:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 12:01:23 +0000 Subject: dovecot-2.2: Remove now-unnecessary direct stdlib.h #includes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9e120590e0ef changeset: 19191:9e120590e0ef user: Teemu Huovila date: Wed Sep 23 14:52:45 2015 +0300 description: Remove now-unnecessary direct stdlib.h #includes. diffstat: src/anvil/anvil-connection.c | 1 - src/anvil/main.c | 1 - src/auth/auth-client-connection.c | 1 - src/auth/auth-master-connection.c | 1 - src/auth/auth-request-handler.c | 1 - src/auth/auth-request.c | 1 - src/auth/auth-token.c | 1 - src/auth/auth-worker-client.c | 1 - src/auth/auth-worker-server.c | 1 - src/auth/checkpassword-reply.c | 1 - src/auth/db-checkpassword.c | 1 - src/auth/db-dict.c | 1 - src/auth/db-ldap.c | 1 - src/auth/db-passwd-file.c | 1 - src/auth/db-sql.c | 1 - src/auth/mech-cram-md5.c | 1 - src/auth/mech-digest-md5.c | 1 - src/auth/mech-gssapi.c | 1 - src/auth/mech-scram-sha1.c | 1 - src/auth/mech-winbind.c | 1 - src/auth/mech.c | 1 - src/auth/passdb-blocking.c | 1 - src/auth/passdb-cache.c | 1 - src/auth/passdb-dict.c | 1 - src/auth/passdb-ldap.c | 1 - src/auth/passdb-pam.c | 1 - src/auth/passdb-sql.c | 1 - src/auth/passdb-vpopmail.c | 1 - src/auth/passdb.c | 1 - src/auth/password-scheme-scram.c | 1 - src/auth/userdb-blocking.c | 1 - src/auth/userdb-dict.c | 1 - src/auth/userdb-ldap.c | 1 - src/auth/userdb-prefetch.c | 1 - src/auth/userdb-sql.c | 1 - src/auth/userdb-static.c | 1 - src/auth/userdb.c | 1 - src/config/config-connection.c | 1 - src/config/config-parser.c | 1 - src/config/doveconf.c | 1 - src/dict/dict-commands.c | 1 - src/dict/dict-connection.c | 1 - src/director/director-test.c | 1 - src/director/test-user-directory.c | 1 - src/doveadm/doveadm-auth.c | 1 - src/doveadm/doveadm-director.c | 1 - src/doveadm/doveadm-dsync.c | 1 - src/doveadm/doveadm-dump-index.c | 1 - src/doveadm/doveadm-kick.c | 1 - src/doveadm/doveadm-mail.c | 1 - src/doveadm/doveadm-penalty.c | 1 - src/doveadm/doveadm-pw.c | 1 - src/doveadm/doveadm-replicator.c | 1 - src/doveadm/doveadm-stats.c | 1 - src/doveadm/doveadm-who.c | 1 - src/doveadm/doveadm.c | 1 - src/doveadm/dsync/dsync-ibc-stream.c | 1 - src/doveadm/dsync/dsync-mailbox-tree.c | 1 - src/doveadm/dsync/test-dsync-mailbox-tree-sync.c | 1 - src/imap-hibernate/imap-hibernate-settings.c | 1 - src/imap-login/client-authenticate.c | 1 - src/imap-login/imap-proxy.c | 1 - src/imap-urlauth/imap-urlauth-client.c | 1 - src/imap-urlauth/imap-urlauth-settings.c | 1 - src/imap-urlauth/imap-urlauth-worker-settings.c | 1 - src/imap-urlauth/imap-urlauth-worker.c | 1 - src/imap-urlauth/imap-urlauth.c | 1 - src/imap/cmd-fetch.c | 1 - src/imap/cmd-select.c | 1 - src/imap/cmd-store.c | 1 - src/imap/imap-client.c | 1 - src/imap/imap-commands.c | 1 - src/imap/imap-fetch-body.c | 1 - src/imap/imap-fetch.c | 1 - src/imap/imap-search-args.c | 1 - src/imap/imap-search.c | 1 - src/imap/imap-settings.c | 1 - src/imap/main.c | 1 - src/indexer/indexer-client.c | 1 - src/lda/main.c | 1 - src/lib-auth/auth-client-request.c | 1 - src/lib-auth/auth-master.c | 1 - src/lib-auth/auth-server-connection.c | 1 - src/lib-compression/test-compression.c | 1 - src/lib-dict/dict-client.c | 1 - src/lib-dict/dict-db.c | 1 - src/lib-dict/dict-file.c | 1 - src/lib-fs/fs-posix.c | 1 - src/lib-fs/fs-randomfail.c | 1 - src/lib-fts/test-fts-tokenizer.c | 1 - src/lib-index/mail-cache-lookup.c | 1 - src/lib-index/mail-index-sync-ext.c | 1 - src/lib-index/mail-index-sync.c | 1 - src/lib-index/mail-index-transaction-sort-appends.c | 1 - src/lib-index/mail-index-transaction.c | 1 - src/lib-index/mail-index-view-sync.c | 1 - src/lib-index/test-mail-index-sync-ext.c | 1 - src/lib-index/test-mail-index-transaction-finish.c | 1 - src/lib-index/test-mail-index-transaction-update.c | 1 - src/lib-index/test-mail-transaction-log-append.c | 1 - src/lib-lda/duplicate.c | 1 - src/lib-lda/mail-send.c | 1 - src/lib-mail/istream-header-filter.c | 1 - src/lib-mail/rfc2231-parser.c | 1 - src/lib-mail/test-message-header-decode.c | 1 - src/lib-master/master-auth.c | 1 - src/lib-master/master-login-auth.c | 1 - src/lib-master/master-service-settings.c | 1 - src/lib-master/master-service.c | 1 - src/lib-master/test-master-service-settings-cache.c | 1 - src/lib-otp/otp-parse.c | 1 - src/lib-settings/settings-parser.c | 1 - src/lib-sql/driver-mysql.c | 1 - src/lib-sql/driver-pgsql.c | 1 - src/lib-sql/sql-api.c | 1 - src/lib-ssl-iostream/iostream-ssl.c | 1 - src/lib-storage/index/dbox-common/dbox-file.c | 1 - src/lib-storage/index/dbox-common/dbox-mail.c | 1 - src/lib-storage/index/dbox-multi/mdbox-file.c | 1 - src/lib-storage/index/dbox-multi/mdbox-mail.c | 1 - src/lib-storage/index/dbox-multi/mdbox-map.c | 1 - src/lib-storage/index/dbox-multi/mdbox-purge.c | 1 - src/lib-storage/index/dbox-multi/mdbox-save.c | 1 - src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 1 - src/lib-storage/index/dbox-multi/mdbox-sync.c | 1 - src/lib-storage/index/dbox-single/sdbox-mail.c | 1 - src/lib-storage/index/dbox-single/sdbox-save.c | 1 - src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c | 1 - src/lib-storage/index/index-search.c | 1 - src/lib-storage/index/index-sort-string.c | 1 - src/lib-storage/index/index-sort.c | 1 - src/lib-storage/index/index-storage.c | 1 - src/lib-storage/index/index-thread-finish.c | 1 - src/lib-storage/index/index-thread.c | 1 - src/lib-storage/index/maildir/maildir-copy.c | 1 - src/lib-storage/index/maildir/maildir-filename-flags.c | 1 - src/lib-storage/index/maildir/maildir-keywords.c | 1 - src/lib-storage/index/maildir/maildir-mail.c | 1 - src/lib-storage/index/maildir/maildir-save.c | 1 - src/lib-storage/index/maildir/maildir-sync-index.c | 1 - src/lib-storage/index/maildir/maildir-sync.c | 1 - src/lib-storage/index/maildir/maildir-uidlist.c | 1 - src/lib-storage/index/mbox/mbox-lock.c | 1 - src/lib-storage/index/mbox/mbox-md5-all.c | 1 - src/lib-storage/index/mbox/mbox-md5-apop3d.c | 1 - src/lib-storage/index/mbox/mbox-save.c | 1 - src/lib-storage/index/mbox/mbox-sync-parse.c | 1 - src/lib-storage/index/mbox/mbox-sync.c | 1 - src/lib-storage/index/shared/shared-storage.c | 1 - src/lib-storage/mail-namespace.c | 1 - src/lib-storage/mail-search-build.c | 1 - src/lib-storage/mail-search-register-imap.c | 1 - src/lib-storage/mail-storage-service.c | 1 - src/lib-storage/mail-storage.c | 1 - src/lib-storage/mail-user.c | 1 - src/lib-storage/mailbox-header.c | 1 - src/lib-storage/mailbox-uidvalidity.c | 1 - src/lib-storage/mailbox-watch.c | 1 - src/lib-test/test-common.c | 1 - src/lib/abspath.c | 1 - src/lib/array.c | 1 - src/lib/backtrace-string.c | 1 - src/lib/compat.c | 1 - src/lib/data-stack.c | 1 - src/lib/env-util.c | 1 - src/lib/failures.c | 1 - src/lib/file-dotlock.c | 1 - src/lib/home-expand.c | 1 - src/lib/hostpid.c | 1 - src/lib/lib.c | 1 - src/lib/mempool-alloconly.c | 1 - src/lib/mempool-datastack.c | 1 - src/lib/mempool-system.c | 1 - src/lib/mempool-unsafe-datastack.c | 1 - src/lib/mmap-anon.c | 1 - src/lib/module-dir.c | 1 - src/lib/net.c | 1 - src/lib/process-title.c | 1 - src/lib/rand.c | 1 - src/lib/randgen.c | 1 - src/lib/restrict-access.c | 1 - src/lib/test-array.c | 1 - src/lib/test-base32.c | 1 - src/lib/test-base64.c | 1 - src/lib/test-buffer.c | 1 - src/lib/test-data-stack.c | 1 - src/lib/test-hash.c | 1 - src/lib/test-istream-concat.c | 1 - src/lib/test-istream-seekable.c | 1 - src/lib/test-istream-tee.c | 1 - src/lib/test-llist.c | 1 - src/lib/test-numpack.c | 1 - src/lib/test-ostream-file.c | 1 - src/lib/test-priorityq.c | 1 - src/lib/test-seq-range-array.c | 1 - src/lib/test-strfuncs.c | 1 - src/lib/test-strnum.c | 1 - src/lib/test-timing.c | 1 - src/lib/test-unichar.c | 1 - src/lib/timing.c | 1 - src/lib/unlink-old-files.c | 1 - src/lib/var-expand.c | 1 - src/lmtp/main.c | 1 - src/log/log-connection.c | 1 - src/login-common/client-common.c | 1 - src/login-common/login-proxy.c | 1 - src/login-common/main.c | 1 - src/login-common/sasl-server.c | 1 - src/login-common/ssl-proxy-gnutls.c | 1 - src/master/main.c | 1 - src/master/sd-daemon.c | 1 - src/master/service-monitor.c | 1 - src/master/service-process.c | 1 - src/plugins/acl/acl-backend.c | 1 - src/plugins/acl/acl-lookup-dict.c | 1 - src/plugins/acl/acl-plugin.c | 1 - src/plugins/acl/acl-storage.c | 1 - src/plugins/expire/expire-plugin.c | 1 - src/plugins/expire/expire-set.c | 1 - src/plugins/fts-solr/fts-solr-plugin.c | 1 - src/plugins/fts-squat/fts-backend-squat.c | 1 - src/plugins/fts-squat/squat-trie.c | 1 - src/plugins/fts/fts-plugin.c | 1 - src/plugins/fts/fts-storage.c | 1 - src/plugins/imap-acl/imap-acl-plugin.c | 1 - src/plugins/imap-quota/imap-quota-plugin.c | 1 - src/plugins/imap-zlib/imap-zlib-plugin.c | 1 - src/plugins/lazy-expunge/lazy-expunge-plugin.c | 1 - src/plugins/mail-log/mail-log-plugin.c | 1 - src/plugins/notify/notify-plugin.c | 1 - src/plugins/quota/quota-dict.c | 1 - src/plugins/quota/quota-dirsize.c | 1 - src/plugins/quota/quota-fs.c | 1 - src/plugins/quota/quota-maildir.c | 1 - src/plugins/quota/quota-plugin.c | 1 - src/plugins/quota/quota-util.c | 1 - src/plugins/quota/quota.c | 1 - src/plugins/replication/replication-plugin.c | 1 - src/plugins/trash/trash-plugin.c | 1 - src/plugins/virtual/virtual-search.c | 1 - src/plugins/virtual/virtual-storage.c | 1 - src/plugins/virtual/virtual-sync.c | 1 - src/plugins/zlib/zlib-plugin.c | 1 - src/pop3-login/client-authenticate.c | 1 - src/pop3/main.c | 1 - src/pop3/pop3-client.c | 1 - src/pop3/pop3-settings.c | 1 - src/ssl-params/ssl-params-settings.c | 1 - src/ssl-params/ssl-params.c | 1 - src/util/gdbhelper.c | 1 - src/util/maildirlock.c | 1 - src/util/rawlog.c | 1 - src/util/script-login.c | 1 - src/util/script.c | 1 - src/util/tcpwrap.c | 1 - 255 files changed, 0 insertions(+), 255 deletions(-) diffs (truncated from 2805 to 300 lines): diff -r 55e6910aed18 -r 9e120590e0ef src/anvil/anvil-connection.c --- a/src/anvil/anvil-connection.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/anvil/anvil-connection.c Wed Sep 23 14:52:45 2015 +0300 @@ -10,7 +10,6 @@ #include "penalty.h" #include "anvil-connection.h" -#include #include #define MAX_INBUF_SIZE 1024 diff -r 55e6910aed18 -r 9e120590e0ef src/anvil/main.c --- a/src/anvil/main.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/anvil/main.c Wed Sep 23 14:52:45 2015 +0300 @@ -13,7 +13,6 @@ #include "penalty.h" #include "anvil-connection.h" -#include #include struct connect_limit *connect_limit; diff -r 55e6910aed18 -r 9e120590e0ef src/auth/auth-client-connection.c --- a/src/auth/auth-client-connection.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/auth-client-connection.c Wed Sep 23 14:52:45 2015 +0300 @@ -20,7 +20,6 @@ #include "auth-client-connection.h" #include "auth-master-connection.h" -#include #define OUTBUF_THROTTLE_SIZE (1024*50) diff -r 55e6910aed18 -r 9e120590e0ef src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/auth-master-connection.c Wed Sep 23 14:52:45 2015 +0300 @@ -24,7 +24,6 @@ #include "auth-master-connection.h" #include -#include #define MAX_INBUF_SIZE 1024 #define MAX_OUTBUF_SIZE (1024*50) diff -r 55e6910aed18 -r 9e120590e0ef src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/auth-request-handler.c Wed Sep 23 14:52:45 2015 +0300 @@ -17,7 +17,6 @@ #include "auth-master-connection.h" #include "auth-request-handler.h" -#include #define AUTH_FAILURE_DELAY_CHECK_MSECS 500 diff -r 55e6910aed18 -r 9e120590e0ef src/auth/auth-request.c --- a/src/auth/auth-request.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/auth-request.c Wed Sep 23 14:52:45 2015 +0300 @@ -25,7 +25,6 @@ #include "userdb-template.h" #include "password-scheme.h" -#include #include #define AUTH_SUBSYS_PROXY "proxy" diff -r 55e6910aed18 -r 9e120590e0ef src/auth/auth-token.c --- a/src/auth/auth-token.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/auth-token.c Wed Sep 23 14:52:45 2015 +0300 @@ -20,7 +20,6 @@ #include "auth-settings.h" #include "auth-token.h" -#include #include #include #include diff -r 55e6910aed18 -r 9e120590e0ef src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/auth-worker-client.c Wed Sep 23 14:52:45 2015 +0300 @@ -14,7 +14,6 @@ #include "auth-request.h" #include "auth-worker-client.h" -#include #define AUTH_WORKER_WARN_DISCONNECTED_LONG_CMD_SECS 30 #define OUTBUF_THROTTLE_SIZE (1024*10) diff -r 55e6910aed18 -r 9e120590e0ef src/auth/auth-worker-server.c --- a/src/auth/auth-worker-server.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/auth-worker-server.c Wed Sep 23 14:52:45 2015 +0300 @@ -14,7 +14,6 @@ #include "auth-worker-client.h" #include "auth-worker-server.h" -#include #include /* Initial lookup timeout */ diff -r 55e6910aed18 -r 9e120590e0ef src/auth/checkpassword-reply.c --- a/src/auth/checkpassword-reply.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/checkpassword-reply.c Wed Sep 23 14:52:45 2015 +0300 @@ -5,7 +5,6 @@ #include "strescape.h" #include "write-full.h" -#include #include int main(void) diff -r 55e6910aed18 -r 9e120590e0ef src/auth/db-checkpassword.c --- a/src/auth/db-checkpassword.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/db-checkpassword.c Wed Sep 23 14:52:45 2015 +0300 @@ -17,7 +17,6 @@ #include "child-wait.h" #include "db-checkpassword.h" -#include #include #include diff -r 55e6910aed18 -r 9e120590e0ef src/auth/db-dict.c --- a/src/auth/db-dict.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/db-dict.c Wed Sep 23 14:52:45 2015 +0300 @@ -13,7 +13,6 @@ #include "db-dict.h" #include -#include enum dict_settings_section { DICT_SETTINGS_SECTION_ROOT = 0, diff -r 55e6910aed18 -r 9e120590e0ef src/auth/db-ldap.c --- a/src/auth/db-ldap.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/db-ldap.c Wed Sep 23 14:52:45 2015 +0300 @@ -18,7 +18,6 @@ #include "db-ldap.h" #include -#include #include #define HAVE_LDAP_SASL diff -r 55e6910aed18 -r 9e120590e0ef src/auth/db-passwd-file.c --- a/src/auth/db-passwd-file.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/db-passwd-file.c Wed Sep 23 14:52:45 2015 +0300 @@ -14,7 +14,6 @@ #include "str.h" #include "eacces-error.h" -#include #include #include #include diff -r 55e6910aed18 -r 9e120590e0ef src/auth/db-sql.c --- a/src/auth/db-sql.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/db-sql.c Wed Sep 23 14:52:45 2015 +0300 @@ -10,7 +10,6 @@ #include "db-sql.h" #include -#include #define DEF_STR(name) DEF_STRUCT_STR(name, sql_settings) #define DEF_INT(name) DEF_STRUCT_INT(name, sql_settings) diff -r 55e6910aed18 -r 9e120590e0ef src/auth/mech-cram-md5.c --- a/src/auth/mech-cram-md5.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/mech-cram-md5.c Wed Sep 23 14:52:45 2015 +0300 @@ -15,7 +15,6 @@ #include "passdb.h" #include "hostpid.h" -#include #include struct cram_auth_request { diff -r 55e6910aed18 -r 9e120590e0ef src/auth/mech-digest-md5.c --- a/src/auth/mech-digest-md5.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/mech-digest-md5.c Wed Sep 23 14:52:45 2015 +0300 @@ -13,7 +13,6 @@ #include "mech.h" #include "passdb.h" -#include #define MAX_REALM_LEN 64 diff -r 55e6910aed18 -r 9e120590e0ef src/auth/mech-gssapi.c --- a/src/auth/mech-gssapi.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/mech-gssapi.c Wed Sep 23 14:52:45 2015 +0300 @@ -21,7 +21,6 @@ #include "mech.h" #include "passdb.h" -#include #if defined(BUILTIN_GSSAPI) || defined(PLUGIN_BUILD) diff -r 55e6910aed18 -r 9e120590e0ef src/auth/mech-scram-sha1.c --- a/src/auth/mech-scram-sha1.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/mech-scram-sha1.c Wed Sep 23 14:52:45 2015 +0300 @@ -6,7 +6,6 @@ * This software is released under the MIT license. */ -#include #include #include "auth-common.h" diff -r 55e6910aed18 -r 9e120590e0ef src/auth/mech-winbind.c --- a/src/auth/mech-winbind.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/mech-winbind.c Wed Sep 23 14:52:45 2015 +0300 @@ -17,7 +17,6 @@ #include "istream.h" #include "ostream.h" -#include #include #include diff -r 55e6910aed18 -r 9e120590e0ef src/auth/mech.c --- a/src/auth/mech.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/mech.c Wed Sep 23 14:52:45 2015 +0300 @@ -6,7 +6,6 @@ #include "str.h" #include "passdb.h" -#include #include static struct mech_module_list *mech_modules; diff -r 55e6910aed18 -r 9e120590e0ef src/auth/passdb-blocking.c --- a/src/auth/passdb-blocking.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/passdb-blocking.c Wed Sep 23 14:52:45 2015 +0300 @@ -8,7 +8,6 @@ #include "passdb.h" #include "passdb-blocking.h" -#include static void auth_worker_reply_parse_args(struct auth_request *request, diff -r 55e6910aed18 -r 9e120590e0ef src/auth/passdb-cache.c --- a/src/auth/passdb-cache.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/passdb-cache.c Wed Sep 23 14:52:45 2015 +0300 @@ -6,7 +6,6 @@ #include "passdb.h" #include "passdb-cache.h" -#include struct auth_cache *passdb_cache = NULL; diff -r 55e6910aed18 -r 9e120590e0ef src/auth/passdb-dict.c --- a/src/auth/passdb-dict.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/passdb-dict.c Wed Sep 23 14:52:45 2015 +0300 @@ -11,7 +11,6 @@ #include "auth-cache.h" #include "db-dict.h" -#include #include struct dict_passdb_module { diff -r 55e6910aed18 -r 9e120590e0ef src/auth/passdb-ldap.c --- a/src/auth/passdb-ldap.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/passdb-ldap.c Wed Sep 23 14:52:45 2015 +0300 @@ -13,7 +13,6 @@ #include "db-ldap.h" #include -#include struct ldap_passdb_module { struct passdb_module module; diff -r 55e6910aed18 -r 9e120590e0ef src/auth/passdb-pam.c --- a/src/auth/passdb-pam.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/passdb-pam.c Wed Sep 23 14:52:45 2015 +0300 @@ -18,7 +18,6 @@ #include "safe-memset.h" #include "auth-cache.h" -#include #include #ifdef HAVE_SECURITY_PAM_APPL_H diff -r 55e6910aed18 -r 9e120590e0ef src/auth/passdb-sql.c --- a/src/auth/passdb-sql.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/passdb-sql.c Wed Sep 23 14:52:45 2015 +0300 @@ -10,7 +10,6 @@ #include "auth-cache.h" #include "db-sql.h" -#include #include struct sql_passdb_module { diff -r 55e6910aed18 -r 9e120590e0ef src/auth/passdb-vpopmail.c --- a/src/auth/passdb-vpopmail.c Wed Sep 23 14:52:44 2015 +0300 +++ b/src/auth/passdb-vpopmail.c Wed Sep 23 14:52:45 2015 +0300 From dovecot at dovecot.org Wed Sep 23 13:11:26 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 13:11:26 +0000 Subject: dovecot-2.2: example-config: Added imap_hibernate_timeout Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fd83572bb229 changeset: 19192:fd83572bb229 user: Timo Sirainen date: Wed Sep 23 16:08:33 2015 +0300 description: example-config: Added imap_hibernate_timeout diffstat: doc/example-config/conf.d/20-imap.conf | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 9e120590e0ef -r fd83572bb229 doc/example-config/conf.d/20-imap.conf --- a/doc/example-config/conf.d/20-imap.conf Wed Sep 23 14:52:45 2015 +0300 +++ b/doc/example-config/conf.d/20-imap.conf Wed Sep 23 16:08:33 2015 +0300 @@ -2,6 +2,12 @@ ## IMAP specific settings ## +# If nothing happens for this long while client is IDLEing, move the connection +# to imap-hibernate process and close the old imap process. This saves memory, +# because connections use very little memory in imap-hibernate process. The +# downside is that recreating the imap process back uses some resources. +#imap_hibernate_timeout = 0 + # Maximum IMAP command line length. Some clients generate very long command # lines with huge mailboxes, so you may need to raise this if you get # "Too long argument" or "IMAP command line too large" errors often. From dovecot at dovecot.org Wed Sep 23 13:11:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 13:11:29 +0000 Subject: dovecot-2.2: virtual: The virtual index may have sometimes been ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8408209309ec changeset: 19193:8408209309ec user: Timo Sirainen date: Wed Sep 23 16:09:47 2015 +0300 description: virtual: The virtual index may have sometimes been unnecessarily deleted. Due to uninitialized broken_r value. diffstat: src/plugins/virtual/virtual-sync.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r fd83572bb229 -r 8408209309ec src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Wed Sep 23 16:08:33 2015 +0300 +++ b/src/plugins/virtual/virtual-sync.c Wed Sep 23 16:09:47 2015 +0300 @@ -155,6 +155,8 @@ uint32_t prev_mailbox_id; int ret = 1; + *broken_r = FALSE; + hdr = mail_index_get_header(view); mail_index_get_header_ext(view, mbox->virtual_ext_id, &ext_data, &ext_size); From dovecot at dovecot.org Wed Sep 23 13:32:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 13:32:56 +0000 Subject: dovecot-2.2: Added tag 2.2.19.rc1 for changeset 60057d955db3 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/58be55e8b7aa changeset: 19195:58be55e8b7aa user: Timo Sirainen date: Wed Sep 23 16:10:24 2015 +0300 description: Added tag 2.2.19.rc1 for changeset 60057d955db3 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 60057d955db3 -r 58be55e8b7aa .hgtags --- a/.hgtags Wed Sep 23 16:10:24 2015 +0300 +++ b/.hgtags Wed Sep 23 16:10:24 2015 +0300 @@ -127,3 +127,4 @@ 2c1ce55de520484431a63041602829d21195b4e9 2.2.17.rc2 166106aaefc5e6cf3850dba9534ca90153931ad9 2.2.17 917d027836d0f9d7d4b740dc33ae48cc5beead6f 2.2.18 +60057d955db325d1af199098ff4cfd57f237645c 2.2.19.rc1 From dovecot at dovecot.org Wed Sep 23 13:32:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 13:32:55 +0000 Subject: dovecot-2.2: Released v2.2.19.rc1. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/60057d955db3 changeset: 19194:60057d955db3 user: Timo Sirainen date: Wed Sep 23 16:10:24 2015 +0300 description: Released v2.2.19.rc1. diffstat: NEWS | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ configure.ac | 4 ++-- 2 files changed, 50 insertions(+), 2 deletions(-) diffs (69 lines): diff -r 8408209309ec -r 60057d955db3 NEWS --- a/NEWS Wed Sep 23 16:09:47 2015 +0300 +++ b/NEWS Wed Sep 23 16:10:24 2015 +0300 @@ -1,3 +1,51 @@ +v2.2.19 2015-09-xx Timo Sirainen + + * "doveadm director flush" command has a changed meaning now: + It safely moves users to their wanted backends, instead of simply + forgetting the mapping entirely and leaving the existing connections + untouched. Use -F parameter to get the original unsafe behavior. + + + Added imap-hibernate processes (see imap_hibernate_timeout setting). + IDLEing IMAP connections can be hibernated, which saves memory. + + Optimized tracking mailboxes' vsizes (= sum of all messages' sizes). + If mailbox_list_index=yes, it's also stored in there. This makes it + very efficient to look up vsizes for all mailboxes. + + Added a quota "count" backend, which uses the mailbox vsizes to get + the current quota usage. It requires using the new quota_vsizes=yes + setting, which tracks the messages' "virtual sizes" rather than + "physical sizes". Their distiction is minor and mostly irrelevant + nowadays (if mail sizes should be counted with LF or CRLF newlines). + + "doveadm director up/down" commands added. The monitoring script + should be using these commands instead of changing the vhost count. + This allows admin to manually disable a server by changing the vhost + count to 0 without the monitoring script changing it back. + + Added support for HAProxy protocol: http://wiki2.dovecot.org/HAProxy + + Added push-notification plugin framework, which can be used to + easily implement push notifications to various backends. Implemented + "ox" backend for notifying Open-Xchange via HTTP/json. + + imap_logout_format supports more variables now, e.g. number of + deleted messages. + + pop3: Added pop3_delete_type setting (related to pop3_deleted_flag). + + plugin { fts_enforced=yes } setting now fails body searches unless + it can be done via the full text search engine. + + Added %{passdb:*} and %{userdb:*} variables to various places + + auth: Added ":protected" suffix for passdb and userdb fields. If + used, the field doesn't overwrite an existing field. + + IMAP/POP3 proxy: If a backend server dies, avoid client reconnection + spikes by slowly disconnecting clients over time. This is enabled by + setting login_proxy_max_disconnect_delay=secs passdb extra field. + + imap: Added new read-only METADATA entries: /private/specialuse, + /shared/comment, /shared/admin + + imap: If client disconnects in the middle of a command, log how long + the command had been running. + - mdbox: Rebuilding could have caused message's reference count to + overflow the 16bit number in some situations, causing problems when + trying to expunge the duplicates. + - Various search fixes (fts, solr, tika, lib-charset, indexer) + - Various virtual plugin fixes + - Various fixes and optimizations to dsync, imapc and pop3-migration + - imap: Various RFC compliancy and crash fixes to NOTIFY + v2.2.18 2015-05-15 Timo Sirainen - director: Login UNIX sockets were normally detected as doveadm or diff -r 8408209309ec -r 60057d955db3 configure.ac --- a/configure.ac Wed Sep 23 16:09:47 2015 +0300 +++ b/configure.ac Wed Sep 23 16:10:24 2015 +0300 @@ -2,8 +2,8 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.18],[dovecot at dovecot.org]) -AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv17($PACKAGE_VERSION)", [Dovecot ABI version]) +AC_INIT([Dovecot],[2.2.19.rc1],[dovecot at dovecot.org]) +AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv19($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Wed Sep 23 13:32:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 13:32:56 +0000 Subject: dovecot-2.2: Added signature for changeset 60057d955db3 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6bacd85b1a2e changeset: 19196:6bacd85b1a2e user: Timo Sirainen date: Wed Sep 23 16:10:27 2015 +0300 description: Added signature for changeset 60057d955db3 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 58be55e8b7aa -r 6bacd85b1a2e .hgsigs --- a/.hgsigs Wed Sep 23 16:10:24 2015 +0300 +++ b/.hgsigs Wed Sep 23 16:10:27 2015 +0300 @@ -90,3 +90,4 @@ 2c1ce55de520484431a63041602829d21195b4e9 0 iEYEABECAAYFAlVSXUYACgkQyUhSUUBVislsrwCdEtWDWrQPsJua75RL83lpkeh7DagAoKW+534P0iziiX/nD6QzNmBTHODe 166106aaefc5e6cf3850dba9534ca90153931ad9 0 iEYEABECAAYFAlVTf5MACgkQyUhSUUBViskLHgCeMooopuB7egroMj8w78YarjjRieYAnjVSKcz0sXShlbYRsLRPOnyfq/pS 917d027836d0f9d7d4b740dc33ae48cc5beead6f 0 iEYEABECAAYFAlVV0mIACgkQyUhSUUBVismFtwCeIVNrGFguuXIhDoCSUm86Q4uTpQUAnAi6UfVL9/+QWC5LpR7jvY97QH0c +60057d955db325d1af199098ff4cfd57f237645c 0 iEYEABECAAYFAlYCpMAACgkQyUhSUUBVismRDwCeIE5jzDFBEBFuIf2JgRMsvgErSmkAniUiUmQ8clnWBSqu7fEY7A2ZbHia From pigeonhole at rename-it.nl Wed Sep 23 18:06:20 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 23 Sep 2015 20:06:20 +0200 Subject: dovecot-2.2-pigeonhole: Sieve extprograms plugin: Made line endi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/9f3118148e67 changeset: 2098:9f3118148e67 user: Stephan Bosch date: Wed Sep 23 20:06:15 2015 +0200 description: Sieve extprograms plugin: Made line endings configurable for the input passed to the external program. diffstat: doc/plugins/sieve_extprograms.txt | 9 +++- src/plugins/sieve-extprograms/sieve-extprograms-common.c | 31 +++++++++++--- src/plugins/sieve-extprograms/sieve-extprograms-common.h | 7 +++ tests/plugins/extprograms/bin/crlf | 3 + tests/plugins/extprograms/execute/execute.svtest | 33 +++++++++++++++- 5 files changed, 74 insertions(+), 9 deletions(-) diffs (183 lines): diff -r ef333e5beccd -r 9f3118148e67 doc/plugins/sieve_extprograms.txt --- a/doc/plugins/sieve_extprograms.txt Mon Sep 21 23:34:41 2015 +0200 +++ b/doc/plugins/sieve_extprograms.txt Wed Sep 23 20:06:15 2015 +0200 @@ -106,9 +106,16 @@ execute directly and pipe messages to. sieve__exec_timeout = 10s - Configures the maximum execution time after which the program is forcefully + Configures the maximum execution time after which the program is forcibly terminated. +sieve__input_eol = crlf + Determines the end-of-line character sequence used for the data piped to + external programs. The default is currently "crlf", which represents a + sequence of the carriage return (CR) and line feed (LF) characters. This + matches the Internet Message Format (RFC5322) and what Sieve itself uses as a + line ending. Set this setting to "lf" to use a single LF character instead. + Examples -------- diff -r ef333e5beccd -r 9f3118148e67 src/plugins/sieve-extprograms/sieve-extprograms-common.c --- a/src/plugins/sieve-extprograms/sieve-extprograms-common.c Mon Sep 21 23:34:41 2015 +0200 +++ b/src/plugins/sieve-extprograms/sieve-extprograms-common.c Wed Sep 23 20:06:15 2015 +0200 @@ -62,7 +62,7 @@ struct sieve_instance *svinst = ext->svinst; struct sieve_extprograms_config *ext_config; const char *extname = sieve_extension_name(ext); - const char *bin_dir, *socket_dir; + const char *bin_dir, *socket_dir, *input_eol; sieve_number_t execute_timeout; extname = strrchr(extname, '.'); @@ -73,6 +73,8 @@ (svinst, t_strdup_printf("sieve_%s_bin_dir", extname)); socket_dir = sieve_setting_get (svinst, t_strdup_printf("sieve_%s_socket_dir", extname)); + input_eol = sieve_setting_get + (svinst, t_strdup_printf("sieve_%s_input_eol", extname)); ext_config = i_new(struct sieve_extprograms_config, 1); ext_config->execute_timeout = @@ -94,6 +96,10 @@ &execute_timeout)) { ext_config->execute_timeout = execute_timeout; } + + ext_config->default_input_eol = SIEVE_EXTPROGRAMS_EOL_CRLF; + if (input_eol != NULL && strcasecmp(input_eol, "lf") == 0) + ext_config->default_input_eol = SIEVE_EXTPROGRAMS_EOL_LF; } if ( sieve_extension_is(ext, vnd_pipe_extension) ) @@ -366,6 +372,8 @@ struct sieve_extprogram { struct sieve_instance *svinst; + const struct sieve_extprograms_config *ext_config; + const struct sieve_script_env *scriptenv; struct program_client_settings set; struct program_client *program_client; @@ -515,6 +523,7 @@ sprog = i_new(struct sieve_extprogram, 1); sprog->svinst = ext->svinst; + sprog->ext_config = ext_config; sprog->scriptenv = senv; sprog->set.client_connect_timeout_msecs = @@ -574,7 +583,20 @@ void sieve_extprogram_set_input (struct sieve_extprogram *sprog, struct istream *input) { + switch (sprog->ext_config->default_input_eol) { + case SIEVE_EXTPROGRAMS_EOL_LF: + input = i_stream_create_lf(input); + break; + case SIEVE_EXTPROGRAMS_EOL_CRLF: + input = i_stream_create_crlf(input); + break; + default: + i_unreached(); + } + program_client_set_input(sprog->program_client, input); + + i_stream_unref(&input); } void sieve_extprogram_set_output_seekable @@ -601,12 +623,7 @@ if (mail_get_stream(mail, NULL, NULL, &input) < 0) return -1; - /* Make sure the message contains CRLF consistently */ - input = i_stream_create_crlf(input); - - program_client_set_input(sprog->program_client, input); - i_stream_unref(&input); - + sieve_extprogram_set_input(sprog, input); return 1; } diff -r ef333e5beccd -r 9f3118148e67 src/plugins/sieve-extprograms/sieve-extprograms-common.h --- a/src/plugins/sieve-extprograms/sieve-extprograms-common.h Mon Sep 21 23:34:41 2015 +0200 +++ b/src/plugins/sieve-extprograms/sieve-extprograms-common.h Wed Sep 23 20:06:15 2015 +0200 @@ -10,6 +10,11 @@ * Extension configuration */ +enum sieve_extprograms_eol { + SIEVE_EXTPROGRAMS_EOL_CRLF = 0, + SIEVE_EXTPROGRAMS_EOL_LF +}; + struct sieve_extprograms_config { const struct sieve_extension *copy_ext; const struct sieve_extension *var_ext; @@ -17,6 +22,8 @@ char *socket_dir; char *bin_dir; + enum sieve_extprograms_eol default_input_eol; + unsigned int execute_timeout; }; diff -r ef333e5beccd -r 9f3118148e67 tests/plugins/extprograms/bin/crlf --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/plugins/extprograms/bin/crlf Wed Sep 23 20:06:15 2015 +0200 @@ -0,0 +1,3 @@ +#!/bin/sh + +tr -s '\r' '#' diff -r ef333e5beccd -r 9f3118148e67 tests/plugins/extprograms/execute/execute.svtest --- a/tests/plugins/extprograms/execute/execute.svtest Mon Sep 21 23:34:41 2015 +0200 +++ b/tests/plugins/extprograms/execute/execute.svtest Wed Sep 23 20:06:15 2015 +0200 @@ -4,6 +4,7 @@ require "variables"; require "relational"; require "environment"; +require "encoded-character"; test_set "message" text: From: stephan at example.com @@ -115,4 +116,34 @@ } } - +test_config_set "sieve_execute_input_eol" "crlf"; +test_config_reload :extension "vnd.dovecot.execute"; +test_result_reset; +set "out" ""; + +test "Execute - CRLF" { + execute + :input "FROP${hex:0A}FRIEP${hex:0a}" + :output "out" + "crlf"; + + if not string "${out}" "FROP#${hex:0A}FRIEP#${hex:0a}" { + test_fail "wrong string returned: '${out}'"; + } +} + +test_config_set "sieve_execute_input_eol" "lf"; +test_config_reload :extension "vnd.dovecot.execute"; +test_result_reset; +set "out" ""; + +test "Execute - LF" { + execute + :input "FROP${hex:0D 0A}FRIEP${hex:0d 0a}" + :output "out" + "crlf"; + + if not string "${out}" "FROP${hex:0A}FRIEP${hex:0a}" { + test_fail "wrong string returned: '${out}'"; + } +} From pigeonhole at rename-it.nl Wed Sep 23 19:50:15 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 23 Sep 2015 21:50:15 +0200 Subject: dovecot-2.2-pigeonhole: Sieve vnd.dovecot.environment extension:... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/fc67d3f0db2d changeset: 2099:fc67d3f0db2d user: Stephan Bosch date: Wed Sep 23 21:49:25 2015 +0200 description: Sieve vnd.dovecot.environment extension: Made activation of the environment extension implicit. diffstat: src/lib-sieve/plugins/environment/sieve-ext-environment.h | 8 +++++++ src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c | 11 +++++++++- tests/extensions/vnd.dovecot/environment/basic.svtest | 1 - tests/extensions/vnd.dovecot/environment/variables.svtest | 1 - 4 files changed, 18 insertions(+), 3 deletions(-) diffs (64 lines): diff -r 9f3118148e67 -r fc67d3f0db2d src/lib-sieve/plugins/environment/sieve-ext-environment.h --- a/src/lib-sieve/plugins/environment/sieve-ext-environment.h Wed Sep 23 20:06:15 2015 +0200 +++ b/src/lib-sieve/plugins/environment/sieve-ext-environment.h Wed Sep 23 21:49:25 2015 +0200 @@ -22,6 +22,14 @@ (svinst, &environment_extension, FALSE); } +static inline const struct sieve_extension * +sieve_ext_environment_require_extension +(struct sieve_instance *svinst) +{ + return sieve_extension_require + (svinst, &environment_extension, TRUE); +} + bool sieve_ext_environment_is_active (const struct sieve_extension *env_ext, struct sieve_interpreter *interp); diff -r 9f3118148e67 -r fc67d3f0db2d src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c --- a/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c Wed Sep 23 20:06:15 2015 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c Wed Sep 23 21:49:25 2015 +0200 @@ -62,7 +62,7 @@ ext_vnd_environment_unload(ext); ectx = i_new(struct ext_vnd_environment_context, 1); - ectx->env_ext = sieve_ext_environment_get_extension(ext->svinst); + ectx->env_ext = sieve_ext_environment_require_extension(ext->svinst); ectx->var_ext = sieve_ext_variables_get_extension(ext->svinst); *context = (void *) ectx; @@ -85,6 +85,15 @@ static bool ext_vnd_environment_validator_load (const struct sieve_extension *ext, struct sieve_validator *valdtr) { + const struct sieve_extension *env_ext; + + /* Load environment extension implicitly */ + + env_ext = sieve_validator_extension_load_implicit + (valdtr, environment_extension.name); + if ( env_ext == NULL ) + return FALSE; + ext_environment_variables_init(ext, valdtr); return TRUE; } diff -r 9f3118148e67 -r fc67d3f0db2d tests/extensions/vnd.dovecot/environment/basic.svtest --- a/tests/extensions/vnd.dovecot/environment/basic.svtest Wed Sep 23 20:06:15 2015 +0200 +++ b/tests/extensions/vnd.dovecot/environment/basic.svtest Wed Sep 23 21:49:25 2015 +0200 @@ -1,5 +1,4 @@ require "vnd.dovecot.testsuite"; -require "environment"; require "vnd.dovecot.environment"; require "variables"; diff -r 9f3118148e67 -r fc67d3f0db2d tests/extensions/vnd.dovecot/environment/variables.svtest --- a/tests/extensions/vnd.dovecot/environment/variables.svtest Wed Sep 23 20:06:15 2015 +0200 +++ b/tests/extensions/vnd.dovecot/environment/variables.svtest Wed Sep 23 21:49:25 2015 +0200 @@ -1,5 +1,4 @@ require "vnd.dovecot.testsuite"; -require "environment"; require "vnd.dovecot.environment"; require "variables"; require "relational"; From pigeonhole at rename-it.nl Wed Sep 23 19:50:15 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 23 Sep 2015 21:50:15 +0200 Subject: dovecot-2.2-pigeonhole: Created specification for the vnd.doveco... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b189a305d045 changeset: 2100:b189a305d045 user: Stephan Bosch date: Wed Sep 23 21:50:10 2015 +0200 description: Created specification for the vnd.dovecot.environment extension. diffstat: doc/rfc/spec-bosch-sieve-dovecot-environment.txt | 224 +++++++++++++++++++ doc/rfc/xml/reference.ENVIRONMENT.xml | 12 + doc/rfc/xml/spec-bosch-sieve-dovecot-environment.xml | 160 +++++++++++++ 3 files changed, 396 insertions(+), 0 deletions(-) diffs (truncated from 408 to 300 lines): diff -r fc67d3f0db2d -r b189a305d045 doc/rfc/spec-bosch-sieve-dovecot-environment.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/rfc/spec-bosch-sieve-dovecot-environment.txt Wed Sep 23 21:50:10 2015 +0200 @@ -0,0 +1,224 @@ + + + +Pigeonhole Project S. Bosch + September 23, 2015 + + + Sieve Email Filtering: Logging Debug Messages + +Abstract + + This document defines new vendor-defined items for the "environment" + extension of the Sieve mail filtering language. + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 2. Conventions Used in This Document . . . . . . . . . . . . . . . 2 + 3. Dovecot Environment Items . . . . . . . . . . . . . . . . . . . 2 + 4. Variables Namespace env . . . . . . . . . . . . . . . . . . . . 2 + 5. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 3 + 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 3 + 7. Normative References . . . . . . . . . . . . . . . . . . . . . 3 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Bosch [Page 1] + + Sieve: Debug Logging September 2015 + + +1. Introduction + + This document describes an extension to the Sieve filtering language + defined by RFC 5228 [SIEVE]. This vendor-defined extension builds + upon the existing standard "environment" extension [ENVIRONMENT], + which allows Sieve scripts to access information about their + execution context, such as the name and version of the Sieve + interpreter implementation. The new "vnd.dovecot.environment" + extension adds a few more environment items that can be accessed by + Sieve scripts. Additionally, it makes the environment items + available directly as variables [VARIABLES]. + + This extension is specific to the Pigeonhole Sieve implementation for + the Dovecot Secure IMAP server. It will therefore most likely not be + supported by web interfaces and GUI-based Sieve editors. + + +2. Conventions Used in This Document + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in [KEYWORDS]. + + Conventions for notations are as in [SIEVE] Section 1.1, including + use of the "Usage:" label for the definition of action and tagged + arguments syntax. + + +3. Dovecot Environment Items + + The set of Dovecot-specific environment items is as follows: + + "vnd.dovecot.default-mailbox" + The default mailbox that is used by the (implicit) keep action. + Normally, this will always yield "INBOX", unless this is + changed in the configuration. + + "vnd.dovecot.username" + The username of the recipient for this particular message + delivery. + + +4. Variables Namespace env + + This document defines the variables namespace "env", in accordance + with the "variables" extension document [VARIABLES], Section 3. It + makes environment items directly available as variables. Since + environment items are always read-only, variables in the "env" + + + +Bosch [Page 2] + + Sieve: Debug Logging September 2015 + + + namespace can never be modified. + + Basically, the variable name for an environment item is composed of + the "env." namespace prefix followed by the full name of the + environment item. This is also true when the environment item name + contains "." characters; those have no special meaning within the + "env" variables namespace. However, since a variable name cannot + contain "-" characters, any "-" character in the name of the + environment item is mapped to an "_" character. The environment item + called "vnd.dovecot.default-mailbox" is therefore available as the + variable "${env.vnd.dovecot.default_mailbox}". + + Example: + + require ["variables", "fileinto", "mailbox"]; + require "vnd.dovecot.environment"; + + fileinto :create "User\${env.vnd.dovecot.username}"; + + + +5. Sieve Capability Strings + + A Sieve implementation that supports the Dovecot-specific environment + items outlined in this document will advertise the capability string + "vnd.dovecot.environment". Note that "vnd.dovecot.environment" + implies "environment", and a script with "vnd.dovecot.environment" in + a "require" list can omit "environment" from that list. + + +6. Security Considerations + + All of the security considerations given in the base Sieve + specification [SIEVE] and the specification of the "environment" + extension [ENVIRONMENT] also apply to this extension. Some of the + environment items added by this extension are somewhat more sensitive + than the standard ones, so enabling this extension should be + considered carefully. + + +7. Normative References + + [ENVIRONMENT] + Freed, N., "Sieve Email Filtering: Environment Extension", + RFC 5183, DOI 10.17487/RFC5183, May 2008, + . + + [KEYWORDS] + + + +Bosch [Page 3] + + Sieve: Debug Logging September 2015 + + + Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, March 1997. + + [SIEVE] Guenther, P. and T. Showalter, "Sieve: An Email Filtering + Language", RFC 5228, January 2008. + + [VARIABLES] + Homme, K., "Sieve Email Filtering: Variables Extension", + RFC 5229, January 2008. + + +Author's Address + + Stephan Bosch + Enschede + NL + + Email: stephan at rename-it.nl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Bosch [Page 4] + diff -r fc67d3f0db2d -r b189a305d045 doc/rfc/xml/reference.ENVIRONMENT.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/rfc/xml/reference.ENVIRONMENT.xml Wed Sep 23 21:50:10 2015 +0200 @@ -0,0 +1,12 @@ + + + + +Sieve Email Filtering: Environment Extension + + +This document describes the "environment" extension to the Sieve email filtering language. The "environment" extension gives a Sieve script access to information about the Sieve interpreter itself, where it is running, and about any transport connection currently involved in transferring the message. [STANDARDS-TRACK] + + + + diff -r fc67d3f0db2d -r b189a305d045 doc/rfc/xml/spec-bosch-sieve-dovecot-environment.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/rfc/xml/spec-bosch-sieve-dovecot-environment.xml Wed Sep 23 21:50:10 2015 +0200 @@ -0,0 +1,160 @@ + + + + + + + + + + + + + + + + + + + + +Sieve Email Filtering: Logging Debug Messages + + + + +
+ + + Enschede + NL + + stephan at rename-it.nl +
+ + + + +General +Pigeonhole Project +sieve +environment +dovecot + + + +This document defines new vendor-defined items for the "environment" extension +of the Sieve mail filtering language. + + + + + From pigeonhole at rename-it.nl Wed Sep 23 19:54:49 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 23 Sep 2015 21:54:49 +0200 Subject: dovecot-2.2-pigeonhole: doc: Removed obsolete vnd.dovecot.duplic... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/44fa661402b7 changeset: 2101:44fa661402b7 user: Stephan Bosch date: Wed Sep 23 21:54:39 2015 +0200 description: doc: Removed obsolete vnd.dovecot.duplicate specification sources. diffstat: doc/rfc/xml/reference.IMAP4FLAGS.xml | 15 - doc/rfc/xml/reference.MAILBOX.xml | 15 - doc/rfc/xml/reference.VACATION.xml | 17 - doc/rfc/xml/spec-bosch-sieve-duplicate.xml | 260 ----------------------------- 4 files changed, 0 insertions(+), 307 deletions(-) diffs (truncated from 323 to 300 lines): diff -r b189a305d045 -r 44fa661402b7 doc/rfc/xml/reference.IMAP4FLAGS.xml --- a/doc/rfc/xml/reference.IMAP4FLAGS.xml Wed Sep 23 21:50:10 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ - - - - - -Sieve Email Filtering: Imap4flags Extension - - - - -Recent discussions have shown that it is desirable to set different IMAP (RFC 3501) flags on message delivery. This can be done, for example, by a Sieve interpreter that works as a part of a Mail Delivery Agent.</t><t> This document describes an extension to the Sieve mail filtering language for setting IMAP flags. The extension allows setting of both IMAP system flags and IMAP keywords. [STANDARDS-TRACK] - - - - diff -r b189a305d045 -r 44fa661402b7 doc/rfc/xml/reference.MAILBOX.xml --- a/doc/rfc/xml/reference.MAILBOX.xml Wed Sep 23 21:50:10 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ - - - - - -The Sieve Mail-Filtering Language -- Extensions for Checking Mailbox Status and Accessing Mailbox Metadata - - - - -This memo defines an extension to the Sieve mail filtering language (RFC 5228) for accessing mailbox and server annotations, checking for mailbox existence, and controlling mailbox creation on "fileinto" action. [STANDARDS-TRACK] - - - - diff -r b189a305d045 -r 44fa661402b7 doc/rfc/xml/reference.VACATION.xml --- a/doc/rfc/xml/reference.VACATION.xml Wed Sep 23 21:50:10 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,17 +0,0 @@ - - - - - -Sieve Email Filtering: Vacation Extension - - - - - - -This document describes an extension to the Sieve email filtering language for an autoresponder similar to that of the Unix "vacation" command for replying to messages. Various safety features are included to prevent problems such as message loops. [STANDARDS-TRACK] - - - - diff -r b189a305d045 -r 44fa661402b7 doc/rfc/xml/spec-bosch-sieve-duplicate.xml --- a/doc/rfc/xml/spec-bosch-sieve-duplicate.xml Wed Sep 23 21:50:10 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,260 +0,0 @@ - - - - - - - - - - - - - - - - - - - - -Sieve Email Filtering: Detecting Duplicate Deliveries - - - - -
- - - Enschede - NL - - stephan at rename-it.nl -
- - - - -General -Pigeonhole Project -sieve -duplicate deliveries - - -This document defines a new vendor-defined test command "duplicate" for the -"Sieve" email filtering language. It can be used to test whether a particular -string value is a duplicate, i.e. whether it was seen before by the delivery -agent that is executing the Sieve script. The main application for this new test -is detecting duplicate message deliveries commonly caused by mailing list -subscriptions or redirected mail addresses. - - - - - -
-This is an extension to the Sieve filtering language defined by -RFC 5228. It adds a test to determine whether a -certain string value was seen before by the delivery agent in an earlier -execution of the Sieve script. This can be used to detect and handle duplicate -message deliveries. - -Duplicate deliveries are a common side-effect of being subscribed to a -mailing list. For example, if a member of the list decides to reply to both the -user and the mailing list itself, the user will get a copy of the message -directly and through mailing list. Also, if someone cross-posts over several -mailing lists to which the user is subscribed, the user will receive a copy from -each of those lists. In another scenario, the user has several redirected -mail addresses all pointing to his main mail account. If one of the user's -contacts sends the message to more than one of those addresses, the user will -receive more than a single copy. Using the "vnd.dovecot.duplicate" extension, -users have the means to detect and handle such duplicates, e.g. by discarding -them or putting them in a special folder. - -Duplicate messages are normally detected using the Message-ID header field, -which is required to be unique for each message. However, the "duplicate" test -is flexible enough to use different (weaker) criteria for defining what makes -a message a duplicate, for example based on the subject line. Also, other -applications of this new test command are possible, as long as the tracked -value is a string. - -This extension is specific to the Pigeonhole Sieve implementation for the -Dovecot Secure IMAP server. It will therefore most likely not be supported by -web interfaces and GUI-based Sieve editors. -
- -
-The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", -"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this -document are to be interpreted as described in . -Conventions for notations are as in Section 1.1, -including use of the "Usage:" label for the definition of action and tagged -arguments syntax. -
- -
- -
-] - [":header" / - ":value" ] - [":handle" ] -]]> -
- -The "duplicate" test keeps track of which values were seen before by this -test in an earlier execution of this Sieve script. In its basic form, the tested -value is the content of the Message-ID header of the message. This way, this -test can be used to detect duplicate deliveries of the same message. It can also -detect duplicate deliveries based on other message header fields if requested -and it can even use a user-provided string value, e.g. as composed from text -extracted from the message using the "variables" -extension. - -The "duplicate" test evaluates to "true" when the provided value was seen -before in an earlier Sieve execution for a previous message delivery. If the -value was not seen earlier, the test evaluates to "false". - -As a side-effect, the "duplicate" test adds the evaluated value to an -internal duplicate tracking list, so that the test will evaluate to "true" the -next time the Sieve script is executed and the same value is encountered. -Note that the "duplicate" test MUST only check for duplicates amongst values -encountered in previous executions of the Sieve script; it MUST NOT consider -values encountered earlier in the current Sieve script execution as potential -duplicates. This means that all "duplicate" tests in a Sieve script execution, -including those located in scripts included using the "include" - extension, MUST yield the same result if the arguments -are identical. - -Implementations MUST prevent adding values to the internal duplicate tracking -list when the Sieve script execution fails. For example, this can be implemented -by deferring the definitive modification of the tracking list to the end of the -Sieve script execution. If failed script executions would add values to the -duplicate tracking list, all "duplicate" tests would erroneously yield "true" -for the next delivery attempt of the same message, which can -- depending on the -action taken for a duplicate -- easily lead to discarding the message without -further notice. - -Implementations SHOULD limit the number of values (and thereby messages) that -are tracked. Also, implementations SHOULD let entries in the value tracking list -expire after a short period of time. The user can explicitly control the length -of this expiration time by means of the ":seconds" argument. If the ":seconds" -argument is omitted, an appropriate default MUST be used. Sites SHOULD impose a -maximum limit on the expiration time. If that limit is exceeded, the maximum -value MUST silently be substituted; exceeding the limit MUST NOT produce an -error. - -By default, the tracked value is the content of the message's Message-ID -header field. For more advanced purposes, the content of another header can be -chosen for tracking by specifying the ":header" argument. The tracked string -value can also be specified explicitly using the ":value" argument. The -":header" and ":value" arguments are mutually exclusive and specifying both for -a single "duplicate" test command MUST trigger an error at compile time. If the -value is extracted from a header, i.e. when the ":value" argument is not used, -leading and trailing whitespace (see Section 2.2 of -RFC 5228) MUST first be trimmed from the value -before performing the actual duplicate verification. - -Using the ":handle" argument, the duplicate test can be employed for multiple -independent purposes. Only when the tracked value was seen before in an earlier -script execution by a "duplicate" test with the same ":handle" argument, it is -recognized as a duplicate. - - -NOTE: The necessary mechanism to track duplicate messages is very similar to -the mechanism that is needed for tracking duplicate responses for the "vacation" - action. One way to implement the necessary mechanism -for the "duplicate" test is therefore to store a hash of the tracked value and, -if provided, the ":handle" argument. -
- -
-A Sieve implementation that defines the "duplicate" test command -will advertise the capability string "vnd.dovecot.duplicate". - -
- -
-In the following basic example, message duplicates are detected by tracking -the Message-ID header. Duplicate deliveries are stored in a special folder -contained in the user's Trash folder. If the folder does not exist, it is -created automatically using the "mailbox" extension. -This way, the user has a chance to recover messages when necessary. Messages -that are not recognized as duplicates are stored in the user's inbox as normal. - - - -
- -
- -The next example shows a more complex use of the "duplicate" test. The user -gets network alerts from a set of remote automated monitoring systems. Multiple -notifications can be received about the same event from different monitoring -systems. The Message-ID of these messages is different, because these are all -distinct messages from different senders. To avoid being notified multiple times -about the same event the user writes the following script: - - -
- -
- -The subjects of the notification message are structured with a predictable -pattern which includes a description of the event. In the script above the -"duplicate" test is used to detect duplicate alert events. The message subject -is matched against a pattern and the event description is extracted using the -"variables" extension. If a message with that -event in the subject was received before, but more than a minute ago, it is not -detected as a duplicate due to the specified ":seconds" argument. In the the -event of a duplicate, the message is marked as seen using the "imap4flags" - extension. All alert messages are put into the -"Alerts" mailbox irrespective of whether those messages are duplicates or not. - - -
- -
-A flood of unique messages could cause the list of tracked values to -grow indefinitely. Implementations therefore SHOULD implement limits on the -number and lifespan of entries in that list. From pigeonhole at rename-it.nl Wed Sep 23 19:57:40 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 23 Sep 2015 21:57:40 +0200 Subject: dovecot-2.2-pigeonhole: Sieve vnd.dovecot.environment extension:... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c1e233848ecd changeset: 2102:c1e233848ecd user: Stephan Bosch date: Wed Sep 23 21:57:31 2015 +0200 description: Sieve vnd.dovecot.environment extension: Referenced specification from extension source. diffstat: src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 44fa661402b7 -r c1e233848ecd src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c --- a/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c Wed Sep 23 21:54:39 2015 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c Wed Sep 23 21:57:31 2015 +0200 @@ -5,7 +5,8 @@ * --------------------------------- * * Authors: Stephan Bosch - * Specification: vendor-defined + * Specification: vendor-defined; + * spec-bosch-sieve-dovecot-environment * Implementation: preliminary * Status: experimental * From dovecot at dovecot.org Wed Sep 23 20:01:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 20:01:52 +0000 Subject: dovecot-2.2: dict-client: Catch more invalid protocol replies. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7363b1c77795 changeset: 19198:7363b1c77795 user: Timo Sirainen date: Wed Sep 23 22:57:55 2015 +0300 description: dict-client: Catch more invalid protocol replies. diffstat: src/lib-dict/dict-client.c | 47 +++++++++++++++++++++++++++++++++++---------- 1 files changed, 36 insertions(+), 11 deletions(-) diffs (91 lines): diff -r 891ff25af7ce -r 7363b1c77795 src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Wed Sep 23 22:49:41 2015 +0300 +++ b/src/lib-dict/dict-client.c Wed Sep 23 22:57:55 2015 +0300 @@ -542,18 +542,22 @@ { struct client_dict *dict = (struct client_dict *)_dict; char *line; - int ret = 0; + int ret; if (!dict->handshaked) return -1; while (dict->async_commits > 0) { - if (client_dict_read_one_line(dict, &line) < 0) { - ret = -1; - break; + if ((ret = client_dict_read_one_line(dict, &line)) < 0) + return -1; + + if (ret > 0) { + i_error("dict-client: Unexpected reply waiting waiting for async commits: %s", line); + client_dict_disconnect(dict); + return -1; } } - return ret; + return 0; } static int client_dict_lookup(struct dict *_dict, pool_t pool, @@ -578,12 +582,19 @@ if (line == NULL) return -1; - if (*line == DICT_PROTOCOL_REPLY_OK) { + switch (*line) { + case DICT_PROTOCOL_REPLY_OK: *value_r = p_strdup(pool, dict_client_unescape(line + 1)); return 1; - } else { + case DICT_PROTOCOL_REPLY_NOTFOUND: *value_r = NULL; - return *line == DICT_PROTOCOL_REPLY_NOTFOUND ? 0 : -1; + return 0; + case DICT_PROTOCOL_REPLY_FAIL: + return -1; + default: + i_error("dict-client: Invalid lookup '%s' reply: %s", key, line); + client_dict_disconnect(dict); + return -1; } } @@ -710,6 +721,10 @@ if (ret < 0) io_remove(&dict->io); + else if (ret > 0) { + i_error("dict-client: Unexpected reply waiting waiting for async commits: %s", line); + client_dict_disconnect(dict); + } } static int @@ -746,12 +761,22 @@ line = client_dict_read_line(dict); if (line == NULL) ret = -1; - else if (*line == DICT_PROTOCOL_REPLY_OK) + else switch (*line) { + case DICT_PROTOCOL_REPLY_OK: ret = 1; - else if (*line == DICT_PROTOCOL_REPLY_NOTFOUND) + break; + case DICT_PROTOCOL_REPLY_NOTFOUND: ret = 0; - else + break; + case DICT_PROTOCOL_REPLY_FAIL: ret = -1; + break; + default: + i_error("dict-client: Invalid commit reply: %s", line); + client_dict_disconnect(dict); + ret = -1; + break; + } } } T_END; From dovecot at dovecot.org Wed Sep 23 20:01:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 20:01:52 +0000 Subject: dovecot-2.2: dict-client: Disconnect from server on unexpected e... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/891ff25af7ce changeset: 19197:891ff25af7ce user: Timo Sirainen date: Wed Sep 23 22:49:41 2015 +0300 description: dict-client: Disconnect from server on unexpected errors. diffstat: src/lib-dict/dict-client.c | 17 ++++++++++++++--- 1 files changed, 14 insertions(+), 3 deletions(-) diffs (51 lines): diff -r 6bacd85b1a2e -r 891ff25af7ce src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Wed Sep 23 16:10:27 2015 +0300 +++ b/src/lib-dict/dict-client.c Wed Sep 23 22:49:41 2015 +0300 @@ -180,6 +180,7 @@ if (o_stream_send_str(dict->output, query) < 0 || o_stream_flush(dict->output) < 0) { i_error("write(%s) failed: %m", dict->path); + client_dict_disconnect(dict); return -1; } } @@ -311,7 +312,8 @@ return ret; } -static int client_dict_read_one_line(struct client_dict *dict, char **line_r) +static int +client_dict_read_one_line_real(struct client_dict *dict, char **line_r) { unsigned int id; char *line; @@ -355,11 +357,11 @@ default: i_error("dict-client: Invalid async commit line: %s", line); - return 0; + return -1; } if (str_to_uint(line+2, &id) < 0) { i_error("dict-client: Invalid ID"); - return 0; + return -1; } client_dict_finish_transaction(dict, id, ret); return 0; @@ -368,6 +370,15 @@ return 1; } +static int client_dict_read_one_line(struct client_dict *dict, char **line_r) +{ + int ret; + + if ((ret = client_dict_read_one_line_real(dict, line_r)) < 0) + client_dict_disconnect(dict); + return ret; +} + static bool client_dict_is_finished(struct client_dict *dict) { return dict->transactions == NULL && !dict->in_iteration && From dovecot at dovecot.org Wed Sep 23 20:01:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 20:01:52 +0000 Subject: dovecot-2.2: dict-client: If dict iteration was aborted, the res... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bc9b61338985 changeset: 19199:bc9b61338985 user: Timo Sirainen date: Wed Sep 23 22:59:55 2015 +0300 description: dict-client: If dict iteration was aborted, the rest of the connection state became broken. We never continued skipping over the rest of the iteration, instead the iteration replies were treated as replies to the following commands. diffstat: src/lib-dict/dict-client.c | 20 ++++++++++++++++++++ 1 files changed, 20 insertions(+), 0 deletions(-) diffs (65 lines): diff -r 7363b1c77795 -r bc9b61338985 src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Wed Sep 23 22:57:55 2015 +0300 +++ b/src/lib-dict/dict-client.c Wed Sep 23 22:59:55 2015 +0300 @@ -46,6 +46,7 @@ unsigned int connect_counter; unsigned int transaction_id_counter; unsigned int async_commits; + unsigned int iter_replies_skip; unsigned int in_iteration:1; unsigned int handshaked:1; @@ -56,6 +57,7 @@ pool_t pool; bool failed; + bool finished; }; struct client_dict_transaction_context { @@ -366,6 +368,19 @@ client_dict_finish_transaction(dict, id, ret); return 0; } + if (dict->iter_replies_skip > 0) { + /* called aborted the iteration before finishing it. + skip over the iteration reply */ + if (*line == DICT_PROTOCOL_REPLY_OK) + return 0; + if (*line != '\0' && *line != DICT_PROTOCOL_REPLY_FAIL) { + i_error("dict-client: Invalid iteration reply line: %s", + line); + return -1; + } + dict->iter_replies_skip--; + return 0; + } *line_r = line; return 1; } @@ -463,6 +478,7 @@ dict->connect_counter++; dict->handshaked = FALSE; + dict->iter_replies_skip = 0; /* abort all pending async commits */ for (ctx = dict->transactions; ctx != NULL; ctx = next) { @@ -649,6 +665,7 @@ if (*line == '\0') { /* end of iteration */ + ctx->finished = TRUE; return FALSE; } @@ -686,6 +703,9 @@ (struct client_dict_iterate_context *)_ctx; int ret = ctx->failed ? -1 : 0; + if (!ctx->finished) + dict->iter_replies_skip++; + pool_unref(&ctx->pool); i_free(ctx); dict->in_iteration = FALSE; From dovecot at dovecot.org Wed Sep 23 20:01:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 20:01:52 +0000 Subject: dovecot-2.2: dict-client: Improved broken iterate reply logging. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/471383fa1284 changeset: 19200:471383fa1284 user: Timo Sirainen date: Wed Sep 23 23:00:30 2015 +0300 description: dict-client: Improved broken iterate reply logging. diffstat: src/lib-dict/dict-client.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diffs (42 lines): diff -r bc9b61338985 -r 471383fa1284 src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Wed Sep 23 22:59:55 2015 +0300 +++ b/src/lib-dict/dict-client.c Wed Sep 23 23:00:30 2015 +0300 @@ -651,7 +651,7 @@ struct client_dict_iterate_context *ctx = (struct client_dict_iterate_context *)_ctx; struct client_dict *dict = (struct client_dict *)_ctx->dict; - char *line, *value; + char *line, *key, *value; if (ctx->failed) return FALSE; @@ -674,24 +674,26 @@ switch (*line) { case DICT_PROTOCOL_REPLY_OK: - value = strchr(++line, '\t'); + key = line+1; + value = strchr(key, '\t'); break; case DICT_PROTOCOL_REPLY_FAIL: ctx->failed = TRUE; return FALSE; default: + key = NULL; value = NULL; break; } if (value == NULL) { /* broken protocol */ - i_error("dict client (%s) sent broken reply", dict->path); + i_error("dict client (%s) sent broken iterate reply: %s", dict->path, line); ctx->failed = TRUE; return FALSE; } *value++ = '\0'; - *key_r = p_strdup(ctx->pool, dict_client_unescape(line)); + *key_r = p_strdup(ctx->pool, dict_client_unescape(key)); *value_r = p_strdup(ctx->pool, dict_client_unescape(value)); return TRUE; } From dovecot at dovecot.org Wed Sep 23 20:02:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 20:02:49 +0000 Subject: dovecot-2.2: dict: Include PID in the log prefix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b8d2cbd6583b changeset: 19201:b8d2cbd6583b user: Timo Sirainen date: Wed Sep 23 23:01:27 2015 +0300 description: dict: Include PID in the log prefix. diffstat: src/dict/main.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (20 lines): diff -r 471383fa1284 -r b8d2cbd6583b src/dict/main.c --- a/src/dict/main.c Wed Sep 23 23:00:30 2015 +0300 +++ b/src/dict/main.c Wed Sep 23 23:01:27 2015 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "restrict-access.h" #include "randgen.h" +#include "hostpid.h" #include "env-util.h" #include "module-dir.h" #include "master-service.h" @@ -96,7 +97,7 @@ &error) < 0) i_fatal("Error reading configuration: %s", error); - master_service_init_log(master_service, "dict: "); + master_service_init_log(master_service, t_strdup_printf("dict(%s): ", my_pid)); main_preinit(); master_service_set_die_callback(master_service, dict_die); From pigeonhole at rename-it.nl Wed Sep 23 20:54:47 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 23 Sep 2015 22:54:47 +0200 Subject: dovecot-2.2-pigeonhole: Remove now-unnecessary direct stdlib.h #... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/7038e943e853 changeset: 2103:7038e943e853 user: Stephan Bosch date: Wed Sep 23 22:54:11 2015 +0200 description: Remove now-unnecessary direct stdlib.h #includes. diffstat: src/lib-sieve-tool/mail-raw.c | 1 - src/lib-sieve-tool/sieve-tool.c | 1 - src/lib-sieve/plugins/include/ext-include-common.c | 1 - src/lib-sieve/plugins/subaddress/ext-subaddress.c | 1 - src/lib-sieve/sieve-common.h | 1 - src/lib-sieve/sieve-error.c | 1 - src/lib-sieve/sieve-lexer.c | 1 - src/lib-sieve/sieve-settings.c | 1 - src/lib-sieve/sieve.c | 1 - src/lib-sieve/storage/file/sieve-file-script-sequence.c | 1 - src/lib-sieve/storage/file/sieve-file-script.c | 1 - src/lib-sieve/storage/file/sieve-file-storage-list.c | 1 - src/lib-sieve/storage/file/sieve-file-storage-quota.c | 1 - src/lib-sieve/storage/file/sieve-file-storage-save.c | 1 - src/lib-sieve/storage/file/sieve-file-storage.c | 1 - src/lib-sieve/storage/ldap/sieve-ldap-db.c | 1 - src/lib-sieve/util/realpath.c | 1 - src/managesieve-login/client-authenticate.c | 1 - src/managesieve-login/client.c | 1 - src/managesieve-login/managesieve-login-settings-plugin.c | 1 - src/managesieve/cmd-noop.c | 1 - src/managesieve/main.c | 1 - src/managesieve/managesieve-client.c | 1 - src/managesieve/managesieve-commands.c | 1 - src/managesieve/managesieve-settings.c | 1 - src/plugins/lda-sieve/lda-sieve-plugin.c | 1 - src/plugins/sieve-extprograms/sieve-extprograms-common.c | 1 - src/sieve-tools/sieve-dump.c | 1 - src/sieve-tools/sieve-filter.c | 1 - src/sieve-tools/sieve-test.c | 1 - src/sieve-tools/sievec.c | 1 - src/testsuite/testsuite-common.c | 1 - src/testsuite/testsuite.c | 1 - 33 files changed, 0 insertions(+), 33 deletions(-) diffs (truncated from 363 to 300 lines): diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve-tool/mail-raw.c --- a/src/lib-sieve-tool/mail-raw.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve-tool/mail-raw.c Wed Sep 23 22:54:11 2015 +0200 @@ -20,7 +20,6 @@ #include "mail-raw.h" #include -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve-tool/sieve-tool.c --- a/src/lib-sieve-tool/sieve-tool.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve-tool/sieve-tool.c Wed Sep 23 22:54:11 2015 +0200 @@ -24,7 +24,6 @@ #include "sieve-tool.h" #include -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/plugins/include/ext-include-common.c --- a/src/lib-sieve/plugins/include/ext-include-common.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/plugins/include/ext-include-common.c Wed Sep 23 22:54:11 2015 +0200 @@ -23,7 +23,6 @@ #include "ext-include-binary.h" #include "ext-include-variables.h" -#include /* * Forward declarations diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/plugins/subaddress/ext-subaddress.c --- a/src/lib-sieve/plugins/subaddress/ext-subaddress.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/plugins/subaddress/ext-subaddress.c Wed Sep 23 22:54:11 2015 +0200 @@ -23,7 +23,6 @@ #include "sieve-generator.h" #include "sieve-interpreter.h" -#include #include /* diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/sieve-common.h --- a/src/lib-sieve/sieve-common.h Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/sieve-common.h Wed Sep 23 22:54:11 2015 +0200 @@ -10,7 +10,6 @@ #include "sieve-types.h" #include -#include /* * Types diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/sieve-error.c --- a/src/lib-sieve/sieve-error.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/sieve-error.c Wed Sep 23 22:54:11 2015 +0200 @@ -15,7 +15,6 @@ #include #include #include -#include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/sieve-lexer.c --- a/src/lib-sieve/sieve-lexer.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/sieve-lexer.c Wed Sep 23 22:54:11 2015 +0200 @@ -18,7 +18,6 @@ #include #include #include -#include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/sieve-settings.c --- a/src/lib-sieve/sieve-settings.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/sieve-settings.c Wed Sep 23 22:54:11 2015 +0200 @@ -8,7 +8,6 @@ #include "sieve-error.h" #include "sieve-settings.h" -#include #include // FIXME: add to dovecot diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/sieve.c --- a/src/lib-sieve/sieve.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/sieve.c Wed Sep 23 22:54:11 2015 +0200 @@ -32,7 +32,6 @@ #include #include #include -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/storage/file/sieve-file-script-sequence.c --- a/src/lib-sieve/storage/file/sieve-file-script-sequence.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-script-sequence.c Wed Sep 23 22:54:11 2015 +0200 @@ -12,7 +12,6 @@ #include "sieve-file-storage.h" #include -#include #include /* diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/storage/file/sieve-file-script.c --- a/src/lib-sieve/storage/file/sieve-file-script.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-script.c Wed Sep 23 22:54:11 2015 +0200 @@ -14,7 +14,6 @@ #include "sieve-file-storage.h" #include -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/storage/file/sieve-file-storage-list.c --- a/src/lib-sieve/storage/file/sieve-file-storage-list.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-storage-list.c Wed Sep 23 22:54:11 2015 +0200 @@ -10,7 +10,6 @@ #include "sieve-file-storage.h" #include -#include #include struct sieve_file_list_context { diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/storage/file/sieve-file-storage-quota.c --- a/src/lib-sieve/storage/file/sieve-file-storage-quota.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-storage-quota.c Wed Sep 23 22:54:11 2015 +0200 @@ -10,7 +10,6 @@ #include "sieve-file-storage.h" #include -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/storage/file/sieve-file-storage-save.c --- a/src/lib-sieve/storage/file/sieve-file-storage-save.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-storage-save.c Wed Sep 23 22:54:11 2015 +0200 @@ -14,7 +14,6 @@ #include "sieve-file-storage.h" #include -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/storage/file/sieve-file-storage.c --- a/src/lib-sieve/storage/file/sieve-file-storage.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-storage.c Wed Sep 23 22:54:11 2015 +0200 @@ -21,7 +21,6 @@ #include "sieve-file-storage.h" #include -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/storage/ldap/sieve-ldap-db.c --- a/src/lib-sieve/storage/ldap/sieve-ldap-db.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/storage/ldap/sieve-ldap-db.c Wed Sep 23 22:54:11 2015 +0200 @@ -24,7 +24,6 @@ #include "istream.h" #include -#include #include struct db_ldap_result { diff -r c1e233848ecd -r 7038e943e853 src/lib-sieve/util/realpath.c --- a/src/lib-sieve/util/realpath.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/lib-sieve/util/realpath.c Wed Sep 23 22:54:11 2015 +0200 @@ -5,7 +5,6 @@ #include "realpath.h" -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/managesieve-login/client-authenticate.c --- a/src/managesieve-login/client-authenticate.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/managesieve-login/client-authenticate.c Wed Sep 23 22:54:11 2015 +0200 @@ -19,7 +19,6 @@ #include "client-authenticate.h" #include "managesieve-proxy.h" -#include const char *client_authenticate_get_capabilities (struct client *client) diff -r c1e233848ecd -r 7038e943e853 src/managesieve-login/client.c --- a/src/managesieve-login/client.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/managesieve-login/client.c Wed Sep 23 22:54:11 2015 +0200 @@ -23,7 +23,6 @@ #include "managesieve-login-settings.h" #include "managesieve-proxy.h" -#include /* Disconnect client when it sends too many bad commands */ #define CLIENT_MAX_BAD_COMMANDS 3 diff -r c1e233848ecd -r 7038e943e853 src/managesieve-login/managesieve-login-settings-plugin.c --- a/src/managesieve-login/managesieve-login-settings-plugin.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/managesieve-login/managesieve-login-settings-plugin.c Wed Sep 23 22:54:11 2015 +0200 @@ -16,7 +16,6 @@ #include #include #include -#include typedef enum { CAP_SIEVE, CAP_NOTIFY } capability_type_t; diff -r c1e233848ecd -r 7038e943e853 src/managesieve/cmd-noop.c --- a/src/managesieve/cmd-noop.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/managesieve/cmd-noop.c Wed Sep 23 22:54:11 2015 +0200 @@ -9,7 +9,6 @@ #include "managesieve-common.h" #include "managesieve-commands.h" -#include bool cmd_noop(struct client_command_context *cmd) { diff -r c1e233848ecd -r 7038e943e853 src/managesieve/main.c --- a/src/managesieve/main.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/managesieve/main.c Wed Sep 23 22:54:11 2015 +0200 @@ -24,7 +24,6 @@ #include "managesieve-capabilities.h" #include -#include #include #define IS_STANDALONE() \ diff -r c1e233848ecd -r 7038e943e853 src/managesieve/managesieve-client.c --- a/src/managesieve/managesieve-client.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/managesieve/managesieve-client.c Wed Sep 23 22:54:11 2015 +0200 @@ -23,7 +23,6 @@ #include "managesieve-commands.h" #include "managesieve-client.h" -#include #include extern struct mail_storage_callbacks mail_storage_callbacks; diff -r c1e233848ecd -r 7038e943e853 src/managesieve/managesieve-commands.c --- a/src/managesieve/managesieve-commands.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/managesieve/managesieve-commands.c Wed Sep 23 22:54:11 2015 +0200 @@ -7,7 +7,6 @@ #include "managesieve-common.h" #include "managesieve-commands.h" -#include /* Might want to combine this somewhere in a commands-common.c * to avoid duplicate code diff -r c1e233848ecd -r 7038e943e853 src/managesieve/managesieve-settings.c --- a/src/managesieve/managesieve-settings.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/managesieve/managesieve-settings.c Wed Sep 23 22:54:11 2015 +0200 @@ -12,7 +12,6 @@ #include "managesieve-settings.h" #include -#include #include static bool managesieve_settings_verify(void *_set, pool_t pool, diff -r c1e233848ecd -r 7038e943e853 src/plugins/lda-sieve/lda-sieve-plugin.c --- a/src/plugins/lda-sieve/lda-sieve-plugin.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/plugins/lda-sieve/lda-sieve-plugin.c Wed Sep 23 22:54:11 2015 +0200 @@ -20,7 +20,6 @@ #include "lda-sieve-log.h" #include "lda-sieve-plugin.h" -#include #include #include diff -r c1e233848ecd -r 7038e943e853 src/plugins/sieve-extprograms/sieve-extprograms-common.c --- a/src/plugins/sieve-extprograms/sieve-extprograms-common.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/plugins/sieve-extprograms/sieve-extprograms-common.c Wed Sep 23 22:54:11 2015 +0200 @@ -36,7 +36,6 @@ #include "sieve-extprograms-common.h" -#include #include #include #include diff -r c1e233848ecd -r 7038e943e853 src/sieve-tools/sieve-dump.c --- a/src/sieve-tools/sieve-dump.c Wed Sep 23 21:57:31 2015 +0200 +++ b/src/sieve-tools/sieve-dump.c Wed Sep 23 22:54:11 2015 +0200 From dovecot at dovecot.org Wed Sep 23 22:05:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 22:05:22 +0000 Subject: dovecot-2.2: lib-sql: sqlpool needs to change transaction querie... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/59e4fcaa0f76 changeset: 19202:59e4fcaa0f76 user: Timo Sirainen date: Thu Sep 24 01:00:45 2015 +0300 description: lib-sql: sqlpool needs to change transaction queries' transaction pointer Otherwise query->trans points to the sqlpool's transaction, while our backends want it to point to the backend's transaction. diffstat: src/lib-sql/driver-sqlpool.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (19 lines): diff -r b8d2cbd6583b -r 59e4fcaa0f76 src/lib-sql/driver-sqlpool.c --- a/src/lib-sql/driver-sqlpool.c Wed Sep 23 23:01:27 2015 +0300 +++ b/src/lib-sql/driver-sqlpool.c Thu Sep 24 01:00:45 2015 +0300 @@ -126,12 +126,15 @@ struct sql_db *conndb) { struct sql_transaction_context *conn_trans; + struct sql_transaction_query *query; conn_trans = sql_transaction_begin(conndb); /* backend will use our queries list (we might still append more queries to the list) */ conn_trans->head = trans->ctx.head; conn_trans->tail = trans->ctx.tail; + for (query = conn_trans->head; query != NULL; query = query->next) + query->trans = conn_trans; return conn_trans; } From dovecot at dovecot.org Wed Sep 23 22:05:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 22:05:24 +0000 Subject: dovecot-2.2: pgsql: Fixed committing multiple transactions. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9ceeb1a5c492 changeset: 19203:9ceeb1a5c492 user: Timo Sirainen date: Thu Sep 24 01:02:32 2015 +0300 description: pgsql: Fixed committing multiple transactions. This code is quite horrible and could use a larger redesign. But it appears to be working for now.. diffstat: src/lib-sql/driver-pgsql.c | 121 +++++++++++++++++++++++++++++--------------- 1 files changed, 80 insertions(+), 41 deletions(-) diffs (236 lines): diff -r 59e4fcaa0f76 -r 9ceeb1a5c492 src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Thu Sep 24 01:00:45 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Thu Sep 24 01:02:32 2015 +0300 @@ -29,6 +29,9 @@ struct ioloop *ioloop, *orig_ioloop; struct sql_result *sync_result; + bool (*next_callback)(void *); + void *next_context; + char *error; const char *connect_state; @@ -68,8 +71,6 @@ pool_t query_pool; const char *error; - unsigned int begin_succeeded:1; - unsigned int begin_failed:1; unsigned int failed:1; }; @@ -77,6 +78,9 @@ extern const struct sql_result driver_pgsql_result; static void result_finish(struct pgsql_result *result); +static void +transaction_update_callback(struct sql_result *result, + struct sql_transaction_query *query); static const char *pgsql_prefix(struct pgsql_db *db) { @@ -97,6 +101,19 @@ io_loop_set_current(db->ioloop); } +static bool driver_pgsql_next_callback(struct pgsql_db *db) +{ + bool (*next_callback)(void *) = db->next_callback; + void *next_context = db->next_context; + + if (next_callback == NULL) + return FALSE; + + db->next_callback = NULL; + db->next_context = NULL; + return next_callback(next_context); +} + static void driver_pgsql_stop_io(struct pgsql_db *db) { if (db->io != NULL) { @@ -124,6 +141,7 @@ /* running a sync query, stop it */ io_loop_stop(db->ioloop); } + driver_pgsql_next_callback(db); } static const char *last_error(struct pgsql_db *db) @@ -293,7 +311,7 @@ if (db->fatal_error) driver_pgsql_close(db); - else + else if (!driver_pgsql_next_callback(db)) driver_pgsql_set_state(db, SQL_DB_STATE_IDLE); } @@ -838,7 +856,6 @@ ctx = i_new(struct pgsql_transaction_context, 1); ctx->ctx.db = db; - ctx->refcount = 1; /* we need to be able to handle multiple open transactions, so at least for now just keep them in memory until commit time. */ ctx->query_pool = pool_alloconly_create("pgsql transaction", 1024); @@ -846,31 +863,13 @@ } static void -driver_pgsql_transaction_unref(struct pgsql_transaction_context *ctx) +driver_pgsql_transaction_free(struct pgsql_transaction_context *ctx) { - i_assert(ctx->refcount > 0); - if (--ctx->refcount > 0) - return; - pool_unref(&ctx->query_pool); i_free(ctx); } static void -transaction_begin_callback(struct sql_result *result, - struct pgsql_transaction_context *ctx) -{ - if (sql_result_next_row(result) < 0) { - ctx->begin_failed = TRUE; - ctx->failed = TRUE; - ctx->error = sql_result_get_error(result); - } else { - ctx->begin_succeeded = TRUE; - } - driver_pgsql_transaction_unref(ctx); -} - -static void transaction_commit_callback(struct sql_result *result, struct pgsql_transaction_context *ctx) { @@ -878,7 +877,50 @@ ctx->callback(sql_result_get_error(result), ctx->context); else ctx->callback(NULL, ctx->context); - driver_pgsql_transaction_unref(ctx); + driver_pgsql_transaction_free(ctx); +} + +static bool transaction_send_next(void *context) +{ + struct pgsql_transaction_context *ctx = context; + + i_assert(!ctx->failed); + + if (ctx->ctx.db->state == SQL_DB_STATE_BUSY) { + /* kludgy.. */ + ctx->ctx.db->state = SQL_DB_STATE_IDLE; + } else if (!SQL_DB_IS_READY(ctx->ctx.db)) { + ctx->callback("Not connected", ctx->context); + return FALSE; + } + + if (ctx->ctx.head != NULL) { + sql_query(ctx->ctx.db, ctx->ctx.head->query, + transaction_update_callback, ctx->ctx.head); + ctx->ctx.head = ctx->ctx.head->next; + } else { + sql_query(ctx->ctx.db, "COMMIT", + transaction_commit_callback, ctx); + } + return TRUE; +} + +static void +transaction_begin_callback(struct sql_result *result, + struct pgsql_transaction_context *ctx) +{ + struct pgsql_db *db = (struct pgsql_db *)result->db; + + i_assert(result->db == ctx->ctx.db); + + if (sql_result_next_row(result) < 0) { + ctx->callback(sql_result_get_error(result), ctx->context); + driver_pgsql_transaction_free(ctx); + return; + } + i_assert(db->next_callback == NULL); + db->next_callback = transaction_send_next; + db->next_context = ctx; } static void @@ -887,18 +929,24 @@ { struct pgsql_transaction_context *ctx = (struct pgsql_transaction_context *)query->trans; + struct pgsql_db *db = (struct pgsql_db *)result->db; if (sql_result_next_row(result) < 0) { - ctx->failed = TRUE; - ctx->error = sql_result_get_error(result); - } else if (query->affected_rows != NULL) { + ctx->callback(sql_result_get_error(result), ctx->context); + driver_pgsql_transaction_free(ctx); + return; + } + + if (query->affected_rows != NULL) { struct pgsql_result *pg_result = (struct pgsql_result *)result; if (str_to_uint(PQcmdTuples(pg_result->pgres), query->affected_rows) < 0) i_unreached(); } - driver_pgsql_transaction_unref(ctx); + i_assert(db->next_callback == NULL); + db->next_callback = transaction_send_next; + db->next_context = ctx; } static void @@ -913,22 +961,15 @@ if (ctx->failed || _ctx->head == NULL) { callback(ctx->failed ? ctx->error : NULL, context); - driver_pgsql_transaction_unref(ctx); + driver_pgsql_transaction_free(ctx); } else if (_ctx->head->next == NULL) { /* just a single query, send it */ sql_query(_ctx->db, _ctx->head->query, transaction_commit_callback, ctx); } else { /* multiple queries, use a transaction */ - ctx->refcount++; + i_assert(_ctx->db->v.query == driver_pgsql_query); sql_query(_ctx->db, "BEGIN", transaction_begin_callback, ctx); - while (_ctx->head != NULL) { - ctx->refcount++; - sql_query(_ctx->db, _ctx->head->query, - transaction_update_callback, _ctx->head); - _ctx->head = _ctx->head->next; - } - sql_query(_ctx->db, "COMMIT", transaction_commit_callback, ctx); } } @@ -1042,8 +1083,7 @@ } } - i_assert(ctx->refcount == 1); - driver_pgsql_transaction_unref(ctx); + driver_pgsql_transaction_free(ctx); return *error_r == NULL ? 0 : -1; } @@ -1053,8 +1093,7 @@ struct pgsql_transaction_context *ctx = (struct pgsql_transaction_context *)_ctx; - i_assert(ctx->refcount == 1); - driver_pgsql_transaction_unref(ctx); + driver_pgsql_transaction_free(ctx); } static void From dovecot at dovecot.org Wed Sep 23 22:05:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 22:05:24 +0000 Subject: dovecot-2.2: pgsql: Fixed committing a single sql_update_get_row... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/beb3b8496b7d changeset: 19204:beb3b8496b7d user: Timo Sirainen date: Thu Sep 24 01:04:00 2015 +0300 description: pgsql: Fixed committing a single sql_update_get_rows() transaction. The rows weren't being set. diffstat: src/lib-sql/driver-pgsql.c | 26 +++++++++++++++++++++++++- 1 files changed, 25 insertions(+), 1 deletions(-) diffs (43 lines): diff -r 9ceeb1a5c492 -r beb3b8496b7d src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Thu Sep 24 01:02:32 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Thu Sep 24 01:04:00 2015 +0300 @@ -950,6 +950,30 @@ } static void +transaction_trans_query_callback(struct sql_result *result, + struct sql_transaction_query *query) +{ + struct pgsql_transaction_context *ctx = + (struct pgsql_transaction_context *)query->trans; + + if (sql_result_next_row(result) < 0) { + ctx->callback(sql_result_get_error(result), ctx->context); + driver_pgsql_transaction_free(ctx); + return; + } + + if (query->affected_rows != NULL) { + struct pgsql_result *pg_result = (struct pgsql_result *)result; + + if (str_to_uint(PQcmdTuples(pg_result->pgres), + query->affected_rows) < 0) + i_unreached(); + } + ctx->callback(NULL, ctx->context); + driver_pgsql_transaction_free(ctx); +} + +static void driver_pgsql_transaction_commit(struct sql_transaction_context *_ctx, sql_commit_callback_t *callback, void *context) { @@ -965,7 +989,7 @@ } else if (_ctx->head->next == NULL) { /* just a single query, send it */ sql_query(_ctx->db, _ctx->head->query, - transaction_commit_callback, ctx); + transaction_trans_query_callback, _ctx->head); } else { /* multiple queries, use a transaction */ i_assert(_ctx->db->v.query == driver_pgsql_query); From dovecot at dovecot.org Wed Sep 23 22:22:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Sep 2015 22:22:24 +0000 Subject: dovecot-2.2: master: stdlib.h wasn't supposed to be removed from... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c4b9e6606cbf changeset: 19205:c4b9e6606cbf user: Timo Sirainen date: Thu Sep 24 01:13:36 2015 +0300 description: master: stdlib.h wasn't supposed to be removed from systemd's sd-daemon.c It's not using our lib.h diffstat: src/master/sd-daemon.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r beb3b8496b7d -r c4b9e6606cbf src/master/sd-daemon.c --- a/src/master/sd-daemon.c Thu Sep 24 01:04:00 2015 +0300 +++ b/src/master/sd-daemon.c Thu Sep 24 01:13:36 2015 +0300 @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include From dovecot at dovecot.org Thu Sep 24 07:12:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 07:12:53 +0000 Subject: dovecot-2.2: push-notification: Added missing newlines to end of... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa23e99981d7 changeset: 19206:aa23e99981d7 user: Timo Sirainen date: Thu Sep 24 10:11:29 2015 +0300 description: push-notification: Added missing newlines to end of files to prevent compiler warnings. diffstat: src/plugins/push-notification/push-notification-driver-dlog.c | 2 +- src/plugins/push-notification/push-notification-drivers.c | 2 +- src/plugins/push-notification/push-notification-event-flagsset.c | 2 +- src/plugins/push-notification/push-notification-event-messageappend.c | 2 +- src/plugins/push-notification/push-notification-event-messageexpunge.c | 2 +- src/plugins/push-notification/push-notification-event-messageread.h | 2 +- src/plugins/push-notification/push-notification-events-rfc5423.c | 2 +- src/plugins/push-notification/push-notification-events.c | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diffs (80 lines): diff -r c4b9e6606cbf -r aa23e99981d7 src/plugins/push-notification/push-notification-driver-dlog.c --- a/src/plugins/push-notification/push-notification-driver-dlog.c Thu Sep 24 01:13:36 2015 +0300 +++ b/src/plugins/push-notification/push-notification-driver-dlog.c Thu Sep 24 10:11:29 2015 +0300 @@ -113,4 +113,4 @@ .deinit = push_notification_driver_dlog_deinit, .cleanup = push_notification_driver_dlog_cleanup } -}; \ No newline at end of file +}; diff -r c4b9e6606cbf -r aa23e99981d7 src/plugins/push-notification/push-notification-drivers.c --- a/src/plugins/push-notification/push-notification-drivers.c Thu Sep 24 01:13:36 2015 +0300 +++ b/src/plugins/push-notification/push-notification-drivers.c Thu Sep 24 10:11:29 2015 +0300 @@ -186,4 +186,4 @@ array_free(&push_notification_drivers); } } -} \ No newline at end of file +} diff -r c4b9e6606cbf -r aa23e99981d7 src/plugins/push-notification/push-notification-event-flagsset.c --- a/src/plugins/push-notification/push-notification-event-flagsset.c Thu Sep 24 01:13:36 2015 +0300 +++ b/src/plugins/push-notification/push-notification-event-flagsset.c Thu Sep 24 10:11:29 2015 +0300 @@ -171,4 +171,4 @@ .flagchange = push_notification_event_flagsset_flags_event, .keywordchange = push_notification_event_flagsset_keywords_event } -}; \ No newline at end of file +}; diff -r c4b9e6606cbf -r aa23e99981d7 src/plugins/push-notification/push-notification-event-messageappend.c --- a/src/plugins/push-notification/push-notification-event-messageappend.c Thu Sep 24 01:13:36 2015 +0300 +++ b/src/plugins/push-notification/push-notification-event-messageappend.c Thu Sep 24 10:11:29 2015 +0300 @@ -112,4 +112,4 @@ .msg_triggers = { .append = push_notification_event_messageappend_event } -}; \ No newline at end of file +}; diff -r c4b9e6606cbf -r aa23e99981d7 src/plugins/push-notification/push-notification-event-messageexpunge.c --- a/src/plugins/push-notification/push-notification-event-messageexpunge.c Thu Sep 24 01:13:36 2015 +0300 +++ b/src/plugins/push-notification/push-notification-event-messageexpunge.c Thu Sep 24 10:11:29 2015 +0300 @@ -52,4 +52,4 @@ .msg_triggers = { .expunge = push_notification_event_messageexpunge_event } -}; \ No newline at end of file +}; diff -r c4b9e6606cbf -r aa23e99981d7 src/plugins/push-notification/push-notification-event-messageread.h --- a/src/plugins/push-notification/push-notification-event-messageread.h Thu Sep 24 01:13:36 2015 +0300 +++ b/src/plugins/push-notification/push-notification-event-messageread.h Thu Sep 24 10:11:29 2015 +0300 @@ -10,4 +10,4 @@ }; -#endif /* PUSH_NOTIFICATION_EVENT_MESSAGEREAD_H */ \ No newline at end of file +#endif /* PUSH_NOTIFICATION_EVENT_MESSAGEREAD_H */ diff -r c4b9e6606cbf -r aa23e99981d7 src/plugins/push-notification/push-notification-events-rfc5423.c --- a/src/plugins/push-notification/push-notification-events-rfc5423.c Thu Sep 24 01:13:36 2015 +0300 +++ b/src/plugins/push-notification/push-notification-events-rfc5423.c Thu Sep 24 10:11:29 2015 +0300 @@ -43,4 +43,4 @@ push_notification_event_register(&push_notification_event_messagenew); push_notification_event_register(&push_notification_event_messageread); push_notification_event_register(&push_notification_event_messagetrash); -} \ No newline at end of file +} diff -r c4b9e6606cbf -r aa23e99981d7 src/plugins/push-notification/push-notification-events.c --- a/src/plugins/push-notification/push-notification-events.c Thu Sep 24 01:13:36 2015 +0300 +++ b/src/plugins/push-notification/push-notification-events.c Thu Sep 24 10:11:29 2015 +0300 @@ -102,4 +102,4 @@ array_free(&push_notification_events); } } -} \ No newline at end of file +} From dovecot at dovecot.org Thu Sep 24 09:33:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 09:33:45 +0000 Subject: dovecot-2.2: Don't try to pass through void function's return value Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ae88b2978e0 changeset: 19207:0ae88b2978e0 user: Timo Sirainen date: Thu Sep 24 12:32:24 2015 +0300 description: Don't try to pass through void function's return value I wish gcc or clang complained about these. diffstat: src/lib-dict/dict.c | 2 +- src/lib-settings/settings-parser.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r aa23e99981d7 -r 0ae88b2978e0 src/lib-dict/dict.c --- a/src/lib-dict/dict.c Thu Sep 24 10:11:29 2015 +0300 +++ b/src/lib-dict/dict.c Thu Sep 24 12:32:24 2015 +0300 @@ -133,7 +133,7 @@ callback(&result, context); return; } - return dict->v.lookup_async(dict, key, callback, context); + dict->v.lookup_async(dict, key, callback, context); } struct dict_iterate_context * diff -r aa23e99981d7 -r 0ae88b2978e0 src/lib-settings/settings-parser.c --- a/src/lib-settings/settings-parser.c Thu Sep 24 10:11:29 2015 +0300 +++ b/src/lib-settings/settings-parser.c Thu Sep 24 12:32:24 2015 +0300 @@ -1296,7 +1296,7 @@ void *set, pool_t pool, const struct var_expand_table *table) { - return settings_var_expand_with_funcs(info, set, pool, table, NULL, NULL); + settings_var_expand_with_funcs(info, set, pool, table, NULL, NULL); } void settings_var_expand_with_funcs(const struct setting_parser_info *info, From dovecot at dovecot.org Thu Sep 24 12:21:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 12:21:13 +0000 Subject: dovecot-2.2: lib: Fixed __builtin_object_size() usage in buffer.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/81db26c26d6d changeset: 19208:81db26c26d6d user: Timo Sirainen date: Thu Sep 24 15:19:53 2015 +0300 description: lib: Fixed __builtin_object_size() usage in buffer.h With type=3 size for unknown objects is set to 0, which causes a compiler failure. With type=1 it's (size_t)-1, which passes the check. diffstat: src/lib/buffer.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 0ae88b2978e0 -r 81db26c26d6d src/lib/buffer.h --- a/src/lib/buffer.h Thu Sep 24 12:32:24 2015 +0300 +++ b/src/lib/buffer.h Thu Sep 24 15:19:53 2015 +0300 @@ -20,10 +20,10 @@ const void *data, size_t size); #if defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__) > 401 #define buffer_create_from_data(b,d,s) ({ \ - (void)COMPILE_ERROR_IF_TRUE(__builtin_object_size((d),3) < ((s)?(s):1)); \ + (void)COMPILE_ERROR_IF_TRUE(__builtin_object_size((d),1) < ((s)?(s):1)); \ buffer_create_from_data((b), (d), (s)); }) #define buffer_create_from_const_data(b,d,s) ({ \ - (void)COMPILE_ERROR_IF_TRUE(__builtin_object_size((d),3) < ((s)?(s):1)); \ + (void)COMPILE_ERROR_IF_TRUE(__builtin_object_size((d),1) < ((s)?(s):1)); \ buffer_create_from_const_data((b), (d), (s)); }) #endif /* Creates a dynamically growing buffer. Whenever write would exceed the From dovecot at dovecot.org Thu Sep 24 13:18:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 13:18:08 +0000 Subject: dovecot-2.2: doveconf: Unless -P parameter is used, hide all key... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a706bdec0200 changeset: 19209:a706bdec0200 user: Timo Sirainen date: Thu Sep 24 16:16:43 2015 +0300 description: doveconf: Unless -P parameter is used, hide all keys ending with "_password" A little bit ugly way, but implementing a proper SET_PASSWORD type was getting too difficult with the current config code. Then again as a bonus this also hides plugin { *_password } settings, if there are any, which wouldn't have been possible with SET_PASSWORD. diffstat: src/config/doveconf.c | 34 +++++++++++++++++++++------------- 1 files changed, 21 insertions(+), 13 deletions(-) diffs (131 lines): diff -r 81db26c26d6d -r a706bdec0200 src/config/doveconf.c --- a/src/config/doveconf.c Thu Sep 24 15:19:53 2015 +0300 +++ b/src/config/doveconf.c Thu Sep 24 16:16:43 2015 +0300 @@ -169,7 +169,7 @@ static int ATTR_NULL(4) config_dump_human_output(struct config_dump_human_context *ctx, struct ostream *output, unsigned int indent, - const char *setting_name_filter) + const char *setting_name_filter, bool hide_passwords) { ARRAY_TYPE(const_string) prefixes_arr; ARRAY_TYPE(prefix_stack) prefix_stack; @@ -302,7 +302,10 @@ value = strchr(key, '='); o_stream_nsend(output, key, value-key); o_stream_nsend_str(output, " = "); - if (!value_need_quote(value+1)) + if (hide_passwords && + value-key > 9 && strncmp(value-9, "_password", 9) == 0) { + o_stream_nsend_str(output, " # hidden, use -P to show it"); + } else if (!value_need_quote(value+1)) o_stream_nsend_str(output, value+1); else { o_stream_nsend(output, "\"", 1); @@ -393,7 +396,7 @@ static int config_dump_human_sections(struct ostream *output, const struct config_filter *filter, - const char *const *modules) + const char *const *modules, bool hide_passwords) { struct config_filter_parser *const *filters; static struct config_dump_human_context *ctx; @@ -412,7 +415,7 @@ indent = config_dump_filter_begin(ctx->list_prefix, &(*filters)->filter); config_export_parsers(ctx->export_ctx, (*filters)->parsers); - if (config_dump_human_output(ctx, output, indent, NULL) < 0) + if (config_dump_human_output(ctx, output, indent, NULL, hide_passwords) < 0) ret = -1; if (ctx->list_prefix_sent) config_dump_filter_end(output, indent); @@ -423,7 +426,8 @@ static int ATTR_NULL(4) config_dump_human(const struct config_filter *filter, const char *const *modules, - enum config_dump_scope scope, const char *setting_name_filter) + enum config_dump_scope scope, const char *setting_name_filter, + bool hide_passwords) { static struct config_dump_human_context *ctx; struct ostream *output; @@ -435,11 +439,11 @@ ctx = config_dump_human_init(modules, scope, TRUE); config_export_by_filter(ctx->export_ctx, filter); - ret = config_dump_human_output(ctx, output, 0, setting_name_filter); + ret = config_dump_human_output(ctx, output, 0, setting_name_filter, hide_passwords); config_dump_human_deinit(ctx); if (setting_name_filter == NULL) - ret = config_dump_human_sections(output, filter, modules); + ret = config_dump_human_sections(output, filter, modules, hide_passwords); o_stream_uncork(output); o_stream_destroy(&output); @@ -448,7 +452,8 @@ static int config_dump_one(const struct config_filter *filter, bool hide_key, - enum config_dump_scope scope, const char *setting_name_filter) + enum config_dump_scope scope, const char *setting_name_filter, + bool hide_passwords) { static struct config_dump_human_context *ctx; const char *const *str; @@ -481,7 +486,7 @@ config_dump_human_deinit(ctx); if (dump_section) - (void)config_dump_human(filter, NULL, scope, setting_name_filter); + (void)config_dump_human(filter, NULL, scope, setting_name_filter, hide_passwords); return 0; } @@ -706,7 +711,7 @@ bool config_path_specified, expand_vars = FALSE, hide_key = FALSE; bool parse_full_config = FALSE, simple_output = FALSE; bool dump_defaults = FALSE, host_verify = FALSE; - bool print_plugin_banner = FALSE; + bool print_plugin_banner = FALSE, hide_passwords = TRUE; if (getenv("USE_SYSEXITS") != NULL) { /* we're coming from (e.g.) LDA */ @@ -716,7 +721,7 @@ memset(&filter, 0, sizeof(filter)); master_service = master_service_init("config", MASTER_SERVICE_FLAG_STANDALONE, - &argc, &argv, "adf:hHm:nNpexS"); + &argc, &argv, "adf:hHm:nNpPexS"); orig_config_path = master_service_get_config_path(master_service); i_set_failure_prefix("doveconf: "); @@ -754,6 +759,9 @@ case 'p': parse_full_config = TRUE; break; + case 'P': + hide_passwords = FALSE; + break; case 'S': simple_output = TRUE; break; @@ -834,7 +842,7 @@ ret = 0; for (i = 0; setting_name_filters[i] != NULL; i++) { if (config_dump_one(&filter, hide_key, scope, - setting_name_filters[i]) < 0) + setting_name_filters[i], hide_passwords) < 0) ret2 = -1; } } else if (exec_args == NULL) { @@ -848,7 +856,7 @@ if (scope == CONFIG_DUMP_SCOPE_ALL) printf("# NOTE: Send doveconf -n output instead when asking for help.\n"); fflush(stdout); - ret2 = config_dump_human(&filter, wanted_modules, scope, NULL); + ret2 = config_dump_human(&filter, wanted_modules, scope, NULL, hide_passwords); } else { struct config_export_context *ctx; From dovecot at dovecot.org Thu Sep 24 13:31:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 13:31:46 +0000 Subject: dovecot-2.2: indexer: Use a slightly different session ID than t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e94781b25c83 changeset: 19210:e94781b25c83 user: Timo Sirainen date: Thu Sep 24 16:30:23 2015 +0300 description: indexer: Use a slightly different session ID than the session that requested indexing. Required to avoid "duplicate session ID" errors in stats process. Alternative would have been to change the stats protocol a bit and always send the service name along with session ID, and use those two together as the primary key. Maybe this could be done in future if we get more of these kind of processes. diffstat: src/indexer/master-connection.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (16 lines): diff -r a706bdec0200 -r e94781b25c83 src/indexer/master-connection.c --- a/src/indexer/master-connection.c Thu Sep 24 16:16:43 2015 +0300 +++ b/src/indexer/master-connection.c Thu Sep 24 16:30:23 2015 +0300 @@ -212,7 +212,11 @@ input.module = "mail"; input.service = "indexer-worker"; input.username = args[0]; - input.session_id = args[2][0] == '\0' ? NULL : args[2]; + /* if session-id is given, add -idx suffix to it so that stats process + doesn't complain about duplicates. also it's nicer to keep the stats + separate for the indexer and the caller. */ + input.session_id = args[2][0] == '\0' ? NULL : + t_strconcat(args[2], "-idx", NULL); if (mail_storage_service_lookup_next(conn->storage_service, &input, &service_user, &user, &error) <= 0) { From dovecot at dovecot.org Thu Sep 24 13:42:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 13:42:36 +0000 Subject: dovecot-2.2: pgsql: Avoid crash in multi-command transaction if ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/68dd32223db7 changeset: 19211:68dd32223db7 user: Timo Sirainen date: Thu Sep 24 16:41:10 2015 +0300 description: pgsql: Avoid crash in multi-command transaction if one of the queries couldn't be sent to server. diffstat: src/lib-sql/driver-pgsql.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (17 lines): diff -r e94781b25c83 -r 68dd32223db7 src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Thu Sep 24 16:30:23 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Thu Sep 24 16:41:10 2015 +0300 @@ -895,9 +895,11 @@ } if (ctx->ctx.head != NULL) { - sql_query(ctx->ctx.db, ctx->ctx.head->query, - transaction_update_callback, ctx->ctx.head); + struct sql_transaction_query *query = ctx->ctx.head; + ctx->ctx.head = ctx->ctx.head->next; + sql_query(ctx->ctx.db, query->query, + transaction_update_callback, query); } else { sql_query(ctx->ctx.db, "COMMIT", transaction_commit_callback, ctx); From dovecot at dovecot.org Thu Sep 24 15:44:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 15:44:20 +0000 Subject: dovecot-2.2: quota: Avoid triggering quota recalc in dict transa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/168ca81df3fe changeset: 19212:168ca81df3fe user: Timo Sirainen date: Thu Sep 24 18:41:48 2015 +0300 description: quota: Avoid triggering quota recalc in dict transaction commit callback. This mainly causes trouble if it again does more dict accesses. diffstat: src/plugins/quota/quota-dict.c | 20 ++++++++++++++++++-- 1 files changed, 18 insertions(+), 2 deletions(-) diffs (64 lines): diff -r 68dd32223db7 -r 168ca81df3fe src/plugins/quota/quota-dict.c --- a/src/plugins/quota/quota-dict.c Thu Sep 24 16:41:10 2015 +0300 +++ b/src/plugins/quota/quota-dict.c Thu Sep 24 18:41:48 2015 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2005-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ioloop.h" #include "str.h" #include "dict.h" #include "mail-user.h" @@ -15,6 +16,7 @@ struct dict_quota_root { struct quota_root root; struct dict *dict; + struct timeout *to_update; }; extern struct quota_backend quota_backend_dict; @@ -94,6 +96,8 @@ { struct dict_quota_root *root = (struct dict_quota_root *)_root; + i_assert(root->to_update == NULL); + if (root->dict != NULL) dict_deinit(&root->dict); i_free(root); @@ -175,14 +179,22 @@ return ret; } +static void dict_quota_recalc_timeout(struct dict_quota_root *root) +{ + uint64_t value; + + timeout_remove(&root->to_update); + (void)dict_quota_count(root, TRUE, &value); +} + static void dict_quota_update_callback(int ret, void *context) { struct dict_quota_root *root = context; - uint64_t value; if (ret == 0) { /* row doesn't exist, need to recalculate it */ - (void)dict_quota_count(root, TRUE, &value); + if (root->to_update == NULL) + root->to_update = timeout_add_short(0, dict_quota_recalc_timeout, root); } else if (ret < 0) { i_error("dict quota: Quota update failed, it's now desynced"); } @@ -220,6 +232,10 @@ struct dict_quota_root *root = (struct dict_quota_root *)_root; (void)dict_wait(root->dict); + if (root->to_update != NULL) { + dict_quota_recalc_timeout(root); + (void)dict_wait(root->dict); + } } struct quota_backend quota_backend_dict = { From dovecot at dovecot.org Thu Sep 24 15:44:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 15:44:21 +0000 Subject: dovecot-2.2: dict-client: Verify that the correct transaction ID... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3599f7b9ea09 changeset: 19213:3599f7b9ea09 user: Timo Sirainen date: Thu Sep 24 18:42:58 2015 +0300 description: dict-client: Verify that the correct transaction ID is returned for commit reply. diffstat: src/lib-dict/dict-client.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diffs (24 lines): diff -r 168ca81df3fe -r 3599f7b9ea09 src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Thu Sep 24 18:41:48 2015 +0300 +++ b/src/lib-dict/dict-client.c Thu Sep 24 18:42:58 2015 +0300 @@ -758,6 +758,7 @@ struct client_dict_transaction_context *ctx = (struct client_dict_transaction_context *)_ctx; struct client_dict *dict = (struct client_dict *)_ctx->dict; + unsigned int id; int ret = ctx->failed ? -1 : 1; ctx->committed = TRUE; @@ -799,6 +800,12 @@ ret = -1; break; } + if (str_to_uint(line+1, &id) < 0 || ctx->id != id) { + i_error("dict-client: Invalid commit reply, " + "expected id=%u: %s", ctx->id, line); + client_dict_disconnect(dict); + ret = -1; + } } } T_END; From dovecot at dovecot.org Thu Sep 24 16:08:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 16:08:32 +0000 Subject: dovecot-2.2: virtual plugin: Fixed assert-crash when closing a v... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4b5b8e6a4a32 changeset: 19214:4b5b8e6a4a32 user: Timo Sirainen date: Thu Sep 24 19:07:12 2015 +0300 description: virtual plugin: Fixed assert-crash when closing a virtual mailbox consisting of many real mailboxes. diffstat: src/plugins/virtual/virtual-storage.c | 22 +++++++++++++++++----- src/plugins/virtual/virtual-storage.h | 1 + 2 files changed, 18 insertions(+), 5 deletions(-) diffs (57 lines): diff -r 3599f7b9ea09 -r 4b5b8e6a4a32 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Thu Sep 24 18:42:58 2015 +0300 +++ b/src/plugins/virtual/virtual-storage.c Thu Sep 24 19:07:12 2015 +0300 @@ -325,12 +325,15 @@ void virtual_backend_box_opened(struct virtual_mailbox *mbox, struct virtual_backend_box *bbox) { + i_assert(!bbox->open_tracked); + /* the backend mailbox was already opened. if we didn't get here from virtual_backend_box_open() we may need to close a mailbox */ while (mbox->backends_open_count > mbox->storage->max_open_mailboxes && virtual_backend_box_close_any_except(mbox, bbox)) ; + bbox->open_tracked = TRUE; mbox->backends_open_count++; DLLIST2_APPEND_FULL(&mbox->open_backend_boxes_head, &mbox->open_backend_boxes_tail, bbox, @@ -367,12 +370,21 @@ mail_search_args_deinit(bbox->search_args); bbox->search_args_initialized = FALSE; } - i_assert(mbox->backends_open_count > 0); - mbox->backends_open_count--; + if (bbox->open_tracked) { + i_assert(mbox->backends_open_count > 0); + mbox->backends_open_count--; + bbox->open_tracked = FALSE; - DLLIST2_REMOVE_FULL(&mbox->open_backend_boxes_head, - &mbox->open_backend_boxes_tail, bbox, - prev_open, next_open); + DLLIST2_REMOVE_FULL(&mbox->open_backend_boxes_head, + &mbox->open_backend_boxes_tail, bbox, + prev_open, next_open); + } else { + /* mailbox can be leaked outside our code via + virtual_get_virtual_backend_boxes() and it could have + been opened there. FIXME: maybe we could hook into the + backend open/close functions to do the tracking and + auto-closing. */ + } mailbox_close(bbox->box); } diff -r 3599f7b9ea09 -r 4b5b8e6a4a32 src/plugins/virtual/virtual-storage.h --- a/src/plugins/virtual/virtual-storage.h Thu Sep 24 18:42:58 2015 +0300 +++ b/src/plugins/virtual/virtual-storage.h Thu Sep 24 19:07:12 2015 +0300 @@ -97,6 +97,7 @@ struct imap_match_glob *glob; struct mail_namespace *ns; + unsigned int open_tracked:1; unsigned int open_failed:1; unsigned int sync_seen:1; unsigned int wildcard:1; From dovecot at dovecot.org Thu Sep 24 16:28:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 16:28:32 +0000 Subject: dovecot-2.2: virtual plugin: Auto-close mailboxes by hooking int... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/055d1e2d894c changeset: 19215:055d1e2d894c user: Timo Sirainen date: Thu Sep 24 19:27:10 2015 +0300 description: virtual plugin: Auto-close mailboxes by hooking into backend's mailbox_open() This is a more reliable way of doing it. The earlier version had problems that all the backend mailboxes could be opened when doing a search on the virtual mailbox (for getting the GUID of the mailboxes). diffstat: src/plugins/virtual/virtual-config.c | 1 + src/plugins/virtual/virtual-plugin.c | 6 +++++ src/plugins/virtual/virtual-storage.c | 38 ++++++++++++++++------------------ src/plugins/virtual/virtual-storage.h | 7 ++++- src/plugins/virtual/virtual-sync.c | 9 ++----- 5 files changed, 33 insertions(+), 28 deletions(-) diffs (159 lines): diff -r 4b5b8e6a4a32 -r 055d1e2d894c src/plugins/virtual/virtual-config.c --- a/src/plugins/virtual/virtual-config.c Thu Sep 24 19:07:12 2015 +0300 +++ b/src/plugins/virtual/virtual-config.c Thu Sep 24 19:27:10 2015 +0300 @@ -138,6 +138,7 @@ /* new mailbox. the search args are added to it later. */ bbox = p_new(ctx->pool, struct virtual_backend_box, 1); + bbox->virtual_mbox = ctx->mbox; if (strcasecmp(line, "INBOX") == 0) line = "INBOX"; bbox->name = p_strdup(ctx->pool, line); diff -r 4b5b8e6a4a32 -r 055d1e2d894c src/plugins/virtual/virtual-plugin.c --- a/src/plugins/virtual/virtual-plugin.c Thu Sep 24 19:07:12 2015 +0300 +++ b/src/plugins/virtual/virtual-plugin.c Thu Sep 24 19:27:10 2015 +0300 @@ -7,12 +7,18 @@ const char *virtual_plugin_version = DOVECOT_ABI_VERSION; +static struct mail_storage_hooks acl_mail_storage_hooks = { + .mailbox_opened = virtual_mailbox_opened_hook +}; + void virtual_plugin_init(struct module *module ATTR_UNUSED) { mail_storage_class_register(&virtual_storage); + mail_storage_hooks_add(module, &acl_mail_storage_hooks); } void virtual_plugin_deinit(void) { mail_storage_class_unregister(&virtual_storage); + mail_storage_hooks_remove(&acl_mail_storage_hooks); } diff -r 4b5b8e6a4a32 -r 055d1e2d894c src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Thu Sep 24 19:07:12 2015 +0300 +++ b/src/plugins/virtual/virtual-storage.c Thu Sep 24 19:27:10 2015 +0300 @@ -182,6 +182,7 @@ mailbox = bbox->name; ns = mail_namespace_find(user->namespaces, mailbox); bbox->box = mailbox_alloc(ns->list, mailbox, flags); + MODULE_CONTEXT_SET(bbox->box, virtual_storage_module, bbox); if (mailbox_exists(bbox->box, TRUE, &existence) < 0) return virtual_backend_box_open_failed(mbox, bbox); @@ -322,10 +323,17 @@ return FALSE; } -void virtual_backend_box_opened(struct virtual_mailbox *mbox, - struct virtual_backend_box *bbox) +void virtual_mailbox_opened_hook(struct mailbox *box) { + struct virtual_backend_box *bbox = VIRTUAL_CONTEXT(box); + struct virtual_mailbox *mbox; + + if (bbox == NULL) { + /* not a backend for a virtual mailbox */ + return; + } i_assert(!bbox->open_tracked); + mbox = bbox->virtual_mbox; /* the backend mailbox was already opened. if we didn't get here from virtual_backend_box_open() we may need to close a mailbox */ @@ -351,16 +359,14 @@ virtual_backend_box_close_any_except(mbox, bbox)) ; - if (mailbox_open(bbox->box) < 0) - return -1; - virtual_backend_box_opened(mbox, bbox); - return 0; + return mailbox_open(bbox->box); } void virtual_backend_box_close(struct virtual_mailbox *mbox, struct virtual_backend_box *bbox) { i_assert(bbox->box->opened); + i_assert(bbox->open_tracked); if (bbox->search_result != NULL) mailbox_search_result_free(&bbox->search_result); @@ -370,21 +376,13 @@ mail_search_args_deinit(bbox->search_args); bbox->search_args_initialized = FALSE; } - if (bbox->open_tracked) { - i_assert(mbox->backends_open_count > 0); - mbox->backends_open_count--; - bbox->open_tracked = FALSE; + i_assert(mbox->backends_open_count > 0); + mbox->backends_open_count--; + bbox->open_tracked = FALSE; - DLLIST2_REMOVE_FULL(&mbox->open_backend_boxes_head, - &mbox->open_backend_boxes_tail, bbox, - prev_open, next_open); - } else { - /* mailbox can be leaked outside our code via - virtual_get_virtual_backend_boxes() and it could have - been opened there. FIXME: maybe we could hook into the - backend open/close functions to do the tracking and - auto-closing. */ - } + DLLIST2_REMOVE_FULL(&mbox->open_backend_boxes_head, + &mbox->open_backend_boxes_tail, bbox, + prev_open, next_open); mailbox_close(bbox->box); } diff -r 4b5b8e6a4a32 -r 055d1e2d894c src/plugins/virtual/virtual-storage.h --- a/src/plugins/virtual/virtual-storage.h Thu Sep 24 19:07:12 2015 +0300 +++ b/src/plugins/virtual/virtual-storage.h Thu Sep 24 19:27:10 2015 +0300 @@ -62,6 +62,9 @@ }; struct virtual_backend_box { + union mailbox_module_context module_ctx; + struct virtual_mailbox *virtual_mbox; + /* linked list for virtual_mailbox->open_backend_boxes_{head,tail} */ struct virtual_backend_box *prev_open, *next_open; @@ -188,8 +191,6 @@ struct virtual_backend_box *bbox); void virtual_backend_box_accessed(struct virtual_mailbox *mbox, struct virtual_backend_box *bbox); -void virtual_backend_box_opened(struct virtual_mailbox *mbox, - struct virtual_backend_box *bbox); void virtual_backend_box_sync_mail_unset(struct virtual_backend_box *bbox); struct mail_search_context * @@ -224,4 +225,6 @@ void virtual_box_copy_error(struct mailbox *dest, struct mailbox *src); +void virtual_mailbox_opened_hook(struct mailbox *box); + #endif diff -r 4b5b8e6a4a32 -r 055d1e2d894c src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Thu Sep 24 19:07:12 2015 +0300 +++ b/src/plugins/virtual/virtual-sync.c Thu Sep 24 19:27:10 2015 +0300 @@ -1131,12 +1131,9 @@ bbox->open_failed = FALSE; } - ret = mailbox_get_status(bbox->box, STATUS_UIDVALIDITY | - STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ, - &status); - if (!bbox_index_opened && bbox->box->opened) - virtual_backend_box_opened(ctx->mbox, bbox); - if (ret < 0) { + if (mailbox_get_status(bbox->box, STATUS_UIDVALIDITY | + STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ, + &status) < 0) { if (mailbox_get_last_mail_error(bbox->box) != MAIL_ERROR_NOTFOUND) return -1; /* mailbox was deleted */ From dovecot at dovecot.org Thu Sep 24 16:42:02 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 16:42:02 +0000 Subject: dovecot-2.2: cassandra: Use a local monotonic timestamp generator. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1e4d2306f1f3 changeset: 19216:1e4d2306f1f3 user: Timo Sirainen date: Thu Sep 24 19:40:41 2015 +0300 description: cassandra: Use a local monotonic timestamp generator. Otherwise we run into race conditions on server side timestamps, which cause problems. diffstat: src/lib-sql/driver-cassandra.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (29 lines): diff -r 055d1e2d894c -r 1e4d2306f1f3 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Sep 24 19:27:10 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Thu Sep 24 19:40:41 2015 +0300 @@ -42,6 +42,7 @@ CassCluster *cluster; CassSession *session; + CassTimestampGen *timestamp_gen; int fd_pipe[2]; struct io *io_pipe; @@ -408,7 +409,9 @@ } T_END; cass_log_set_level(db->log_level); + db->timestamp_gen = cass_timestamp_gen_server_side_new(); db->cluster = cass_cluster_new(); + cass_cluster_set_timestamp_gen(db->cluster, db->timestamp_gen); cass_cluster_set_connect_timeout(db->cluster, SQL_CONNECT_TIMEOUT_SECS * 1000); cass_cluster_set_request_timeout(db->cluster, SQL_QUERY_TIMEOUT_SECS * 1000); cass_cluster_set_contact_points(db->cluster, db->hosts); @@ -431,6 +434,7 @@ cass_session_free(db->session); cass_cluster_free(db->cluster); + cass_timestamp_gen_free(db->timestamp_gen); i_free(db->hosts); i_free(db->error); i_free(db->keyspace); From dovecot at dovecot.org Thu Sep 24 17:30:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 17:30:11 +0000 Subject: dovecot-2.2: doveconf: Don't if _password value is empty, don't ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bf64724225e9 changeset: 19217:bf64724225e9 user: Timo Sirainen date: Thu Sep 24 20:28:51 2015 +0300 description: doveconf: Don't if _password value is empty, don't hide it. diffstat: src/config/doveconf.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1e4d2306f1f3 -r bf64724225e9 src/config/doveconf.c --- a/src/config/doveconf.c Thu Sep 24 19:40:41 2015 +0300 +++ b/src/config/doveconf.c Thu Sep 24 20:28:51 2015 +0300 @@ -302,7 +302,7 @@ value = strchr(key, '='); o_stream_nsend(output, key, value-key); o_stream_nsend_str(output, " = "); - if (hide_passwords && + if (hide_passwords && value[1] != '\0' && value-key > 9 && strncmp(value-9, "_password", 9) == 0) { o_stream_nsend_str(output, " # hidden, use -P to show it"); } else if (!value_need_quote(value+1)) From dovecot at dovecot.org Thu Sep 24 17:52:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 17:52:39 +0000 Subject: dovecot-2.2: cassandra: Added "version" parameter to specify a p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e92af8f36ed0 changeset: 19218:e92af8f36ed0 user: Timo Sirainen date: Thu Sep 24 20:51:16 2015 +0300 description: cassandra: Added "version" parameter to specify a protocol version. Mainly to avoid warnings logged when connecting to an older Cassandra server. diffstat: src/lib-sql/driver-cassandra.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (30 lines): diff -r bf64724225e9 -r e92af8f36ed0 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Sep 24 20:28:51 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Thu Sep 24 20:51:16 2015 +0300 @@ -39,6 +39,7 @@ char *hosts, *keyspace; CassConsistency read_consistency, write_consistency, delete_consistency; CassLogLevel log_level; + unsigned int protocol_version; CassCluster *cluster; CassSession *session; @@ -384,6 +385,9 @@ } else if (strcmp(key, "log_level") == 0) { if (log_level_parse(value, &db->log_level) < 0) i_fatal("cassandra: Unknown log_level: %s", value); + } else if (strcmp(key, "version") == 0) { + if (str_to_uint(value, &db->protocol_version) < 0) + i_fatal("cassandra: Invalid version: %s", value); } else { i_fatal("cassandra: Unknown connect string: %s", key); } @@ -415,6 +419,8 @@ cass_cluster_set_connect_timeout(db->cluster, SQL_CONNECT_TIMEOUT_SECS * 1000); cass_cluster_set_request_timeout(db->cluster, SQL_QUERY_TIMEOUT_SECS * 1000); cass_cluster_set_contact_points(db->cluster, db->hosts); + if (db->protocol_version != 0) + cass_cluster_set_protocol_version(db->cluster, db->protocol_version); db->session = cass_session_new(); i_array_init(&db->results, 16); i_array_init(&db->callbacks, 16); From dovecot at dovecot.org Thu Sep 24 17:59:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Sep 2015 17:59:42 +0000 Subject: dovecot-2.2: cassandra: copy&paste mistake - use monotonic times... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8183663ad7c0 changeset: 19219:8183663ad7c0 user: Timo Sirainen date: Thu Sep 24 20:58:04 2015 +0300 description: cassandra: copy&paste mistake - use monotonic timestamp generator, not server side.. diffstat: src/lib-sql/driver-cassandra.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e92af8f36ed0 -r 8183663ad7c0 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Sep 24 20:51:16 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Thu Sep 24 20:58:04 2015 +0300 @@ -413,7 +413,7 @@ } T_END; cass_log_set_level(db->log_level); - db->timestamp_gen = cass_timestamp_gen_server_side_new(); + db->timestamp_gen = cass_timestamp_gen_monotonic_new(); db->cluster = cass_cluster_new(); cass_cluster_set_timestamp_gen(db->cluster, db->timestamp_gen); cass_cluster_set_connect_timeout(db->cluster, SQL_CONNECT_TIMEOUT_SECS * 1000); From pigeonhole at rename-it.nl Thu Sep 24 21:57:48 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 24 Sep 2015 23:57:48 +0200 Subject: dovecot-2.2-pigeonhole: Released v0.4.9.rc1 for Dovecot v2.2.19.... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b2d5b1a5fa25 changeset: 2104:b2d5b1a5fa25 user: Stephan Bosch date: Thu Sep 24 23:33:17 2015 +0200 description: Released v0.4.9.rc1 for Dovecot v2.2.19.rc1. diffstat: NEWS | 27 +++++++++++++++++++++++++++ configure.ac | 2 +- 2 files changed, 28 insertions(+), 1 deletions(-) diffs (43 lines): diff -r 7038e943e853 -r b2d5b1a5fa25 NEWS --- a/NEWS Wed Sep 23 22:54:11 2015 +0200 +++ b/NEWS Thu Sep 24 23:33:17 2015 +0200 @@ -1,3 +1,30 @@ +v0.4.9 xx-10-2015 Stephan Bosch + + * Properly implemented checking of ABI version for Sieve interpreter plugins, + much like Dovecot itself does for plugins. This will prevent plugin ABI + mismatches. + + Implemented a vnd.dovecot.environment extension. This builds upon the + standard environment extension and adds a few more environment items, such + as username and default mailbox. It also creates a variables namespace so + that environment items can be accessed directly. I am still thinking about + more environment items that can be added. + + Sieve extprograms plugin: Made line endings of the input passed to the + external programs configurable. This can be configured separately for each + of the three extensions. + + ManageSieve: Implemented proxy XCLIENT support. This allows the proxy to + pass client information to the back-end. + - ManageSieve: Fixed an assert failure occurring when a client disconnects + during the GETSCRIPT command. + - doveadm sieve plugin: Fixed incorrect initialization of mail user. This + caused a few memory leaks. + - sieve-filter command line tool: Fixed handling of failure-related implicit + keep when there is an explicit default destination folder. This caused + message duplication. + - lib-sieve: Fixed bug in RFC5322 header folding. Words longer than the + optimal line length caused empty lines in the output, which would break the + resulting message header. This surfaced in References: headers with very + long message IDs. + v0.4.8 15-05-2015 Stephan Bosch * LDA Sieve plugin: Dovecot changed the deliver_log_format setting to include diff -r 7038e943e853 -r b2d5b1a5fa25 configure.ac --- a/configure.ac Wed Sep 23 22:54:11 2015 +0200 +++ b/configure.ac Thu Sep 24 23:33:17 2015 +0200 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.4.8], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_INIT([Pigeonhole], [0.4.9.rc1], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) AC_DEFINE_UNQUOTED([PIGEONHOLE_ABI_VERSION], "0.4.ABIv1($PACKAGE_VERSION)", [Pigeonhole ABI version]) AC_CONFIG_AUX_DIR([.]) From pigeonhole at rename-it.nl Thu Sep 24 21:57:48 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 24 Sep 2015 23:57:48 +0200 Subject: dovecot-2.2-pigeonhole: Added tag 0.4.9.rc1 for changeset b2d5b1... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/af0fa9a48c90 changeset: 2105:af0fa9a48c90 user: Stephan Bosch date: Thu Sep 24 23:35:00 2015 +0200 description: Added tag 0.4.9.rc1 for changeset b2d5b1a5fa25 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r b2d5b1a5fa25 -r af0fa9a48c90 .hgtags --- a/.hgtags Thu Sep 24 23:33:17 2015 +0200 +++ b/.hgtags Thu Sep 24 23:35:00 2015 +0200 @@ -31,3 +31,4 @@ 44d41235776de692f1f4ef9843ee6f3901fa3138 0.4.8.rc2 7b154d69394fcdbbc0b066978d864e551b736ce9 0.4.8.rc3 baa15dd77f9e2a1b7794771ce7deb6f961ed0360 0.4.8 +b2d5b1a5fa25766eb09d5a823b5d79b37b8ecefc 0.4.9.rc1 From pigeonhole at rename-it.nl Thu Sep 24 21:59:46 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 24 Sep 2015 23:59:46 +0200 Subject: dovecot-2.2-pigeonhole: Added signature for changeset b2d5b1a5fa25 Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/87dbcf33c13c changeset: 2106:87dbcf33c13c user: Stephan Bosch date: Fri Sep 25 00:01:42 2015 +0200 description: Added signature for changeset b2d5b1a5fa25 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r af0fa9a48c90 -r 87dbcf33c13c .hgsigs --- a/.hgsigs Thu Sep 24 23:35:00 2015 +0200 +++ b/.hgsigs Fri Sep 25 00:01:42 2015 +0200 @@ -25,3 +25,4 @@ 44d41235776de692f1f4ef9843ee6f3901fa3138 0 iQEcBAABAgAGBQJVUP1CAAoJEATWKx49+7T0+GMH/3QKZewiy+jeeHmb8wYExW6eiAEprb7lq2U8UjN5iOuID7OnDbkZ0/jr/oU0zsxpch/lacpAa0cIvgvujhGuXzeR2XNO3MWyiOPe4t0WcFaoP4Hl+EV7faZipCdPx7mGbxFYSEx9rz4Kp0MBnG/I+yE2+I1AbBsHqQMC1meAA3llu1n/Wf9BOhyzUP4/AL9CSNUtO/6tUZriXQOrRZpwkcyc7mrH6Trpyy0YXkEVWAZ8jdusiVuLmwMzEHUaF4SXc2daHhRRRiwp3wCeIkyVvqm8BbxFGZ7ZMM7fKnhj3LGmSLfeloHHmCC3HU6SfzvNmv8Fcieah/thpl/EOdKiWeI= 7b154d69394fcdbbc0b066978d864e551b736ce9 0 iQEcBAABAgAGBQJVU7n1AAoJEATWKx49+7T0sOYH/jL46oo70cwImQN5pyAHudSarM9Mtls8//vX7JmJ/b8OPftwwx4ExBNYmvHSv315CPjq6ntMA7GMZ1tAd9lHk+ZB2WFL/Y7MQZUGgowo6K/F36MW1Z6tGN2rtnooqglyamKUPqrLIurs5Hc9444VrGMcdeMecGn9mEpxULy1jsMmo0AVVsyf50PHbYRwSgq960onz93lKhTcaD7/Sc76FPbJenqstP9iPQBuXugVxv/p9gN31Xx/V8RlxmHIjeHljx33m76sTGQ/8ca2XKvBwft8dZU4lTsrSmcvt+0wxATurjtfJ+aqPlH5pPXmqydbsEe6F3fjZTblwexc4kFNDsQ= baa15dd77f9e2a1b7794771ce7deb6f961ed0360 0 iQEcBAABAgAGBQJVVkC/AAoJEATWKx49+7T0nhoIAInNseUHHa4TUCeKH5NOr4f8d5Sj1arnxr1IGPuNlhz/Naeu2999cyVLHWGfOyWN7/B3SCmbJ/Vb/41lJLikC7Ma01IEskMDIcvJca/Nx8ZR5mcDxwpxma7k0III+gxf0pP2HWvqYXOZD+JXEGmvjCXSJyMAYPkEUEO/aVH83fNQVaPAqvvrl3cZz53TF8U9l01DecduRwLm+od2zHiFByB4bJuKhIvcAq2ksrjiHyfLJv9MXLZnhSbTLbWr4SXHs8bctzgBg6TXl6Fgp6ehOR3UQTR+k3xFOfXJY2GBfNPzBUi+rY8g8DgirNDDYjSwrc1W1+QRD5d9ANwmppeDDzg= +b2d5b1a5fa25766eb09d5a823b5d79b37b8ecefc 0 iQEcBAABAgAGBQJWBHLBAAoJEATWKx49+7T0MmwH/A6QURy8nLougjKBPzdISMhk75frCr9XlKwBz5DxxPU3ed1dD2I/Bgb6xoW+KIlvf5SO1zPhhkeD9MInImKXZuQbT47Ain0Q8mMJE7jXQbx8lkMpPbOtlsthjcxMnnIYQhE0wWSe40xtFRy217ry9NCWqByNyZrNOjKLOF27rO3h538xD31xflkAslqSY1enMVeJJsAzT7tGxithEufESDW4toBgPLFlgSmaKS5c9VxMWCQ04YWJVkO9IeIac8vK5crRpzCVUjd8lu5ucucAV5PoCNGunKCu0svsDkCL5X6wzZAglZHxfs5pM3LBDOerT4DnOMXQHOzdsKl9o5HDkXg= From dovecot at dovecot.org Fri Sep 25 15:18:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 25 Sep 2015 15:18:20 +0000 Subject: dovecot-2.2: dict-sql: If value isn't a hexblob as expected, log... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fbff1e801300 changeset: 19220:fbff1e801300 user: Timo Sirainen date: Fri Sep 25 18:16:56 2015 +0300 description: dict-sql: If value isn't a hexblob as expected, log an error instead of killing the whole process. diffstat: src/lib-dict/dict-sql.c | 184 +++++++++++++++++++++++++++++++---------------- 1 files changed, 120 insertions(+), 64 deletions(-) diffs (truncated from 445 to 300 lines): diff -r 8183663ad7c0 -r fbff1e801300 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Thu Sep 24 20:58:04 2015 +0300 +++ b/src/lib-dict/dict-sql.c Fri Sep 25 18:16:56 2015 +0300 @@ -211,44 +211,48 @@ return NULL; } -static void +static int sql_dict_value_escape(string_t *str, struct sql_dict *dict, bool value_is_hexblob, const char *field_name, - const char *value, const char *value_suffix) + const char *value, const char *value_suffix, + const char **error_r) { buffer_t *buf; if (!value_is_hexblob) { str_printfa(str, "'%s%s'", sql_escape_string(dict->db, value), value_suffix); - return; + return 0; } buf = buffer_create_dynamic(pool_datastack_create(), strlen(value)/2); if (hex_to_binary(value, buf) < 0) { /* we shouldn't get untrusted input here. it's also a bit annoying to handle this error. */ - i_fatal("dict-sql: field %s value isn't hexblob: %s", - field_name, value); + *error_r = t_strdup_printf("field %s value isn't hexblob: %s", + field_name, value); + return -1; } str_append(buf, value_suffix); str_append(str, sql_escape_blob(dict->db, buf->data, buf->used)); + return 0; } -static void +static int sql_dict_field_escape_value(string_t *str, struct sql_dict *dict, const struct dict_sql_field *field, - const char *value, const char *value_suffix) + const char *value, const char *value_suffix, + const char **error_r) { - sql_dict_value_escape(str, dict, field->value_is_hexblob, - field->name, value, value_suffix); + return sql_dict_value_escape(str, dict, field->value_is_hexblob, + field->name, value, value_suffix, error_r); } -static void +static int sql_dict_where_build(struct sql_dict *dict, const struct dict_sql_map *map, const ARRAY_TYPE(const_string) *values_arr, char key1, enum sql_recurse_type recurse_type, - string_t *query) + string_t *query, const char **error_r) { const struct dict_sql_field *sql_fields; const char *const *values; @@ -262,7 +266,7 @@ if (count2 == 0 && !priv) { /* we want everything */ - return; + return 0; } str_append(query, " WHERE"); @@ -272,7 +276,9 @@ if (i > 0) str_append(query, " AND"); str_printfa(query, " %s = ", sql_fields[i].name); - sql_dict_field_escape_value(query, dict, &sql_fields[i], values[i], ""); + if (sql_dict_field_escape_value(query, dict, &sql_fields[i], + values[i], "", error_r) < 0) + return -1; } switch (recurse_type) { case SQL_DICT_RECURSE_NONE: @@ -282,11 +288,13 @@ str_append(query, " AND"); if (i < count2) { str_printfa(query, " %s LIKE ", sql_fields[i].name); - sql_dict_field_escape_value(query, dict, &sql_fields[i], - values[i], "/%"); + if (sql_dict_field_escape_value(query, dict, &sql_fields[i], + values[i], "/%", error_r) < 0) + return -1; str_printfa(query, " AND %s NOT LIKE ", sql_fields[i].name); - sql_dict_field_escape_value(query, dict, &sql_fields[i], - values[i], "/%/%"); + if (sql_dict_field_escape_value(query, dict, &sql_fields[i], + values[i], "/%/%", error_r) < 0) + return -1; } else { str_printfa(query, " %s LIKE '%%' AND " "%s NOT LIKE '%%/%%'", @@ -299,8 +307,9 @@ str_append(query, " AND"); str_printfa(query, " %s LIKE ", sql_fields[i].name); - sql_dict_field_escape_value(query, dict, &sql_fields[i], - values[i], "/%"); + if (sql_dict_field_escape_value(query, dict, &sql_fields[i], + values[i], "/%", error_r) < 0) + return -1; } break; } @@ -310,6 +319,7 @@ str_printfa(query, " %s = '%s'", map->username_field, sql_escape_string(dict->db, dict->username)); } + return 0; } static int @@ -318,6 +328,7 @@ { const struct dict_sql_map *map; ARRAY_TYPE(const_string) values; + const char *error; map = *map_r = sql_dict_find_map(dict, key, &values); if (map == NULL) { @@ -326,8 +337,11 @@ } str_printfa(query, "SELECT %s FROM %s", map->value_field, map->table); - sql_dict_where_build(dict, map, &values, key[0], - SQL_DICT_RECURSE_NONE, query); + if (sql_dict_where_build(dict, map, &values, key[0], + SQL_DICT_RECURSE_NONE, query, &error) < 0) { + i_error("sql dict lookup: Failed to lookup key %s: %s", key, error); + return -1; + } return 0; } @@ -483,9 +497,9 @@ return NULL; } -static bool +static int sql_dict_iterate_build_next_query(struct sql_dict_iterate_context *ctx, - string_t *query) + string_t *query, const char **error_r) { struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; const struct dict_sql_map *map; @@ -495,8 +509,10 @@ unsigned int i, count; map = sql_dict_iterate_find_next_map(ctx, &values); - if (map == NULL) - return FALSE; + if (map == NULL) { + *error_r = "Invalid/unmapped path"; + return 0; + } if (ctx->result != NULL) { sql_result_unref(ctx->result); @@ -529,9 +545,10 @@ recurse_type = SQL_DICT_RECURSE_NONE; else recurse_type = SQL_DICT_RECURSE_ONE; - sql_dict_where_build(dict, map, &values, - ctx->paths[ctx->path_idx][0], - recurse_type, query); + if (sql_dict_where_build(dict, map, &values, + ctx->paths[ctx->path_idx][0], + recurse_type, query, error_r) < 0) + return -1; if ((ctx->flags & DICT_ITERATE_FLAG_SORT_BY_KEY) != 0) { str_append(query, " ORDER BY "); @@ -544,7 +561,7 @@ str_printfa(query, " ORDER BY %s", map->value_field); ctx->map = map; - return TRUE; + return 1; } static void sql_dict_iterate_callback(struct sql_result *result, @@ -556,17 +573,20 @@ ctx->ctx.async_callback(ctx->ctx.async_context); } -static bool sql_dict_iterate_next_query(struct sql_dict_iterate_context *ctx) +static int sql_dict_iterate_next_query(struct sql_dict_iterate_context *ctx, + const char **error_r) { struct sql_dict *dict = (struct sql_dict *)ctx->ctx.dict; - bool ret; + char *error = NULL; + int ret; T_BEGIN { string_t *query = t_str_new(256); - ret = sql_dict_iterate_build_next_query(ctx, query); - if (!ret) { + ret = sql_dict_iterate_build_next_query(ctx, query, error_r); + if (ret <= 0) { /* failed */ + error = i_strdup(*error_r); } else if ((ctx->flags & DICT_ITERATE_FLAG_ASYNC) == 0) { ctx->result = sql_query_s(dict->db, str_c(query)); } else { @@ -575,6 +595,8 @@ sql_dict_iterate_callback, ctx); } } T_END; + *error_r = t_strdup(error); + i_free(error); return ret; } @@ -584,6 +606,7 @@ { struct sql_dict_iterate_context *ctx; unsigned int i, path_count; + const char *error; pool_t pool; pool = pool_alloconly_create("sql dict iterate", 512); @@ -598,9 +621,9 @@ ctx->paths[i] = p_strdup(pool, paths[i]); ctx->key = str_new(pool, 256); - if (!sql_dict_iterate_next_query(ctx)) { - i_error("sql dict iterate: Invalid/unmapped path: %s", - paths[0]); + if (sql_dict_iterate_next_query(ctx, &error) <= 0) { + i_error("sql dict iterate failed for %s: %s", + paths[0], error); ctx->result = NULL; ctx->failed = TRUE; return &ctx->ctx; @@ -613,7 +636,7 @@ { struct sql_dict_iterate_context *ctx = (struct sql_dict_iterate_context *)_ctx; - const char *p, *value; + const char *p, *value, *error; unsigned int i, sql_field_i, count; int ret; @@ -635,8 +658,9 @@ /* see if there are more results in the next map. don't do it if we're looking for an exact match, since we already should have handled it. */ - if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0 || - !sql_dict_iterate_next_query(ctx)) + if ((ctx->flags & DICT_ITERATE_FLAG_EXACT_KEY) != 0) + return FALSE; + if ((ret = sql_dict_iterate_next_query(ctx, &error)) == 0) return FALSE; } if (ret < 0) { @@ -806,7 +830,8 @@ bool inc; }; -static const char *sql_dict_set_query(const struct dict_sql_build_query *build) +static int sql_dict_set_query(const struct dict_sql_build_query *build, + const char **query_r, const char **error_r) { struct sql_dict *dict = build->dict; const struct dict_sql_build_query_field *fields; @@ -831,9 +856,10 @@ if (build->inc) str_append(suffix, fields[i].value); else { - sql_dict_value_escape(suffix, dict, + if (sql_dict_value_escape(suffix, dict, fields[i].map->value_hexblob, - "value", fields[i].value, ""); + "value", fields[i].value, "", error_r) < 0) + return -1; } } if (build->key1 == DICT_PATH_PRIVATE[0]) { @@ -849,14 +875,17 @@ for (i = 0; i < count; i++) { str_printfa(prefix, ",%s", sql_fields[i].name); str_append_c(suffix, ','); - sql_dict_field_escape_value(suffix, dict, &sql_fields[i], - extra_values[i], ""); + if (sql_dict_field_escape_value(suffix, dict, &sql_fields[i], + extra_values[i], "", error_r) < 0) + return -1; } From dovecot at dovecot.org Sun Sep 27 15:59:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 15:59:34 +0000 Subject: dovecot-2.2: lib-http: Removed "pure" attribute from http_header... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5e18e1386600 changeset: 19221:5e18e1386600 user: Timo Sirainen date: Sun Sep 27 18:58:08 2015 +0300 description: lib-http: Removed "pure" attribute from http_header_field_find_unique(). It's a bit questionable if returning a pointer via parameter is allowed for a pure function. Also having this attribute seems to confuse clang static analyzer, because it gives false positives for code using http_header_field_find_unique(). It's not important for performance anyway, so better to avoid using it. diffstat: src/lib-http/http-header.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (11 lines): diff -r fbff1e801300 -r 5e18e1386600 src/lib-http/http-header.h --- a/src/lib-http/http-header.h Fri Sep 25 18:16:56 2015 +0300 +++ b/src/lib-http/http-header.h Sun Sep 27 18:58:08 2015 +0300 @@ -40,6 +40,6 @@ http_header_field_get(const struct http_header *header, const char *name) ATTR_PURE; int http_header_field_find_unique(const struct http_header *header, - const char *name, const struct http_header_field **hfield_r) ATTR_PURE; + const char *name, const struct http_header_field **hfield_r); #endif From dovecot at dovecot.org Sun Sep 27 16:05:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 16:05:00 +0000 Subject: dovecot-2.2: Make static analyzer happier. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e97a8dd8d2b6 changeset: 19222:e97a8dd8d2b6 user: Timo Sirainen date: Sun Sep 27 19:03:31 2015 +0300 description: Make static analyzer happier. diffstat: src/lib-mail/test-message-id.c | 3 ++- src/lib-storage/list/mailbox-list-index-status.c | 2 ++ src/lib-storage/mail-namespace.c | 2 ++ 3 files changed, 6 insertions(+), 1 deletions(-) diffs (37 lines): diff -r 5e18e1386600 -r e97a8dd8d2b6 src/lib-mail/test-message-id.c --- a/src/lib-mail/test-message-id.c Sun Sep 27 18:58:08 2015 +0300 +++ b/src/lib-mail/test-message-id.c Sun Sep 27 19:03:31 2015 +0300 @@ -27,7 +27,8 @@ while ((next_msgid = message_id_get_next(&msgid)) != NULL) { if (output[j] == NULL) break; - test_assert(strcmp(output[j++], next_msgid) == 0); + test_assert(strcmp(output[j], next_msgid) == 0); + j++; } test_assert(output[j++] == NULL && next_msgid == NULL); } diff -r 5e18e1386600 -r e97a8dd8d2b6 src/lib-storage/list/mailbox-list-index-status.c --- a/src/lib-storage/list/mailbox-list-index-status.c Sun Sep 27 18:58:08 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-status.c Sun Sep 27 19:03:31 2015 +0300 @@ -275,6 +275,8 @@ enum mailbox_metadata_items noncached_items; int ret; + i_assert(metadata_r != NULL); + if (box->opened) { /* if mailbox is already opened, don't bother using the values in mailbox list index. they have a higher chance of being diff -r 5e18e1386600 -r e97a8dd8d2b6 src/lib-storage/mail-namespace.c --- a/src/lib-storage/mail-namespace.c Sun Sep 27 18:58:08 2015 +0300 +++ b/src/lib-storage/mail-namespace.c Sun Sep 27 19:03:31 2015 +0300 @@ -321,6 +321,8 @@ struct mail_namespace *ns; bool prefixless_found = FALSE; + i_assert(namespaces != NULL); + for (ns = namespaces; ns != NULL; ns = ns->next) { if (ns->prefix_len == 0) prefixless_found = TRUE; From dovecot at dovecot.org Sun Sep 27 17:48:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 17:48:51 +0000 Subject: dovecot-2.2: lib-http: Avoid compiler warnings in printf() strings. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7010dd10ccc6 changeset: 19224:7010dd10ccc6 user: Timo Sirainen date: Sun Sep 27 20:47:20 2015 +0300 description: lib-http: Avoid compiler warnings in printf() strings. Cast fields explicitly as long, which should be large enough. diffstat: src/lib-http/http-server-connection.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (16 lines): diff -r ca0916045005 -r 7010dd10ccc6 src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Sun Sep 27 20:46:29 2015 +0300 +++ b/src/lib-http/http-server-connection.c Sun Sep 27 20:47:20 2015 +0300 @@ -806,10 +806,10 @@ if (net_getunixcred(fd_in, &cred) < 0) { name = t_strdup_printf("[%u]", id); } else if (cred.pid == (pid_t)-1) { - name = t_strdup_printf("unix:uid=%u [%u]", cred.uid, id); + name = t_strdup_printf("unix:uid=%ld [%u]", (long)cred.uid, id); } else { name = t_strdup_printf - ("unix:pid=%u,uid=%u [%u]", cred.pid, cred.uid, id); + ("unix:pid=%ld,uid=%ld [%u]", (long)cred.pid, (long)cred.uid, id); } } else if (addr.family == AF_INET6) { name = t_strdup_printf("[%s]:%u [%u]", net_ip2addr(&addr), port, id); From dovecot at dovecot.org Sun Sep 27 17:48:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 17:48:49 +0000 Subject: dovecot-2.2: Fixed compiler warnings in some systems. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ca0916045005 changeset: 19223:ca0916045005 user: Timo Sirainen date: Sun Sep 27 20:46:29 2015 +0300 description: Fixed compiler warnings in some systems. major() and minor() are unsigned ints nowadays, but in Solaris it's major_t. Cast them explicitly to avoid compiler warnings, and use unsigned longs instead of unsigned ints just in case. diffstat: src/imap-hibernate/imap-client.c | 5 +++-- src/imap/imap-client-hibernate.c | 5 +++-- src/imap/imap-master-client.c | 11 ++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diffs (51 lines): diff -r e97a8dd8d2b6 -r ca0916045005 src/imap-hibernate/imap-client.c --- a/src/imap-hibernate/imap-client.c Sun Sep 27 19:03:31 2015 +0300 +++ b/src/imap-hibernate/imap-client.c Sun Sep 27 20:46:29 2015 +0300 @@ -118,8 +118,9 @@ str_append_tabescaped(str, state->userdb_fields); } if (major(state->peer_dev) != 0 || minor(state->peer_dev) != 0) { - str_printfa(str, "\tpeer_dev_major=%u\tpeer_dev_minor=%u", - major(state->peer_dev), minor(state->peer_dev)); + str_printfa(str, "\tpeer_dev_major=%lu\tpeer_dev_minor=%lu", + (unsigned long)major(state->peer_dev), + (unsigned long)minor(state->peer_dev)); } if (state->peer_ino != 0) str_printfa(str, "\tpeer_ino=%llu", (unsigned long long)state->peer_ino); diff -r e97a8dd8d2b6 -r ca0916045005 src/imap/imap-client-hibernate.c --- a/src/imap/imap-client-hibernate.c Sun Sep 27 19:03:31 2015 +0300 +++ b/src/imap/imap-client-hibernate.c Sun Sep 27 20:46:29 2015 +0300 @@ -51,8 +51,9 @@ str_printfa(cmd, "\tidle_notify_interval=%u", client->set->imap_idle_notify_interval); if (fstat(client->fd_in, &peer_st) == 0) { - str_printfa(cmd, "\tpeer_dev_major=%u\tpeer_dev_minor=%u\tpeer_ino=%llu", - major(peer_st.st_dev), minor(peer_st.st_dev), + str_printfa(cmd, "\tpeer_dev_major=%lu\tpeer_dev_minor=%lu\tpeer_ino=%llu", + (unsigned long)major(peer_st.st_dev), + (unsigned long)minor(peer_st.st_dev), (unsigned long long)peer_st.st_ino); } diff -r e97a8dd8d2b6 -r ca0916045005 src/imap/imap-master-client.c --- a/src/imap/imap-master-client.c Sun Sep 27 19:03:31 2015 +0300 +++ b/src/imap/imap-master-client.c Sun Sep 27 20:46:29 2015 +0300 @@ -165,11 +165,12 @@ if (peer_st.st_ino != master_input->peer_ino || !CMP_DEV_T(peer_st.st_dev, master_input->peer_dev)) { *error_r = t_strdup_printf( - "BUG: Expected peer device=%u,%u inode=%s doesn't match " - "client fd's actual device=%u,%u inode=%s", - major(peer_st.st_dev), minor(peer_st.st_dev), dec2str(peer_st.st_ino), - major(master_input->peer_dev), - minor(master_input->peer_dev), + "BUG: Expected peer device=%lu,%lu inode=%s doesn't match " + "client fd's actual device=%lu,%lu inode=%s", + (unsigned long)major(peer_st.st_dev), + (unsigned long)minor(peer_st.st_dev), dec2str(peer_st.st_ino), + (unsigned long)major(master_input->peer_dev), + (unsigned long)minor(master_input->peer_dev), dec2str(master_input->peer_ino)); return -1; } From dovecot at dovecot.org Sun Sep 27 18:27:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 18:27:51 +0000 Subject: dovecot-2.2: *-login: Session ID generator wasn't encoding remot... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/038efefc7e70 changeset: 19225:038efefc7e70 user: Timo Sirainen date: Sun Sep 27 21:25:59 2015 +0300 description: *-login: Session ID generator wasn't encoding remote port number correctly to it. The upper 8bits of the port number were always written as 0. This could have lead to duplicate session ID strings in some rare cases. Found by Coverity. diffstat: src/login-common/client-common.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7010dd10ccc6 -r 038efefc7e70 src/login-common/client-common.c --- a/src/login-common/client-common.c Sun Sep 27 20:47:20 2015 +0300 +++ b/src/login-common/client-common.c Sun Sep 27 21:25:59 2015 +0300 @@ -453,7 +453,7 @@ buffer_append_c(buf, (timestamp >> i) & 0xff); buffer_append_c(buf, client->remote_port & 0xff); - buffer_append_c(buf, (client->remote_port >> 16) & 0xff); + buffer_append_c(buf, (client->remote_port >> 8) & 0xff); #ifdef HAVE_IPV6 if (IPADDR_IS_V6(&client->ip)) buffer_append(buf, &client->ip.u.ip6, sizeof(client->ip.u.ip6)); From dovecot at dovecot.org Sun Sep 27 18:51:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 18:51:03 +0000 Subject: dovecot-2.2: imapc: Fixed crash if imapc_host or imapc_password ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a887dc2b3fb9 changeset: 19227:a887dc2b3fb9 user: Timo Sirainen date: Sun Sep 27 21:30:56 2015 +0300 description: imapc: Fixed crash if imapc_host or imapc_password setting was missing. Instead of giving a nice error message about it. Found by Coverity. diffstat: src/lib-storage/index/imapc/imapc-list.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (16 lines): diff -r e5164f14723f -r a887dc2b3fb9 src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Sun Sep 27 21:28:25 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.c Sun Sep 27 21:30:56 2015 +0300 @@ -87,11 +87,10 @@ { struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; - list->client->destroying = TRUE; - /* make sure all pending commands are aborted before anything is deinitialized */ if (list->client != NULL) { + list->client->destroying = TRUE; imapc_client_disconnect(list->client->client); imapc_storage_client_unref(&list->client); } From dovecot at dovecot.org Sun Sep 27 18:50:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 18:50:54 +0000 Subject: dovecot-2.2: dict-client: Avoid crash if server disconnects duri... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e5164f14723f changeset: 19226:e5164f14723f user: Timo Sirainen date: Sun Sep 27 21:28:25 2015 +0300 description: dict-client: Avoid crash if server disconnects during transaction commit. Found by Coverity. diffstat: src/lib-dict/dict-client.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (17 lines): diff -r 038efefc7e70 -r e5164f14723f src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Sun Sep 27 21:25:59 2015 +0300 +++ b/src/lib-dict/dict-client.c Sun Sep 27 21:28:25 2015 +0300 @@ -797,10 +797,12 @@ default: i_error("dict-client: Invalid commit reply: %s", line); client_dict_disconnect(dict); + line = NULL; ret = -1; break; } - if (str_to_uint(line+1, &id) < 0 || ctx->id != id) { + if (line != NULL && + (str_to_uint(line+1, &id) < 0 || ctx->id != id)) { i_error("dict-client: Invalid commit reply, " "expected id=%u: %s", ctx->id, line); client_dict_disconnect(dict); From dovecot at dovecot.org Sun Sep 27 18:51:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 18:51:03 +0000 Subject: dovecot-2.2: dsync: Removed unnecessary code - cur_mail is never... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/83ccad730ea3 changeset: 19228:83ccad730ea3 user: Timo Sirainen date: Sun Sep 27 21:33:30 2015 +0300 description: dsync: Removed unnecessary code - cur_mail is never NULL at this point. Found by Coverity. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 9 ++------- 1 files changed, 2 insertions(+), 7 deletions(-) diffs (19 lines): diff -r a887dc2b3fb9 -r 83ccad730ea3 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Sun Sep 27 21:30:56 2015 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Sun Sep 27 21:33:30 2015 +0300 @@ -1628,13 +1628,8 @@ } else { (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r); } - if (importer->cur_mail != NULL) { - *result_r = t_strdup_printf("%s (next local mail UID=%u)", - *result_r, importer->cur_mail->uid); - } else { - *result_r = t_strdup_printf("%s (no more local mails)", - *result_r); - } + *result_r = t_strdup_printf("%s (next local mail UID=%u)", + *result_r, importer->cur_mail->uid); } int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, From dovecot at dovecot.org Sun Sep 27 18:51:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 18:51:03 +0000 Subject: dovecot-2.2: lib-storage: Removed unnecessary code - nnode is ne... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5af9b91c38c0 changeset: 19229:5af9b91c38c0 user: Timo Sirainen date: Sun Sep 27 21:34:48 2015 +0300 description: lib-storage: Removed unnecessary code - nnode is never NULL at this point. Found by Coverity. diffstat: src/lib-storage/list/mailbox-list-index-notify.c | 12 +++++------- 1 files changed, 5 insertions(+), 7 deletions(-) diffs (22 lines): diff -r 83ccad730ea3 -r 5af9b91c38c0 src/lib-storage/list/mailbox-list-index-notify.c --- a/src/lib-storage/list/mailbox-list-index-notify.c Sun Sep 27 21:33:30 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-notify.c Sun Sep 27 21:34:48 2015 +0300 @@ -722,13 +722,11 @@ } /* update internal state */ - if (nnode != NULL) { - nnode->uidvalidity = status.uidvalidity; - nnode->uidnext = status.uidnext; - nnode->messages = status.messages; - nnode->unseen = status.unseen; - nnode->highest_modseq = status.highest_modseq; - } + nnode->uidvalidity = status.uidvalidity; + nnode->uidnext = status.uidnext; + nnode->messages = status.messages; + nnode->unseen = status.unseen; + nnode->highest_modseq = status.highest_modseq; return TRUE; } From dovecot at dovecot.org Sun Sep 27 18:51:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 18:51:03 +0000 Subject: dovecot-2.2: Added NULL-check asserts to make static analyzer ha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8d2f1c363853 changeset: 19230:8d2f1c363853 user: Timo Sirainen date: Sun Sep 27 21:42:17 2015 +0300 description: Added NULL-check asserts to make static analyzer happier. These shouldn't be happening. diffstat: src/config/doveconf.c | 1 + src/plugins/push-notification/push-notification-driver-ox.c | 1 + 2 files changed, 2 insertions(+), 0 deletions(-) diffs (22 lines): diff -r 5af9b91c38c0 -r 8d2f1c363853 src/config/doveconf.c --- a/src/config/doveconf.c Sun Sep 27 21:34:48 2015 +0300 +++ b/src/config/doveconf.c Sun Sep 27 21:42:17 2015 +0300 @@ -300,6 +300,7 @@ key = strings[i] + skip_len; if (unique_key) key++; value = strchr(key, '='); + i_assert(value != NULL); o_stream_nsend(output, key, value-key); o_stream_nsend_str(output, " = "); if (hide_passwords && value[1] != '\0' && diff -r 5af9b91c38c0 -r 8d2f1c363853 src/plugins/push-notification/push-notification-driver-ox.c --- a/src/plugins/push-notification/push-notification-driver-ox.c Sun Sep 27 21:34:48 2015 +0300 +++ b/src/plugins/push-notification/push-notification-driver-ox.c Sun Sep 27 21:42:17 2015 +0300 @@ -265,6 +265,7 @@ push_notification_driver_ox_init_global(user); messagenew = push_notification_txn_msg_get_eventdata(msg, "MessageNew"); + i_assert(messagenew != NULL); http_req = http_client_request_url(ox_global->http_client, "PUT", dconfig->http_url, From dovecot at dovecot.org Sun Sep 27 18:51:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 27 Sep 2015 18:51:03 +0000 Subject: dovecot-2.2: push-notification: Log about unexpectedly failing m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/30f4615efa10 changeset: 19231:30f4615efa10 user: Timo Sirainen date: Sun Sep 27 21:48:34 2015 +0300 description: push-notification: Log about unexpectedly failing mailbox functions. Found by Coverity. diffstat: src/plugins/push-notification/push-notification-driver-ox.c | 6 +++++- src/plugins/push-notification/push-notification-event-mailboxcreate.c | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diffs (33 lines): diff -r 8d2f1c363853 -r 30f4615efa10 src/plugins/push-notification/push-notification-driver-ox.c --- a/src/plugins/push-notification/push-notification-driver-ox.c Sun Sep 27 21:42:17 2015 +0300 +++ b/src/plugins/push-notification/push-notification-driver-ox.c Sun Sep 27 21:48:34 2015 +0300 @@ -128,7 +128,11 @@ success = TRUE; } - mailbox_transaction_commit(&mctx); + if (mailbox_transaction_commit(&mctx) < 0) { + i_error(OX_LOG_LABEL "Transaction commit failed: %s", + mailbox_get_last_error(inbox, NULL)); + /* the commit doesn't matter though. */ + } } mailbox_free(&inbox); diff -r 8d2f1c363853 -r 30f4615efa10 src/plugins/push-notification/push-notification-event-mailboxcreate.c --- a/src/plugins/push-notification/push-notification-event-mailboxcreate.c Sun Sep 27 21:42:17 2015 +0300 +++ b/src/plugins/push-notification/push-notification-event-mailboxcreate.c Sun Sep 27 21:48:34 2015 +0300 @@ -26,7 +26,12 @@ struct push_notification_event_mailboxcreate_data *data; struct mailbox_status status; - mailbox_get_status(ptxn->mbox, STATUS_UIDVALIDITY, &status); + if (mailbox_get_status(ptxn->mbox, STATUS_UIDVALIDITY, &status) < 0) { + i_error(EVENT_NAME "Failed to get created mailbox '%s' uidvalidity: %s", + mailbox_get_vname(ptxn->mbox), + mailbox_get_last_error(ptxn->mbox, NULL)); + status.uidvalidity = 0; + } data = p_new(ptxn->pool, struct push_notification_event_mailboxcreate_data, 1); From dovecot at dovecot.org Mon Sep 28 09:20:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 09:20:50 +0000 Subject: dovecot-2.2: quota-clone: Memory leak fix when mailbox had no qu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9b2be8702e59 changeset: 19232:9b2be8702e59 user: Timo Sirainen date: Mon Sep 28 12:19:20 2015 +0300 description: quota-clone: Memory leak fix when mailbox had no quota roots. diffstat: src/plugins/quota-clone/quota-clone-plugin.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (17 lines): diff -r 30f4615efa10 -r 9b2be8702e59 src/plugins/quota-clone/quota-clone-plugin.c --- a/src/plugins/quota-clone/quota-clone-plugin.c Sun Sep 27 21:48:34 2015 +0300 +++ b/src/plugins/quota-clone/quota-clone-plugin.c Mon Sep 28 12:19:20 2015 +0300 @@ -45,12 +45,12 @@ /* we'll clone the first quota root */ iter = quota_root_iter_init(box); root = quota_root_iter_next(iter); + quota_root_iter_deinit(&iter); if (root == NULL) { /* no quota roots defined for this mailbox - ignore */ qbox->quota_changed = FALSE; return; } - quota_root_iter_deinit(&iter); trans = dict_transaction_begin(quser->dict); /* update bytes */ From dovecot at dovecot.org Mon Sep 28 09:35:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 09:35:20 +0000 Subject: dovecot-2.2: quota: Avoid crash in quota_root_iter_init() if mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/14f670053196 changeset: 19233:14f670053196 user: Timo Sirainen date: Mon Sep 28 12:33:33 2015 +0300 description: quota: Avoid crash in quota_root_iter_init() if mailbox has no quota enabled. This allows quota_get_mail_user_quota() to be called even when quota doesn't exist. Cleaned up all the code using it to now check for the NULL result. This fixes a crash in quota_clone plugin. diffstat: src/plugins/quota/quota-storage.c | 13 ++++++------- src/plugins/quota/quota.c | 5 +++++ 2 files changed, 11 insertions(+), 7 deletions(-) diffs (69 lines): diff -r 9b2be8702e59 -r 14f670053196 src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Mon Sep 28 12:19:20 2015 +0300 +++ b/src/plugins/quota/quota-storage.c Mon Sep 28 12:33:33 2015 +0300 @@ -501,7 +501,7 @@ { struct quota_user *quser = QUOTA_USER_CONTEXT(user); - return quser->quota; + return quser == NULL ? NULL : quser->quota; } static void quota_user_deinit(struct mail_user *user) @@ -572,11 +572,10 @@ struct mail_user *quota_user; bool add; - if (QUOTA_USER_CONTEXT(list->ns->user) == NULL) - return; - /* see if we have a quota explicitly defined for this namespace */ quota = quota_get_mail_user_quota(list->ns->user); + if (quota == NULL) + return; root = quota_find_root_for_ns(quota, list->ns); if (root != NULL) { /* explicit quota root */ @@ -609,6 +608,7 @@ MODULE_CONTEXT_SET(list, quota_mailbox_list_module, qlist); quota = quota_get_mail_user_quota(quota_user); + i_assert(quota != NULL); quota_add_user_namespace(quota, list->ns); } } @@ -646,10 +646,9 @@ struct quota_root *const *roots; unsigned int i, count; - if (QUOTA_USER_CONTEXT(namespaces->user) == NULL) + quota = quota_get_mail_user_quota(namespaces->user); + if (quota == NULL) return; - - quota = quota_get_mail_user_quota(namespaces->user); roots = array_get("a->roots, &count); for (i = 0; i < count; i++) quota_root_set_namespace(roots[i], namespaces); diff -r 9b2be8702e59 -r 14f670053196 src/plugins/quota/quota.c --- a/src/plugins/quota/quota.c Mon Sep 28 12:19:20 2015 +0300 +++ b/src/plugins/quota/quota.c Mon Sep 28 12:33:33 2015 +0300 @@ -590,6 +590,9 @@ uint64_t value, limit; int ret; + if (iter->quota == NULL) + return NULL; + roots = array_get(&iter->quota->roots, &count); if (iter->i >= count) return NULL; @@ -635,6 +638,8 @@ unsigned int i, count; quota = quota_get_mail_user_quota(user); + if (quota == NULL) + return NULL; roots = array_get("a->roots, &count); for (i = 0; i < count; i++) { if (strcmp(roots[i]->set->name, name) == 0) From dovecot at dovecot.org Mon Sep 28 10:56:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 10:56:06 +0000 Subject: dovecot-2.2: lib-http: Avoid crashes when server is already disc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/77486e114423 changeset: 19235:77486e114423 user: Timo Sirainen date: Mon Sep 28 13:53:22 2015 +0300 description: lib-http: Avoid crashes when server is already disconnected. I'm not sure if this is the nicest fix, but seems to work for now at least. diffstat: src/lib-http/http-client-connection.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 44dd7e81a38b -r 77486e114423 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Mon Sep 28 13:51:49 2015 +0300 +++ b/src/lib-http/http-client-connection.c Mon Sep 28 13:53:22 2015 +0300 @@ -551,7 +551,7 @@ /* request is dereferenced in payload destroy callback */ i_stream_unref(&payload); - if (conn->to_input != NULL) { + if (conn->to_input != NULL && conn->conn.input != NULL) { /* already finished reading the payload */ http_client_payload_finished(conn); } @@ -561,7 +561,7 @@ http_client_request_unref(&req); } - if (conn->incoming_payload == NULL) { + if (conn->incoming_payload == NULL && conn->conn.input != NULL) { i_assert(conn->conn.io != NULL || conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW); ret = TRUE; From dovecot at dovecot.org Mon Sep 28 10:56:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 10:56:04 +0000 Subject: dovecot-2.2: lib-http: Disconnection from HTTP server could have... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/44dd7e81a38b changeset: 19234:44dd7e81a38b user: Timo Sirainen date: Mon Sep 28 13:51:49 2015 +0300 description: lib-http: Disconnection from HTTP server could have caused accessing already freed memory. Freeing a request could have freed the connection. diffstat: src/lib-http/http-client-connection.c | 11 +++++++---- 1 files changed, 7 insertions(+), 4 deletions(-) diffs (36 lines): diff -r 14f670053196 -r 44dd7e81a38b src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Mon Sep 28 12:33:33 2015 +0300 +++ b/src/lib-http/http-client-connection.c Mon Sep 28 13:51:49 2015 +0300 @@ -487,7 +487,7 @@ struct http_client_request *req, struct http_response *response) { struct istream *payload; - bool retrying; + bool retrying, ret; i_assert(!conn->in_req_callback); i_assert(conn->incoming_payload == NULL); @@ -541,6 +541,7 @@ return TRUE; } + http_client_connection_ref(conn); if (response->payload != NULL) { req->state = HTTP_REQUEST_STATE_PAYLOAD_IN; payload = response->payload; @@ -563,10 +564,12 @@ if (conn->incoming_payload == NULL) { i_assert(conn->conn.io != NULL || conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW); - return TRUE; + ret = TRUE; + } else { + ret = FALSE; } - - return FALSE; + http_client_connection_unref(&conn); + return ret; } static void http_client_connection_input(struct connection *_conn) From dovecot at dovecot.org Mon Sep 28 10:56:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 10:56:06 +0000 Subject: dovecot-2.2: lib-http: Added asserts to all refcount increments. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c30c30c1e642 changeset: 19236:c30c30c1e642 user: Timo Sirainen date: Mon Sep 28 13:54:28 2015 +0300 description: lib-http: Added asserts to all refcount increments. These could catch situations where an already freed object's refcount is attempted to be increased. diffstat: src/lib-http/http-client-connection.c | 1 + src/lib-http/http-client-request.c | 1 + src/lib-http/http-server-connection.c | 1 + src/lib-http/http-server-request.c | 1 + 4 files changed, 4 insertions(+), 0 deletions(-) diffs (44 lines): diff -r 77486e114423 -r c30c30c1e642 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Mon Sep 28 13:53:22 2015 +0300 +++ b/src/lib-http/http-client-connection.c Mon Sep 28 13:54:28 2015 +0300 @@ -1294,6 +1294,7 @@ void http_client_connection_ref(struct http_client_connection *conn) { + i_assert(conn->refcount > 0); conn->refcount++; } diff -r 77486e114423 -r c30c30c1e642 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Mon Sep 28 13:53:22 2015 +0300 +++ b/src/lib-http/http-client-request.c Mon Sep 28 13:54:28 2015 +0300 @@ -150,6 +150,7 @@ void http_client_request_ref(struct http_client_request *req) { + i_assert(req->refcount > 0); req->refcount++; } diff -r 77486e114423 -r c30c30c1e642 src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Mon Sep 28 13:53:22 2015 +0300 +++ b/src/lib-http/http-server-connection.c Mon Sep 28 13:54:28 2015 +0300 @@ -831,6 +831,7 @@ void http_server_connection_ref(struct http_server_connection *conn) { + i_assert(conn->refcount > 0); conn->refcount++; } diff -r 77486e114423 -r c30c30c1e642 src/lib-http/http-server-request.c --- a/src/lib-http/http-server-request.c Mon Sep 28 13:53:22 2015 +0300 +++ b/src/lib-http/http-server-request.c Mon Sep 28 13:54:28 2015 +0300 @@ -23,6 +23,7 @@ void http_server_request_ref(struct http_server_request *req) { + i_assert(req->refcount > 0); req->refcount++; } From dovecot at dovecot.org Mon Sep 28 12:01:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 12:01:33 +0000 Subject: dovecot-2.2: lib-http: When receiving unexpected input in HTTP r... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cfee3a8b35ec changeset: 19237:cfee3a8b35ec user: Timo Sirainen date: Mon Sep 28 14:34:05 2015 +0300 description: lib-http: When receiving unexpected input in HTTP response, include a longer snippet of it in the error. diffstat: src/lib-http/http-response-parser.c | 49 +++++++++++++++++++++++++++--------- 1 files changed, 37 insertions(+), 12 deletions(-) diffs (95 lines): diff -r c30c30c1e642 -r cfee3a8b35ec src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Mon Sep 28 13:54:28 2015 +0300 +++ b/src/lib-http/http-response-parser.c Mon Sep 28 14:34:05 2015 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2013-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "str.h" #include "istream.h" #include "http-parser.h" #include "http-date.h" @@ -92,15 +93,37 @@ return 1; } -static inline const char *_chr_sanitize(unsigned char c) +static const char *_reply_sanitize(struct http_message_parser *parser) { - if (c >= 0x20 && c < 0x7F) - return t_strdup_printf("`%c'", c); - if (c == 0x0a) - return ""; - if (c == 0x0d) - return ""; - return t_strdup_printf("<0x%02x>", c); + string_t *str = t_str_new(32); + const unsigned char *p; + unsigned int i; + bool quote_open = FALSE; + + i_assert(parser->cur < parser->end); + for (p = parser->cur, i = 0; p < parser->end && i < 20; p++, i++) { + if (*p >= 0x20 && *p < 0x7F) { + if (!quote_open) { + str_append_c(str, '`'); + quote_open = TRUE; + } + str_append_c(str, *p); + } else { + if (quote_open) { + str_append_c(str, '\''); + quote_open = FALSE; + } + if (*p == 0x0a) + str_append(str, ""); + else if (*p == 0x0d) + str_append(str, ""); + else + str_printfa(str, "<0x%02x>", *p); + } + } + if (quote_open) + str_append_c(str, '\''); + return str_c(str); } static int http_response_parse(struct http_response_parser *parser) @@ -122,7 +145,9 @@ case HTTP_RESPONSE_PARSE_STATE_VERSION: if ((ret=http_message_parse_version(_parser)) <= 0) { if (ret < 0) - _parser->error = "Invalid HTTP version in response"; + _parser->error = t_strdup_printf( + "Invalid HTTP version in response: %s", + _reply_sanitize(_parser)); return ret; } parser->state = HTTP_RESPONSE_PARSE_STATE_SP1; @@ -133,7 +158,7 @@ if (*_parser->cur != ' ') { _parser->error = t_strdup_printf ("Expected ' ' after response version, but found %s", - _chr_sanitize(*_parser->cur)); + _reply_sanitize(_parser)); return -1; } _parser->cur++; @@ -155,7 +180,7 @@ if (*_parser->cur != ' ') { _parser->error = t_strdup_printf ("Expected ' ' after response status code, but found %s", - _chr_sanitize(*_parser->cur)); + _reply_sanitize(_parser)); return -1; } _parser->cur++; @@ -183,7 +208,7 @@ if (*_parser->cur != '\n') { _parser->error = t_strdup_printf ("Expected line end after response, but found %s", - _chr_sanitize(*_parser->cur)); + _reply_sanitize(_parser)); return -1; } _parser->cur++; From dovecot at dovecot.org Mon Sep 28 12:01:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 12:01:34 +0000 Subject: dovecot-2.2: lib: When destroying istream-sized stream, it seeks... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ea56193d05b3 changeset: 19238:ea56193d05b3 user: Timo Sirainen date: Mon Sep 28 14:41:15 2015 +0300 description: lib: When destroying istream-sized stream, it seeks the parent input at the end offset. This is also how istream-limit works. diffstat: src/lib/istream-sized.c | 16 ++++++++++++++++ 1 files changed, 16 insertions(+), 0 deletions(-) diffs (33 lines): diff -r cfee3a8b35ec -r ea56193d05b3 src/lib/istream-sized.c --- a/src/lib/istream-sized.c Mon Sep 28 14:34:05 2015 +0300 +++ b/src/lib/istream-sized.c Mon Sep 28 14:41:15 2015 +0300 @@ -11,6 +11,21 @@ bool failed; }; +static void i_stream_sized_destroy(struct iostream_private *stream) +{ + struct sized_istream *sstream = (struct sized_istream *)stream; + uoff_t v_offset; + + v_offset = sstream->istream.parent_start_offset + + sstream->istream.istream.v_offset; + if (sstream->istream.parent->seekable || + v_offset > sstream->istream.parent->v_offset) { + /* get to same position in parent stream */ + i_stream_seek(sstream->istream.parent, v_offset); + } + i_stream_unref(&sstream->istream.parent); +} + static ssize_t i_stream_sized_read(struct istream_private *stream) { struct sized_istream *sstream = @@ -116,6 +131,7 @@ sstream->size = size; sstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + sstream->istream.iostream.destroy = i_stream_sized_destroy; sstream->istream.read = i_stream_sized_read; sstream->istream.stat = i_stream_sized_stat; From dovecot at dovecot.org Mon Sep 28 12:01:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 12:01:34 +0000 Subject: dovecot-2.2: lib: Added i_stream_create_sized_with_callback(). T... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5fa5cbdf65a3 changeset: 19239:5fa5cbdf65a3 user: Timo Sirainen date: Mon Sep 28 14:59:04 2015 +0300 description: lib: Added i_stream_create_sized_with_callback(). The callback returns the wanted error string. diffstat: src/lib/istream-sized.c | 72 ++++++++++++++++++++++++++++++++++++++++-------- src/lib/istream-sized.h | 26 +++++++++++++++++ 2 files changed, 86 insertions(+), 12 deletions(-) diffs (161 lines): diff -r ea56193d05b3 -r 5fa5cbdf65a3 src/lib/istream-sized.c --- a/src/lib/istream-sized.c Mon Sep 28 14:41:15 2015 +0300 +++ b/src/lib/istream-sized.c Mon Sep 28 14:59:04 2015 +0300 @@ -7,6 +7,9 @@ struct sized_istream { struct istream_private istream; + istream_sized_callback_t *error_callback; + void *error_context; + uoff_t size; bool failed; }; @@ -26,10 +29,28 @@ i_stream_unref(&sstream->istream.parent); } +static const char * +i_stream_create_sized_default_error_callback( + const struct istream_sized_error_data *data, void *context ATTR_UNUSED) +{ + if (data->v_offset + data->new_bytes < data->wanted_size) { + return t_strdup_printf("Stream is smaller than expected " + "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", + data->v_offset + data->new_bytes, data->wanted_size); + } else { + return t_strdup_printf("Stream is larger than expected " + "(%"PRIuUOFF_T" > %"PRIuUOFF_T", eof=%d)", + data->v_offset + data->new_bytes, data->wanted_size, + data->eof); + } +} + static ssize_t i_stream_sized_read(struct istream_private *stream) { struct sized_istream *sstream = (struct sized_istream *)stream; + struct istream_sized_error_data data; + const char *error; uoff_t left; ssize_t ret; size_t pos; @@ -64,15 +85,18 @@ stream->buffer = i_stream_get_data(stream->parent, &pos); } while (pos <= stream->pos && ret > 0); + memset(&data, 0, sizeof(data)); + data.v_offset = stream->istream.v_offset; + data.new_bytes = pos; + data.wanted_size = sstream->size; + data.eof = stream->istream.eof; + left = sstream->size - stream->istream.v_offset; if (pos == left) stream->istream.eof = TRUE; else if (pos > left) { - io_stream_set_error(&stream->iostream, - "Stream is larger than expected " - "(%"PRIuUOFF_T" > %"PRIuUOFF_T", eof=%d)", - stream->istream.v_offset+pos, sstream->size, - stream->istream.eof); + error = sstream->error_callback(&data, sstream->error_context); + io_stream_set_error(&stream->iostream, "%s", error); i_error("read(%s) failed: %s", i_stream_get_name(stream->parent), stream->iostream.error); @@ -86,10 +110,8 @@ } else if (stream->istream.stream_errno == ENOENT) { /* lost the file */ } else { - io_stream_set_error(&stream->iostream, - "Stream is smaller than expected " - "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", - stream->istream.v_offset+pos, sstream->size); + error = sstream->error_callback(&data, sstream->error_context); + io_stream_set_error(&stream->iostream, "%s", error); i_error("read(%s) failed: %s", i_stream_get_name(stream->parent), stream->iostream.error); @@ -123,7 +145,8 @@ return 0; } -struct istream *i_stream_create_sized(struct istream *input, uoff_t size) +static struct sized_istream * +i_stream_create_sized_common(struct istream *input, uoff_t size) { struct sized_istream *sstream; @@ -138,6 +161,31 @@ sstream->istream.istream.readable_fd = input->readable_fd; sstream->istream.istream.blocking = input->blocking; sstream->istream.istream.seekable = input->seekable; - return i_stream_create(&sstream->istream, input, - i_stream_get_fd(input)); + (void)i_stream_create(&sstream->istream, input, + i_stream_get_fd(input)); + return sstream; } + +struct istream *i_stream_create_sized(struct istream *input, uoff_t size) +{ + struct sized_istream *sstream; + + sstream = i_stream_create_sized_common(input, size); + sstream->error_callback = i_stream_create_sized_default_error_callback; + sstream->error_context = sstream; + return &sstream->istream.istream; +} + +#undef i_stream_create_sized_with_callback +struct istream * +i_stream_create_sized_with_callback(struct istream *input, uoff_t size, + istream_sized_callback_t *error_callback, + void *context) +{ + struct sized_istream *sstream; + + sstream = i_stream_create_sized_common(input, size); + sstream->error_callback = error_callback; + sstream->error_context = context; + return &sstream->istream.istream; +} diff -r ea56193d05b3 -r 5fa5cbdf65a3 src/lib/istream-sized.h --- a/src/lib/istream-sized.h Mon Sep 28 14:41:15 2015 +0300 +++ b/src/lib/istream-sized.h Mon Sep 28 14:59:04 2015 +0300 @@ -1,8 +1,34 @@ #ifndef ISTREAM_SIZED_H #define ISTREAM_SIZED_H +struct istream_sized_error_data { + /* Stream's current v_offset */ + uoff_t v_offset; + /* How many more bytes are being added within this read() */ + size_t new_bytes; + /* What's the original wanted size. */ + uoff_t wanted_size; + /* TRUE if we're at EOF now */ + bool eof; +}; + +typedef const char * +istream_sized_callback_t(const struct istream_sized_error_data *data, + void *context); + /* Assume that input is exactly the given size. If it's smaller, log an error and fail with EINVAL error. If it's larger, log an error but don't fail. */ struct istream *i_stream_create_sized(struct istream *input, uoff_t size); +/* Same as i_stream_create_sized(), but set the error message via the + callback. */ +struct istream * +i_stream_create_sized_with_callback(struct istream *input, uoff_t size, + istream_sized_callback_t *error_callback, + void *context); +#define i_stream_create_sized_with_callback(input, size, error_callback, context) \ + i_stream_create_sized_with_callback(input, size + \ + CALLBACK_TYPECHECK(error_callback, \ + const char *(*)(const struct istream_sized_error_data *, typeof(context))), \ + (istream_sized_callback_t *)error_callback, context) #endif From dovecot at dovecot.org Mon Sep 28 12:01:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 12:01:35 +0000 Subject: dovecot-2.2: lib-http: Don't silently truncate response payload ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ecad47a83d1d changeset: 19240:ecad47a83d1d user: Timo Sirainen date: Mon Sep 28 15:00:04 2015 +0300 description: lib-http: Don't silently truncate response payload if server disconnects during it. diffstat: src/lib-http/http-message-parser.c | 25 +++++++++++++++++++++++-- 1 files changed, 23 insertions(+), 2 deletions(-) diffs (52 lines): diff -r 5fa5cbdf65a3 -r ecad47a83d1d src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Mon Sep 28 14:59:04 2015 +0300 +++ b/src/lib-http/http-message-parser.c Mon Sep 28 15:00:04 2015 +0300 @@ -3,6 +3,8 @@ #include "lib.h" #include "array.h" #include "istream.h" +#include "iostream.h" +#include "istream-sized.h" #include "http-parser.h" #include "http-header.h" #include "http-header-parser.h" @@ -383,8 +385,22 @@ return ret; } +static const char * +http_istream_error_callback(const struct istream_sized_error_data *data, + struct istream *input) +{ + i_assert(data->eof); + i_assert(data->v_offset + data->new_bytes < data->wanted_size); + + return t_strdup_printf("Disconnected from server at offset %"PRIuUOFF_T + " (wanted %"PRIuUOFF_T"): %s", data->v_offset + data->new_bytes, + data->wanted_size, io_stream_get_disconnect_reason(input, NULL)); +} + int http_message_parse_body(struct http_message_parser *parser, bool request) { + struct istream *input; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; parser->error = NULL; @@ -472,9 +488,14 @@ } /* Got explicit message size from Content-Length: header */ - parser->payload = - i_stream_create_limit(parser->input, + input = i_stream_create_limit(parser->input, parser->msg.content_length); + /* Make sure we return failure if HTTP connection closes before + we've finished reading the full input. */ + parser->payload = i_stream_create_sized_with_callback(input, + parser->msg.content_length, + http_istream_error_callback, input); + i_stream_unref(&input); } else if (!parser->msg.have_content_length && !request) { /* RFC 7230, Section 3.3.3: Message Body Length From dovecot at dovecot.org Mon Sep 28 12:12:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 12:12:06 +0000 Subject: dovecot-2.2: lib-storage: Added assert - mailbox has to be open ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/60814596aff2 changeset: 19242:60814596aff2 user: Timo Sirainen date: Mon Sep 28 15:10:29 2015 +0300 description: lib-storage: Added assert - mailbox has to be open when starting vsize header update. diffstat: src/lib-storage/index/index-mailbox-size.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 12205b780996 -r 60814596aff2 src/lib-storage/index/index-mailbox-size.c --- a/src/lib-storage/index/index-mailbox-size.c Mon Sep 28 15:10:02 2015 +0300 +++ b/src/lib-storage/index/index-mailbox-size.c Mon Sep 28 15:10:29 2015 +0300 @@ -102,6 +102,8 @@ { struct mailbox_vsize_update *update; + i_assert(box->opened); + update = i_new(struct mailbox_vsize_update, 1); update->box = box; update->lock_fd = -1; From dovecot at dovecot.org Mon Sep 28 12:12:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 12:12:00 +0000 Subject: dovecot-2.2: lib-storage: Don't try to update index vsize if syn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/12205b780996 changeset: 19241:12205b780996 user: Timo Sirainen date: Mon Sep 28 15:10:02 2015 +0300 description: lib-storage: Don't try to update index vsize if syncing failed. diffstat: src/lib-storage/index/index-sync.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r ecad47a83d1d -r 12205b780996 src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Mon Sep 28 15:00:04 2015 +0300 +++ b/src/lib-storage/index/index-sync.c Mon Sep 28 15:10:02 2015 +0300 @@ -336,7 +336,8 @@ array_free(&ctx->all_flag_update_uids); /* update vsize header if wanted */ - index_mailbox_vsize_update_appends(_ctx->box); + if (ret == 0) + index_mailbox_vsize_update_appends(_ctx->box); i_free(ctx); return ret; } From dovecot at dovecot.org Mon Sep 28 12:53:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 12:53:16 +0000 Subject: dovecot-2.2: lib: Using istream-timeout with timeout disabled ca... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7a0b0d86651 changeset: 19243:b7a0b0d86651 user: Timo Sirainen date: Mon Sep 28 15:51:45 2015 +0300 description: lib: Using istream-timeout with timeout disabled caused excessive CPU usage. diffstat: src/lib/istream-timeout.c | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diffs (20 lines): diff -r 60814596aff2 -r b7a0b0d86651 src/lib/istream-timeout.c --- a/src/lib/istream-timeout.c Mon Sep 28 15:10:29 2015 +0300 +++ b/src/lib/istream-timeout.c Mon Sep 28 15:51:45 2015 +0300 @@ -101,13 +101,12 @@ "%s (opened %d secs ago)", i_stream_get_error(stream->parent), diff); } - } else if (tstream->to == NULL) { + } else if (tstream->to == NULL && tstream->timeout_msecs > 0) { /* first read. add the timeout here instead of in init in case the stream is created long before it's actually read from. */ - tstream->to = tstream->timeout_msecs == 0 ? NULL : - timeout_add(tstream->timeout_msecs, - i_stream_timeout, tstream); + tstream->to = timeout_add(tstream->timeout_msecs, + i_stream_timeout, tstream); i_stream_timeout_set_pending(tstream); } else if (ret > 0 && tstream->to != NULL) { /* we read something, reset the timeout */ From dovecot at dovecot.org Mon Sep 28 13:10:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 13:10:00 +0000 Subject: dovecot-2.2: quota: doveadm quota recalc works now for "count" b... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/687dd4db9373 changeset: 19245:687dd4db9373 user: Timo Sirainen date: Mon Sep 28 16:07:12 2015 +0300 description: quota: doveadm quota recalc works now for "count" backend by rebuilding vsize header. diffstat: src/plugins/quota/quota-count.c | 67 +++++++++++++++++++++++++++++++++++++++- 1 files changed, 64 insertions(+), 3 deletions(-) diffs (80 lines): diff -r 78a027250652 -r 687dd4db9373 src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Mon Sep 28 16:06:43 2015 +0300 +++ b/src/plugins/quota/quota-count.c Mon Sep 28 16:07:12 2015 +0300 @@ -197,12 +197,73 @@ return 1; } +static int quota_count_recalculate_box(struct mailbox *box) +{ + struct mail_index_transaction *trans; + struct mailbox_metadata metadata; + struct mailbox_index_vsize vsize_hdr; + const char *errstr; + enum mail_error error; + + if (mailbox_open(box) < 0) { + errstr = mailbox_get_last_error(box, &error); + if (error != MAIL_ERROR_TEMP) { + /* non-temporary error, e.g. ACLs denied access. */ + return 0; + } + i_error("Couldn't open mailbox %s: %s", box->vname, errstr); + return -1; + } + + /* reset the vsize header first */ + trans = mail_index_transaction_begin(box->view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + memset(&vsize_hdr, 0, sizeof(vsize_hdr)); + mail_index_update_header_ext(trans, box->vsize_hdr_ext_id, + 0, &vsize_hdr, sizeof(vsize_hdr)); + if (mail_index_transaction_commit(&trans) < 0) + return -1; + /* getting the vsize now forces its recalculation */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_VIRTUAL_SIZE, + &metadata) < 0) { + i_error("Couldn't get mailbox %s vsize: %s", box->vname, + mailbox_get_last_error(box, NULL)); + return -1; + } + /* call sync to write the change to mailbox list index */ + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) { + i_error("Couldn't sync mailbox %s: %s", box->vname, + mailbox_get_last_error(box, NULL)); + return -1; + } + return 0; +} + +static int quota_count_recalculate(struct quota_root *root) +{ + struct quota_mailbox_iter *iter; + const struct mailbox_info *info; + struct mailbox *box; + int ret = 0; + + iter = quota_mailbox_iter_begin(root); + while ((info = quota_mailbox_iter_next(iter)) != NULL) { + box = mailbox_alloc(info->ns->list, info->vname, 0); + if (quota_count_recalculate_box(box) < 0) + ret = -1; + mailbox_free(&box); + } + quota_mailbox_iter_deinit(&iter); + return ret; +} + static int -count_quota_update(struct quota_root *root ATTR_UNUSED, - struct quota_transaction_context *ctx ATTR_UNUSED) +count_quota_update(struct quota_root *root, + struct quota_transaction_context *ctx) { if (ctx->recalculate) { - //FIXME: remove cached values from index + if (quota_count_recalculate(root) < 0) + return -1; } return 0; } From dovecot at dovecot.org Mon Sep 28 13:09:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 13:09:55 +0000 Subject: dovecot-2.2: quota: Code cleanup - abstract out iteration of mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/78a027250652 changeset: 19244:78a027250652 user: Timo Sirainen date: Mon Sep 28 16:06:43 2015 +0300 description: quota: Code cleanup - abstract out iteration of mailboxes related to a quota root. diffstat: src/plugins/quota/quota-count.c | 110 +++++++++++++++++++++++++++------------ 1 files changed, 76 insertions(+), 34 deletions(-) diffs (148 lines): diff -r b7a0b0d86651 -r 78a027250652 src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Mon Sep 28 15:51:45 2015 +0300 +++ b/src/plugins/quota/quota-count.c Mon Sep 28 16:06:43 2015 +0300 @@ -4,6 +4,15 @@ #include "mailbox-list-iter.h" #include "quota-private.h" +struct quota_mailbox_iter { + struct quota_root *root; + struct mail_namespace *ns; + unsigned int ns_idx; + struct mailbox_list_iterate_context *iter; + struct mailbox_info info; + bool failed; +}; + extern struct quota_backend quota_backend_count; static int @@ -49,44 +58,81 @@ return ret; } +static struct quota_mailbox_iter * +quota_mailbox_iter_begin(struct quota_root *root) +{ + struct quota_mailbox_iter *iter; + + iter = i_new(struct quota_mailbox_iter, 1); + iter->root = root; + return iter; +} + static int -quota_count_namespace(struct quota_root *root, struct mail_namespace *ns, - uint64_t *bytes, uint64_t *count) +quota_mailbox_iter_deinit(struct quota_mailbox_iter **_iter) { - struct mailbox_list_iterate_context *ctx; - const struct mailbox_info *info; - int ret = 0; + struct quota_mailbox_iter *iter = *_iter; + int ret = iter->failed ? -1 : 0; - ctx = mailbox_list_iter_init(ns->list, "*", - MAILBOX_LIST_ITER_SKIP_ALIASES | - MAILBOX_LIST_ITER_RETURN_NO_FLAGS); - while ((info = mailbox_list_iter_next(ctx)) != NULL) { - if ((info->flags & (MAILBOX_NONEXISTENT | - MAILBOX_NOSELECT)) == 0) { - ret = quota_count_mailbox(root, ns, info->vname, - bytes, count); - if (ret < 0) - break; + *_iter = NULL; + + if (iter->iter != NULL) { + if (mailbox_list_iter_deinit(&iter->iter) < 0) { + i_error("quota: Listing namespace '%s' failed: %s", + iter->ns->prefix, + mailbox_list_get_last_error(iter->ns->list, NULL)); + ret = -1; } } - if (mailbox_list_iter_deinit(&ctx) < 0) { + i_free(iter); + return ret; +} + +static const struct mailbox_info * +quota_mailbox_iter_next(struct quota_mailbox_iter *iter) +{ + struct mail_namespace *const *namespaces; + const struct mailbox_info *info; + unsigned int count; + + if (iter->iter == NULL) { + namespaces = array_get(&iter->root->quota->namespaces, &count); + if (iter->ns_idx >= count) + return NULL; + + iter->ns = namespaces[iter->ns_idx++]; + iter->iter = mailbox_list_iter_init(iter->ns->list, "*", + MAILBOX_LIST_ITER_SKIP_ALIASES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS); + } + while ((info = mailbox_list_iter_next(iter->iter)) != NULL) { + if ((info->flags & (MAILBOX_NONEXISTENT | + MAILBOX_NOSELECT)) == 0) + return info; + } + if (mailbox_list_iter_deinit(&iter->iter) < 0) { i_error("quota: Listing namespace '%s' failed: %s", - ns->prefix, mailbox_list_get_last_error(ns->list, NULL)); - ret = -1; + iter->ns->prefix, + mailbox_list_get_last_error(iter->ns->list, NULL)); + iter->failed = TRUE; } - if (ns->prefix_len > 0 && ret == 0 && - (ns->prefix_len != 6 || strncasecmp(ns->prefix, "INBOX", 5) != 0)) { + if (iter->ns->prefix_len > 0 && + (iter->ns->prefix_len != 6 || + strncasecmp(iter->ns->prefix, "INBOX", 5) != 0)) { /* if the namespace prefix itself exists, count it also */ - const char *name = t_strndup(ns->prefix, ns->prefix_len-1); - ret = quota_count_mailbox(root, ns, name, bytes, count); + iter->info.ns = iter->ns; + iter->info.vname = t_strndup(iter->ns->prefix, + iter->ns->prefix_len-1); + return &iter->info; } - return ret; + /* try the next namespace */ + return quota_mailbox_iter_next(iter); } int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r) { - struct mail_namespace *const *namespaces; - unsigned int i, count; + struct quota_mailbox_iter *iter; + const struct mailbox_info *info; int ret = 0; *bytes_r = *count_r = 0; @@ -94,16 +140,12 @@ return 0; root->recounting = TRUE; - namespaces = array_get(&root->quota->namespaces, &count); - for (i = 0; i < count; i++) { - if (!quota_root_is_namespace_visible(root, namespaces[i])) - continue; - - ret = quota_count_namespace(root, namespaces[i], - bytes_r, count_r); - if (ret < 0) - break; + iter = quota_mailbox_iter_begin(root); + while (ret >= 0 && (info = quota_mailbox_iter_next(iter)) != NULL) { + ret = quota_count_mailbox(root, info->ns, info->vname, + bytes_r, count_r); } + quota_mailbox_iter_deinit(&iter); root->recounting = FALSE; return ret; } From dovecot at dovecot.org Mon Sep 28 13:18:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 13:18:53 +0000 Subject: dovecot-2.2: virtual plugin: Fixed crash on mailbox_status() if ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1041fb241512 changeset: 19246:1041fb241512 user: Timo Sirainen date: Mon Sep 28 16:13:00 2015 +0300 description: virtual plugin: Fixed crash on mailbox_status() if virtual mailbox wasn't opened yet. mbox->backend_boxes is initialized only when mailbox is opened. diffstat: src/plugins/virtual/virtual-storage.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 687dd4db9373 -r 1041fb241512 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Mon Sep 28 16:07:12 2015 +0300 +++ b/src/plugins/virtual/virtual-storage.c Mon Sep 28 16:13:00 2015 +0300 @@ -508,6 +508,11 @@ struct mailbox_status status; bool opened; + if (!mbox->box.opened) { + if (mailbox_open(&mbox->box) < 0) + return -1; + } + mbox->have_guids = TRUE; mbox->have_save_guids = TRUE; From dovecot at dovecot.org Mon Sep 28 13:18:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 13:18:57 +0000 Subject: dovecot-2.2: virtual plugin: Fixed assert-crash on mailbox_statu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f2c25d5b6280 changeset: 19247:f2c25d5b6280 user: Timo Sirainen date: Mon Sep 28 16:15:00 2015 +0300 description: virtual plugin: Fixed assert-crash on mailbox_status() if virtual mailbox wasn't opened yet. We no longer require that mailboxes must not be auto-opened. diffstat: src/plugins/virtual/virtual-storage.c | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diffs (24 lines): diff -r 1041fb241512 -r f2c25d5b6280 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Mon Sep 28 16:13:00 2015 +0300 +++ b/src/plugins/virtual/virtual-storage.c Mon Sep 28 16:15:00 2015 +0300 @@ -506,7 +506,6 @@ struct virtual_backend_box *const *bboxes; unsigned int i, count; struct mailbox_status status; - bool opened; if (!mbox->box.opened) { if (mailbox_open(&mbox->box) < 0) @@ -518,12 +517,10 @@ bboxes = array_get(&mbox->backend_boxes, &count); for (i = 0; i < count; i++) { - opened = bboxes[i]->box->opened; if (mailbox_get_status(bboxes[i]->box, 0, &status) < 0) { virtual_box_copy_error(&mbox->box, bboxes[i]->box); return -1; } - i_assert(bboxes[i]->box->opened == opened); if (!status.have_guids) mbox->have_guids = FALSE; if (!status.have_save_guids) From dovecot at dovecot.org Mon Sep 28 13:19:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 13:19:00 +0000 Subject: dovecot-2.2: quota: The new quota counting code was counting onl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/08725018dedd changeset: 19248:08725018dedd user: Timo Sirainen date: Mon Sep 28 16:17:13 2015 +0300 description: quota: The new quota counting code was counting only the last mailbox. diffstat: src/plugins/quota/quota-count.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (24 lines): diff -r f2c25d5b6280 -r 08725018dedd src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Mon Sep 28 16:15:00 2015 +0300 +++ b/src/plugins/quota/quota-count.c Mon Sep 28 16:17:13 2015 +0300 @@ -17,7 +17,7 @@ static int quota_count_mailbox(struct quota_root *root, struct mail_namespace *ns, - const char *vname, uint64_t *bytes_r, uint64_t *count_r) + const char *vname, uint64_t *bytes, uint64_t *count) { struct quota_rule *rule; struct mailbox *box; @@ -50,9 +50,9 @@ } } else { ret = 1; - *bytes_r = root->quota->set->vsizes ? + *bytes += root->quota->set->vsizes ? metadata.virtual_size : metadata.physical_size; - *count_r = status.messages; + *count += status.messages; } mailbox_free(&box); return ret; From dovecot at dovecot.org Mon Sep 28 13:21:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 13:21:28 +0000 Subject: dovecot-2.2: quota: Fixed "count" backend to ignore storages tha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2db9595b302e changeset: 19249:2db9595b302e user: Timo Sirainen date: Mon Sep 28 16:20:02 2015 +0300 description: quota: Fixed "count" backend to ignore storages that have quota disabled. For example virtual mailboxes shouldn't be counted in quota. diffstat: src/plugins/quota/quota-count.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diffs (20 lines): diff -r 08725018dedd -r 2db9595b302e src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Mon Sep 28 16:17:13 2015 +0300 +++ b/src/plugins/quota/quota-count.c Mon Sep 28 16:20:02 2015 +0300 @@ -34,10 +34,12 @@ } box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); - if (mailbox_get_metadata(box, root->quota->set->vsizes ? - MAILBOX_METADATA_VIRTUAL_SIZE : - MAILBOX_METADATA_PHYSICAL_SIZE, - &metadata) < 0 || + if ((box->storage->class_flags & MAIL_STORAGE_CLASS_FLAG_NOQUOTA) != 0) { + /* quota doesn't exist for this mailbox/storage */ + } else if (mailbox_get_metadata(box, root->quota->set->vsizes ? + MAILBOX_METADATA_VIRTUAL_SIZE : + MAILBOX_METADATA_PHYSICAL_SIZE, + &metadata) < 0 || mailbox_get_status(box, STATUS_MESSAGES, &status) < 0) { errstr = mailbox_get_last_error(box, &error); if (error == MAIL_ERROR_TEMP) { From dovecot at dovecot.org Mon Sep 28 13:39:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 13:39:04 +0000 Subject: dovecot-2.2: quota: Fix to previous commit - counting quota-disa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/15be6c2e26c0 changeset: 19250:15be6c2e26c0 user: Timo Sirainen date: Mon Sep 28 16:37:37 2015 +0300 description: quota: Fix to previous commit - counting quota-disabled-storages may have returned random failures. diffstat: src/plugins/quota/quota-count.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 2db9595b302e -r 15be6c2e26c0 src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Mon Sep 28 16:20:02 2015 +0300 +++ b/src/plugins/quota/quota-count.c Mon Sep 28 16:37:37 2015 +0300 @@ -36,6 +36,7 @@ box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); if ((box->storage->class_flags & MAIL_STORAGE_CLASS_FLAG_NOQUOTA) != 0) { /* quota doesn't exist for this mailbox/storage */ + ret = 0; } else if (mailbox_get_metadata(box, root->quota->set->vsizes ? MAILBOX_METADATA_VIRTUAL_SIZE : MAILBOX_METADATA_PHYSICAL_SIZE, From dovecot at dovecot.org Mon Sep 28 14:36:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 14:36:15 +0000 Subject: dovecot-2.2: Released v2.2.19.rc2. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7a5726201e40 changeset: 19251:7a5726201e40 user: Timo Sirainen date: Mon Sep 28 17:25:59 2015 +0300 description: Released v2.2.19.rc2. diffstat: configure.ac | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 15be6c2e26c0 -r 7a5726201e40 configure.ac --- a/configure.ac Mon Sep 28 16:37:37 2015 +0300 +++ b/configure.ac Mon Sep 28 17:25:59 2015 +0300 @@ -2,7 +2,7 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.19.rc1],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.2.19.rc2],[dovecot at dovecot.org]) AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv19($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Mon Sep 28 14:36:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 14:36:16 +0000 Subject: dovecot-2.2: Added tag 2.2.19.rc2 for changeset 7a5726201e40 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8d86e2d2f049 changeset: 19252:8d86e2d2f049 user: Timo Sirainen date: Mon Sep 28 17:26:00 2015 +0300 description: Added tag 2.2.19.rc2 for changeset 7a5726201e40 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 7a5726201e40 -r 8d86e2d2f049 .hgtags --- a/.hgtags Mon Sep 28 17:25:59 2015 +0300 +++ b/.hgtags Mon Sep 28 17:26:00 2015 +0300 @@ -128,3 +128,4 @@ 166106aaefc5e6cf3850dba9534ca90153931ad9 2.2.17 917d027836d0f9d7d4b740dc33ae48cc5beead6f 2.2.18 60057d955db325d1af199098ff4cfd57f237645c 2.2.19.rc1 +7a5726201e40ac299b677cebd73326ec92b8fdd8 2.2.19.rc2 From dovecot at dovecot.org Mon Sep 28 14:36:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 14:36:16 +0000 Subject: dovecot-2.2: Added signature for changeset 7a5726201e40 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/460cee8121ff changeset: 19253:460cee8121ff user: Timo Sirainen date: Mon Sep 28 17:26:04 2015 +0300 description: Added signature for changeset 7a5726201e40 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 8d86e2d2f049 -r 460cee8121ff .hgsigs --- a/.hgsigs Mon Sep 28 17:26:00 2015 +0300 +++ b/.hgsigs Mon Sep 28 17:26:04 2015 +0300 @@ -91,3 +91,4 @@ 166106aaefc5e6cf3850dba9534ca90153931ad9 0 iEYEABECAAYFAlVTf5MACgkQyUhSUUBViskLHgCeMooopuB7egroMj8w78YarjjRieYAnjVSKcz0sXShlbYRsLRPOnyfq/pS 917d027836d0f9d7d4b740dc33ae48cc5beead6f 0 iEYEABECAAYFAlVV0mIACgkQyUhSUUBVismFtwCeIVNrGFguuXIhDoCSUm86Q4uTpQUAnAi6UfVL9/+QWC5LpR7jvY97QH0c 60057d955db325d1af199098ff4cfd57f237645c 0 iEYEABECAAYFAlYCpMAACgkQyUhSUUBVismRDwCeIE5jzDFBEBFuIf2JgRMsvgErSmkAniUiUmQ8clnWBSqu7fEY7A2ZbHia +7a5726201e40ac299b677cebd73326ec92b8fdd8 0 iEYEABECAAYFAlYJTfgACgkQyUhSUUBVislrewCfZRDY+7eAKD4vPigNVUlVU/fU1CUAoIbplWaRSdUFByC4A4tb1OdU1uWc From dovecot at dovecot.org Mon Sep 28 20:40:58 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Sep 2015 20:40:58 +0000 Subject: dovecot-2.2: lib: s/EDEADLOCK/EDEADLK/ Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7633bca91551 changeset: 19254:7633bca91551 user: Timo Sirainen date: Mon Sep 28 23:39:31 2015 +0300 description: lib: s/EDEADLOCK/EDEADLK/ It felt wrong since the beginning, but my first attempt at EDEADLCK didn't work and EDEADLOCK did. Of course, not everywhere. diffstat: src/lib/file-lock.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 460cee8121ff -r 7633bca91551 src/lib/file-lock.c --- a/src/lib/file-lock.c Mon Sep 28 17:26:04 2015 +0300 +++ b/src/lib/file-lock.c Mon Sep 28 23:39:31 2015 +0300 @@ -211,7 +211,7 @@ } *error_r = t_strdup_printf("fcntl(%s, %s, %s) locking failed: %m", path, lock_type_str, timeout_secs == 0 ? "F_SETLK" : "F_SETLKW"); - if (errno == EDEADLOCK) + if (errno == EDEADLK) i_panic("%s%s", *error_r, file_lock_find(fd, lock_method, lock_type)); return -1; #endif @@ -259,7 +259,7 @@ } *error_r = t_strdup_printf("flock(%s, %s) failed: %m", path, lock_type_str); - if (errno == EDEADLOCK) + if (errno == EDEADLK) i_panic("%s%s", *error_r, file_lock_find(fd, lock_method, lock_type)); return -1; #endif From dovecot at dovecot.org Tue Sep 29 18:07:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Sep 2015 18:07:46 +0000 Subject: dovecot-2.2: fts-solr: Removed Content-Type header from "select"... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/202c236f7581 changeset: 19255:202c236f7581 user: Timo Sirainen date: Tue Sep 29 21:02:37 2015 +0300 description: fts-solr: Removed Content-Type header from "select" requests to fix Solr v5.1. diffstat: src/plugins/fts-solr/solr-connection.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 7633bca91551 -r 202c236f7581 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Mon Sep 28 23:39:31 2015 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Tue Sep 29 21:02:37 2015 +0300 @@ -431,7 +431,6 @@ solr_connection_select_response, conn); http_client_request_set_port(http_req, conn->http_port); http_client_request_set_ssl(http_req, conn->http_ssl); - http_client_request_add_header(http_req, "Content-Type", "text/xml"); http_client_request_submit(http_req); conn->request_status = 0; From pigeonhole at rename-it.nl Tue Sep 29 22:55:09 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 30 Sep 2015 00:55:09 +0200 Subject: dovecot-2.2-pigeonhole: Sieve vnd.dovecot.environment extension:... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/9ac3914392a4 changeset: 2107:9ac3914392a4 user: Stephan Bosch date: Wed Sep 30 00:54:40 2015 +0200 description: Sieve vnd.dovecot.environment extension: Forgot 'extern' in extension declaration. This causes link failures for certain platforms. diffstat: src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-common.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 87dbcf33c13c -r 9ac3914392a4 src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-common.h --- a/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-common.h Fri Sep 25 00:01:42 2015 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-common.h Wed Sep 30 00:54:40 2015 +0200 @@ -15,7 +15,7 @@ const struct sieve_extension *var_ext; }; -const struct sieve_extension_def vnd_environment_extension; +extern const struct sieve_extension_def vnd_environment_extension; /* * Operands From dovecot at dovecot.org Wed Sep 30 15:13:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Sep 2015 15:13:39 +0000 Subject: dovecot-2.2: dict-sql: Added support for "uint" field type. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22015b472147 changeset: 19257:22015b472147 user: Timo Sirainen date: Wed Sep 30 18:11:53 2015 +0300 description: dict-sql: Added support for "uint" field type. diffstat: src/lib-dict/dict-sql-settings.c | 5 +++++ src/lib-dict/dict-sql-settings.h | 1 + src/lib-dict/dict-sql.c | 22 ++++++++++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diffs (70 lines): diff -r 6785752f1e25 -r 22015b472147 src/lib-dict/dict-sql-settings.c --- a/src/lib-dict/dict-sql-settings.c Wed Sep 30 17:38:12 2015 +0300 +++ b/src/lib-dict/dict-sql-settings.c Wed Sep 30 18:11:53 2015 +0300 @@ -177,6 +177,11 @@ field->variable = p_strndup(ctx->pool, value + 10, value_len-10-1); field->sql_field.value_type = DICT_SQL_TYPE_HEXBLOB; + } else if (strncmp(value, "${uint:", 7) == 0 && + value[value_len-1] == '}') { + field->variable = p_strndup(ctx->pool, value + 7, + value_len-7-1); + field->sql_field.value_type = DICT_SQL_TYPE_UINT; } else { field->variable = p_strdup(ctx->pool, value + 1); } diff -r 6785752f1e25 -r 22015b472147 src/lib-dict/dict-sql-settings.h --- a/src/lib-dict/dict-sql-settings.h Wed Sep 30 17:38:12 2015 +0300 +++ b/src/lib-dict/dict-sql-settings.h Wed Sep 30 18:11:53 2015 +0300 @@ -3,6 +3,7 @@ enum dict_sql_type { DICT_SQL_TYPE_STRING = 0, + DICT_SQL_TYPE_UINT, DICT_SQL_TYPE_HEXBLOB }; diff -r 6785752f1e25 -r 22015b472147 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Wed Sep 30 17:38:12 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 30 18:11:53 2015 +0300 @@ -218,11 +218,24 @@ const char **error_r) { buffer_t *buf; + unsigned int num; - if (value_type == DICT_SQL_TYPE_STRING) { + switch (value_type) { + case DICT_SQL_TYPE_STRING: str_printfa(str, "'%s%s'", sql_escape_string(dict->db, value), value_suffix); return 0; + case DICT_SQL_TYPE_UINT: + if (value_suffix[0] != '\0' || str_to_uint(value, &num) < 0) { + *error_r = t_strdup_printf( + "field %s value isn't unsigned integer: %s%s", + field_name, value, value_suffix); + return -1; + } + str_printfa(str, "%u", num); + return 0; + case DICT_SQL_TYPE_HEXBLOB: + break; } buf = buffer_create_dynamic(pool_datastack_create(), strlen(value)/2); @@ -353,8 +366,13 @@ size_t size; string_t *str; - if (type == DICT_SQL_TYPE_STRING) + switch (type) { + case DICT_SQL_TYPE_STRING: + case DICT_SQL_TYPE_UINT: return p_strdup(pool, sql_result_get_field_value(result, result_idx)); + case DICT_SQL_TYPE_HEXBLOB: + break; + } data = sql_result_get_field_value_binary(result, result_idx, &size); str = str_new(pool, size*2 + 1); From dovecot at dovecot.org Wed Sep 30 15:13:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Sep 2015 15:13:37 +0000 Subject: dovecot-2.2: dict-sql: Added support for extensible field types. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6785752f1e25 changeset: 19256:6785752f1e25 user: Timo Sirainen date: Wed Sep 30 17:38:12 2015 +0300 description: dict-sql: Added support for extensible field types. diffstat: src/lib-dict/dict-sql-settings.c | 2 +- src/lib-dict/dict-sql-settings.h | 7 ++++++- src/lib-dict/dict-sql.c | 28 ++++++++++++++++------------ 3 files changed, 23 insertions(+), 14 deletions(-) diffs (121 lines): diff -r 202c236f7581 -r 6785752f1e25 src/lib-dict/dict-sql-settings.c --- a/src/lib-dict/dict-sql-settings.c Tue Sep 29 21:02:37 2015 +0300 +++ b/src/lib-dict/dict-sql-settings.c Wed Sep 30 17:38:12 2015 +0300 @@ -176,7 +176,7 @@ value[value_len-1] == '}') { field->variable = p_strndup(ctx->pool, value + 10, value_len-10-1); - field->sql_field.value_is_hexblob = TRUE; + field->sql_field.value_type = DICT_SQL_TYPE_HEXBLOB; } else { field->variable = p_strdup(ctx->pool, value + 1); } diff -r 202c236f7581 -r 6785752f1e25 src/lib-dict/dict-sql-settings.h --- a/src/lib-dict/dict-sql-settings.h Tue Sep 29 21:02:37 2015 +0300 +++ b/src/lib-dict/dict-sql-settings.h Wed Sep 30 17:38:12 2015 +0300 @@ -1,9 +1,14 @@ #ifndef DICT_SQL_SETTINGS_H #define DICT_SQL_SETTINGS_H +enum dict_sql_type { + DICT_SQL_TYPE_STRING = 0, + DICT_SQL_TYPE_HEXBLOB +}; + struct dict_sql_field { const char *name; - bool value_is_hexblob; + enum dict_sql_type value_type; }; struct dict_sql_map { diff -r 202c236f7581 -r 6785752f1e25 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Tue Sep 29 21:02:37 2015 +0300 +++ b/src/lib-dict/dict-sql.c Wed Sep 30 17:38:12 2015 +0300 @@ -213,13 +213,13 @@ static int sql_dict_value_escape(string_t *str, struct sql_dict *dict, - bool value_is_hexblob, const char *field_name, + enum dict_sql_type value_type, const char *field_name, const char *value, const char *value_suffix, const char **error_r) { buffer_t *buf; - if (!value_is_hexblob) { + if (value_type == DICT_SQL_TYPE_STRING) { str_printfa(str, "'%s%s'", sql_escape_string(dict->db, value), value_suffix); return 0; @@ -244,7 +244,7 @@ const char *value, const char *value_suffix, const char **error_r) { - return sql_dict_value_escape(str, dict, field->value_is_hexblob, + return sql_dict_value_escape(str, dict, field->value_type, field->name, value, value_suffix, error_r); } @@ -346,14 +346,14 @@ } static const char * -sql_dict_result_unescape(bool hexblob, pool_t pool, struct sql_result *result, - unsigned int result_idx) +sql_dict_result_unescape(enum dict_sql_type type, pool_t pool, + struct sql_result *result, unsigned int result_idx) { const unsigned char *data; size_t size; string_t *str; - if (!hexblob) + if (type == DICT_SQL_TYPE_STRING) return p_strdup(pool, sql_result_get_field_value(result, result_idx)); data = sql_result_get_field_value_binary(result, result_idx, &size); @@ -366,7 +366,9 @@ sql_dict_result_unescape_value(const struct dict_sql_map *map, pool_t pool, struct sql_result *result) { - return sql_dict_result_unescape(map->value_hexblob, pool, result, 0); + enum dict_sql_type value_type = map->value_hexblob ? + DICT_SQL_TYPE_HEXBLOB : DICT_SQL_TYPE_STRING; + return sql_dict_result_unescape(value_type, pool, result, 0); } static const char * @@ -377,7 +379,7 @@ const struct dict_sql_field *sql_field; sql_field = array_idx(&map->sql_fields, sql_field_idx); - return sql_dict_result_unescape(sql_field->value_is_hexblob, pool, + return sql_dict_result_unescape(sql_field->value_type, pool, result, result_idx); } @@ -856,8 +858,9 @@ if (build->inc) str_append(suffix, fields[i].value); else { - if (sql_dict_value_escape(suffix, dict, - fields[i].map->value_hexblob, + enum dict_sql_type value_type = fields[i].map->value_hexblob ? + DICT_SQL_TYPE_HEXBLOB : DICT_SQL_TYPE_STRING; + if (sql_dict_value_escape(suffix, dict, value_type, "value", fields[i].value, "", error_r) < 0) return -1; } @@ -898,8 +901,9 @@ fields[i].map->value_field, fields[i].value); } else { - if (sql_dict_value_escape(prefix, dict, - fields[i].map->value_hexblob, + enum dict_sql_type value_type = fields[i].map->value_hexblob ? + DICT_SQL_TYPE_HEXBLOB : DICT_SQL_TYPE_STRING; + if (sql_dict_value_escape(prefix, dict, value_type, "value", fields[i].value, "", error_r) < 0) return -1; } From dovecot at dovecot.org Wed Sep 30 15:51:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Sep 2015 15:51:39 +0000 Subject: dovecot-2.2: lib-storage: Renamed MAIL_FETCH_UIDL_FILE_NAME to M... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b631f67f56d3 changeset: 19258:b631f67f56d3 user: Timo Sirainen date: Wed Sep 30 18:49:39 2015 +0300 description: lib-storage: Renamed MAIL_FETCH_UIDL_FILE_NAME to MAIL_FETCH_STORAGE_ID. Keep MAIL_FETCH_UIDL_FILE_NAME for backwards compatibility for now. diffstat: src/lib-storage/index/index-mail.c | 2 +- src/lib-storage/index/maildir/maildir-copy.c | 2 +- src/lib-storage/index/maildir/maildir-mail.c | 10 +++++----- src/lib-storage/index/raw/raw-mail.c | 2 +- src/lib-storage/mail-storage.h | 3 ++- src/pop3/pop3-commands.c | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diffs (106 lines): diff -r 22015b472147 -r b631f67f56d3 src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Sep 30 18:11:53 2015 +0300 +++ b/src/lib-storage/index/index-mail.c Wed Sep 30 18:49:39 2015 +0300 @@ -1397,7 +1397,7 @@ return 0; case MAIL_FETCH_BODY_SNIPPET: return index_mail_fetch_body_snippet(mail, value_r); - case MAIL_FETCH_UIDL_FILE_NAME: + case MAIL_FETCH_STORAGE_ID: case MAIL_FETCH_UIDL_BACKEND: case MAIL_FETCH_SEARCH_RELEVANCY: case MAIL_FETCH_GUID: diff -r 22015b472147 -r b631f67f56d3 src/lib-storage/index/maildir/maildir-copy.c --- a/src/lib-storage/index/maildir/maildir-copy.c Wed Sep 30 18:11:53 2015 +0300 +++ b/src/lib-storage/index/maildir/maildir-copy.c Wed Sep 30 18:49:39 2015 +0300 @@ -91,7 +91,7 @@ return -1; } else { /* raw / lda */ - if (mail_get_special(mail, MAIL_FETCH_UIDL_FILE_NAME, + if (mail_get_special(mail, MAIL_FETCH_STORAGE_ID, &path) < 0 || *path == '\0') return 0; if (do_hardlink(dest_mbox, path, &do_ctx) < 0) diff -r 22015b472147 -r b631f67f56d3 src/lib-storage/index/maildir/maildir-mail.c --- a/src/lib-storage/index/maildir/maildir-mail.c Wed Sep 30 18:11:53 2015 +0300 +++ b/src/lib-storage/index/maildir/maildir-mail.c Wed Sep 30 18:49:39 2015 +0300 @@ -232,7 +232,7 @@ /* if this mail itself has non-pop3 fields we know we're not pop3-only */ allowed_pop3_fields = MAIL_FETCH_FLAGS | MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY | MAIL_FETCH_UIDL_FILE_NAME | + MAIL_FETCH_STREAM_BODY | MAIL_FETCH_STORAGE_ID | MAIL_FETCH_VIRTUAL_SIZE; if (mail->data.wanted_headers != NULL || @@ -524,12 +524,12 @@ } /* default to base filename: */ - if (maildir_mail_get_special(_mail, MAIL_FETCH_UIDL_FILE_NAME, + if (maildir_mail_get_special(_mail, MAIL_FETCH_STORAGE_ID, value_r) < 0) return -1; mail->data.guid = mail->data.filename; return 0; - case MAIL_FETCH_UIDL_FILE_NAME: + case MAIL_FETCH_STORAGE_ID: if (mail->data.filename != NULL) { *value_r = mail->data.filename; return 0; @@ -561,7 +561,7 @@ } else if (*uidl == '\0') { /* special optimization case: use the base file name */ return maildir_mail_get_special(_mail, - MAIL_FETCH_UIDL_FILE_NAME, value_r); + MAIL_FETCH_STORAGE_ID, value_r); } else { *value_r = p_strdup(mail->mail.data_pool, uidl); } @@ -621,7 +621,7 @@ struct maildir_mailbox *mbox = (struct maildir_mailbox *)_mail->box; const char *fname; - if (maildir_mail_get_special(_mail, MAIL_FETCH_UIDL_FILE_NAME, + if (maildir_mail_get_special(_mail, MAIL_FETCH_STORAGE_ID, &fname) == 0 && strcmp(uidl, fname) == 0) { /* special case optimization: empty UIDL means the same diff -r 22015b472147 -r b631f67f56d3 src/lib-storage/index/raw/raw-mail.c --- a/src/lib-storage/index/raw/raw-mail.c Wed Sep 30 18:11:53 2015 +0300 +++ b/src/lib-storage/index/raw/raw-mail.c Wed Sep 30 18:49:39 2015 +0300 @@ -105,7 +105,7 @@ *value_r = mbox->envelope_sender != NULL ? mbox->envelope_sender : ""; return 0; - case MAIL_FETCH_UIDL_FILE_NAME: + case MAIL_FETCH_STORAGE_ID: *value_r = mbox->have_filename ? mailbox_get_path(_mail->box) : ""; return 0; diff -r 22015b472147 -r b631f67f56d3 src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Wed Sep 30 18:11:53 2015 +0300 +++ b/src/lib-storage/mail-storage.h Wed Sep 30 18:49:39 2015 +0300 @@ -150,7 +150,8 @@ MAIL_FETCH_IMAP_ENVELOPE = 0x00004000, MAIL_FETCH_FROM_ENVELOPE = 0x00008000, MAIL_FETCH_HEADER_MD5 = 0x00010000, - MAIL_FETCH_UIDL_FILE_NAME = 0x00020000, + MAIL_FETCH_STORAGE_ID = 0x00020000, +#define MAIL_FETCH_UIDL_FILE_NAME MAIL_FETCH_STORAGE_ID /* FIXME: remove in v2.3 */ MAIL_FETCH_UIDL_BACKEND = 0x00040000, MAIL_FETCH_MAILBOX_NAME = 0x00080000, MAIL_FETCH_SEARCH_RELEVANCY = 0x00100000, diff -r 22015b472147 -r b631f67f56d3 src/pop3/pop3-commands.c --- a/src/pop3/pop3-commands.c Wed Sep 30 18:11:53 2015 +0300 +++ b/src/pop3/pop3-commands.c Wed Sep 30 18:49:39 2015 +0300 @@ -620,7 +620,7 @@ } } if ((client->uidl_keymask & UIDL_FILE_NAME) != 0) { - if (mail_get_special(mail, MAIL_FETCH_UIDL_FILE_NAME, + if (mail_get_special(mail, MAIL_FETCH_STORAGE_ID, &tab[3].value) < 0) { i_error("UIDL: File name lookup failed: %s", mailbox_get_last_error(mail->box, NULL)); From dovecot at dovecot.org Wed Sep 30 15:51:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Sep 2015 15:51:40 +0000 Subject: dovecot-2.2: doveadm fetch: Added "storageid" field. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2ceca0becf8e changeset: 19259:2ceca0becf8e user: Timo Sirainen date: Wed Sep 30 18:50:09 2015 +0300 description: doveadm fetch: Added "storageid" field. diffstat: src/doveadm/doveadm-mail-fetch.c | 13 ++++++++++++- 1 files changed, 12 insertions(+), 1 deletions(-) diffs (30 lines): diff -r b631f67f56d3 -r 2ceca0becf8e src/doveadm/doveadm-mail-fetch.c --- a/src/doveadm/doveadm-mail-fetch.c Wed Sep 30 18:49:39 2015 +0300 +++ b/src/doveadm/doveadm-mail-fetch.c Wed Sep 30 18:50:09 2015 +0300 @@ -386,6 +386,16 @@ return 0; } +static int fetch_storageid(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_STORAGE_ID, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + static const struct fetch_field fetch_fields[] = { { "user", 0, fetch_user }, { "mailbox", 0, fetch_mailbox }, @@ -412,7 +422,8 @@ { "imap.bodystructure", MAIL_FETCH_IMAP_BODYSTRUCTURE, fetch_imap_bodystructure }, { "pop3.uidl", MAIL_FETCH_UIDL_BACKEND, fetch_pop3_uidl }, { "pop3.order", MAIL_FETCH_POP3_ORDER, fetch_pop3_order }, - { "refcount", MAIL_FETCH_REFCOUNT, fetch_refcount } + { "refcount", MAIL_FETCH_REFCOUNT, fetch_refcount }, + { "storageid", MAIL_FETCH_STORAGE_ID, fetch_storageid } }; static const struct fetch_field *fetch_field_find(const char *name) From pigeonhole at rename-it.nl Wed Sep 30 22:11:12 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 01 Oct 2015 00:11:12 +0200 Subject: dovecot-2.2-pigeonhole: Sieve extprograms plugin: Fixed Coverity... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/3928e08d9340 changeset: 2111:3928e08d9340 user: Stephan Bosch date: Thu Oct 01 00:10:31 2015 +0200 description: Sieve extprograms plugin: Fixed Coverity warning. diffstat: src/plugins/sieve-extprograms/cmd-pipe.c | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diffs (20 lines): diff -r 0f43eed67bd5 -r 3928e08d9340 src/plugins/sieve-extprograms/cmd-pipe.c --- a/src/plugins/sieve-extprograms/cmd-pipe.c Thu Oct 01 00:09:51 2015 +0200 +++ b/src/plugins/sieve-extprograms/cmd-pipe.c Thu Oct 01 00:10:31 2015 +0200 @@ -163,7 +163,6 @@ /* Dump optional operands */ for (;;) { int opt; - bool opok = TRUE; if ( (opt=sieve_action_opr_optional_dump(denv, address, &opt_code)) < 0 ) return FALSE; @@ -177,8 +176,6 @@ default: return FALSE; } - - if ( !opok ) return FALSE; } if ( !sieve_opr_string_dump(denv, address, "program-name") ) From pigeonhole at rename-it.nl Wed Sep 30 22:11:12 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 01 Oct 2015 00:11:12 +0200 Subject: dovecot-2.2-pigeonhole: Sieve metadata extension: Fixed couple o... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/51e2872510a8 changeset: 2108:51e2872510a8 user: Stephan Bosch date: Thu Oct 01 00:02:44 2015 +0200 description: Sieve metadata extension: Fixed couple of Coverity warnings. diffstat: src/lib-sieve/plugins/metadata/tst-metadata.c | 3 +-- src/lib-sieve/plugins/metadata/tst-metadataexists.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diffs (28 lines): diff -r 9ac3914392a4 -r 51e2872510a8 src/lib-sieve/plugins/metadata/tst-metadata.c --- a/src/lib-sieve/plugins/metadata/tst-metadata.c Wed Sep 30 00:54:40 2015 +0200 +++ b/src/lib-sieve/plugins/metadata/tst-metadata.c Thu Oct 01 00:02:44 2015 +0200 @@ -378,9 +378,8 @@ if ( metadata ) { if ( !sieve_mailbox_check_name(str_c(mailbox), &error) ) { - sieve_runtime_warning(renv, NULL, "%s test: " + sieve_runtime_warning(renv, NULL, "metadata test: " "invalid mailbox name `%s' specified: %s", - (metadata ? "metadata" : "servermetadata"), str_sanitize(str_c(mailbox), 256), error); sieve_interpreter_set_test_result(renv->interp, FALSE); return SIEVE_EXEC_OK; diff -r 9ac3914392a4 -r 51e2872510a8 src/lib-sieve/plugins/metadata/tst-metadataexists.c --- a/src/lib-sieve/plugins/metadata/tst-metadataexists.c Wed Sep 30 00:54:40 2015 +0200 +++ b/src/lib-sieve/plugins/metadata/tst-metadataexists.c Thu Oct 01 00:02:44 2015 +0200 @@ -364,9 +364,8 @@ */ if ( metadata && !sieve_mailbox_check_name(str_c(mailbox), &error) ) { - sieve_runtime_warning(renv, NULL, "%s test: " + sieve_runtime_warning(renv, NULL, "metadata test: " "invalid mailbox name `%s' specified: %s", - (metadata ? "metadataexists" : "servermetadataexists"), str_sanitize(str_c(mailbox), 256), error); sieve_interpreter_set_test_result(renv->interp, FALSE); return SIEVE_EXEC_OK; From pigeonhole at rename-it.nl Wed Sep 30 22:11:12 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 01 Oct 2015 00:11:12 +0200 Subject: dovecot-2.2-pigeonhole: sieve-test: Fixed couple of Coverity war... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/47e8e82c98d3 changeset: 2112:47e8e82c98d3 user: Stephan Bosch date: Thu Oct 01 00:11:04 2015 +0200 description: sieve-test: Fixed couple of Coverity warnings. diffstat: src/sieve-tools/sieve-test.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (20 lines): diff -r 3928e08d9340 -r 47e8e82c98d3 src/sieve-tools/sieve-test.c --- a/src/sieve-tools/sieve-test.c Thu Oct 01 00:10:31 2015 +0200 +++ b/src/sieve-tools/sieve-test.c Thu Oct 01 00:11:04 2015 +0200 @@ -378,7 +378,7 @@ sbin = main_sbin; main_sbin = NULL; - sieve_multiscript_run(mscript, sbin, + (void)sieve_multiscript_run(mscript, sbin, ehandler, ehandler, 0, TRUE); } @@ -395,6 +395,7 @@ case SIEVE_EXEC_BIN_CORRUPT: i_info("corrupt binary deleted."); (void) unlink(sieve_binary_path(sbin)); + /* fall through */ case SIEVE_EXEC_FAILURE: i_info("final result: failed; resolved with successful implicit keep"); exit_status = EXIT_FAILURE; From pigeonhole at rename-it.nl Wed Sep 30 22:11:12 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 01 Oct 2015 00:11:12 +0200 Subject: dovecot-2.2-pigeonhole: doveadm sieve plugin: Addressed spurious... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/0f43eed67bd5 changeset: 2110:0f43eed67bd5 user: Stephan Bosch date: Thu Oct 01 00:09:51 2015 +0200 description: doveadm sieve plugin: Addressed spurious Coverity warning. diffstat: src/plugins/doveadm-sieve/doveadm-sieve-cmd.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 35e2fcff8e7a -r 0f43eed67bd5 src/plugins/doveadm-sieve/doveadm-sieve-cmd.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-cmd.c Thu Oct 01 00:04:37 2015 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-cmd.c Thu Oct 01 00:09:51 2015 +0200 @@ -21,6 +21,7 @@ switch ( error ) { case SIEVE_ERROR_NONE: i_unreached(); + return; case SIEVE_ERROR_TEMP_FAILURE: exit_code = EX_TEMPFAIL; break; From pigeonhole at rename-it.nl Wed Sep 30 22:11:12 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 01 Oct 2015 00:11:12 +0200 Subject: dovecot-2.2-pigeonhole: Sieve file storage: Fixed couple of Cove... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/35e2fcff8e7a changeset: 2109:35e2fcff8e7a user: Stephan Bosch date: Thu Oct 01 00:04:37 2015 +0200 description: Sieve file storage: Fixed couple of Coverity warnings. diffstat: src/lib-sieve/storage/file/sieve-file-script.c | 8 +------- src/lib-sieve/storage/file/sieve-file-storage.c | 2 ++ 2 files changed, 3 insertions(+), 7 deletions(-) diffs (30 lines): diff -r 51e2872510a8 -r 35e2fcff8e7a src/lib-sieve/storage/file/sieve-file-script.c --- a/src/lib-sieve/storage/file/sieve-file-script.c Thu Oct 01 00:02:44 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-script.c Thu Oct 01 00:04:37 2015 +0200 @@ -354,13 +354,7 @@ if ( ret < 0 ) { /* Make sure we have a script name for the error */ if ( name == NULL ) { - if ( basename == NULL ) { - if ( filename == NULL ) - filename = path_split_filename(path, &dirpath); - basename = sieve_script_file_get_scriptname(filename); - if ( basename == NULL ) - basename = filename; - } + i_assert( basename != NULL ); name = basename; } sieve_file_script_handle_error diff -r 51e2872510a8 -r 35e2fcff8e7a src/lib-sieve/storage/file/sieve-file-storage.c --- a/src/lib-sieve/storage/file/sieve-file-storage.c Thu Oct 01 00:02:44 2015 +0200 +++ b/src/lib-sieve/storage/file/sieve-file-storage.c Thu Oct 01 00:04:37 2015 +0200 @@ -298,6 +298,8 @@ bool have_link = FALSE; int ret; + i_assert( storage_path != NULL || active_path != NULL ); + fstorage->prev_mtime = (time_t)-1; /* Get active script path */