From dovecot at dovecot.org Mon Jun 1 17:50:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 17:50:14 +0000 Subject: dovecot-2.2: lib-fts: Change TR29 tokenizer to break at full sto... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8973a5837b48 changeset: 18766:8973a5837b48 user: Teemu Huovila date: Mon Jun 01 18:35:58 2015 +0300 description: lib-fts: Change TR29 tokenizer to break at full stop (and others). Diverge from the TR29 rules and always break at MidNumLet letters. This fixes tokenizing first.last at domain.tld email addresses. diffstat: src/lib-fts/fts-tokenizer-generic-private.h | 1 + src/lib-fts/fts-tokenizer-generic.c | 55 ++++++++++++++-------------- 2 files changed, 28 insertions(+), 28 deletions(-) diffs (137 lines): diff -r bb1522e10108 -r 8973a5837b48 src/lib-fts/fts-tokenizer-generic-private.h --- a/src/lib-fts/fts-tokenizer-generic-private.h Fri May 29 21:39:33 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic-private.h Mon Jun 01 18:35:58 2015 +0300 @@ -25,6 +25,7 @@ LETTER_TYPE_EXTENDNUMLET, LETTER_TYPE_SOT, LETTER_TYPE_EOT, + LETTER_TYPE_APOSTROPHE, /* Own modification to TR29 */ LETTER_TYPE_OTHER /* WB14 "any" */ }; diff -r bb1522e10108 -r 8973a5837b48 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Fri May 29 21:39:33 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 18:35:58 2015 +0300 @@ -268,6 +268,8 @@ { unsigned int idx; + if (IS_APOSTROPHE(c)) + return LETTER_TYPE_APOSTROPHE; if (uint32_find(CR, N_ELEMENTS(CR), c, &idx)) return LETTER_TYPE_CR; if (uint32_find(LF, N_ELEMENTS(LF), c, &idx)) @@ -349,10 +351,10 @@ if (tok->prev_letter == LETTER_TYPE_HEBREW_LETTER) return FALSE; - /* WB7 WB7c */ + /* WB7 WB7c, except MidNumLet */ if (tok->prev_prev_letter == LETTER_TYPE_HEBREW_LETTER && (tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE || - tok->prev_letter == LETTER_TYPE_MIDNUMLET || + tok->prev_letter == LETTER_TYPE_APOSTROPHE || tok->prev_letter == LETTER_TYPE_MIDLETTER || tok->prev_letter == LETTER_TYPE_DOUBLE_QUOTE)) return FALSE; @@ -374,10 +376,10 @@ if (tok->prev_letter == LETTER_TYPE_ALETTER) return FALSE; - /* WB7 */ + /* WB7, except MidNumLet */ if (tok->prev_prev_letter == LETTER_TYPE_ALETTER && (tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE || - tok->prev_letter == LETTER_TYPE_MIDNUMLET || + tok->prev_letter == LETTER_TYPE_APOSTROPHE || tok->prev_letter == LETTER_TYPE_MIDLETTER)) return FALSE; @@ -416,18 +418,11 @@ return TRUE; /* Any / Any */ } -static bool letter_midnumlet(struct generic_fts_tokenizer *tok) +static bool letter_midnumlet(struct generic_fts_tokenizer *tok ATTR_UNUSED) { - /* WB6 */ - if (tok->prev_letter == LETTER_TYPE_ALETTER || - tok->prev_letter == LETTER_TYPE_HEBREW_LETTER) - return FALSE; - /* WB12 */ - if (tok->prev_letter == LETTER_TYPE_NUMERIC) - return FALSE; - - return TRUE; /* Any / Any */ + /* Break at MidNumLet, non-conformant with WB6/WB7 */ + return TRUE; } static bool letter_midletter(struct generic_fts_tokenizer *tok) @@ -488,6 +483,15 @@ return TRUE; /* Any / Any */ } +static bool letter_apostrophe(struct generic_fts_tokenizer *tok) +{ + + if (tok->prev_letter == LETTER_TYPE_ALETTER || + tok->prev_letter == LETTER_TYPE_HEBREW_LETTER) + return FALSE; + + return TRUE; /* Any / Any */ +} static bool letter_other(struct generic_fts_tokenizer *tok ATTR_UNUSED) { @@ -536,12 +540,14 @@ /* WB6/7 false positive detected at one past end. */ if (tok->prev_letter == LETTER_TYPE_MIDLETTER || tok->prev_letter == LETTER_TYPE_MIDNUMLET || + tok->prev_letter == LETTER_TYPE_APOSTROPHE || tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE ) return TRUE; /* WB12/12 false positive detected at one past end. */ if (tok->prev_letter == LETTER_TYPE_MIDNUM || tok->prev_letter == LETTER_TYPE_MIDNUMLET || + tok->prev_letter == LETTER_TYPE_APOSTROPHE || tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE) return TRUE; @@ -577,7 +583,7 @@ {letter_single_quote}, {letter_double_quote}, {letter_midnumlet}, {letter_midletter}, {letter_midnum}, {letter_numeric}, {letter_extendnumlet}, {letter_panic}, - {letter_panic}, {letter_other} + {letter_panic}, {letter_apostrophe}, {letter_other} }; /* @@ -585,19 +591,12 @@ #29, but tailored for FTS purposes. http://www.unicode.org/reports/tr29/ - Adaptions: No word boundary at Start-Of-Text or End-of-Text (Wb1 and - WB2). Break just once, not before and after. Other things also - (e.g. is_nonword(), not really pure tr29. Meant to assist in finding - individual words. - - TODO: If this letter_fns based approach is too kludgy, do a FSM with function - pointers and transition tables. - - TODO: Alternative idea: Replace everything with a super simplistic - "lt != ALETTER, HEBREW, NUMERIC, ... --> word break" - - TODO: Rules get split up over several functions. Is it too - confusing? + Adaptions: + * No word boundary at Start-Of-Text or End-of-Text (Wb1 and WB2). + * Break just once, not before and after. + * Break at MidNumLet, except apostrophes (diverging from WB6/WB7). + * Other things also (e.g. is_nonword(), not really pure tr29. Meant + to assist in finding individual words. */ static bool uni_found_word_boundary(struct generic_fts_tokenizer *tok, enum letter_type lt) From dovecot at dovecot.org Mon Jun 1 17:50:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 17:50:20 +0000 Subject: dovecot-2.2: lib-fts: Fix unit tests for TR29 full stop change. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1f609b2ad80c changeset: 18767:1f609b2ad80c user: Teemu Huovila date: Mon Jun 01 18:35:58 2015 +0300 description: lib-fts: Fix unit tests for TR29 full stop change. diffstat: src/lib-fts/test-fts-tokenizer.c | 26 +++++++++++++++++++------- 1 files changed, 19 insertions(+), 7 deletions(-) diffs (68 lines): diff -r 8973a5837b48 -r 1f609b2ad80c src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Mon Jun 01 18:35:58 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Mon Jun 01 18:35:58 2015 +0300 @@ -165,13 +165,13 @@ static const char *const expected_output[] = { "hello", "world", "And", "there", "was", "text", "galore", - "abc", "example.com", "Bar", "Baz", - "bar", "example.org", "foo", "domain", + "abc", "example", "com", "Bar", "Baz", + "bar", "example", "org", "foo", "domain", "1234567890123456789012345678?", "12345678901234567890123456789", "123456789012345678901234567890", "and", "longlonglongabcdefghijklmnopqr", - "more", "Hello", "world", "3.14", "3,14", "last", NULL, + "more", "Hello", "world", "3", "14", "3,14", "last", NULL, "1", NULL, @@ -213,7 +213,7 @@ test_end(); } -static void test_fts_tokenizer_address_parent(void) +static void test_fts_tokenizer_address_parent(const char *name, const char * const *settings) { static const char input[] = TEST_INPUT_ADDRESS; static const char *const expected_output[] = { @@ -225,8 +225,8 @@ struct fts_tokenizer *tok, *gen_tok; const char *error; - test_begin("fts tokenizer email address + parent"); - test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0); + test_begin(t_strdup_printf("fts tokenizer email address + parent %s", name)); + test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, settings, &gen_tok, &error) == 0); test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, NULL, &tok, &error) == 0); test_tokenizer_inputoutput(tok, input, expected_output, 0); fts_tokenizer_unref(&tok); @@ -234,6 +234,17 @@ test_end(); } +const char *const simple_settings[] = {"algorithm", "simple", NULL}; +static void test_fts_tokenizer_address_parent_simple(void) +{ + test_fts_tokenizer_address_parent("simple", simple_settings); +} + +static void test_fts_tokenizer_address_parent_tr29(void) +{ + test_fts_tokenizer_address_parent("tr29", tr29_settings); +} + static void test_fts_tokenizer_address_search(void) { static const char input[] = TEST_INPUT_ADDRESS; @@ -288,7 +299,8 @@ test_fts_tokenizer_generic_only, test_fts_tokenizer_generic_tr29_only, test_fts_tokenizer_address_only, - test_fts_tokenizer_address_parent, + test_fts_tokenizer_address_parent_simple, + test_fts_tokenizer_address_parent_tr29, test_fts_tokenizer_address_search, NULL }; From dovecot at dovecot.org Mon Jun 1 17:50:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 17:50:20 +0000 Subject: dovecot-2.2: lib-fts: Correct internal helper function for tr29. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6e459e8c3a5b changeset: 18768:6e459e8c3a5b user: Teemu Huovila date: Mon Jun 01 18:35:58 2015 +0300 description: lib-fts: Correct internal helper function for tr29. diffstat: src/lib-fts/fts-tokenizer-generic.c | 7 +------ 1 files changed, 1 insertions(+), 6 deletions(-) diffs (24 lines): diff -r 1f609b2ad80c -r 6e459e8c3a5b src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 18:35:58 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 18:35:58 2015 +0300 @@ -532,11 +532,6 @@ */ static bool is_one_past_end(struct generic_fts_tokenizer *tok) { - - /* Short circuit for simple algorithm. */ - if (tok->prev_letter == LETTER_TYPE_NONE) - return FALSE; - /* WB6/7 false positive detected at one past end. */ if (tok->prev_letter == LETTER_TYPE_MIDLETTER || tok->prev_letter == LETTER_TYPE_MIDNUMLET || @@ -544,7 +539,7 @@ tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE ) return TRUE; - /* WB12/12 false positive detected at one past end. */ + /* WB11/12 false positive detected at one past end. */ if (tok->prev_letter == LETTER_TYPE_MIDNUM || tok->prev_letter == LETTER_TYPE_MIDNUMLET || tok->prev_letter == LETTER_TYPE_APOSTROPHE || From dovecot at dovecot.org Mon Jun 1 18:31:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 18:31:45 +0000 Subject: dovecot-2.2: lib-fts: Optimization for tr29 - we don't need to t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3ff93cabcac3 changeset: 18769:3ff93cabcac3 user: Timo Sirainen date: Mon Jun 01 21:08:27 2015 +0300 description: lib-fts: Optimization for tr29 - we don't need to track last_size explicitly diffstat: src/lib-fts/fts-tokenizer-generic-private.h | 1 - src/lib-fts/fts-tokenizer-generic.c | 20 +++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diffs (60 lines): diff -r 6e459e8c3a5b -r 3ff93cabcac3 src/lib-fts/fts-tokenizer-generic-private.h --- a/src/lib-fts/fts-tokenizer-generic-private.h Mon Jun 01 18:35:58 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic-private.h Mon Jun 01 21:08:27 2015 +0300 @@ -43,7 +43,6 @@ enum boundary_algorithm algorithm; enum letter_type prev_letter; enum letter_type prev_prev_letter; - size_t last_size; /* Bytes in latest utf8 character. */ buffer_t *token; }; diff -r 6e459e8c3a5b -r 3ff93cabcac3 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 18:35:58 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:08:27 2015 +0300 @@ -184,7 +184,6 @@ tok->prev_letter = LETTER_TYPE_NONE; tok->prev_prev_letter = LETTER_TYPE_NONE; - tok->last_size = 0; buffer_set_used_size(tok->token, 0); } @@ -552,17 +551,21 @@ fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { - size_t end_skip = 0; - ssize_t len; + const unsigned char *data = tok->token->data; + ssize_t len = tok->token->used; - if (is_one_past_end(tok)) - end_skip = tok->last_size; + if (is_one_past_end(tok)) { + /* delete the last character */ + while ((data[len-1] & 0x80) != 0) + len--; + i_assert(len > 0); + len--; + } tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; - len = tok->token->used - end_skip; - *token_r = len == 0 ? "" : fts_uni_strndup(tok->token->data, len); + *token_r = len == 0 ? "" : fts_uni_strndup(data, len); buffer_set_used_size(tok->token, 0); return len > 0; } @@ -629,8 +632,7 @@ char_start_i = i; if (uni_utf8_get_char_n(data + i, size - i, &c) <= 0) i_unreached(); - tok->last_size = uni_utf8_char_bytes(data[i]); - i += tok->last_size - 1; /* Utf8 bytes > 1, for() handles the 1 byte increment. */ + i += uni_utf8_char_bytes(data[i]) - 1; /* Utf8 bytes > 1, for() handles the 1 byte increment. */ lt = letter_type(c); if (tok->prev_letter == LETTER_TYPE_NONE && is_nonword(lt)) { /* TODO: test that start_skip works with multibyte utf8 chars */ From dovecot at dovecot.org Mon Jun 1 18:31:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 18:31:46 +0000 Subject: dovecot-2.2: lib-fts: tr29 cleanup - token can never be empty by... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c9b2d4e228f5 changeset: 18770:c9b2d4e228f5 user: Timo Sirainen date: Mon Jun 01 21:10:11 2015 +0300 description: lib-fts: tr29 cleanup - token can never be empty by the time it's being returned. diffstat: src/lib-fts/fts-tokenizer-generic.c | 17 ++++++++++------- 1 files changed, 10 insertions(+), 7 deletions(-) diffs (53 lines): diff -r 3ff93cabcac3 -r c9b2d4e228f5 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:08:27 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:10:11 2015 +0300 @@ -547,7 +547,7 @@ return FALSE; } -static bool +static void fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { @@ -561,13 +561,16 @@ i_assert(len > 0); len--; } + /* we're skipping all non-text at the beginning of the word, + so by this point we must have something here - even if we just + deleted the last character */ + i_assert(len > 0); tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; - *token_r = len == 0 ? "" : fts_uni_strndup(data, len); + *token_r = fts_uni_strndup(data, len); buffer_set_used_size(tok->token, 0); - return len > 0; } struct letter_fn { @@ -644,8 +647,8 @@ tok_append_truncated(tok, data + start_skip, char_start_i - start_skip); *skip_r = i + 1; - if (fts_tokenizer_generic_tr29_current_token(tok, token_r)) - return 1; + fts_tokenizer_generic_tr29_current_token(tok, token_r); + return 1; } } i_assert(i >= start_skip && size >= start_skip); @@ -655,8 +658,8 @@ if (size == 0 && tok->token->used > 0) { /* return the last token */ *skip_r = 0; - if (fts_tokenizer_generic_tr29_current_token(tok, token_r)) - return 1; + fts_tokenizer_generic_tr29_current_token(tok, token_r); + return 1; } return 0; } From dovecot at dovecot.org Mon Jun 1 18:31:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 18:31:56 +0000 Subject: dovecot-2.2: lib-fts: tr29 cleanup - Avoid i++ in the for loop t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/04ea590951c1 changeset: 18771:04ea590951c1 user: Timo Sirainen date: Mon Jun 01 21:11:55 2015 +0300 description: lib-fts: tr29 cleanup - Avoid i++ in the for loop to avoid extra calculations diffstat: src/lib-fts/fts-tokenizer-generic.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (30 lines): diff -r c9b2d4e228f5 -r 04ea590951c1 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:10:11 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:11:55 2015 +0300 @@ -631,22 +631,22 @@ enum letter_type lt; /* TODO: Process 8bit chars separately, to speed things up. */ - for (i = 0; i < size; i++) { + for (i = 0; i < size; ) { char_start_i = i; if (uni_utf8_get_char_n(data + i, size - i, &c) <= 0) i_unreached(); - i += uni_utf8_char_bytes(data[i]) - 1; /* Utf8 bytes > 1, for() handles the 1 byte increment. */ + i += uni_utf8_char_bytes(data[i]); lt = letter_type(c); if (tok->prev_letter == LETTER_TYPE_NONE && is_nonword(lt)) { /* TODO: test that start_skip works with multibyte utf8 chars */ - start_skip = i + 1; /* Skip non-token chars at start of data */ + start_skip = i; /* Skip non-token chars at start of data */ continue; } if (uni_found_word_boundary(tok, lt)) { i_assert(char_start_i >= start_skip && size >= start_skip); tok_append_truncated(tok, data + start_skip, char_start_i - start_skip); - *skip_r = i + 1; + *skip_r = i; fts_tokenizer_generic_tr29_current_token(tok, token_r); return 1; } From dovecot at dovecot.org Mon Jun 1 18:32:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 18:32:01 +0000 Subject: dovecot-2.2: lib-fts: tr29 cleanup - consistently call valid cha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/62b201a1ee06 changeset: 18772:62b201a1ee06 user: Timo Sirainen date: Mon Jun 01 21:16:35 2015 +0300 description: lib-fts: tr29 cleanup - consistently call valid chars "token" and "non-token" chars. Instead of word/token/text. diffstat: src/lib-fts/fts-tokenizer-generic.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (46 lines): diff -r 04ea590951c1 -r 62b201a1ee06 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:11:55 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:16:35 2015 +0300 @@ -513,9 +513,8 @@ TODO: Does this "reverse approach" include too much in "whitespace"? TODO: Possibly use is_word_break()? */ -static bool is_nonword(enum letter_type lt) +static bool is_nontoken(enum letter_type lt) { - if (lt == LETTER_TYPE_REGIONAL_INDICATOR || lt == LETTER_TYPE_KATAKANA || lt == LETTER_TYPE_HEBREW_LETTER || lt == LETTER_TYPE_ALETTER || lt == LETTER_TYPE_NUMERIC) @@ -561,7 +560,7 @@ i_assert(len > 0); len--; } - /* we're skipping all non-text at the beginning of the word, + /* we're skipping all non-token chars at the beginning of the word, so by this point we must have something here - even if we just deleted the last character */ i_assert(len > 0); @@ -596,7 +595,7 @@ * No word boundary at Start-Of-Text or End-of-Text (Wb1 and WB2). * Break just once, not before and after. * Break at MidNumLet, except apostrophes (diverging from WB6/WB7). - * Other things also (e.g. is_nonword(), not really pure tr29. Meant + * Other things also (e.g. is_nontoken(), not really pure tr29. Meant to assist in finding individual words. */ static bool @@ -637,9 +636,10 @@ i_unreached(); i += uni_utf8_char_bytes(data[i]); lt = letter_type(c); - if (tok->prev_letter == LETTER_TYPE_NONE && is_nonword(lt)) { - /* TODO: test that start_skip works with multibyte utf8 chars */ - start_skip = i; /* Skip non-token chars at start of data */ + if (tok->prev_letter == LETTER_TYPE_NONE && is_nontoken(lt)) { + /* Skip non-token chars at the beginning of token */ + i_assert(tok->token->used == 0); + start_skip = i; continue; } if (uni_found_word_boundary(tok, lt)) { From dovecot at dovecot.org Mon Jun 1 18:32:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 18:32:01 +0000 Subject: dovecot-2.2: lib-fts: simple tokenizer cleanup - removed unneces... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b239f075147b changeset: 18773:b239f075147b user: Timo Sirainen date: Mon Jun 01 21:19:47 2015 +0300 description: lib-fts: simple tokenizer cleanup - removed unnecessary variables diffstat: src/lib-fts/fts-tokenizer-generic.c | 9 +++------ 1 files changed, 3 insertions(+), 6 deletions(-) diffs (35 lines): diff -r 62b201a1ee06 -r b239f075147b src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:16:35 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:19:47 2015 +0300 @@ -220,19 +220,17 @@ { struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; - size_t i, char_start_i, len, start = 0; + size_t i, start = 0; unsigned int char_size; unichar_t c; for (i = 0; i < size; i += char_size) { - char_start_i = i; if (uni_utf8_get_char_n(data + i, size - i, &c) <= 0) i_unreached(); char_size = uni_utf8_char_bytes(data[i]); if (fts_ascii_word_break(data[i]) || fts_uni_word_break(c) || fts_apostrophe_word_break(tok, c)) { - len = char_start_i - start; - tok_append_truncated(tok, data + start, len); + tok_append_truncated(tok, data + start, i - start); if (tok->token->used > 0 && fts_tokenizer_generic_simple_current_token(tok, token_r)) { *skip_r = i + char_size; @@ -242,8 +240,7 @@ } } /* word boundary not found yet */ - len = i - start; - tok_append_truncated(tok, data + start, len); + tok_append_truncated(tok, data + start, i - start); *skip_r = i; /* return the last token */ From dovecot at dovecot.org Mon Jun 1 18:32:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 18:32:01 +0000 Subject: dovecot-2.2: lib-fts: simple tokenizer cleanup - make prev_lette... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/33547eaa0cac changeset: 18774:33547eaa0cac user: Timo Sirainen date: Mon Jun 01 21:27:09 2015 +0300 description: lib-fts: simple tokenizer cleanup - make prev_letter updating more explicit. It was now hidden inside one of the functions, which didn't make the prev_letter very consistent when a word break was found. It didn't actually matter what the prev_letter was at that point, but now the behavior is more consistent. diffstat: src/lib-fts/fts-tokenizer-generic.c | 32 +++++++++++++++++++------------- 1 files changed, 19 insertions(+), 13 deletions(-) diffs (62 lines): diff -r b239f075147b -r 33547eaa0cac src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:19:47 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:27:09 2015 +0300 @@ -163,18 +163,14 @@ return FALSE; } -static bool -fts_apostrophe_word_break(struct generic_fts_tokenizer *tok, unichar_t c) +static inline bool +fts_simple_is_word_break(struct generic_fts_tokenizer *tok, + unichar_t c, bool apostrophe) { - if (IS_APOSTROPHE(c)) { - if (tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE) - return TRUE; - else - tok->prev_letter = LETTER_TYPE_SINGLE_QUOTE; - } else { - tok->prev_letter = LETTER_TYPE_NONE; - } - return FALSE; + if (apostrophe) + return tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE; + else + return fts_ascii_word_break(c) || fts_uni_word_break(c); } static void fts_tokenizer_generic_reset(struct fts_tokenizer *_tok) @@ -223,13 +219,15 @@ size_t i, start = 0; unsigned int char_size; unichar_t c; + bool apostrophe; for (i = 0; i < size; i += char_size) { if (uni_utf8_get_char_n(data + i, size - i, &c) <= 0) i_unreached(); char_size = uni_utf8_char_bytes(data[i]); - if (fts_ascii_word_break(data[i]) || fts_uni_word_break(c) || - fts_apostrophe_word_break(tok, c)) { + + apostrophe = IS_APOSTROPHE(c); + if (fts_simple_is_word_break(tok, c, apostrophe)) { tok_append_truncated(tok, data + start, i - start); if (tok->token->used > 0 && fts_tokenizer_generic_simple_current_token(tok, token_r)) { @@ -237,6 +235,14 @@ return 1; } start = i + char_size; + /* it doesn't actually matter at this point how whether + subsequent apostrophes are handled by prefix + skipping or by ignoring empty tokens - they will be + dropped in any case. */ + tok->prev_letter = LETTER_TYPE_NONE; + } else { + tok->prev_letter = apostrophe ? + LETTER_TYPE_SINGLE_QUOTE : LETTER_TYPE_NONE; } } /* word boundary not found yet */ From dovecot at dovecot.org Mon Jun 1 18:32:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 18:32:06 +0000 Subject: dovecot-2.2: lib-fts: simple tokenizer optimization - don't chec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7e3f4fa82dfe changeset: 18775:7e3f4fa82dfe user: Timo Sirainen date: Mon Jun 01 21:28:42 2015 +0300 description: lib-fts: simple tokenizer optimization - don't check unicode word breaks for ASCII chars. diffstat: src/lib-fts/fts-tokenizer-generic.c | 15 +++------------ 1 files changed, 3 insertions(+), 12 deletions(-) diffs (37 lines): diff -r 33547eaa0cac -r 7e3f4fa82dfe src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:27:09 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:28:42 2015 +0300 @@ -129,21 +129,10 @@ BINARY_NUMBER_SEARCH(data, count, value, idx_r); } -static bool fts_ascii_word_break(unsigned char c) -{ - if (c < 0x80) - return fts_ascii_word_breaks[c] != 0; - return FALSE; -} - static bool fts_uni_word_break(unichar_t c) { unsigned int idx; - /* Override some apostrophes, which get special treatment. */ - if (IS_APOSTROPHE(c)) - return FALSE; - /* Unicode General Punctuation, including deprecated characters. */ if (c >= 0x2000 && c <= 0x206f) return TRUE; @@ -169,8 +158,10 @@ { if (apostrophe) return tok->prev_letter == LETTER_TYPE_SINGLE_QUOTE; + else if (c < 0x80) + return fts_ascii_word_breaks[c] != 0; else - return fts_ascii_word_break(c) || fts_uni_word_break(c); + return fts_uni_word_break(c); } static void fts_tokenizer_generic_reset(struct fts_tokenizer *_tok) From dovecot at dovecot.org Mon Jun 1 19:01:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:01:24 +0000 Subject: dovecot-2.2: lib-fts: tr29 tokenizer cleanup - Avoid unnecessary... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4a72ef26dbad changeset: 18776:4a72ef26dbad user: Timo Sirainen date: Mon Jun 01 21:33:11 2015 +0300 description: lib-fts: tr29 tokenizer cleanup - Avoid unnecessary goto. diffstat: src/lib-fts/fts-tokenizer-generic.c | 19 +++++++++---------- 1 files changed, 9 insertions(+), 10 deletions(-) diffs (30 lines): diff -r 7e3f4fa82dfe -r 4a72ef26dbad src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:28:42 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:33:11 2015 +0300 @@ -597,17 +597,16 @@ { /* No rule knows what to do with just one char, except the linebreaks we eat away (above) anyway. */ - if (tok->prev_letter == LETTER_TYPE_NONE) - goto false_out; + if (tok->prev_letter != LETTER_TYPE_NONE) { + if (letter_fns[lt].fn(tok)) + return TRUE; + } - if (letter_fns[lt].fn(tok)) - return TRUE; - - false_out: - /* Extend and format types are ignored. */ - if (lt == LETTER_TYPE_EXTEND || lt == LETTER_TYPE_FORMAT) - return FALSE; - add_prev_letter(tok,lt); + if (lt == LETTER_TYPE_EXTEND || lt == LETTER_TYPE_FORMAT) { + /* These types are completely ignored. */ + } else { + add_prev_letter(tok,lt); + } return FALSE; } From dovecot at dovecot.org Mon Jun 1 19:01:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:01:24 +0000 Subject: dovecot-2.2: lib-fts: simple tokenizer minor cleanup - removed u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f44961c66a48 changeset: 18777:f44961c66a48 user: Timo Sirainen date: Mon Jun 01 21:35:39 2015 +0300 description: lib-fts: simple tokenizer minor cleanup - removed unnecessary token length > 0 check fts_tokenizer_generic_simple_current_token() will check it in any case. diffstat: src/lib-fts/fts-tokenizer-generic.c | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diffs (22 lines): diff -r 4a72ef26dbad -r f44961c66a48 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:33:11 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:35:39 2015 +0300 @@ -220,8 +220,7 @@ apostrophe = IS_APOSTROPHE(c); if (fts_simple_is_word_break(tok, c, apostrophe)) { tok_append_truncated(tok, data + start, i - start); - if (tok->token->used > 0 && - fts_tokenizer_generic_simple_current_token(tok, token_r)) { + if (fts_tokenizer_generic_simple_current_token(tok, token_r)) { *skip_r = i + char_size; return 1; } @@ -241,7 +240,7 @@ *skip_r = i; /* return the last token */ - if (size == 0 && tok->token->used > 0) { + if (size == 0) { if (fts_tokenizer_generic_simple_current_token(tok, token_r)) return 1; } From dovecot at dovecot.org Mon Jun 1 19:01:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:01:24 +0000 Subject: dovecot-2.2: lib-fts: tokenizers - don't include removed apostro... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e3f9e4c8a338 changeset: 18778:e3f9e4c8a338 user: Timo Sirainen date: Mon Jun 01 21:48:59 2015 +0300 description: lib-fts: tokenizers - don't include removed apostrophes as part of the token size diffstat: src/lib-fts/fts-tokenizer-generic.c | 13 +++++++++++++ src/lib-fts/test-fts-tokenizer.c | 6 ++++++ 2 files changed, 19 insertions(+), 0 deletions(-) diffs (53 lines): diff -r f44961c66a48 -r e3f9e4c8a338 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:35:39 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:48:59 2015 +0300 @@ -180,6 +180,19 @@ size_t append_len, pos = 0, appended = 0; unichar_t c; + if (size == 0) + return; + if (data[0] == '\'' && tok->token->used == 0) { + /* Skip apostrophes in the beginning of the token. + We need to do it here so that we don't truncate the + token too early. */ + data++; + size--; + if (size == 0) + return; + i_assert(data[0] != '\''); + } + i_assert(tok->max_length >= tok->token->used); append_len = I_MIN(size, tok->max_length - tok->token->used); diff -r f44961c66a48 -r e3f9e4c8a338 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Mon Jun 01 21:35:39 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Mon Jun 01 21:48:59 2015 +0300 @@ -31,6 +31,8 @@ "' ' '' ''' 'quoted text' 'word' 'hlo words' you're bad'''word '''pre post'''", + "'1234567890123456789012345678?," + /* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 81 9f) */ "hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text " @@ -136,6 +138,8 @@ "quoted", "text", "word", "hlo", "words", "you're", "bad", "word", "pre", "post", NULL, + "1234567890123456789012345678?", + "hello", "world", "And", "there", "was", "text", "galore", "and", "more", NULL, @@ -178,6 +182,8 @@ "quoted", "text", "word", "hlo", "words", "you're", "bad", "word", "pre", "post", NULL, + "1234567890123456789012345678?", + "hello", "world", "And", "there", "was", "text", "galore", "and", "more", NULL, From dovecot at dovecot.org Mon Jun 1 19:01:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:01:24 +0000 Subject: dovecot-2.2: lib-fts: tr29 tokenizer - rename variable in prepar... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/218c3988e105 changeset: 18779:218c3988e105 user: Timo Sirainen date: Mon Jun 01 21:49:18 2015 +0300 description: lib-fts: tr29 tokenizer - rename variable in preparation for the next patch diffstat: src/lib-fts/fts-tokenizer-generic.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (39 lines): diff -r e3f9e4c8a338 -r 218c3988e105 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:48:59 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:49:18 2015 +0300 @@ -631,7 +631,7 @@ struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; unichar_t c; - size_t i, char_start_i, start_skip = 0; + size_t i, char_start_i, start_pos = 0; enum letter_type lt; /* TODO: Process 8bit chars separately, to speed things up. */ @@ -644,20 +644,20 @@ if (tok->prev_letter == LETTER_TYPE_NONE && is_nontoken(lt)) { /* Skip non-token chars at the beginning of token */ i_assert(tok->token->used == 0); - start_skip = i; + start_pos = i; continue; } if (uni_found_word_boundary(tok, lt)) { - i_assert(char_start_i >= start_skip && size >= start_skip); - tok_append_truncated(tok, data + start_skip, - char_start_i - start_skip); + i_assert(char_start_i >= start_pos && size >= start_pos); + tok_append_truncated(tok, data + start_pos, + char_start_i - start_pos); *skip_r = i; fts_tokenizer_generic_tr29_current_token(tok, token_r); return 1; } } - i_assert(i >= start_skip && size >= start_skip); - tok_append_truncated(tok, data + start_skip, i - start_skip); + i_assert(i >= start_pos && size >= start_pos); + tok_append_truncated(tok, data + start_pos, i - start_pos); *skip_r = i; if (size == 0 && tok->token->used > 0) { From dovecot at dovecot.org Mon Jun 1 19:01:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:01:29 +0000 Subject: dovecot-2.2: lib-fts: Optimize tokenizers - Rewrite of apostroph... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b6510dfd396f changeset: 18780:b6510dfd396f user: Timo Sirainen date: Mon Jun 01 21:51:33 2015 +0300 description: lib-fts: Optimize tokenizers - Rewrite of apostrophe handling. We parse through the UTF8 characters only once now, not twice. diffstat: src/lib-fts/fts-tokenizer-generic.c | 85 ++++++++++++++++-------------------- 1 files changed, 38 insertions(+), 47 deletions(-) diffs (127 lines): diff -r 218c3988e105 -r b6510dfd396f src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:49:18 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:51:33 2015 +0300 @@ -105,21 +105,26 @@ const char **token_r) { const unsigned char *data; - size_t start = 0, len; + size_t len = tok->token->used; - /* clean trailing and starting apostrophes. they were all made - into U+0027 earlier. */ - data = tok->token->data; - len = tok->token->used; - while (len > 0 && data[len - 1] == '\'') - len--; - while (start < len && data[start] == '\'') - start++; + if (len > 0) { + /* Remove the trailing apostrophe - it was made + into U+0027 earlier. There can be only a single such + apostrophe, because otherwise the token would have already + been split. We also want to remove the trailing apostrophe + only if it's the the last character in the nontruncated + token - a truncated token may end with apostrophe. */ + data = tok->token->data; + if (data[len-1] == '\'') { + len--; + i_assert(len > 0 && data[len-1] != '\''); + } + } - *token_r = len - start == 0 ? "" : - fts_uni_strndup(CONST_PTR_OFFSET(tok->token->data, start), - len - start); + *token_r = len == 0 ? "" : + fts_uni_strndup(tok->token->data, len); buffer_set_used_size(tok->token, 0); + tok->prev_letter = LETTER_TYPE_NONE; return (*token_r)[0] != '\0'; } @@ -177,39 +182,8 @@ static void tok_append_truncated(struct generic_fts_tokenizer *tok, const unsigned char *data, size_t size) { - size_t append_len, pos = 0, appended = 0; - unichar_t c; - - if (size == 0) - return; - if (data[0] == '\'' && tok->token->used == 0) { - /* Skip apostrophes in the beginning of the token. - We need to do it here so that we don't truncate the - token too early. */ - data++; - size--; - if (size == 0) - return; - i_assert(data[0] != '\''); - } - - i_assert(tok->max_length >= tok->token->used); - append_len = I_MIN(size, tok->max_length - tok->token->used); - - /* Append only one kind of apostrophes. Simplifies things when returning - token. */ - while (pos < append_len) { - if (uni_utf8_get_char_n(data + pos, size - pos, &c) <= 0) - i_unreached(); - if (IS_NONASCII_APOSTROPHE(c)) { - buffer_append(tok->token, data, pos); - buffer_append_c(tok->token, '\''); - appended = pos + 1; - } - pos += uni_utf8_char_bytes(data[pos]); - } - if (appended < append_len) - buffer_append(tok->token, data + appended, append_len - appended); + buffer_append(tok->token, data, + I_MIN(size, tok->max_length - tok->token->used)); } static int @@ -243,9 +217,17 @@ skipping or by ignoring empty tokens - they will be dropped in any case. */ tok->prev_letter = LETTER_TYPE_NONE; + } else if (apostrophe) { + /* all apostrophes require special handling */ + const unsigned char apostrophe_char = '\''; + + tok_append_truncated(tok, data + start, i - start); + if (tok->token->used > 0) + tok_append_truncated(tok, &apostrophe_char, 1); + start = i + char_size; + tok->prev_letter = LETTER_TYPE_SINGLE_QUOTE; } else { - tok->prev_letter = apostrophe ? - LETTER_TYPE_SINGLE_QUOTE : LETTER_TYPE_NONE; + tok->prev_letter = LETTER_TYPE_NONE; } } /* word boundary not found yet */ @@ -641,6 +623,7 @@ i_unreached(); i += uni_utf8_char_bytes(data[i]); lt = letter_type(c); + if (tok->prev_letter == LETTER_TYPE_NONE && is_nontoken(lt)) { /* Skip non-token chars at the beginning of token */ i_assert(tok->token->used == 0); @@ -654,6 +637,14 @@ *skip_r = i; fts_tokenizer_generic_tr29_current_token(tok, token_r); return 1; + } else if (lt == LETTER_TYPE_APOSTROPHE || + lt == LETTER_TYPE_SINGLE_QUOTE) { + /* all apostrophes require special handling */ + const unsigned char apostrophe_char = '\''; + tok_append_truncated(tok, data + start_pos, + char_start_i - start_pos); + tok_append_truncated(tok, &apostrophe_char, 1); + start_pos = i; } } i_assert(i >= start_pos && size >= start_pos); From dovecot at dovecot.org Mon Jun 1 19:01:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:01:30 +0000 Subject: dovecot-2.2: lib-fts: tokenizers - Fixed removal of trailing cha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/78d473873e12 changeset: 18781:78d473873e12 user: Timo Sirainen date: Mon Jun 01 21:58:30 2015 +0300 description: lib-fts: tokenizers - Fixed removal of trailing character in truncated tokens. If the token is truncated, we don't want to remove the trailing character since it's not actually there. Also we don't want to remove trailing apostrophes from a truncated word, because they're not actually at the end of the (untruncated) token there. This doesn't make a big difference, but it's slightly more correct. diffstat: src/lib-fts/fts-tokenizer-generic-private.h | 1 + src/lib-fts/fts-tokenizer-generic.c | 9 +++++++-- src/lib-fts/test-fts-tokenizer.c | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diffs (113 lines): diff -r b6510dfd396f -r 78d473873e12 src/lib-fts/fts-tokenizer-generic-private.h --- a/src/lib-fts/fts-tokenizer-generic-private.h Mon Jun 01 21:51:33 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic-private.h Mon Jun 01 21:58:30 2015 +0300 @@ -43,6 +43,7 @@ enum boundary_algorithm algorithm; enum letter_type prev_letter; enum letter_type prev_prev_letter; + size_t untruncated_length; buffer_t *token; }; diff -r b6510dfd396f -r 78d473873e12 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:51:33 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 21:58:30 2015 +0300 @@ -107,7 +107,7 @@ const unsigned char *data; size_t len = tok->token->used; - if (len > 0) { + if (len > 0 && tok->untruncated_length <= tok->max_length) { /* Remove the trailing apostrophe - it was made into U+0027 earlier. There can be only a single such apostrophe, because otherwise the token would have already @@ -124,6 +124,7 @@ *token_r = len == 0 ? "" : fts_uni_strndup(tok->token->data, len); buffer_set_used_size(tok->token, 0); + tok->untruncated_length = 0; tok->prev_letter = LETTER_TYPE_NONE; return (*token_r)[0] != '\0'; } @@ -176,6 +177,7 @@ tok->prev_letter = LETTER_TYPE_NONE; tok->prev_prev_letter = LETTER_TYPE_NONE; + tok->untruncated_length = 0; buffer_set_used_size(tok->token, 0); } @@ -184,6 +186,7 @@ { buffer_append(tok->token, data, I_MIN(size, tok->max_length - tok->token->used)); + tok->untruncated_length += size; } static int @@ -541,7 +544,8 @@ const unsigned char *data = tok->token->data; ssize_t len = tok->token->used; - if (is_one_past_end(tok)) { + if (is_one_past_end(tok) && + tok->untruncated_length <= tok->max_length) { /* delete the last character */ while ((data[len-1] & 0x80) != 0) len--; @@ -558,6 +562,7 @@ *token_r = fts_uni_strndup(data, len); buffer_set_used_size(tok->token, 0); + tok->untruncated_length = 0; } struct letter_fn { diff -r b6510dfd396f -r 78d473873e12 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Mon Jun 01 21:51:33 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Mon Jun 01 21:58:30 2015 +0300 @@ -32,6 +32,14 @@ "' ' '' ''' 'quoted text' 'word' 'hlo words' you're bad'''word '''pre post'''", "'1234567890123456789012345678?," + "123456789012345678901234567x'?," + "1234567890123456789012345678x're," + "1234567890123456789012345678x'," + "1234567890123456789012345678x''," + "12345678901234567890123456789x'," + "12345678901234567890123456789x''," + "123456789012345678901234567890x'," + "123456789012345678901234567890x''," /* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 81 9f) */ @@ -139,6 +147,14 @@ "word", "pre", "post", NULL, "1234567890123456789012345678?", + "123456789012345678901234567x'", + "1234567890123456789012345678x'", + "1234567890123456789012345678x", + "1234567890123456789012345678x", + "12345678901234567890123456789x", + "12345678901234567890123456789x", + "123456789012345678901234567890", + "123456789012345678901234567890", "hello", "world", "And", "there", "was", "text", "galore", @@ -183,6 +199,14 @@ "word", "pre", "post", NULL, "1234567890123456789012345678?", + "123456789012345678901234567x'", + "1234567890123456789012345678x'", + "1234567890123456789012345678x", + "1234567890123456789012345678x", + "12345678901234567890123456789x", + "12345678901234567890123456789x", + "123456789012345678901234567890", + "123456789012345678901234567890", "hello", "world", "And", "there", "was", "text", "galore", From dovecot at dovecot.org Mon Jun 1 19:01:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:01:30 +0000 Subject: dovecot-2.2: lib-fts: Added unit testing for unicode apostrophe ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff791143d1b4 changeset: 18782:ff791143d1b4 user: Timo Sirainen date: Mon Jun 01 21:59:02 2015 +0300 description: lib-fts: Added unit testing for unicode apostrophe handling. diffstat: src/lib-fts/test-fts-tokenizer.c | 15 +++++++++++++++ 1 files changed, 15 insertions(+), 0 deletions(-) diffs (39 lines): diff -r 78d473873e12 -r ff791143d1b4 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Mon Jun 01 21:58:30 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Mon Jun 01 21:59:02 2015 +0300 @@ -41,6 +41,11 @@ "123456789012345678901234567890x'," "123456789012345678901234567890x''," + /* \xe28099 = U+2019 is a smart quote, sometimes used as an apostrophe */ + "\xE2\x80\x99 \xE2\x80\x99 \xE2\x80\x99\xE2\x80\x99 \xE2\x80\x99\xE2\x80\x99\xE2\x80\x99 \xE2\x80\x99quoted text\xE2\x80\x99\xE2\x80\x99word\xE2\x80\x99 \xE2\x80\x99hlo words\xE2\x80\x99 you\xE2\x80\x99re78901234567890123456789012 bad\xE2\x80\x99\xE2\x80\x99\xE2\x80\x99word\xE2\x80\x99\xE2\x80\x99\xE2\x80\x99pre post\xE2\x80\x99\xE2\x80\x99\xE2\x80\x99", + + "you\xE2\x80\x99re\xE2\x80\x99xyz", + /* whitespace: with Unicode(utf8) U+FF01(ef bc 81)(U+2000(e2 80 80) and U+205A(e2 81 9a) and U+205F(e2 81 9f) */ "hello\xEF\xBC\x81world\r\nAnd\xE2\x80\x80there\twas: text " @@ -156,6 +161,11 @@ "123456789012345678901234567890", "123456789012345678901234567890", + "quoted", "text", "word", "hlo", "words", "you're789012345678901234567890", "bad", + "word", "pre", "post", NULL, + + "you're'xyz", NULL, + "hello", "world", "And", "there", "was", "text", "galore", "and", "more", NULL, @@ -208,6 +218,11 @@ "123456789012345678901234567890", "123456789012345678901234567890", + "quoted", "text", "word", "hlo", "words", "you're789012345678901234567890", "bad", + "word", "pre", "post", NULL, + + "you're'xyz", NULL, + "hello", "world", "And", "there", "was", "text", "galore", "and", "more", NULL, From dovecot at dovecot.org Mon Jun 1 19:18:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:18:47 +0000 Subject: dovecot-2.2: lib: test-unichar - streamline the unichars test Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa2f464daa66 changeset: 18783:aa2f464daa66 user: Phil Carmody date: Mon Jun 01 22:06:44 2015 +0300 description: lib: test-unichar - streamline the unichars test It's doing 2 kinds of tests, split them into separate test cases. And the first part has started to get expensive, so just make sure all code paths are tested by skipping most values. Only 3 from each set of 64 (lowest 6 bits) are tested. Signed-off-by: Phil Carmody diffstat: src/lib/test-unichar.c | 12 +++++++++++- 1 files changed, 11 insertions(+), 1 deletions(-) diffs (37 lines): diff -r ff791143d1b4 -r aa2f464daa66 src/lib/test-unichar.c --- a/src/lib/test-unichar.c Mon Jun 01 21:59:02 2015 +0300 +++ b/src/lib/test-unichar.c Mon Jun 01 22:06:44 2015 +0300 @@ -4,6 +4,7 @@ #include "str.h" #include "buffer.h" #include "unichar.h" +#include static void test_unichar_uni_utf8_strlen(void) { @@ -45,15 +46,24 @@ unichar_t chr, chr2; string_t *str = t_str_new(16); - test_begin("unichars"); + test_begin("unichars encode/decode"); for (chr = 0; chr <= 0x10ffff; chr++) { + /* The bottom 6 bits should be irrelevant to code coverage, + only test 000000, 111111, and something in between. */ + if ((chr & 63) == 1) + chr += rand() % 62; /* After 0, somewhere between 1 and 62 */ + else if ((chr & 63) > 0 && (chr & 63) < 63) + chr |= 63; /* After random, straight to 63 */ + str_truncate(str, 0); uni_ucs4_to_utf8_c(chr, str); test_assert(uni_utf8_str_is_valid(str_c(str))); test_assert(uni_utf8_get_char(str_c(str), &chr2) > 0); test_assert(chr2 == chr); } + test_end(); + test_begin("unichar collation"); collate_out = buffer_create_dynamic(default_pool, 32); uni_utf8_to_decomposed_titlecase(collate_in, sizeof(collate_in), collate_out); From dovecot at dovecot.org Mon Jun 1 19:18:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:18:47 +0000 Subject: dovecot-2.2: lib: test-unichar - test invalid utf8 encodings Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3468de815591 changeset: 18784:3468de815591 user: Phil Carmody date: Mon Jun 01 22:07:44 2015 +0300 description: lib: test-unichar - test invalid utf8 encodings Chop trailing characters off valid encodings, and watch them fail. (There's no need to do this on most of the test characters, as they're truncated to the same byte sequence - only do 1 in 64.) Signed-off-by: Phil Carmody diffstat: src/lib/test-unichar.c | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diffs (27 lines): diff -r aa2f464daa66 -r 3468de815591 src/lib/test-unichar.c --- a/src/lib/test-unichar.c Mon Jun 01 22:06:44 2015 +0300 +++ b/src/lib/test-unichar.c Mon Jun 01 22:07:44 2015 +0300 @@ -60,6 +60,23 @@ test_assert(uni_utf8_str_is_valid(str_c(str))); test_assert(uni_utf8_get_char(str_c(str), &chr2) > 0); test_assert(chr2 == chr); + + if ((chr & 0x63) == 0) { + unsigned int utf8len = uni_utf8_char_bytes(*str_c(str)); + + /* virtually truncate the byte string */ + while (--utf8len > 0) + test_assert(uni_utf8_get_char_n(str_c(str), utf8len, &chr2) == 0); + + utf8len = uni_utf8_char_bytes(*str_c(str)); + + /* actually truncate the byte stream */ + while (--utf8len > 0) { + str_truncate(str, utf8len); + test_assert(!uni_utf8_str_is_valid(str_c(str))); + test_assert(uni_utf8_get_char(str_c(str), &chr2) == 0); + } + } } test_end(); From dovecot at dovecot.org Mon Jun 1 19:19:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:19:03 +0000 Subject: dovecot-2.2: fts-solr: laxer check of uni_utf8_get_char_n() retu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7bd2f9be5493 changeset: 18785:7bd2f9be5493 user: Phil Carmody date: Mon Jun 01 22:08:27 2015 +0300 description: fts-solr: laxer check of uni_utf8_get_char_n() return value If uni_utf8_get_char*() were changed to return the number of bytes in the character on success, then all we care about is it being > 0 (i.e. not error, not truncated). Signed-off-by: Phil Carmody diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 2 +- src/plugins/fts-solr/fts-backend-solr.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 3468de815591 -r 7bd2f9be5493 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Mon Jun 01 22:07:44 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Mon Jun 01 22:08:27 2015 +0300 @@ -91,7 +91,7 @@ unsigned int char_len = uni_utf8_char_bytes(data[i]); if (i + char_len <= len && - uni_utf8_get_char_n(data + i, char_len, &chr) == 1 && + uni_utf8_get_char_n(data + i, char_len, &chr) > 0 && is_valid_xml_char(chr)) str_append_n(dest, data + i, char_len); else { diff -r 3468de815591 -r 7bd2f9be5493 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Mon Jun 01 22:07:44 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Mon Jun 01 22:08:27 2015 +0300 @@ -112,7 +112,7 @@ unsigned int char_len = uni_utf8_char_bytes(data[i]); if (i + char_len <= len && - uni_utf8_get_char_n(data + i, char_len, &chr) == 1 && + uni_utf8_get_char_n(data + i, char_len, &chr) > 0 && is_valid_xml_char(chr)) str_append_n(dest, data + i, char_len); else { From dovecot at dovecot.org Mon Jun 1 19:19:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:19:08 +0000 Subject: dovecot-2.2: lib: API change - have uni_utf8_get_char*() return ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8b7be28bd518 changeset: 18786:8b7be28bd518 user: Phil Carmody date: Mon Jun 01 22:08:43 2015 +0300 description: lib: API change - have uni_utf8_get_char*() return _char_bytes Often the two functions are called in close proximity (both ways round). As _get_char*() calls _char_bytes() early on the success path, we may as well return that value to the caller for immediate use. The callers which call _char_bytes() first are simply rejecting the truncated case quickly - all other invalid cases still call both functions, and all other valid cases (which should be the fast path) likewise call both. Signed-off-by: Phil Carmody diffstat: src/lib/unichar.c | 2 +- src/lib/unichar.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 7bd2f9be5493 -r 8b7be28bd518 src/lib/unichar.c --- a/src/lib/unichar.c Mon Jun 01 22:08:27 2015 +0300 +++ b/src/lib/unichar.c Mon Jun 01 22:08:43 2015 +0300 @@ -79,7 +79,7 @@ if (len <= max_len) { lowest_valid_chr = lowest_valid_chr_table[len]; - ret = 1; + ret = len; } else { /* check first if the input is invalid before returning 0 */ lowest_valid_chr = 0; diff -r 7bd2f9be5493 -r 8b7be28bd518 src/lib/unichar.h --- a/src/lib/unichar.h Mon Jun 01 22:08:27 2015 +0300 +++ b/src/lib/unichar.h Mon Jun 01 22:08:43 2015 +0300 @@ -48,7 +48,7 @@ void uni_ucs4_to_utf8(const unichar_t *input, size_t len, buffer_t *output); void uni_ucs4_to_utf8_c(unichar_t chr, buffer_t *output); -/* Returns 1 if *chr_r is set, 0 for incomplete trailing character, +/* Returns char_bytes (>0) if *chr_r is set, 0 for incomplete trailing character, -1 for invalid input. */ int uni_utf8_get_char(const char *input, unichar_t *chr_r); int uni_utf8_get_char_n(const void *input, size_t max_len, unichar_t *chr_r); From dovecot at dovecot.org Mon Jun 1 19:19:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:19:08 +0000 Subject: dovecot-2.2: lib: Improved unit test for uni_utf8_get_char() ret... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5f4742b69e7e changeset: 18787:5f4742b69e7e user: Timo Sirainen date: Mon Jun 01 22:11:55 2015 +0300 description: lib: Improved unit test for uni_utf8_get_char() return value check. diffstat: src/lib/test-unichar.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 8b7be28bd518 -r 5f4742b69e7e src/lib/test-unichar.c --- a/src/lib/test-unichar.c Mon Jun 01 22:08:43 2015 +0300 +++ b/src/lib/test-unichar.c Mon Jun 01 22:11:55 2015 +0300 @@ -58,7 +58,7 @@ str_truncate(str, 0); uni_ucs4_to_utf8_c(chr, str); test_assert(uni_utf8_str_is_valid(str_c(str))); - test_assert(uni_utf8_get_char(str_c(str), &chr2) > 0); + test_assert(uni_utf8_get_char(str_c(str), &chr2) == (int)uni_utf8_char_bytes(*str_data(str))); test_assert(chr2 == chr); if ((chr & 0x63) == 0) { From dovecot at dovecot.org Mon Jun 1 19:19:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:19:13 +0000 Subject: dovecot-2.2: lib-fts: Use new uni_utf8_get_char*() interface Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3a5ea8cf2233 changeset: 18788:3a5ea8cf2233 user: Timo Sirainen date: Mon Jun 01 22:14:19 2015 +0300 description: lib-fts: Use new uni_utf8_get_char*() interface diffstat: src/lib-fts/fts-tokenizer-generic.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (39 lines): diff -r 5f4742b69e7e -r 3a5ea8cf2233 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 22:11:55 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Jun 01 22:14:19 2015 +0300 @@ -198,14 +198,13 @@ struct generic_fts_tokenizer *tok = (struct generic_fts_tokenizer *)_tok; size_t i, start = 0; - unsigned int char_size; + int char_size; unichar_t c; bool apostrophe; for (i = 0; i < size; i += char_size) { - if (uni_utf8_get_char_n(data + i, size - i, &c) <= 0) - i_unreached(); - char_size = uni_utf8_char_bytes(data[i]); + char_size = uni_utf8_get_char_n(data + i, size - i, &c); + i_assert(char_size > 0); apostrophe = IS_APOSTROPHE(c); if (fts_simple_is_word_break(tok, c, apostrophe)) { @@ -620,13 +619,14 @@ unichar_t c; size_t i, char_start_i, start_pos = 0; enum letter_type lt; + int char_size; /* TODO: Process 8bit chars separately, to speed things up. */ for (i = 0; i < size; ) { char_start_i = i; - if (uni_utf8_get_char_n(data + i, size - i, &c) <= 0) - i_unreached(); - i += uni_utf8_char_bytes(data[i]); + char_size = uni_utf8_get_char_n(data + i, size - i, &c); + i_assert(char_size > 0); + i += char_size; lt = letter_type(c); if (tok->prev_letter == LETTER_TYPE_NONE && is_nontoken(lt)) { From dovecot at dovecot.org Mon Jun 1 19:19:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 19:19:13 +0000 Subject: dovecot-2.2: various: use new uni_utf8_get_char*() interface Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/778b57788590 changeset: 18789:778b57788590 user: Phil Carmody date: Mon Jun 01 22:16:19 2015 +0300 description: various: use new uni_utf8_get_char*() interface No need for additional uni_utf8_char_bytes() calls if you can parse and know the size with one call. Signed-off-by: Phil Carmody diffstat: src/lib/str-sanitize.c | 26 ++++++++++---------------- src/lib/unichar.c | 26 ++++++++++---------------- src/plugins/fts-solr/fts-backend-solr-old.c | 6 ++---- src/plugins/fts-solr/fts-backend-solr.c | 6 ++---- 4 files changed, 24 insertions(+), 40 deletions(-) diffs (155 lines): diff -r 3a5ea8cf2233 -r 778b57788590 src/lib/str-sanitize.c --- a/src/lib/str-sanitize.c Mon Jun 01 22:14:19 2015 +0300 +++ b/src/lib/str-sanitize.c Mon Jun 01 22:16:19 2015 +0300 @@ -7,13 +7,12 @@ static size_t str_sanitize_skip_start(const char *src, size_t max_bytes) { - unsigned int len; unichar_t chr; size_t i; - for (i = 0; src[i] != '\0'; ) { - len = uni_utf8_char_bytes(src[i]); - if (i + len > max_bytes || uni_utf8_get_char(src+i, &chr) <= 0) + for (i = 0; i < max_bytes && src[i] != '\0'; ) { + int len = uni_utf8_get_char_n(src+i, max_bytes-i, &chr); + if (len <= 0) break; if ((unsigned char)src[i] < 32) break; @@ -45,23 +44,18 @@ void str_sanitize_append(string_t *dest, const char *src, size_t max_bytes) { - unsigned int len, initial_pos = str_len(dest); + unsigned int initial_pos = str_len(dest); unichar_t chr; size_t i; - int ret; - for (i = 0; src[i] != '\0'; ) { - len = uni_utf8_char_bytes(src[i]); - if (i + len > max_bytes) - break; - ret = uni_utf8_get_char(src+i, &chr); - if (ret <= 0) { + for (i = 0; i < max_bytes && src[i] != '\0'; ) { + int len = uni_utf8_get_char_n(src+i, max_bytes-i, &chr); + if (len == 0) + break; /* input ended too early */ + + if (len < 0) { /* invalid UTF-8 */ str_append_c(dest, '?'); - if (ret == 0) { - /* input ended too early */ - return; - } i++; continue; } diff -r 3a5ea8cf2233 -r 778b57788590 src/lib/unichar.c --- a/src/lib/unichar.c Mon Jun 01 22:14:19 2015 +0300 +++ b/src/lib/unichar.c Mon Jun 01 22:16:19 2015 +0300 @@ -109,11 +109,12 @@ unichar_t chr; while (*input != '\0') { - if (uni_utf8_get_char(input, &chr) <= 0) { + int len = uni_utf8_get_char(input, &chr); + if (len <= 0) { /* invalid input */ return -1; } - input += uni_utf8_char_bytes(*input); + input += len; array_append(output, &chr, 1); } @@ -124,15 +125,11 @@ ARRAY_TYPE(unichars) *output) { unichar_t chr; - unsigned int len; while (size > 0) { - if (uni_utf8_get_char_n(input, size, &chr) <= 0) { - /* invalid input */ - return -1; - } - len = uni_utf8_char_bytes(*input); - i_assert(len <= size); + int len = uni_utf8_get_char_n(input, size, &chr); + if (len <= 0) + return -1; /* invalid input */ input += len; size -= len; array_append(output, &chr, 1); @@ -329,19 +326,18 @@ buffer_t *output) { const unsigned char *input = _input; - unsigned int bytes; unichar_t chr; int ret = 0; while (size > 0) { - if (uni_utf8_get_char_n(input, size, &chr) <= 0) { + int bytes = uni_utf8_get_char_n(input, size, &chr); + if (bytes <= 0) { /* invalid input. try the next byte. */ ret = -1; input++; size--; output_add_replacement_char(output); continue; } - bytes = uni_utf8_char_bytes(*input); input += bytes; size -= bytes; @@ -359,10 +355,8 @@ is_valid_utf8_seq(const unsigned char *input, unsigned int size) { unichar_t chr; - - if (uni_utf8_get_char_n(input, size, &chr) <= 0) - return 0; - return uni_utf8_char_bytes(input[0]); + int len = uni_utf8_get_char_n(input, size, &chr); + return len <= 0 ? 0 : len; } static int uni_utf8_find_invalid_pos(const unsigned char *input, size_t size, diff -r 3a5ea8cf2233 -r 778b57788590 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Mon Jun 01 22:14:19 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Mon Jun 01 22:16:19 2015 +0300 @@ -89,10 +89,8 @@ /* make sure the character is valid for XML so we don't get XML parser errors */ unsigned int char_len = - uni_utf8_char_bytes(data[i]); - if (i + char_len <= len && - uni_utf8_get_char_n(data + i, char_len, &chr) > 0 && - is_valid_xml_char(chr)) + uni_utf8_get_char_n(data + i, len - i, &chr); + if (char_len > 0 && is_valid_xml_char(chr)) str_append_n(dest, data + i, char_len); else { str_append_n(dest, utf8_replacement_char, diff -r 3a5ea8cf2233 -r 778b57788590 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Mon Jun 01 22:14:19 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Mon Jun 01 22:16:19 2015 +0300 @@ -110,10 +110,8 @@ /* make sure the character is valid for XML so we don't get XML parser errors */ unsigned int char_len = - uni_utf8_char_bytes(data[i]); - if (i + char_len <= len && - uni_utf8_get_char_n(data + i, char_len, &chr) > 0 && - is_valid_xml_char(chr)) + uni_utf8_get_char_n(data + i, len - i, &chr); + if (char_len > 0 && is_valid_xml_char(chr)) str_append_n(dest, data + i, char_len); else { str_append_n(dest, utf8_replacement_char, From dovecot at dovecot.org Mon Jun 1 23:41:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 23:41:40 +0000 Subject: dovecot-2.2: lib-fts: Call libicu u_cleanup. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/346da98eaa1a changeset: 18790:346da98eaa1a user: Teemu Huovila date: Tue Jun 02 02:26:44 2015 +0300 description: lib-fts: Call libicu u_cleanup. diffstat: src/lib-fts/fts-filter.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (23 lines): diff -r 778b57788590 -r 346da98eaa1a src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Mon Jun 01 22:16:19 2015 +0300 +++ b/src/lib-fts/fts-filter.c Tue Jun 02 02:26:44 2015 +0300 @@ -5,6 +5,9 @@ #include "fts-language.h" #include "fts-filter.h" #include "fts-filter-private.h" +#ifdef HAVE_LIBICU +# include +#endif static ARRAY(const struct fts_filter *) fts_filter_classes; @@ -20,6 +23,9 @@ void fts_filters_deinit(void) { +#ifdef HAVE_LIBICU + u_cleanup(); +#endif array_free(&fts_filter_classes); } From dovecot at dovecot.org Mon Jun 1 23:41:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 23:41:40 +0000 Subject: dovecot-2.2: lib-fts: Correct comment in filter internal API. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4f4cbc63b49 changeset: 18791:e4f4cbc63b49 user: Teemu Huovila date: Tue Jun 02 02:26:44 2015 +0300 description: lib-fts: Correct comment in filter internal API. diffstat: src/lib-fts/fts-filter-private.h | 9 ++++----- 1 files changed, 4 insertions(+), 5 deletions(-) diffs (19 lines): diff -r 346da98eaa1a -r e4f4cbc63b49 src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Tue Jun 02 02:26:44 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Tue Jun 02 02:26:44 2015 +0300 @@ -4,11 +4,10 @@ #define FTS_FILTER_CLASSES_NR 3 /* - API that stemming providers (classes) must provide: The register() - function is called when the class is registered via - fts_filter_register() The create() function is called to get an - instance of a registered filter class. The destroy function is - called to destroy an instance of a filter. + API that stemming providers (classes) must provide: The create() + function is called to get an instance of a registered filter class. + The filter() function is called with tokens for the specific filter. + The destroy function is called to destroy an instance of a filter. */ struct fts_filter_vfuncs { From dovecot at dovecot.org Mon Jun 1 23:41:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 01 Jun 2015 23:41:40 +0000 Subject: dovecot-2.2: fts: Fixed memory leak at deinit when using multipl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/77575024f874 changeset: 18792:77575024f874 user: Timo Sirainen date: Tue Jun 02 02:39:20 2015 +0300 description: fts: Fixed memory leak at deinit when using multiple fts plugins at the same time using libfts. diffstat: src/plugins/fts-lucene/fts-lucene-plugin.c | 3 ++- src/plugins/fts-solr/fts-solr-plugin.c | 3 ++- src/plugins/fts/fts-user.c | 17 ++++++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diffs (67 lines): diff -r e4f4cbc63b49 -r 77575024f874 src/plugins/fts-lucene/fts-lucene-plugin.c --- a/src/plugins/fts-lucene/fts-lucene-plugin.c Tue Jun 02 02:26:44 2015 +0300 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.c Tue Jun 02 02:39:20 2015 +0300 @@ -96,7 +96,8 @@ { struct fts_lucene_user *fuser = FTS_LUCENE_USER_CONTEXT(user); - fts_mail_user_deinit(user); + if (fuser->set.use_libfts) + fts_mail_user_deinit(user); fuser->module_ctx.super.deinit(user); } diff -r e4f4cbc63b49 -r 77575024f874 src/plugins/fts-solr/fts-solr-plugin.c --- a/src/plugins/fts-solr/fts-solr-plugin.c Tue Jun 02 02:26:44 2015 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.c Tue Jun 02 02:39:20 2015 +0300 @@ -54,7 +54,8 @@ { struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(user); - fts_mail_user_deinit(user); + if (fuser->set.use_libfts) + fts_mail_user_deinit(user); fuser->module_ctx.super.deinit(user); } diff -r e4f4cbc63b49 -r 77575024f874 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Tue Jun 02 02:26:44 2015 +0300 +++ b/src/plugins/fts/fts-user.c Tue Jun 02 02:39:20 2015 +0300 @@ -16,6 +16,7 @@ struct fts_user { union mail_user_module_context module_ctx; + int refcount; struct fts_language_list *lang_list; struct fts_tokenizer *index_tokenizer, *search_tokenizer; @@ -335,9 +336,16 @@ int fts_mail_user_init(struct mail_user *user, const char **error_r) { - struct fts_user *fuser; + struct fts_user *fuser = FTS_USER_CONTEXT(user); + + if (fuser != NULL) { + /* multiple fts plugins are loaded */ + fuser->refcount++; + return 0; + } fuser = p_new(user->pool, struct fts_user, 1); + fuser->refcount = 1; p_array_init(&fuser->languages, user->pool, 4); if (fts_user_init_languages(user, fuser, error_r) < 0) { @@ -358,6 +366,9 @@ { struct fts_user *fuser = FTS_USER_CONTEXT(user); - if (fuser != NULL) - fts_user_free(fuser); + if (fuser != NULL) { + i_assert(fuser->refcount > 0); + if (--fuser->refcount == 0) + fts_user_free(fuser); + } } From dovecot at dovecot.org Tue Jun 2 16:54:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 16:54:59 +0000 Subject: dovecot-2.2: fts: Error logging fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f61345596ce6 changeset: 18793:f61345596ce6 user: Timo Sirainen date: Tue Jun 02 19:52:15 2015 +0300 description: fts: Error logging fix. 1) We were logging the error after it was already freed from data stack. 2) We were logging uninitialized error string when fts indexing was the one that failed. diffstat: src/plugins/fts/fts-build-mail.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 77575024f874 -r f61345596ce6 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Tue Jun 02 02:39:20 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Tue Jun 02 19:52:15 2015 +0300 @@ -273,6 +273,8 @@ ret = ret2 = fts_tokenizer_next(tokenizer, data, size, &token, &error); if (ret2 > 0 && filter != NULL) ret2 = fts_filter_filter(filter, &token, &error); + if (ret2 < 0) + i_error("fts: Couldn't create indexable tokens: %s", error); if (ret2 > 0) { if (fts_backend_update_build_more(ctx->update_ctx, (const void *)token, @@ -280,8 +282,6 @@ ret = -1; } } T_END; - if (ret < 0) - i_error("fts: Couldn't create indexable tokens: %s", error); return ret; } From dovecot at dovecot.org Tue Jun 2 16:54:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 16:54:59 +0000 Subject: dovecot-2.2: fts: If we fail while indexing headers, return fail... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dc4826a29990 changeset: 18794:dc4826a29990 user: Timo Sirainen date: Tue Jun 02 19:52:37 2015 +0300 description: fts: If we fail while indexing headers, return failure immediately. diffstat: src/plugins/fts/fts-build-mail.c | 30 +++++++++++++++++++----------- 1 files changed, 19 insertions(+), 11 deletions(-) diffs (99 lines): diff -r f61345596ce6 -r dc4826a29990 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Tue Jun 02 19:52:15 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Tue Jun 02 19:52:37 2015 +0300 @@ -97,13 +97,14 @@ fts_build_parse_content_disposition(ctx, hdr); } -static void +static int fts_build_unstructured_header(struct fts_mail_build_context *ctx, const struct message_header_line *hdr) { const unsigned char *data = hdr->full_value; unsigned char *buf = NULL; unsigned int i; + int ret; /* @UNSAFE: if there are any NULs, replace them with spaces */ for (i = 0; i < hdr->full_value_len; i++) { @@ -118,8 +119,9 @@ buf[i] = data[i]; } } - (void)fts_build_data(ctx, data, hdr->full_value_len, TRUE); + ret = fts_build_data(ctx, data, hdr->full_value_len, TRUE); i_free(buf); + return ret; } static bool data_has_8bit(const unsigned char *data, size_t size) @@ -150,14 +152,15 @@ ctx->cur_user_lang = fts_user_get_data_lang(ctx->update_ctx->backend->ns->user); } -static void fts_build_mail_header(struct fts_mail_build_context *ctx, - const struct message_block *block) +static int fts_build_mail_header(struct fts_mail_build_context *ctx, + const struct message_block *block) { const struct message_header_line *hdr = block->hdr; struct fts_backend_build_key key; + int ret; if (hdr->eoh) - return; + return 0; /* hdr->full_value is always set because we get the block from message_decoder */ @@ -173,11 +176,11 @@ fts_build_tokenized_hdr_update_lang(ctx, hdr); if (!fts_backend_update_set_build_key(ctx->update_ctx, &key)) - return; + return 0; if (!message_header_is_address(hdr->name)) { /* regular unstructured header */ - fts_build_unstructured_header(ctx, hdr); + ret = fts_build_unstructured_header(ctx, hdr); } else T_BEGIN { /* message address. normalize it to give better search results. */ @@ -191,7 +194,7 @@ str = t_str_new(hdr->full_value_len); message_address_write(str, addr); - (void)fts_build_data(ctx, str_data(str), str_len(str), TRUE); + ret = fts_build_data(ctx, str_data(str), str_len(str), TRUE); } T_END; if ((ctx->update_ctx->backend->flags & @@ -199,10 +202,12 @@ /* index the header name itself */ key.hdr_name = ""; if (fts_backend_update_set_build_key(ctx->update_ctx, &key)) { - (void)fts_build_data(ctx, (const void *)hdr->name, - strlen(hdr->name), TRUE); + if (fts_build_data(ctx, (const void *)hdr->name, + strlen(hdr->name), TRUE) < 0) + ret = -1; } } + return ret; } static bool @@ -543,7 +548,10 @@ if (block.hdr != NULL) { fts_parse_mail_header(&ctx, &raw_block); - fts_build_mail_header(&ctx, &block); + if (fts_build_mail_header(&ctx, &block) < 0) { + ret = -1; + break; + } } else if (block.size == 0) { /* end of headers */ } else { From dovecot at dovecot.org Tue Jun 2 17:02:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 17:02:07 +0000 Subject: dovecot-2.2: pop3-login: Added support for Zimbra proxy's XOIP c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0bcd3e9e77d4 changeset: 18795:0bcd3e9e77d4 user: Timo Sirainen date: Tue Jun 02 19:59:45 2015 +0300 description: pop3-login: Added support for Zimbra proxy's XOIP command. diffstat: src/pop3-login/client.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r dc4826a29990 -r 0bcd3e9e77d4 src/pop3-login/client.c --- a/src/pop3-login/client.c Tue Jun 02 19:52:37 2015 +0300 +++ b/src/pop3-login/client.c Tue Jun 02 19:59:45 2015 +0300 @@ -99,6 +99,10 @@ return cmd_quit(client); if (strcmp(cmd, "XCLIENT") == 0) return cmd_xclient(client, args); + if (strcmp(cmd, "XOIP") == 0) { + /* Compatibility with Zimbra's patched nginx */ + return cmd_xclient(client, t_strconcat("ADDR=", args, NULL)); + } client_send_reply(&client->common, POP3_CMD_REPLY_ERROR, "Unknown command."); From dovecot at dovecot.org Tue Jun 2 17:52:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 17:52:47 +0000 Subject: dovecot-2.2: lib-fts: Fixed tr29 tokenizer to delete last charac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/97b5c5e88540 changeset: 18796:97b5c5e88540 user: Timo Sirainen date: Tue Jun 02 20:50:23 2015 +0300 description: lib-fts: Fixed tr29 tokenizer to delete last character correctly when it's preceded by non-ASCII diffstat: src/lib-fts/fts-tokenizer-generic.c | 3 ++- src/lib-fts/test-fts-tokenizer.c | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diffs (43 lines): diff -r 0bcd3e9e77d4 -r 97b5c5e88540 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Tue Jun 02 19:59:45 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Tue Jun 02 20:50:23 2015 +0300 @@ -546,7 +546,8 @@ if (is_one_past_end(tok) && tok->untruncated_length <= tok->max_length) { /* delete the last character */ - while ((data[len-1] & 0x80) != 0) + while ((data[len-1] & 0x80) != 0 && + ((data[len-1] & (0x80|0x40)) != (0x80|0x40))) len--; i_assert(len > 0); len--; diff -r 0bcd3e9e77d4 -r 97b5c5e88540 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Tue Jun 02 19:59:45 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Tue Jun 02 20:50:23 2015 +0300 @@ -17,7 +17,7 @@ static const char *test_inputs[] = { /* generic things and word truncation: */ - "hello world\r\n\nAnd there\twas: text galore, " + "hello world\r\n\nAnd there\twas: text galor\xC3\xA9\xE2\x80\xA7 " "abc at example.com, " "Bar Baz , " "foo at domain " @@ -137,7 +137,7 @@ { static const char *const expected_output[] = { "hello", "world", "And", - "there", "was", "text", "galore", + "there", "was", "text", "galor\xC3\xA9", "abc", "example", "com", "Bar", "Baz", "bar", "example", "org", "foo", "domain", "1234567890123456789012345678?", @@ -194,7 +194,7 @@ { static const char *const expected_output[] = { "hello", "world", "And", - "there", "was", "text", "galore", + "there", "was", "text", "galor\xC3\xA9", "abc", "example", "com", "Bar", "Baz", "bar", "example", "org", "foo", "domain", "1234567890123456789012345678?", From dovecot at dovecot.org Tue Jun 2 18:48:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 18:48:08 +0000 Subject: dovecot-2.2: lib-fts: Removed unnecessary array.h include from f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/635e6747c41d changeset: 18797:635e6747c41d user: Timo Sirainen date: Tue Jun 02 21:41:46 2015 +0300 description: lib-fts: Removed unnecessary array.h include from fts-filter.h diffstat: src/lib-fts/fts-filter-normalizer-icu.c | 1 + src/lib-fts/fts-filter.h | 1 - 2 files changed, 1 insertions(+), 1 deletions(-) diffs (21 lines): diff -r 97b5c5e88540 -r 635e6747c41d src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Tue Jun 02 20:50:23 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Tue Jun 02 21:41:46 2015 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "buffer.h" #include "str.h" #include "unichar.h" /* unicode replacement char */ #include "fts-filter.h" diff -r 97b5c5e88540 -r 635e6747c41d src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Tue Jun 02 20:50:23 2015 +0300 +++ b/src/lib-fts/fts-filter.h Tue Jun 02 21:41:46 2015 +0300 @@ -1,6 +1,5 @@ #ifndef FTS_FILTER_H #define FTS_FILTER_H -#include "array.h" struct fts_language; struct fts_filter; From dovecot at dovecot.org Tue Jun 2 18:48:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 18:48:09 +0000 Subject: dovecot-2.2: lib-fts: Renamed variable. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b8665bd3574b changeset: 18798:b8665bd3574b user: Timo Sirainen date: Tue Jun 02 21:43:27 2015 +0300 description: lib-fts: Renamed variable. This is a lowercase-filter, not a normalizer. diffstat: src/lib-fts/fts-filter-lowercase.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 635e6747c41d -r b8665bd3574b src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 21:41:46 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 21:43:27 2015 +0300 @@ -40,7 +40,7 @@ return 1; } -static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { +static const struct fts_filter_vfuncs lowercase_filter_vfuncs = { fts_filter_lowercase_create, fts_filter_lowercase_filter, fts_filter_lowercase_destroy @@ -48,7 +48,7 @@ static const struct fts_filter fts_filter_lowercase_real = { .class_name = "lowercase", - .v = &normalizer_filter_vfuncs + .v = &lowercase_filter_vfuncs }; const struct fts_filter *fts_filter_lowercase = &fts_filter_lowercase_real; From dovecot at dovecot.org Tue Jun 2 18:48:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 18:48:09 +0000 Subject: dovecot-2.2: lib-fts: Include fts-filter.h in fts-filter-private.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/51194b531dd7 changeset: 18799:51194b531dd7 user: Timo Sirainen date: Tue Jun 02 21:45:46 2015 +0300 description: lib-fts: Include fts-filter.h in fts-filter-private.h This avoids having to always include them both. diffstat: src/lib-fts/fts-filter-lowercase.c | 1 - src/lib-fts/fts-filter-normalizer-icu.c | 1 - src/lib-fts/fts-filter-private.h | 2 ++ src/lib-fts/fts-filter-stemmer-snowball.c | 1 - src/lib-fts/fts-filter-stopwords.c | 1 - src/lib-fts/fts-filter.c | 1 - 6 files changed, 2 insertions(+), 5 deletions(-) diffs (67 lines): diff -r b8665bd3574b -r 51194b531dd7 src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 21:43:27 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 21:45:46 2015 +0300 @@ -2,7 +2,6 @@ #include "lib.h" #include "str.h" -#include "fts-filter.h" #include "fts-filter-private.h" #include "fts-language.h" diff -r b8665bd3574b -r 51194b531dd7 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Tue Jun 02 21:43:27 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Tue Jun 02 21:45:46 2015 +0300 @@ -4,7 +4,6 @@ #include "buffer.h" #include "str.h" #include "unichar.h" /* unicode replacement char */ -#include "fts-filter.h" #include "fts-filter-private.h" #include "fts-language.h" diff -r b8665bd3574b -r 51194b531dd7 src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Tue Jun 02 21:43:27 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Tue Jun 02 21:45:46 2015 +0300 @@ -1,6 +1,8 @@ #ifndef FTS_FILTER_PRIVATE_H #define FTS_FILTER_PRIVATE_H +#include "fts-filter.h" + #define FTS_FILTER_CLASSES_NR 3 /* diff -r b8665bd3574b -r 51194b531dd7 src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Tue Jun 02 21:43:27 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Tue Jun 02 21:45:46 2015 +0300 @@ -2,7 +2,6 @@ #include "lib.h" #include "fts-language.h" -#include "fts-filter.h" #include "fts-filter-private.h" #ifdef HAVE_FTS_STEMMER diff -r b8665bd3574b -r 51194b531dd7 src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Tue Jun 02 21:43:27 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Tue Jun 02 21:45:46 2015 +0300 @@ -7,7 +7,6 @@ #include "hash.h" #include "unichar.h" #include "fts-language.h" -#include "fts-filter.h" #include "fts-filter-private.h" #define STOPWORDS_FILE_FORMAT "%s/stopwords_%s.txt" diff -r b8665bd3574b -r 51194b531dd7 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Tue Jun 02 21:43:27 2015 +0300 +++ b/src/lib-fts/fts-filter.c Tue Jun 02 21:45:46 2015 +0300 @@ -3,7 +3,6 @@ #include "lib.h" #include "array.h" #include "fts-language.h" -#include "fts-filter.h" #include "fts-filter-private.h" #ifdef HAVE_LIBICU # include From dovecot at dovecot.org Tue Jun 2 18:52:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 18:52:16 +0000 Subject: dovecot-2.2: lib-fts: Do all the ICU cleanup in fts_icu_deinit(). Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e2d42eab722f changeset: 18800:e2d42eab722f user: Timo Sirainen date: Tue Jun 02 21:49:46 2015 +0300 description: lib-fts: Do all the ICU cleanup in fts_icu_deinit(). diffstat: src/lib-fts/fts-filter.c | 6 ++---- src/lib-fts/fts-icu.c | 7 +++++++ src/lib-fts/fts-icu.h | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diffs (56 lines): diff -r 51194b531dd7 -r e2d42eab722f src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Tue Jun 02 21:45:46 2015 +0300 +++ b/src/lib-fts/fts-filter.c Tue Jun 02 21:49:46 2015 +0300 @@ -3,10 +3,8 @@ #include "lib.h" #include "array.h" #include "fts-language.h" +#include "fts-icu.h" #include "fts-filter-private.h" -#ifdef HAVE_LIBICU -# include -#endif static ARRAY(const struct fts_filter *) fts_filter_classes; @@ -23,7 +21,7 @@ void fts_filters_deinit(void) { #ifdef HAVE_LIBICU - u_cleanup(); + fts_icu_deinit(); #endif array_free(&fts_filter_classes); } diff -r 51194b531dd7 -r e2d42eab722f src/lib-fts/fts-icu.c --- a/src/lib-fts/fts-icu.c Tue Jun 02 21:45:46 2015 +0300 +++ b/src/lib-fts/fts-icu.c Tue Jun 02 21:49:46 2015 +0300 @@ -5,6 +5,8 @@ #include "unichar.h" #include "fts-icu.h" +#include + void fts_icu_utf8_to_utf16(buffer_t *dest_utf16, const char *src_utf8) { UErrorCode err = U_ZERO_ERROR; @@ -108,3 +110,8 @@ buffer_set_used_size(dest_utf16, utf16_len * sizeof(UChar)); return 0; } + +void fts_icu_deinit(void) +{ + u_cleanup(); +} diff -r 51194b531dd7 -r e2d42eab722f src/lib-fts/fts-icu.h --- a/src/lib-fts/fts-icu.h Tue Jun 02 21:45:46 2015 +0300 +++ b/src/lib-fts/fts-icu.h Tue Jun 02 21:49:46 2015 +0300 @@ -14,4 +14,7 @@ unsigned int src_len, UTransliterator *transliterator, const char **error_r); +/* Free all the memory used by ICU functions. */ +void fts_icu_deinit(void); + #endif From dovecot at dovecot.org Tue Jun 2 19:03:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 19:03:31 +0000 Subject: dovecot-2.2: lib-fts: Added fts_icu_lcase() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2bf5c51738d4 changeset: 18801:2bf5c51738d4 user: Timo Sirainen date: Tue Jun 02 21:54:52 2015 +0300 description: lib-fts: Added fts_icu_lcase() diffstat: src/lib-fts/fts-icu.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib-fts/fts-icu.h | 2 + src/lib-fts/test-fts-icu.c | 33 +++++++++++++++++++++++++++++++- 3 files changed, 81 insertions(+), 1 deletions(-) diffs (131 lines): diff -r e2d42eab722f -r 2bf5c51738d4 src/lib-fts/fts-icu.c --- a/src/lib-fts/fts-icu.c Tue Jun 02 21:49:46 2015 +0300 +++ b/src/lib-fts/fts-icu.c Tue Jun 02 21:54:52 2015 +0300 @@ -5,8 +5,26 @@ #include "unichar.h" #include "fts-icu.h" +#include +#include #include +static struct UCaseMap *icu_csm = NULL; + +static struct UCaseMap *fts_icu_csm(void) +{ + UErrorCode err = U_ZERO_ERROR; + + if (icu_csm != NULL) + return icu_csm; + icu_csm = ucasemap_open(NULL, U_FOLD_CASE_DEFAULT, &err); + if (U_FAILURE(err)) { + i_fatal("LibICU ucasemap_open() failed: %s", + u_errorName(err)); + } + return icu_csm; +} + void fts_icu_utf8_to_utf16(buffer_t *dest_utf16, const char *src_utf8) { UErrorCode err = U_ZERO_ERROR; @@ -111,7 +129,36 @@ return 0; } +void fts_icu_lcase(string_t *dest_utf8, const char *src_utf8) +{ + struct UCaseMap *csm = fts_icu_csm(); + size_t avail_bytes, dest_pos = dest_utf8->used; + char *dest_data; + int dest_full_len; + UErrorCode err = 0; + + avail_bytes = buffer_get_writable_size(dest_utf8) - dest_pos; + dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, avail_bytes); + + dest_full_len = ucasemap_utf8ToLower(csm, dest_data, avail_bytes, + src_utf8, -1, &err); + if (err == U_BUFFER_OVERFLOW_ERROR) { + err = U_ZERO_ERROR; + dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, dest_full_len); + dest_full_len = ucasemap_utf8ToLower(csm, dest_data, dest_full_len, + src_utf8, -1, &err); + i_assert(err != U_BUFFER_OVERFLOW_ERROR); + } + if (U_FAILURE(err)) { + i_fatal("LibICU ucasemap_utf8ToLower() failed: %s", + u_errorName(err)); + } + buffer_set_used_size(dest_utf8, dest_full_len); +} + void fts_icu_deinit(void) { + if (icu_csm != NULL) + ucasemap_close(icu_csm); u_cleanup(); } diff -r e2d42eab722f -r 2bf5c51738d4 src/lib-fts/fts-icu.h --- a/src/lib-fts/fts-icu.h Tue Jun 02 21:49:46 2015 +0300 +++ b/src/lib-fts/fts-icu.h Tue Jun 02 21:54:52 2015 +0300 @@ -13,6 +13,8 @@ int fts_icu_translate(buffer_t *dest_utf16, const UChar *src_utf16, unsigned int src_len, UTransliterator *transliterator, const char **error_r); +/* Lowercase the given UTF-8 string. */ +void fts_icu_lcase(string_t *dest_utf8, const char *src_utf8); /* Free all the memory used by ICU functions. */ void fts_icu_deinit(void); diff -r e2d42eab722f -r 2bf5c51738d4 src/lib-fts/test-fts-icu.c --- a/src/lib-fts/test-fts-icu.c Tue Jun 02 21:49:46 2015 +0300 +++ b/src/lib-fts/test-fts-icu.c Tue Jun 02 21:54:52 2015 +0300 @@ -141,6 +141,35 @@ test_end(); } +static void test_fts_icu_lcase(void) +{ + const char *src = "aBcD\xC3\x84\xC3\xA4"; + string_t *dest = t_str_new(64); + + test_begin("fts_icu_lcase"); + fts_icu_lcase(dest, src); + test_assert(strcmp(str_c(dest), "abcd\xC3\xA4\xC3\xA4") == 0); + test_end(); +} + +static void test_fts_icu_lcase_resize(void) +{ + const char *src = "a\xC3\x84"; + string_t *dest; + unsigned int i; + + test_begin("fts_icu_lcase resize"); + for (i = 2; i <= 4; i++) { + dest = t_str_new(i); + test_assert(buffer_get_size(dest) == i); + fts_icu_lcase(dest, src); + test_assert(strcmp(str_c(dest), "a\xC3\xA4") == 0); + test_assert(buffer_get_size(dest) == 4); + } + + test_end(); +} + int main(void) { static void (*test_functions[])(void) = { @@ -150,9 +179,11 @@ test_fts_icu_utf16_to_utf8_resize, test_fts_icu_translate, test_fts_icu_translate_resize, + test_fts_icu_lcase, + test_fts_icu_lcase_resize, NULL }; int ret = test_run(test_functions); - u_cleanup(); + fts_icu_deinit(); return ret; } From dovecot at dovecot.org Tue Jun 2 19:03:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 19:03:31 +0000 Subject: dovecot-2.2: lib-fts: Added string_t *token to struct fts_filter Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/26d9a4fcb0d4 changeset: 18802:26d9a4fcb0d4 user: Timo Sirainen date: Tue Jun 02 21:56:03 2015 +0300 description: lib-fts: Added string_t *token to struct fts_filter This makes the work a bit easier for simple filters that don't need any state but want to use a string_t. diffstat: src/lib-fts/fts-filter-private.h | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 2bf5c51738d4 -r 26d9a4fcb0d4 src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Tue Jun 02 21:54:52 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Tue Jun 02 21:56:03 2015 +0300 @@ -28,6 +28,7 @@ const struct fts_filter_vfuncs *v; int refcount; struct fts_filter *parent; + string_t *token; }; #endif From dovecot at dovecot.org Tue Jun 2 19:03:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 19:03:31 +0000 Subject: dovecot-2.2: lib-fts: Use fts_icu_lcase() for fts-filter-lowerca... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f227c2318e02 changeset: 18803:f227c2318e02 user: Timo Sirainen date: Tue Jun 02 21:56:29 2015 +0300 description: lib-fts: Use fts_icu_lcase() for fts-filter-lowercase if possible. diffstat: src/lib-fts/fts-filter-lowercase.c | 13 +++++++++++-- 1 files changed, 11 insertions(+), 2 deletions(-) diffs (45 lines): diff -r 26d9a4fcb0d4 -r f227c2318e02 src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 21:56:03 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 21:56:29 2015 +0300 @@ -2,12 +2,14 @@ #include "lib.h" #include "str.h" +#include "fts-icu.h" +#include "fts-language.h" #include "fts-filter-private.h" -#include "fts-language.h" static void fts_filter_lowercase_destroy(struct fts_filter *filter) { + str_free(&filter->token); i_free(filter); } @@ -25,17 +27,24 @@ } filter = i_new(struct fts_filter, 1); *filter = *fts_filter_lowercase; + filter->token = str_new(default_pool, 64); *filter_r = filter; return 0; } static int -fts_filter_lowercase_filter(struct fts_filter *_filter ATTR_UNUSED, +fts_filter_lowercase_filter(struct fts_filter *filter, const char **token, const char **error_r ATTR_UNUSED) { +#ifdef HAVE_LIBICU + str_truncate(filter->token, 0); + fts_icu_lcase(filter->token, *token); + *token = str_c(filter->token); +#else *token = t_str_lcase(*token); +#endif return 1; } From dovecot at dovecot.org Tue Jun 2 19:03:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 19:03:32 +0000 Subject: dovecot-2.2: lib-fts: Optimized truncation of partial trailing U... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/633ab1934f1f changeset: 18804:633ab1934f1f user: Timo Sirainen date: Tue Jun 02 22:01:07 2015 +0300 description: lib-fts: Optimized truncation of partial trailing UTF-8 characters in tokenizers. diffstat: src/lib-fts/fts-tokenizer-generic.c | 42 +++++++++++++++++++++++++----------- 1 files changed, 29 insertions(+), 13 deletions(-) diffs (101 lines): diff -r f227c2318e02 -r 633ab1934f1f src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Tue Jun 02 21:56:29 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Tue Jun 02 22:01:07 2015 +0300 @@ -90,43 +90,56 @@ i_free(tok); } -static const char *fts_uni_strndup(const unsigned char *data, size_t size) +static void +fts_tokenizer_delete_trailing_partial_char(const unsigned char *data, + size_t *len) { size_t pos; + unsigned int char_bytes; - /* if input is truncated with a partial UTF-8 character, drop it */ - (void)uni_utf8_partial_strlen_n(data, size, &pos); - i_assert(pos > 0); - return t_strndup(data, pos); + /* the token is truncated - make sure the last character + exists entirely in the token */ + for (pos = *len-1; pos > 0; pos--) { + if ((data[pos] & 0x80) == 0 || + ((data[pos] & (0x80|0x40)) == (0x80|0x40))) + break; + } + char_bytes = uni_utf8_char_bytes(data[pos]); + if (char_bytes != *len-pos) { + i_assert(char_bytes > *len-pos); + *len = pos; + } } static bool fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok, const char **token_r) { - const unsigned char *data; + const unsigned char *data = tok->token->data; size_t len = tok->token->used; - if (len > 0 && tok->untruncated_length <= tok->max_length) { + if (tok->untruncated_length <= tok->max_length) { /* Remove the trailing apostrophe - it was made into U+0027 earlier. There can be only a single such apostrophe, because otherwise the token would have already been split. We also want to remove the trailing apostrophe only if it's the the last character in the nontruncated token - a truncated token may end with apostrophe. */ - data = tok->token->data; - if (data[len-1] == '\'') { + if (len > 0 && data[len-1] == '\'') { len--; i_assert(len > 0 && data[len-1] != '\''); } + } else { + fts_tokenizer_delete_trailing_partial_char(data, &len); } + i_assert(len <= tok->max_length); *token_r = len == 0 ? "" : - fts_uni_strndup(tok->token->data, len); + t_strndup(tok->token->data, len); buffer_set_used_size(tok->token, 0); tok->untruncated_length = 0; tok->prev_letter = LETTER_TYPE_NONE; - return (*token_r)[0] != '\0'; + return len > 0; } static bool uint32_find(const uint32_t *data, unsigned int count, @@ -541,7 +554,7 @@ const char **token_r) { const unsigned char *data = tok->token->data; - ssize_t len = tok->token->used; + size_t len = tok->token->used; if (is_one_past_end(tok) && tok->untruncated_length <= tok->max_length) { @@ -551,16 +564,19 @@ len--; i_assert(len > 0); len--; + } else if (tok->untruncated_length > tok->max_length) { + fts_tokenizer_delete_trailing_partial_char(data, &len); } /* we're skipping all non-token chars at the beginning of the word, so by this point we must have something here - even if we just deleted the last character */ i_assert(len > 0); + i_assert(len <= tok->max_length); tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; - *token_r = fts_uni_strndup(data, len); + *token_r = t_strndup(data, len); buffer_set_used_size(tok->token, 0); tok->untruncated_length = 0; } From dovecot at dovecot.org Tue Jun 2 19:19:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 19:19:08 +0000 Subject: dovecot-2.2: lib-fts: Added more supported languages. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e3603730b2df changeset: 18805:e3603730b2df user: Timo Sirainen date: Tue Jun 02 22:16:46 2015 +0300 description: lib-fts: Added more supported languages. This includes now all the ones that are currently supported by both snowball and exttextcat. diffstat: src/lib-fts/fts-language.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 633ab1934f1f -r e3603730b2df src/lib-fts/fts-language.c --- a/src/lib-fts/fts-language.c Tue Jun 02 22:01:07 2015 +0300 +++ b/src/lib-fts/fts-language.c Tue Jun 02 22:16:46 2015 +0300 @@ -30,10 +30,17 @@ }; const struct fts_language fts_languages[] = { + { "da" }, + { "de" }, { "en" }, + { "es" }, { "fi" }, { "fr" }, - { "de" } + { "it" }, + { "nl" }, + { "pt" }, + { "ro" }, + { "ru" }, }; const struct fts_language fts_language_data = { From dovecot at dovecot.org Tue Jun 2 19:22:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 19:22:25 +0000 Subject: dovecot-2.2: lib-fts: Fixed compiling without libicu Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/75d7bf7560d3 changeset: 18806:75d7bf7560d3 user: Timo Sirainen date: Tue Jun 02 22:20:03 2015 +0300 description: lib-fts: Fixed compiling without libicu diffstat: src/lib-fts/fts-filter-lowercase.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (18 lines): diff -r e3603730b2df -r 75d7bf7560d3 src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 22:16:46 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 22:20:03 2015 +0300 @@ -2,10 +2,13 @@ #include "lib.h" #include "str.h" -#include "fts-icu.h" #include "fts-language.h" #include "fts-filter-private.h" +#ifdef HAVE_LIBICU +# include "fts-icu.h" +#endif + static void fts_filter_lowercase_destroy(struct fts_filter *filter) { From dovecot at dovecot.org Tue Jun 2 19:49:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 19:49:48 +0000 Subject: dovecot-2.2: imapc: If auth fails due to connection failure, don... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4f66f717fe87 changeset: 18807:4f66f717fe87 user: Timo Sirainen date: Tue Jun 02 22:47:21 2015 +0300 description: imapc: If auth fails due to connection failure, don't treat it the same as failed login. diffstat: src/lib-storage/index/imapc/imapc-storage.c | 16 ++++++++++++---- 1 files changed, 12 insertions(+), 4 deletions(-) diffs (28 lines): diff -r 75d7bf7560d3 -r 4f66f717fe87 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Tue Jun 02 22:20:03 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.c Tue Jun 02 22:47:21 2015 +0300 @@ -212,12 +212,20 @@ client->auth_failed = TRUE; if (client->_storage != NULL) { - mail_storage_set_error(&client->_storage->storage, - MAIL_ERROR_PERM, reply->text_full); + if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) + mail_storage_set_internal_error(&client->_storage->storage); + else { + mail_storage_set_error(&client->_storage->storage, + MAIL_ERROR_PERM, reply->text_full); + } } if (client->_list != NULL) { - mailbox_list_set_error(&client->_list->list, - MAIL_ERROR_PERM, reply->text_full); + if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) + mailbox_list_set_internal_error(&client->_list->list); + else { + mailbox_list_set_error(&client->_list->list, + MAIL_ERROR_PERM, reply->text_full); + } } } From dovecot at dovecot.org Tue Jun 2 20:30:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 20:30:56 +0000 Subject: dovecot-2.2: lib-fts: Avoid compiler warning when building witho... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1972b0acce17 changeset: 18808:1972b0acce17 user: Timo Sirainen date: Tue Jun 02 23:28:35 2015 +0300 description: lib-fts: Avoid compiler warning when building without libicu diffstat: src/lib-fts/fts-filter-lowercase.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 4f66f717fe87 -r 1972b0acce17 src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 22:47:21 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Tue Jun 02 23:28:35 2015 +0300 @@ -37,7 +37,7 @@ } static int -fts_filter_lowercase_filter(struct fts_filter *filter, +fts_filter_lowercase_filter(struct fts_filter *filter ATTR_UNUSED, const char **token, const char **error_r ATTR_UNUSED) { From dovecot at dovecot.org Tue Jun 2 20:37:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 20:37:43 +0000 Subject: dovecot-2.2: lib-fts: Compiling fix without libicu Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/357fe95d487f changeset: 18809:357fe95d487f user: Timo Sirainen date: Tue Jun 02 23:35:23 2015 +0300 description: lib-fts: Compiling fix without libicu diffstat: src/lib-fts/fts-filter.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (17 lines): diff -r 1972b0acce17 -r 357fe95d487f src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Tue Jun 02 23:28:35 2015 +0300 +++ b/src/lib-fts/fts-filter.c Tue Jun 02 23:35:23 2015 +0300 @@ -3,9 +3,12 @@ #include "lib.h" #include "array.h" #include "fts-language.h" -#include "fts-icu.h" #include "fts-filter-private.h" +#ifdef HAVE_LIBICU +# include "fts-icu.h" +#endif + static ARRAY(const struct fts_filter *) fts_filter_classes; void fts_filters_init(void) From dovecot at dovecot.org Tue Jun 2 20:47:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 20:47:42 +0000 Subject: dovecot-2.2: lib-lda: Added %{to_envelope} to deliver_log_format Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d79006a6ea65 changeset: 18810:d79006a6ea65 user: Timo Sirainen date: Tue Jun 02 23:43:05 2015 +0300 description: lib-lda: Added %{to_envelope} to deliver_log_format diffstat: src/lib-lda/mail-deliver.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (19 lines): diff -r 357fe95d487f -r d79006a6ea65 src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Tue Jun 02 23:35:23 2015 +0300 +++ b/src/lib-lda/mail-deliver.c Tue Jun 02 23:43:05 2015 +0300 @@ -67,6 +67,7 @@ { 'w', NULL, "vsize" }, { '\0', NULL, "delivery_time" }, { '\0', NULL, "session_time" }, + { '\0', NULL, "to_envelope" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -97,6 +98,7 @@ mail_deliver_log_var_expand_table_update_times(ctx, tab); tab[8].value = dec2str(ctx->session_time_msecs); } + tab[9].value = ctx->dest_addr; return tab; } From dovecot at dovecot.org Tue Jun 2 20:48:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 20:48:50 +0000 Subject: dovecot-2.2: example-config: Updated deliver_log_format comments Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8c5fce2ff6f8 changeset: 18811:8c5fce2ff6f8 user: Timo Sirainen date: Tue Jun 02 23:46:28 2015 +0300 description: example-config: Updated deliver_log_format comments diffstat: doc/example-config/conf.d/10-logging.conf | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r d79006a6ea65 -r 8c5fce2ff6f8 doc/example-config/conf.d/10-logging.conf --- a/doc/example-config/conf.d/10-logging.conf Tue Jun 02 23:43:05 2015 +0300 +++ b/doc/example-config/conf.d/10-logging.conf Tue Jun 02 23:46:28 2015 +0300 @@ -74,7 +74,8 @@ # possible variables you can use. #mail_log_prefix = "%s(%u): " -# Format to use for logging mail deliveries. You can use variables: +# Format to use for logging mail deliveries. See doc/wiki/Variables.txt for +# list of all variables you can use. Some of the common ones include: # %$ - Delivery status message (e.g. "saved to INBOX") # %m - Message-ID # %s - Subject From dovecot at dovecot.org Tue Jun 2 22:07:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 22:07:10 +0000 Subject: dovecot-2.2: lib-fts: Moved IS_APOSTROPHE() to fts-common.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a13ce2bce9cd changeset: 18812:a13ce2bce9cd user: Timo Sirainen date: Wed Jun 03 00:39:11 2015 +0300 description: lib-fts: Moved IS_APOSTROPHE() to fts-common.h diffstat: src/lib-fts/Makefile.am | 1 + src/lib-fts/fts-common.h | 9 +++++++++ src/lib-fts/fts-tokenizer-generic.c | 6 +----- 3 files changed, 11 insertions(+), 5 deletions(-) diffs (47 lines): diff -r 8c5fce2ff6f8 -r a13ce2bce9cd src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue Jun 02 23:46:28 2015 +0300 +++ b/src/lib-fts/Makefile.am Wed Jun 03 00:39:11 2015 +0300 @@ -73,6 +73,7 @@ $(ICU_SOURCES) noinst_HEADERS = \ + fts-common.h \ fts-filter.h \ fts-filter-private.h \ fts-icu.h \ diff -r 8c5fce2ff6f8 -r a13ce2bce9cd src/lib-fts/fts-common.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-common.h Wed Jun 03 00:39:11 2015 +0300 @@ -0,0 +1,9 @@ +#ifndef FTS_COMMON_H +#define FTS_COMMON_H + +#define IS_NONASCII_APOSTROPHE(c) \ + ((c) == 0x2019 || (c) == 0xFF07) +#define IS_APOSTROPHE(c) \ + ((c) == 0x0027 || IS_NONASCII_APOSTROPHE(c)) + +#endif diff -r 8c5fce2ff6f8 -r a13ce2bce9cd src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Tue Jun 02 23:46:28 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Wed Jun 03 00:39:11 2015 +0300 @@ -4,6 +4,7 @@ #include "buffer.h" #include "unichar.h" #include "bsearch-insert-pos.h" +#include "fts-common.h" #include "fts-tokenizer-private.h" #include "fts-tokenizer-generic-private.h" #include "word-boundary-data.c" @@ -11,11 +12,6 @@ #define FTS_DEFAULT_TOKEN_MAX_LENGTH 30 -#define IS_NONASCII_APOSTROPHE(c) \ - ((c) == 0x2019 || (c) == 0xFF07) -#define IS_APOSTROPHE(c) \ - ((c) == 0x0027 || IS_NONASCII_APOSTROPHE(c)) - static unsigned char fts_ascii_word_breaks[128] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0-15 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 16-31 */ From dovecot at dovecot.org Tue Jun 2 22:07:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 22:07:16 +0000 Subject: dovecot-2.2: lib-fts: Implemented default create/destory functio... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6a710924df27 changeset: 18813:6a710924df27 user: Timo Sirainen date: Wed Jun 03 00:43:28 2015 +0300 description: lib-fts: Implemented default create/destory functions for fts-filters. diffstat: src/lib-fts/fts-filter.c | 26 ++++++++++++++++++++++---- 1 files changed, 22 insertions(+), 4 deletions(-) diffs (50 lines): diff -r a13ce2bce9cd -r 6a710924df27 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Wed Jun 03 00:39:11 2015 +0300 +++ b/src/lib-fts/fts-filter.c Wed Jun 03 00:43:28 2015 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "str.h" #include "fts-language.h" #include "fts-filter-private.h" @@ -62,9 +63,19 @@ if (settings == NULL) settings = &empty_settings; - if (filter_class->v->create(lang, settings, &fp, error_r) < 0) { - *filter_r = NULL; - return -1; + if (filter_class->v->create != NULL) { + if (filter_class->v->create(lang, settings, &fp, error_r) < 0) { + *filter_r = NULL; + return -1; + } + } else { + /* default implementation */ + if (settings[0] != NULL) { + *error_r = t_strdup_printf("Unknown setting: %s", settings[0]); + return -1; + } + fp = i_new(struct fts_filter, 1); + *fp = *filter_class; } fp->refcount = 1; fp->parent = parent; @@ -93,7 +104,14 @@ if (fp->parent != NULL) fts_filter_unref(&fp->parent); - fp->v->destroy(fp); + if (fp->v->destroy != NULL) + fp->v->destroy(fp); + else { + /* default destroy implementation */ + if (fp->token != NULL) + str_free(&fp->token); + i_free(fp); + } } int fts_filter_filter(struct fts_filter *filter, const char **token, From dovecot at dovecot.org Tue Jun 2 22:07:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 22:07:17 +0000 Subject: dovecot-2.2: lib-fts: fts-lowercase can now use the default dest... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/932a0a83514b changeset: 18814:932a0a83514b user: Timo Sirainen date: Wed Jun 03 00:43:51 2015 +0300 description: lib-fts: fts-lowercase can now use the default destroy function. diffstat: src/lib-fts/fts-filter-lowercase.c | 9 +-------- 1 files changed, 1 insertions(+), 8 deletions(-) diffs (26 lines): diff -r 6a710924df27 -r 932a0a83514b src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Wed Jun 03 00:43:28 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Wed Jun 03 00:43:51 2015 +0300 @@ -9,13 +9,6 @@ # include "fts-icu.h" #endif -static void -fts_filter_lowercase_destroy(struct fts_filter *filter) -{ - str_free(&filter->token); - i_free(filter); -} - static int fts_filter_lowercase_create(const struct fts_language *lang ATTR_UNUSED, const char *const *settings, @@ -54,7 +47,7 @@ static const struct fts_filter_vfuncs lowercase_filter_vfuncs = { fts_filter_lowercase_create, fts_filter_lowercase_filter, - fts_filter_lowercase_destroy + NULL }; static const struct fts_filter fts_filter_lowercase_real = { From dovecot at dovecot.org Tue Jun 2 22:07:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 22:07:17 +0000 Subject: dovecot-2.2: lib-fts: fts-filter API changed to have a non-point... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/11c181d51b54 changeset: 18815:11c181d51b54 user: Timo Sirainen date: Wed Jun 03 00:46:23 2015 +0300 description: lib-fts: fts-filter API changed to have a non-pointer vfuncs variable. The main benefit being that the fts-filter implementations can save a few lines of code. diffstat: src/lib-fts/fts-filter-lowercase.c | 12 +++++------- src/lib-fts/fts-filter-normalizer-icu.c | 12 +++++------- src/lib-fts/fts-filter-private.h | 2 +- src/lib-fts/fts-filter-stemmer-snowball.c | 11 +++++------ src/lib-fts/fts-filter-stopwords.c | 12 +++++------- src/lib-fts/fts-filter.c | 10 +++++----- 6 files changed, 26 insertions(+), 33 deletions(-) diffs (141 lines): diff -r 932a0a83514b -r 11c181d51b54 src/lib-fts/fts-filter-lowercase.c --- a/src/lib-fts/fts-filter-lowercase.c Wed Jun 03 00:43:51 2015 +0300 +++ b/src/lib-fts/fts-filter-lowercase.c Wed Jun 03 00:46:23 2015 +0300 @@ -44,15 +44,13 @@ return 1; } -static const struct fts_filter_vfuncs lowercase_filter_vfuncs = { - fts_filter_lowercase_create, - fts_filter_lowercase_filter, - NULL -}; - static const struct fts_filter fts_filter_lowercase_real = { .class_name = "lowercase", - .v = &lowercase_filter_vfuncs + .v = { + fts_filter_lowercase_create, + fts_filter_lowercase_filter, + NULL + } }; const struct fts_filter *fts_filter_lowercase = &fts_filter_lowercase_real; diff -r 932a0a83514b -r 11c181d51b54 src/lib-fts/fts-filter-normalizer-icu.c --- a/src/lib-fts/fts-filter-normalizer-icu.c Wed Jun 03 00:43:51 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Wed Jun 03 00:46:23 2015 +0300 @@ -156,15 +156,13 @@ #endif -static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { - fts_filter_normalizer_icu_create, - fts_filter_normalizer_icu_filter, - fts_filter_normalizer_icu_destroy -}; - static const struct fts_filter fts_filter_normalizer_icu_real = { .class_name = "normalizer-icu", - .v = &normalizer_filter_vfuncs + .v = { + fts_filter_normalizer_icu_create, + fts_filter_normalizer_icu_filter, + fts_filter_normalizer_icu_destroy + } }; const struct fts_filter *fts_filter_normalizer_icu = diff -r 932a0a83514b -r 11c181d51b54 src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Wed Jun 03 00:43:51 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Wed Jun 03 00:46:23 2015 +0300 @@ -25,7 +25,7 @@ struct fts_filter { const char *class_name; /* name of the class this is based on */ - const struct fts_filter_vfuncs *v; + struct fts_filter_vfuncs v; int refcount; struct fts_filter *parent; string_t *token; diff -r 932a0a83514b -r 11c181d51b54 src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Wed Jun 03 00:43:51 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Wed Jun 03 00:46:23 2015 +0300 @@ -116,15 +116,14 @@ } #endif -static const struct fts_filter_vfuncs snowball_stemmer_filter_vfuncs = { - fts_filter_stemmer_snowball_create, - fts_filter_stemmer_snowball_filter, - fts_filter_stemmer_snowball_destroy -}; static const struct fts_filter fts_filter_stemmer_snowball_real = { .class_name = "snowball", - .v = &snowball_stemmer_filter_vfuncs + .v = { + fts_filter_stemmer_snowball_create, + fts_filter_stemmer_snowball_filter, + fts_filter_stemmer_snowball_destroy + } }; const struct fts_filter *fts_filter_stemmer_snowball = &fts_filter_stemmer_snowball_real; diff -r 932a0a83514b -r 11c181d51b54 src/lib-fts/fts-filter-stopwords.c --- a/src/lib-fts/fts-filter-stopwords.c Wed Jun 03 00:43:51 2015 +0300 +++ b/src/lib-fts/fts-filter-stopwords.c Wed Jun 03 00:46:23 2015 +0300 @@ -113,14 +113,12 @@ return hash_table_lookup(sp->stopwords, *token) == NULL ? 1 : 0; } -const struct fts_filter_vfuncs stopwords_filter_vfuncs = { - fts_filter_stopwords_create, - fts_filter_stopwords_filter, - fts_filter_stopwords_destroy -}; - const struct fts_filter fts_filter_stopwords_real = { .class_name = "stopwords", - .v = &stopwords_filter_vfuncs + .v = { + fts_filter_stopwords_create, + fts_filter_stopwords_filter, + fts_filter_stopwords_destroy + } }; const struct fts_filter *fts_filter_stopwords = &fts_filter_stopwords_real; diff -r 932a0a83514b -r 11c181d51b54 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Wed Jun 03 00:43:51 2015 +0300 +++ b/src/lib-fts/fts-filter.c Wed Jun 03 00:46:23 2015 +0300 @@ -63,8 +63,8 @@ if (settings == NULL) settings = &empty_settings; - if (filter_class->v->create != NULL) { - if (filter_class->v->create(lang, settings, &fp, error_r) < 0) { + if (filter_class->v.create != NULL) { + if (filter_class->v.create(lang, settings, &fp, error_r) < 0) { *filter_r = NULL; return -1; } @@ -104,8 +104,8 @@ if (fp->parent != NULL) fts_filter_unref(&fp->parent); - if (fp->v->destroy != NULL) - fp->v->destroy(fp); + if (fp->v.destroy != NULL) + fp->v.destroy(fp); else { /* default destroy implementation */ if (fp->token != NULL) @@ -127,7 +127,7 @@ /* Parent returned token or no parent. */ if (ret > 0 || filter->parent == NULL) - ret = filter->v->filter(filter, token, error_r); + ret = filter->v.filter(filter, token, error_r); if (ret <= 0) *token = NULL; From dovecot at dovecot.org Tue Jun 2 22:07:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 22:07:22 +0000 Subject: dovecot-2.2: lib: Added UTF8_IS_START_SEQ() helper macro Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b95be677f483 changeset: 18816:b95be677f483 user: Timo Sirainen date: Wed Jun 03 01:03:52 2015 +0300 description: lib: Added UTF8_IS_START_SEQ() helper macro diffstat: src/lib/unichar.h | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 11c181d51b54 -r b95be677f483 src/lib/unichar.h --- a/src/lib/unichar.h Wed Jun 03 00:46:23 2015 +0300 +++ b/src/lib/unichar.h Wed Jun 03 01:03:52 2015 +0300 @@ -23,6 +23,11 @@ (UTF16_SURROGATE_LOW_FIRST + \ (((chr) - UTF16_SURROGATE_BASE) & UTF16_SURROGATE_MASK)) +/* Returns TRUE if given byte is ASCII character or the beginning of a + multibyte UTF-8 sequence */ +#define UTF8_IS_START_SEQ(b) \ + (((b) & 0x80) == 0 || ((b) & 0xC0) == 0xC0) + #define UTF8_REPLACEMENT_CHAR_LEN 3 typedef uint32_t unichar_t; From dovecot at dovecot.org Tue Jun 2 22:07:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 22:07:23 +0000 Subject: dovecot-2.2: lib-fts: Use UTF8_IS_START_SEQ() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff79a2178fd4 changeset: 18817:ff79a2178fd4 user: Timo Sirainen date: Wed Jun 03 01:04:07 2015 +0300 description: lib-fts: Use UTF8_IS_START_SEQ() diffstat: src/lib-fts/fts-tokenizer-generic.c | 6 ++---- 1 files changed, 2 insertions(+), 4 deletions(-) diffs (23 lines): diff -r b95be677f483 -r ff79a2178fd4 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Wed Jun 03 01:03:52 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Wed Jun 03 01:04:07 2015 +0300 @@ -96,8 +96,7 @@ /* the token is truncated - make sure the last character exists entirely in the token */ for (pos = *len-1; pos > 0; pos--) { - if ((data[pos] & 0x80) == 0 || - ((data[pos] & (0x80|0x40)) == (0x80|0x40))) + if (UTF8_IS_START_SEQ(data[pos])) break; } char_bytes = uni_utf8_char_bytes(data[pos]); @@ -555,8 +554,7 @@ if (is_one_past_end(tok) && tok->untruncated_length <= tok->max_length) { /* delete the last character */ - while ((data[len-1] & 0x80) != 0 && - ((data[len-1] & (0x80|0x40)) != (0x80|0x40))) + while (!UTF8_IS_START_SEQ(data[len-1])) len--; i_assert(len > 0); len--; From dovecot at dovecot.org Tue Jun 2 22:07:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Jun 2015 22:07:23 +0000 Subject: dovecot-2.2: lib-fts: Added "english-possessive" filter. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c909977ec1a1 changeset: 18818:c909977ec1a1 user: Timo Sirainen date: Wed Jun 03 01:04:49 2015 +0300 description: lib-fts: Added "english-possessive" filter. diffstat: src/lib-fts/Makefile.am | 1 + src/lib-fts/fts-filter-english-possessive.c | 47 +++++++++++++++++++++++ src/lib-fts/fts-filter.c | 1 + src/lib-fts/fts-filter.h | 3 + src/lib-fts/test-fts-filter.c | 57 +++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 0 deletions(-) diffs (160 lines): diff -r ff79a2178fd4 -r c909977ec1a1 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Wed Jun 03 01:04:07 2015 +0300 +++ b/src/lib-fts/Makefile.am Wed Jun 03 01:04:49 2015 +0300 @@ -62,6 +62,7 @@ libfts_la_SOURCES = \ fts-filter.c \ + fts-filter-english-possessive.c \ fts-filter-lowercase.c \ fts-filter-normalizer-icu.c \ fts-filter-stopwords.c \ diff -r ff79a2178fd4 -r c909977ec1a1 src/lib-fts/fts-filter-english-possessive.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-filter-english-possessive.c Wed Jun 03 01:04:49 2015 +0300 @@ -0,0 +1,47 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "unichar.h" +#include "fts-common.h" +#include "fts-filter-private.h" + +static unichar_t get_ending_utf8_char(const char *str, unsigned int *end_pos) +{ + unichar_t c; + + while (!UTF8_IS_START_SEQ(str[*end_pos])) { + i_assert(*end_pos > 0); + *end_pos -= 1; + } + if (uni_utf8_get_char(str + *end_pos, &c) <= 0) + i_unreached(); + return c; +} + +static int +fts_filter_english_possessive_filter(struct fts_filter *filter ATTR_UNUSED, + const char **token, + const char **error_r ATTR_UNUSED) +{ + unsigned int len = strlen(*token); + unichar_t c; + + if (len > 1 && ((*token)[len-1] == 's' || (*token)[len-1] == 'S')) { + len -= 2; + c = get_ending_utf8_char(*token, &len); + if (IS_APOSTROPHE(c)) + *token = t_strndup(*token, len); + } + return 1; +} + +static const struct fts_filter fts_filter_english_possessive_real = { + .class_name = "english-possessive", + .v = { + NULL, + fts_filter_english_possessive_filter, + NULL + } +}; + +const struct fts_filter *fts_filter_english_possessive = &fts_filter_english_possessive_real; diff -r ff79a2178fd4 -r c909977ec1a1 src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Wed Jun 03 01:04:07 2015 +0300 +++ b/src/lib-fts/fts-filter.c Wed Jun 03 01:04:49 2015 +0300 @@ -20,6 +20,7 @@ fts_filter_register(fts_filter_stemmer_snowball); fts_filter_register(fts_filter_normalizer_icu); fts_filter_register(fts_filter_lowercase); + fts_filter_register(fts_filter_english_possessive); } void fts_filters_deinit(void) diff -r ff79a2178fd4 -r c909977ec1a1 src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Wed Jun 03 01:04:07 2015 +0300 +++ b/src/lib-fts/fts-filter.h Wed Jun 03 01:04:49 2015 +0300 @@ -32,6 +32,9 @@ /* Lowecases the input. Currently only ASCII data is lowercased. */ extern const struct fts_filter *fts_filter_lowercase; +/* Removes <'s> suffix from words. */ +extern const struct fts_filter *fts_filter_english_possessive; + /* Register all built-in filters. */ void fts_filters_init(void); void fts_filters_deinit(void); diff -r ff79a2178fd4 -r c909977ec1a1 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Wed Jun 03 01:04:07 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Wed Jun 03 01:04:49 2015 +0300 @@ -572,6 +572,62 @@ #endif #endif +static void test_fts_filter_english_possessive(void) +{ + struct fts_filter *norm = NULL; + const char *input[] = { + "foo'", + + "foo's", + "foo?'s", + "foo'S", + "foos'S", + "foo's's", + "foo'ss", + + "foo\xE2\x80\x99s", + "foo?\xE2\x80\x99s", + "foo\xE2\x80\x99S", + "foos\xE2\x80\x99S", + "foo\xE2\x80\x99s\xE2\x80\x99s", + "foo\xE2\x80\x99ss" + }; + const char *expected_output[] = { + "foo'", + + "foo", + "foo?", + "foo", + "foos", + "foo's", + "foo'ss", + + "foo", + "foo?", + "foo", + "foos", + "foo\xE2\x80\x99s", + "foo\xE2\x80\x99ss" + }; + const char *error = NULL; + const char *token = NULL; + unsigned int i; + + test_begin("fts filter english possessive"); + + T_BEGIN { + test_assert(fts_filter_create(fts_filter_english_possessive, NULL, NULL, NULL, &norm, &error) == 0); + for (i = 0; i < N_ELEMENTS(input); i++) { + token = input[i]; + test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); + test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); + } + fts_filter_unref(&norm); + } T_END; + test_assert(norm == NULL); + test_end(); +} + /* TODO: Functions to test 1. ref-unref pairs 2. multiple registers + an unregister + find */ @@ -600,6 +656,7 @@ test_fts_filter_normalizer_stopwords_stemmer_eng, #endif #endif + test_fts_filter_english_possessive, NULL }; int ret; From dovecot at dovecot.org Wed Jun 3 13:49:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 13:49:59 +0000 Subject: dovecot-2.2: lib-lda: Fixed crash in mail_deliver_get_log_var_ex... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/06505210b25d changeset: 18819:06505210b25d user: Teemu Huovila date: Wed Jun 03 16:47:25 2015 +0300 description: lib-lda: Fixed crash in mail_deliver_get_log_var_expand_table(). Discovered by clang static analyzer. This caused crashes with older versions of Pigeonhole. diffstat: src/lib-lda/mail-deliver.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (13 lines): diff -r c909977ec1a1 -r 06505210b25d src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Wed Jun 03 01:04:49 2015 +0300 +++ b/src/lib-lda/mail-deliver.c Wed Jun 03 16:47:25 2015 +0300 @@ -97,8 +97,8 @@ if (ctx != NULL) { mail_deliver_log_var_expand_table_update_times(ctx, tab); tab[8].value = dec2str(ctx->session_time_msecs); + tab[9].value = ctx->dest_addr; } - tab[9].value = ctx->dest_addr; return tab; } From dovecot at dovecot.org Wed Jun 3 18:53:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 18:53:05 +0000 Subject: dovecot-2.2: fts: If reading mail fails, log an error. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9e7d1e2acd23 changeset: 18820:9e7d1e2acd23 user: Timo Sirainen date: Wed Jun 03 21:49:52 2015 +0300 description: fts: If reading mail fails, log an error. diffstat: src/plugins/fts/fts-build-mail.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 06505210b25d -r 9e7d1e2acd23 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Wed Jun 03 16:47:25 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Wed Jun 03 21:49:52 2015 +0300 @@ -503,6 +503,11 @@ if (ret < 0) { if (input->stream_errno == 0) ret = 0; + else { + i_error("read(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); + } break; } From dovecot at dovecot.org Wed Jun 3 18:53:06 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 18:53:06 +0000 Subject: dovecot-2.2: fts: If mail indexing fails, don't log a "BUG: Unkn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/588e141980e2 changeset: 18821:588e141980e2 user: Timo Sirainen date: Wed Jun 03 21:50:40 2015 +0300 description: fts: If mail indexing fails, don't log a "BUG: Unknown internal error" diffstat: src/plugins/fts/fts-storage.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (23 lines): diff -r 9e7d1e2acd23 -r 588e141980e2 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Wed Jun 03 21:49:52 2015 +0300 +++ b/src/plugins/fts/fts-storage.c Wed Jun 03 21:50:40 2015 +0300 @@ -448,6 +448,7 @@ while (mailbox_search_next(ctx, &mail)) { if (fts_build_mail(update_ctx, mail) < 0) { + mail_storage_set_internal_error(trans->box->storage); ret = -1; break; } @@ -504,8 +505,10 @@ if (ft->next_index_seq == _mail->seq) { fts_backend_update_set_mailbox(flist->update_ctx, _mail->box); - if (fts_build_mail(flist->update_ctx, _mail) < 0) + if (fts_build_mail(flist->update_ctx, _mail) < 0) { + mail_storage_set_internal_error(_mail->box->storage); ft->failed = TRUE; + } ft->next_index_seq = _mail->seq + 1; } } From dovecot at dovecot.org Wed Jun 3 19:27:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 19:27:50 +0000 Subject: dovecot-2.2: indexer: Use array.h API instead of writing our own. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/165564cc0e0e changeset: 18822:165564cc0e0e user: Timo Sirainen date: Wed Jun 03 22:25:28 2015 +0300 description: indexer: Use array.h API instead of writing our own. The space savings aren't worth the extra complexity. diffstat: src/indexer/indexer-queue.c | 27 ++++++++++----------------- src/indexer/indexer-queue.h | 2 +- 2 files changed, 11 insertions(+), 18 deletions(-) diffs (75 lines): diff -r 588e141980e2 -r 165564cc0e0e src/indexer/indexer-queue.c --- a/src/indexer/indexer-queue.c Wed Jun 03 21:50:40 2015 +0300 +++ b/src/indexer/indexer-queue.c Wed Jun 03 22:25:28 2015 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "llist.h" #include "hash.h" #include "indexer-queue.h" @@ -70,22 +71,12 @@ static void request_add_context(struct indexer_request *request, void *context) { - unsigned int count = 0; - if (context == NULL) return; - if (request->contexts == NULL) { - request->contexts = i_new(void *, 2); - } else { - for (; request->contexts[count] != NULL; count++) ; - - request->contexts = - i_realloc(request->contexts, - sizeof(*request->contexts) * (count + 1), - sizeof(*request->contexts) * (count + 2)); - } - request->contexts[count] = context; + if (!array_is_created(&request->contexts)) + i_array_init(&request->contexts, 2); + array_append(&request->contexts, &context, 1); } static struct indexer_request * @@ -173,11 +164,12 @@ struct indexer_request *request, int percentage) { + void *const *contextp; unsigned int i; - if (request->contexts != NULL) { - for (i = 0; request->contexts[i] != NULL; i++) - queue->callback(percentage, request->contexts[i]); + for (i = 0; i < array_count(&request->contexts); i++) { + contextp = array_idx(&request->contexts, i); + queue->callback(percentage, *contextp); } } @@ -199,7 +191,8 @@ *_request = NULL; indexer_queue_request_status_int(queue, request, success ? 100 : -1); - i_free(request->contexts); + if (array_is_created(&request->contexts)) + array_free(&request->contexts); i_free(request->username); i_free(request->mailbox); i_free(request); diff -r 588e141980e2 -r 165564cc0e0e src/indexer/indexer-queue.h --- a/src/indexer/indexer-queue.h Wed Jun 03 21:50:40 2015 +0300 +++ b/src/indexer/indexer-queue.h Wed Jun 03 22:25:28 2015 +0300 @@ -15,7 +15,7 @@ /* optimize this mailbox */ unsigned int optimize:1; - void **contexts; + ARRAY(void *) contexts; }; struct indexer_queue *indexer_queue_init(indexer_status_callback_t *callback); From dovecot at dovecot.org Wed Jun 3 19:28:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 19:28:27 +0000 Subject: dovecot-2.2: indexer: Try to always flush as many requests from ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a5d5d2dd6aea changeset: 18823:a5d5d2dd6aea user: Timo Sirainen date: Wed Jun 03 22:26:05 2015 +0300 description: indexer: Try to always flush as many requests from queue as possible. I don't think this changes the behavior much. It's mainly a code cleanup. diffstat: src/indexer/indexer.c | 35 ++++++++++------------------------- 1 files changed, 10 insertions(+), 25 deletions(-) diffs (57 lines): diff -r 165564cc0e0e -r a5d5d2dd6aea src/indexer/indexer.c --- a/src/indexer/indexer.c Wed Jun 03 22:25:28 2015 +0300 +++ b/src/indexer/indexer.c Wed Jun 03 22:26:05 2015 +0300 @@ -53,7 +53,7 @@ worker_connection_request(conn, request, wrequest); } -static void queue_handle_existing_user_requests(struct indexer_queue *queue) +static void queue_try_send_more(struct indexer_queue *queue) { struct worker_connection *conn; struct indexer_request *request; @@ -61,35 +61,20 @@ while ((request = indexer_queue_request_peek(queue)) != NULL) { conn = worker_pool_find_username_connection(worker_pool, request->username); - if (conn == NULL) - break; - + if (conn != NULL) { + /* there is already a worker handling this user. + it must be the one doing the indexing. use the same + connection for sending this next request. */ + } else { + /* try to find an empty worker */ + if (!worker_pool_get_connection(worker_pool, &conn)) + break; + } indexer_queue_request_remove(queue); - /* there is already a worker handling this user. - it must be the one doing the indexing. */ worker_send_request(conn, request); } } -static void queue_try_send_more(struct indexer_queue *queue) -{ - struct worker_connection *conn; - struct indexer_request *request; - - queue_handle_existing_user_requests(queue); - - request = indexer_queue_request_peek(queue); - if (request == NULL) - return; - - /* okay, we have a request for a new user. */ - if (!worker_pool_get_connection(worker_pool, &conn)) - return; - - indexer_queue_request_remove(queue); - worker_send_request(conn, request); -} - static void queue_listen_callback(struct indexer_queue *queue) { queue_try_send_more(queue); From dovecot at dovecot.org Wed Jun 3 19:32:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 19:32:39 +0000 Subject: dovecot-2.2: indexer: Improved handling multiple indexing reques... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2f5799572a2f changeset: 18824:2f5799572a2f user: Timo Sirainen date: Wed Jun 03 22:30:05 2015 +0300 description: indexer: Improved handling multiple indexing requests to the same mailbox. If a request arrives for a mailbox that we were already indexes, the previous code simply sent the indexing request to the existing worker process. This could have caused a lot of requests to be buffered to the same mailbox if the indexing took a long time, which could have taken a while to process even though they weren't really doing anything indexing work. The new code instead just keeps track in memory that when the earlier indexing is finished, it's done again once to finish any pending changes. diffstat: src/indexer/indexer-queue.c | 40 +++++++++++++++++++++++++++++++++++++--- src/indexer/indexer-queue.h | 13 +++++++++++++ src/indexer/indexer.c | 1 + 3 files changed, 51 insertions(+), 3 deletions(-) diffs (124 lines): diff -r a5d5d2dd6aea -r 2f5799572a2f src/indexer/indexer-queue.c --- a/src/indexer/indexer-queue.c Wed Jun 03 22:26:05 2015 +0300 +++ b/src/indexer/indexer-queue.c Wed Jun 03 22:30:05 2015 +0300 @@ -98,6 +98,14 @@ if (request->max_recent_msgs > max_recent_msgs) request->max_recent_msgs = max_recent_msgs; request_add_context(request, context); + if (request->working) { + /* we're already indexing this mailbox. */ + if (append) + request->reindex_tail = TRUE; + else + request->reindex_head = TRUE; + return request; + } if (append) { /* keep the request in its old position */ return request; @@ -156,8 +164,6 @@ i_assert(request != NULL); DLLIST2_REMOVE(&queue->head, &queue->tail, request); - hash_table_remove(queue->requests, request); - indexer_refresh_proctitle(); } static void indexer_queue_request_status_int(struct indexer_queue *queue, @@ -167,7 +173,7 @@ void *const *contextp; unsigned int i; - for (i = 0; i < array_count(&request->contexts); i++) { + for (i = 0; i < request->working_context_idx; i++) { contextp = array_idx(&request->contexts, i); queue->callback(percentage, *contextp); } @@ -182,6 +188,14 @@ indexer_queue_request_status_int(queue, request, percentage); } +void indexer_queue_request_work(struct indexer_request *request) +{ + request->working = TRUE; + request->working_context_idx = + !array_is_created(&request->contexts) ? 0 : + array_count(&request->contexts); +} + void indexer_queue_request_finish(struct indexer_queue *queue, struct indexer_request **_request, bool success) @@ -191,11 +205,31 @@ *_request = NULL; indexer_queue_request_status_int(queue, request, success ? 100 : -1); + + if (request->reindex_head || request->reindex_tail) { + i_assert(request->working); + request->working = FALSE; + request->reindex_head = FALSE; + request->reindex_tail = FALSE; + if (request->working_context_idx > 0) { + array_delete(&request->contexts, 0, + request->working_context_idx); + } + if (request->reindex_head) + DLLIST2_PREPEND(&queue->head, &queue->tail, request); + else + DLLIST2_APPEND(&queue->head, &queue->tail, request); + return; + } + + hash_table_remove(queue->requests, request); if (array_is_created(&request->contexts)) array_free(&request->contexts); i_free(request->username); i_free(request->mailbox); i_free(request); + + indexer_refresh_proctitle(); } void indexer_queue_cancel_all(struct indexer_queue *queue) diff -r a5d5d2dd6aea -r 2f5799572a2f src/indexer/indexer-queue.h --- a/src/indexer/indexer-queue.h Wed Jun 03 22:26:05 2015 +0300 +++ b/src/indexer/indexer-queue.h Wed Jun 03 22:30:05 2015 +0300 @@ -14,6 +14,17 @@ unsigned int index:1; /* optimize this mailbox */ unsigned int optimize:1; + /* currently indexing this mailbox */ + unsigned int working:1; + /* after indexing is finished, add this request back to the queue and + reindex it (i.e. a new indexing request came while we were + working.) */ + unsigned int reindex_head:1; + unsigned int reindex_tail:1; + + /* when working finished, call this number of contexts and leave the + rest to the reindexing. */ + unsigned int working_context_idx; ARRAY(void *) contexts; }; @@ -45,6 +56,8 @@ void indexer_queue_request_status(struct indexer_queue *queue, struct indexer_request *request, int percentage); +/* Start working on a request */ +void indexer_queue_request_work(struct indexer_request *request); /* Finish the request and free its memory. */ void indexer_queue_request_finish(struct indexer_queue *queue, struct indexer_request **request, diff -r a5d5d2dd6aea -r 2f5799572a2f src/indexer/indexer.c --- a/src/indexer/indexer.c Wed Jun 03 22:26:05 2015 +0300 +++ b/src/indexer/indexer.c Wed Jun 03 22:30:05 2015 +0300 @@ -50,6 +50,7 @@ wrequest->conn = conn; wrequest->request = request; + indexer_queue_request_work(request); worker_connection_request(conn, request, wrequest); } From dovecot at dovecot.org Wed Jun 3 19:42:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 19:42:13 +0000 Subject: dovecot-2.2: lib-fts: fts_tokenizer_reset() didn't reset the ent... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/576267a41570 changeset: 18825:576267a41570 user: Timo Sirainen date: Wed Jun 03 22:39:52 2015 +0300 description: lib-fts: fts_tokenizer_reset() didn't reset the entire state. diffstat: src/lib-fts/fts-tokenizer.c | 16 +++++++++++----- 1 files changed, 11 insertions(+), 5 deletions(-) diffs (47 lines): diff -r 2f5799572a2f -r 576267a41570 src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Wed Jun 03 22:30:05 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Wed Jun 03 22:39:52 2015 +0300 @@ -66,6 +66,14 @@ return tok->name; } +static void fts_tokenizer_self_reset(struct fts_tokenizer *tok) +{ + tok->prev_data = NULL; + tok->prev_size = 0; + tok->prev_skip = 0; + tok->prev_reply_finished = TRUE; +} + int fts_tokenizer_create(const struct fts_tokenizer *tok_class, struct fts_tokenizer *parent, const char *const *settings, @@ -85,7 +93,7 @@ return -1; } tok->refcount = 1; - tok->prev_reply_finished = TRUE; + fts_tokenizer_self_reset(tok); if (parent != NULL) { fts_tokenizer_ref(parent); tok->parent = parent; @@ -150,10 +158,7 @@ tok->prev_reply_finished = FALSE; } else if (ret == 0) { /* we need a new data block */ - tok->prev_data = NULL; - tok->prev_size = 0; - tok->prev_skip = 0; - tok->prev_reply_finished = TRUE; + fts_tokenizer_self_reset(tok); } return ret; } @@ -161,6 +166,7 @@ void fts_tokenizer_reset(struct fts_tokenizer *tok) { tok->v->reset(tok); + fts_tokenizer_self_reset(tok); } int fts_tokenizer_next(struct fts_tokenizer *tok, From dovecot at dovecot.org Wed Jun 3 20:58:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 20:58:56 +0000 Subject: dovecot-2.2: fts: Added "doveadm fts lookup" command. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e222d1265e4e changeset: 18826:e222d1265e4e user: Timo Sirainen date: Wed Jun 03 23:56:32 2015 +0300 description: fts: Added "doveadm fts lookup" command. This is mainly useful for debugging lib-fts. It doesn't perform any of the lib-fts tokenization / filtering so you can do raw lookups. diffstat: src/plugins/fts/Makefile.am | 1 + src/plugins/fts/doveadm-fts.c | 98 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 0 deletions(-) diffs (134 lines): diff -r 576267a41570 -r e222d1265e4e src/plugins/fts/Makefile.am --- a/src/plugins/fts/Makefile.am Wed Jun 03 22:39:52 2015 +0300 +++ b/src/plugins/fts/Makefile.am Wed Jun 03 23:56:32 2015 +0300 @@ -7,6 +7,7 @@ -I$(top_srcdir)/src/lib-fts \ -I$(top_srcdir)/src/lib-http \ -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage \ -I$(top_srcdir)/src/lib-storage/index \ diff -r 576267a41570 -r e222d1265e4e src/plugins/fts/doveadm-fts.c --- a/src/plugins/fts/doveadm-fts.c Wed Jun 03 22:39:52 2015 +0300 +++ b/src/plugins/fts/doveadm-fts.c Wed Jun 03 23:56:32 2015 +0300 @@ -1,14 +1,111 @@ /* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "str.h" +#include "imap-util.h" #include "mail-namespace.h" +#include "mail-search.h" +#include "mailbox-list-iter.h" #include "fts-storage.h" #include "doveadm-mail.h" +#include "doveadm-mailbox-list-iter.h" #include "doveadm-fts.h" const char *doveadm_fts_plugin_version = DOVECOT_ABI_VERSION; static int +cmd_search_box(struct doveadm_mail_cmd_context *ctx, + const struct mailbox_info *info) +{ + struct mailbox *box; + struct fts_backend *backend; + struct fts_result result; + int ret = 0; + + backend = fts_list_backend(info->ns->list); + if (backend == NULL) { + i_error("fts not enabled for %s", info->vname); + return -1; + } + + memset(&result, 0, sizeof(result)); + i_array_init(&result.definite_uids, 16); + i_array_init(&result.maybe_uids, 16); + i_array_init(&result.scores, 16); + + box = mailbox_alloc(info->ns->list, info->vname, 0); + if (fts_backend_lookup(backend, box, ctx->search_args->args, + FTS_LOOKUP_FLAG_AND_ARGS, &result) < 0) { + i_error("fts lookup failed"); + doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP); + ret = -1; + } else { + printf("%s: ", info->vname); + if (array_count(&result.definite_uids) == 0) + printf("no results\n"); + else T_BEGIN { + string_t *str = t_str_new(128); + imap_write_seq_range(str, &result.definite_uids); + printf("%s\n", str_c(str)); + } T_END; + if (array_count(&result.maybe_uids) > 0) T_BEGIN { + string_t *str = t_str_new(128); + imap_write_seq_range(str, &result.maybe_uids); + printf(" - maybe: %s\n", str_c(str)); + } T_END; + fts_backend_lookup_done(backend); + } + mailbox_free(&box); + array_free(&result.definite_uids); + array_free(&result.maybe_uids); + array_free(&result.scores); + return ret; +} + +static int +cmd_fts_lookup_run(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user) +{ + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_search_box(ctx, info) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static void +cmd_fts_lookup_init(struct doveadm_mail_cmd_context *ctx, + const char *const args[]) +{ + if (args[0] == NULL) + doveadm_mail_help_name("fts lookup"); + + ctx->search_args = doveadm_mail_build_search_args(args); +} + +static struct doveadm_mail_cmd_context * +cmd_fts_lookup_alloc(void) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); + ctx->v.run = cmd_fts_lookup_run; + ctx->v.init = cmd_fts_lookup_init; + return ctx; +} + +static int fts_namespace_find(struct mail_user *user, const char *ns_prefix, struct mail_namespace **ns_r) { @@ -113,6 +210,7 @@ } static struct doveadm_mail_cmd fts_commands[] = { + { cmd_fts_lookup_alloc, "fts lookup", "" }, { cmd_fts_optimize_alloc, "fts optimize", "[]" }, { cmd_fts_rescan_alloc, "fts rescan", "[]" } }; From dovecot at dovecot.org Wed Jun 3 21:47:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 21:47:33 +0000 Subject: dovecot-2.2: lib-storage: Added mail_search_args_to_cmdline() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/53824319afb5 changeset: 18827:53824319afb5 user: Timo Sirainen date: Thu Jun 04 00:44:42 2015 +0300 description: lib-storage: Added mail_search_args_to_cmdline() This isn't entirely correct yet, but it's only going to be used for debugging for now. diffstat: src/lib-storage/Makefile.am | 1 + src/lib-storage/mail-search-args-cmdline.c | 100 +++++++++++++++++++++++++++++ src/lib-storage/mail-search.h | 3 + 3 files changed, 104 insertions(+), 0 deletions(-) diffs (128 lines): diff -r e222d1265e4e -r 53824319afb5 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Wed Jun 03 23:56:32 2015 +0300 +++ b/src/lib-storage/Makefile.am Thu Jun 04 00:44:42 2015 +0300 @@ -28,6 +28,7 @@ mail-error.c \ mail-namespace.c \ mail-search.c \ + mail-search-args-cmdline.c \ mail-search-args-imap.c \ mail-search-args-simplify.c \ mail-search-build.c \ diff -r e222d1265e4e -r 53824319afb5 src/lib-storage/mail-search-args-cmdline.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mail-search-args-cmdline.c Thu Jun 04 00:44:42 2015 +0300 @@ -0,0 +1,100 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "imap-quote.h" +#include "mail-search.h" + +static void +mail_search_arg_to_cmdline(string_t *dest, const struct mail_search_arg *arg); + +static void +mail_search_subargs_to_cmdline(string_t *dest, const struct mail_search_arg *args, + const char *middle) +{ + const struct mail_search_arg *arg; + + str_append(dest, "( "); + for (arg = args; arg != NULL; arg = arg->next) { + mail_search_arg_to_cmdline(dest, arg); + if (arg->next != NULL) + str_append(dest, middle); + } + str_append(dest, " )"); +} + +static void +mail_search_arg_to_cmdline(string_t *dest, const struct mail_search_arg *arg) +{ + struct mail_search_arg new_arg; + const char *error; + + if (arg->match_not) + str_append(dest, "NOT "); + switch (arg->type) { + case SEARCH_OR: + mail_search_subargs_to_cmdline(dest, arg->value.subargs, " OR "); + return; + case SEARCH_SUB: + mail_search_subargs_to_cmdline(dest, arg->value.subargs, " "); + return; + case SEARCH_FLAGS: + case SEARCH_KEYWORDS: { + size_t pos = str_len(dest); + + if (!mail_search_arg_to_imap(dest, arg, &error)) + i_unreached(); + str_insert(dest, pos+1, " "); + str_insert(dest, str_len(dest)-2, " "); + return; + } + case SEARCH_INTHREAD: + str_append(dest, "INTHREAD "); + imap_append_astring(dest, mail_thread_type_to_str(arg->value.thread_type)); + str_append_c(dest, ' '); + mail_search_subargs_to_cmdline(dest, arg->value.subargs, " "); + break; + case SEARCH_MAILBOX: + str_append(dest, "MAILBOX "); + imap_append_astring(dest, arg->value.str); + return; + case SEARCH_MAILBOX_GUID: + str_append(dest, "MAILBOX-GUID "); + imap_append_astring(dest, arg->value.str); + return; + case SEARCH_ALL: + case SEARCH_SEQSET: + case SEARCH_UIDSET: + case SEARCH_BEFORE: + case SEARCH_ON: + case SEARCH_SINCE: + case SEARCH_SMALLER: + case SEARCH_LARGER: + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + case SEARCH_BODY: + case SEARCH_TEXT: + case SEARCH_MODSEQ: + case SEARCH_GUID: + case SEARCH_MAILBOX_GLOB: + case SEARCH_REAL_UID: + break; + } + new_arg = *arg; + new_arg.match_not = FALSE; + if (!mail_search_arg_to_imap(dest, &new_arg, &error)) + i_panic("mail_search_args_to_cmdline(): Missing handler: %s", error); +} + +void mail_search_args_to_cmdline(string_t *dest, + const struct mail_search_arg *args) +{ + const struct mail_search_arg *arg; + + for (arg = args; arg != NULL; arg = arg->next) { + mail_search_arg_to_cmdline(dest, arg); + if (arg->next != NULL) + str_append_c(dest, ' '); + } +} diff -r e222d1265e4e -r 53824319afb5 src/lib-storage/mail-search.h --- a/src/lib-storage/mail-search.h Wed Jun 03 23:56:32 2015 +0300 +++ b/src/lib-storage/mail-search.h Thu Jun 04 00:44:42 2015 +0300 @@ -199,6 +199,9 @@ /* Like mail_search_args_to_imap(), but append only a single arg. */ bool mail_search_arg_to_imap(string_t *dest, const struct mail_search_arg *arg, const char **error_r); +/* Write all args to dest string as cmdline/human compatible input. */ +void mail_search_args_to_cmdline(string_t *dest, + const struct mail_search_arg *args); /* Serialization for search args' results. */ void mail_search_args_result_serialize(const struct mail_search_args *args, From dovecot at dovecot.org Wed Jun 3 21:47:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Jun 2015 21:47:33 +0000 Subject: dovecot-2.2: fts: Added "doveadm fts expand" to show lib-fts sea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7cb0bc84914d changeset: 18828:7cb0bc84914d user: Timo Sirainen date: Thu Jun 04 00:45:11 2015 +0300 description: fts: Added "doveadm fts expand" to show lib-fts search query expanded. diffstat: src/plugins/fts/doveadm-fts.c | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 49 insertions(+), 0 deletions(-) diffs (73 lines): diff -r 53824319afb5 -r 7cb0bc84914d src/plugins/fts/doveadm-fts.c --- a/src/plugins/fts/doveadm-fts.c Thu Jun 04 00:44:42 2015 +0300 +++ b/src/plugins/fts/doveadm-fts.c Thu Jun 04 00:45:11 2015 +0300 @@ -7,6 +7,7 @@ #include "mail-search.h" #include "mailbox-list-iter.h" #include "fts-storage.h" +#include "fts-search-args.h" #include "doveadm-mail.h" #include "doveadm-mailbox-list-iter.h" #include "doveadm-fts.h" @@ -106,6 +107,53 @@ } static int +cmd_fts_expand_run(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user) +{ + struct mail_namespace *ns = mail_namespace_find_inbox(user->namespaces); + struct mailbox *box; + struct fts_backend *backend; + string_t *str = t_str_new(128); + + backend = fts_list_backend(ns->list); + if (backend == NULL) { + i_error("fts not enabled for INBOX"); + return -1; + } + + box = mailbox_alloc(ns->list, "INBOX", 0); + mail_search_args_init(ctx->search_args, box, FALSE, NULL); + + if (fts_search_args_expand(backend, ctx->search_args) < 0) + i_fatal("Couldn't expand search args"); + mail_search_args_to_cmdline(str, ctx->search_args->args); + printf("%s\n", str_c(str)); + mailbox_free(&box); + return 0; +} + +static void +cmd_fts_expand_init(struct doveadm_mail_cmd_context *ctx, + const char *const args[]) +{ + if (args[0] == NULL) + doveadm_mail_help_name("fts expand"); + + ctx->search_args = doveadm_mail_build_search_args(args); +} + +static struct doveadm_mail_cmd_context * +cmd_fts_expand_alloc(void) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); + ctx->v.run = cmd_fts_expand_run; + ctx->v.init = cmd_fts_expand_init; + return ctx; +} + +static int fts_namespace_find(struct mail_user *user, const char *ns_prefix, struct mail_namespace **ns_r) { @@ -211,6 +259,7 @@ static struct doveadm_mail_cmd fts_commands[] = { { cmd_fts_lookup_alloc, "fts lookup", "" }, + { cmd_fts_expand_alloc, "fts expand", "" }, { cmd_fts_optimize_alloc, "fts optimize", "[]" }, { cmd_fts_rescan_alloc, "fts rescan", "[]" } }; From dovecot at dovecot.org Thu Jun 4 18:59:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 04 Jun 2015 18:59:56 +0000 Subject: dovecot-2.2: fts: lib-fts search arg expansion wasn't done for n... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ee82bdc63939 changeset: 18829:ee82bdc63939 user: Timo Sirainen date: Thu Jun 04 21:57:32 2015 +0300 description: fts: lib-fts search arg expansion wasn't done for non-TEXT/BODY searches when fts_enforced=no diffstat: src/plugins/fts/fts-search.c | 5 +++++ src/plugins/fts/fts-storage.c | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) diffs (45 lines): diff -r 7cb0bc84914d -r ee82bdc63939 src/plugins/fts/fts-search.c --- a/src/plugins/fts/fts-search.c Thu Jun 04 00:45:11 2015 +0300 +++ b/src/plugins/fts/fts-search.c Thu Jun 04 21:57:32 2015 +0300 @@ -7,6 +7,7 @@ #include "mail-search.h" #include "../virtual/virtual-storage.h" #include "fts-api-private.h" +#include "fts-search-args.h" #include "fts-search-serialize.h" #include "fts-storage.h" @@ -353,6 +354,10 @@ &seq1, &seq2); fctx->first_unindexed_seq = seq1 != 0 ? seq1 : (uint32_t)-1; + if ((fctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) { + if (fts_search_args_expand(fctx->backend, fctx->args) < 0) + return; + } fts_search_serialize(fctx->orig_matches, fctx->args->args); if (fts_search_lookup_level(fctx, fctx->args->args, TRUE) == 0) { diff -r 7cb0bc84914d -r ee82bdc63939 src/plugins/fts/fts-storage.c --- a/src/plugins/fts/fts-storage.c Thu Jun 04 00:45:11 2015 +0300 +++ b/src/plugins/fts/fts-storage.c Thu Jun 04 21:57:32 2015 +0300 @@ -14,7 +14,6 @@ #include "fts-tokenizer.h" #include "fts-indexer.h" #include "fts-build-mail.h" -#include "fts-search-args.h" #include "fts-search-serialize.h" #include "fts-plugin.h" #include "fts-storage.h" @@ -133,11 +132,6 @@ if (ret < 0) return; - if ((fctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) { - if (fts_search_args_expand(fctx->backend, fctx->args) < 0) - return; - } - if (ret == 0) { /* the index was up to date */ fts_search_lookup(fctx); From dovecot at dovecot.org Fri Jun 5 11:59:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 05 Jun 2015 11:59:17 +0000 Subject: dovecot-2.2: man: doveadm-pw.1: Added a note about quoting passw... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/90210d09b83c changeset: 18830:90210d09b83c user: Pascal Volk date: Thu Jun 04 21:03:42 2015 +0000 description: man: doveadm-pw.1: Added a note about quoting password hashes. diffstat: doc/man/doveadm-pw.1.in | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (24 lines): diff -r ee82bdc63939 -r 90210d09b83c doc/man/doveadm-pw.1.in --- a/doc/man/doveadm-pw.1.in Thu Jun 04 21:57:32 2015 +0300 +++ b/doc/man/doveadm-pw.1.in Thu Jun 04 21:03:42 2015 +0000 @@ -1,5 +1,5 @@ .\" Copyright (c) 2010-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-PW 1 "2013-08-17" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-PW 1 "2015-06-05" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-pw \- Dovecot\(aqs password hash generator .\"------------------------------------------------------------------------ @@ -114,8 +114,12 @@ .TP .BI \-t\ hash Test if the given password -.IR hash +.I hash matches a given plain text password. +You should enclose the password +.I hash +in single quotes, if it contains one ore more dollar signs +.RB ( $ ). The plain text password may be passed using the .BR \-p " option." When no password was specified, From dovecot at dovecot.org Fri Jun 5 13:39:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 05 Jun 2015 13:39:52 +0000 Subject: dovecot-2.2: man: doveadm-pw.1: Fix typo. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1cebb87f2f8f changeset: 18831:1cebb87f2f8f user: Teemu Huovila date: Fri Jun 05 16:37:24 2015 +0300 description: man: doveadm-pw.1: Fix typo. diffstat: doc/man/doveadm-pw.1.in | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 90210d09b83c -r 1cebb87f2f8f doc/man/doveadm-pw.1.in --- a/doc/man/doveadm-pw.1.in Thu Jun 04 21:03:42 2015 +0000 +++ b/doc/man/doveadm-pw.1.in Fri Jun 05 16:37:24 2015 +0300 @@ -118,7 +118,7 @@ matches a given plain text password. You should enclose the password .I hash -in single quotes, if it contains one ore more dollar signs +in single quotes, if it contains one or more dollar signs .RB ( $ ). The plain text password may be passed using the .BR \-p " option." From dovecot at dovecot.org Tue Jun 9 16:34:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 09 Jun 2015 16:34:37 +0000 Subject: dovecot-2.2: dict-sql: Don't try to optimize finding a matching ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1a778838d3e1 changeset: 18832:1a778838d3e1 user: Timo Sirainen date: Tue Jun 09 19:32:09 2015 +0300 description: dict-sql: Don't try to optimize finding a matching map by using the previous match. In some setups multiple maps can match and it's important that the matching is done in the same order always, otherwise the results could become somewhat random. diffstat: src/lib-dict/dict-sql.c | 13 ++++--------- 1 files changed, 4 insertions(+), 9 deletions(-) diffs (34 lines): diff -r 1cebb87f2f8f -r 1a778838d3e1 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Fri Jun 05 16:37:24 2015 +0300 +++ b/src/lib-dict/dict-sql.c Tue Jun 09 19:32:09 2015 +0300 @@ -28,7 +28,6 @@ struct sql_db *db; const char *username; const struct dict_sql_settings *set; - unsigned int prev_map_match_idx; unsigned int has_on_duplicate_key:1; }; @@ -190,18 +189,14 @@ ARRAY_TYPE(const_string) *values) { const struct dict_sql_map *maps; - unsigned int i, idx, count, len; + unsigned int i, count, len; t_array_init(values, dict->set->max_field_count); maps = array_get(&dict->set->maps, &count); for (i = 0; i < count; i++) { - /* start matching from the previously successful match */ - idx = (dict->prev_map_match_idx + i) % count; - if (dict_sql_map_match(&maps[idx], path, values, - &len, &len, FALSE, FALSE)) { - dict->prev_map_match_idx = idx; - return &maps[idx]; - } + if (dict_sql_map_match(&maps[i], path, values, + &len, &len, FALSE, FALSE)) + return &maps[i]; } return NULL; } From dovecot at dovecot.org Wed Jun 10 13:11:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Jun 2015 13:11:42 +0000 Subject: dovecot-2.2: cassandra: Don't crash on failed queries. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0cf38b6b055d changeset: 18833:0cf38b6b055d user: Timo Sirainen date: Wed Jun 10 16:08:28 2015 +0300 description: cassandra: Don't crash on failed queries. diffstat: src/lib-sql/driver-cassandra.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 1a778838d3e1 -r 0cf38b6b055d src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Tue Jun 09 19:32:09 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Wed Jun 10 16:08:28 2015 +0300 @@ -437,8 +437,10 @@ db->cur_result = NULL; driver_cassandra_set_idle(db); - cass_result_free(result->result); - cass_iterator_free(result->iterator); + if (result->result != NULL) + cass_result_free(result->result); + if (result->iterator != NULL) + cass_iterator_free(result->iterator); cass_statement_free(result->statement); pool_unref(&result->row_pool); i_free(result->query); From dovecot at dovecot.org Wed Jun 10 13:11:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Jun 2015 13:11:47 +0000 Subject: dovecot-2.2: cassandra: Added extra asserts and sanity checks to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/42cdfb0153d9 changeset: 18834:42cdfb0153d9 user: Timo Sirainen date: Wed Jun 10 16:09:13 2015 +0300 description: cassandra: Added extra asserts and sanity checks to make sure query errors are caught. diffstat: src/lib-sql/driver-cassandra.c | 44 +++++++++++++++++++++++++++++------------ 1 files changed, 31 insertions(+), 13 deletions(-) diffs (123 lines): diff -r 0cf38b6b055d -r 42cdfb0153d9 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Wed Jun 10 16:08:28 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Wed Jun 10 16:09:13 2015 +0300 @@ -301,8 +301,11 @@ { struct cassandra_db *db = (struct cassandra_db *)_db; - if (db->cur_result != NULL && !db->cur_result->finished) - result_finish(db->cur_result); + if (db->cur_result != NULL && !db->cur_result->finished) { + if (db->cur_result->error == NULL) + db->cur_result->error = i_strdup("disconnecting"); + result_finish(db->cur_result); + } driver_cassandra_close(db); } @@ -394,8 +397,11 @@ { struct cassandra_db *db = (struct cassandra_db *)_db; - if (db->cur_result != NULL && !db->cur_result->finished) - result_finish(db->cur_result); + if (db->cur_result != NULL && !db->cur_result->finished) { + if (db->cur_result->error == NULL) + db->cur_result->error = i_strdup("deinitializing"); + result_finish(db->cur_result); + } driver_cassandra_close(db); i_assert(array_count(&db->callbacks) == 0); @@ -455,6 +461,8 @@ result->finished = TRUE; + i_assert((result->error != NULL) == (result->iterator == NULL)); + result->api.callback = TRUE; T_BEGIN { result->callback(&result->api, result->context); @@ -846,17 +854,19 @@ sql_result_unref(result); } -static struct sql_result * -driver_cassandra_transaction_commit_multi(struct cassandra_transaction_context *ctx) +static int +driver_cassandra_transaction_commit_multi(struct cassandra_transaction_context *ctx, + struct sql_result **result_r) { struct cassandra_db *db = (struct cassandra_db *)ctx->ctx.db; struct sql_result *result; struct sql_transaction_query *query; + int ret = 0; result = driver_cassandra_sync_query(db, "BEGIN"); if (sql_result_next_row(result) < 0) { commit_multi_fail(ctx, result, "BEGIN"); - return NULL; + return -1; } sql_result_unref(result); @@ -865,13 +875,15 @@ result = driver_cassandra_sync_query(db, query->query); if (sql_result_next_row(result) < 0) { commit_multi_fail(ctx, result, query->query); + ret = -1; break; } sql_result_unref(result); } - return driver_cassandra_sync_query(db, ctx->failed ? - "ROLLBACK" : "COMMIT"); + *result_r = driver_cassandra_sync_query(db, ctx->failed ? + "ROLLBACK" : "COMMIT"); + return ret; } static void @@ -881,7 +893,8 @@ struct sql_transaction_context *_ctx = &ctx->ctx; struct cassandra_db *db = (struct cassandra_db *)_ctx->db; struct sql_transaction_query *single_query = NULL; - struct sql_result *result; + struct sql_result *result = NULL; + int ret = 0; if (_ctx->head->next == NULL) { /* just a single query, send it */ @@ -890,16 +903,20 @@ } else { /* multiple queries, use a transaction */ driver_cassandra_sync_init(db); - result = driver_cassandra_transaction_commit_multi(ctx); + ret = driver_cassandra_transaction_commit_multi(ctx, &result); + i_assert(ret == 0 || ctx->failed); driver_cassandra_sync_deinit(db); } if (ctx->failed) { i_assert(ctx->error != NULL); *error_r = ctx->error; - } else if (result != NULL) { - if (sql_result_next_row(result) < 0) + } else { + if (sql_result_next_row(result) < 0) { + ctx->failed = TRUE; *error_r = sql_result_get_error(result); + i_assert(*error_r != NULL); + } } *error_r = t_strdup(*error_r); if (result != NULL) @@ -920,6 +937,7 @@ i_assert(ctx->refcount == 1); driver_cassandra_transaction_unref(ctx); + i_assert((*error_r != NULL) == ctx->failed); return *error_r == NULL ? 0 : -1; } From dovecot at dovecot.org Wed Jun 10 13:20:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Jun 2015 13:20:38 +0000 Subject: dovecot-2.2: cassandra: If log_level=debug/trace, log each query... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/539322434f9b changeset: 18835:539322434f9b user: Timo Sirainen date: Wed Jun 10 16:18:10 2015 +0300 description: cassandra: If log_level=debug/trace, log each query and its result. diffstat: src/lib-sql/driver-cassandra.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 42cdfb0153d9 -r 539322434f9b src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Wed Jun 10 16:09:13 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Wed Jun 10 16:18:10 2015 +0300 @@ -461,6 +461,11 @@ result->finished = TRUE; + if (db->log_level >= CASS_LOG_DEBUG) { + i_debug("cassandra: Finished query '%s': %s", result->query, + result->error != NULL ? result->error : "success"); + } + i_assert((result->error != NULL) == (result->iterator == NULL)); result->api.callback = TRUE; From dovecot at dovecot.org Wed Jun 10 16:52:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Jun 2015 16:52:57 +0000 Subject: dovecot-2.2: doveadm fs delete -R: Fixed hang when some of the d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3db134293be8 changeset: 18836:3db134293be8 user: Timo Sirainen date: Wed Jun 10 19:50:29 2015 +0300 description: doveadm fs delete -R: Fixed hang when some of the deletions failed. diffstat: src/doveadm/doveadm-fs.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 539322434f9b -r 3db134293be8 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Wed Jun 10 16:18:10 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Wed Jun 10 19:50:29 2015 +0300 @@ -329,7 +329,7 @@ goto retry; } } T_END; - while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(&ctx) != 0) { + while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(&ctx) > 0) { if (fs_wait_async(fs) < 0) { i_error("fs_wait_async() failed: %s", fs_last_error(fs)); doveadm_exit_code = EX_TEMPFAIL; From dovecot at dovecot.org Wed Jun 10 17:32:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Jun 2015 17:32:03 +0000 Subject: dovecot-2.2: lmtp: Fixed assert-crash on anvil lookup failures. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/457daf0bfbfa changeset: 18837:457daf0bfbfa user: Timo Sirainen date: Wed Jun 10 20:29:35 2015 +0300 description: lmtp: Fixed assert-crash on anvil lookup failures. If anvil_client_query() fails, it immediately calls the callback and returns NULL. So we need to increase anvil_queries even before calling anvil_client_query() diffstat: src/lmtp/commands.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (15 lines): diff -r 3db134293be8 -r 457daf0bfbfa src/lmtp/commands.c --- a/src/lmtp/commands.c Wed Jun 10 19:50:29 2015 +0300 +++ b/src/lmtp/commands.c Wed Jun 10 20:29:35 2015 +0300 @@ -693,10 +693,9 @@ master_service_get_name(master_service), "/", str_tabescape(username), NULL); lmtp_anvil_init(); + client->state.anvil_queries++; rcpt->anvil_query = anvil_client_query(anvil, query, rcpt_anvil_lookup_callback, rcpt); - if (rcpt->anvil_query != NULL) - client->state.anvil_queries++; } return 0; } From dovecot at dovecot.org Thu Jun 11 12:22:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Jun 2015 12:22:22 +0000 Subject: dovecot-2.2: doveadm fs delete -R: More fixes to handling errors... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf05939c3d09 changeset: 18838:cf05939c3d09 user: Timo Sirainen date: Thu Jun 11 15:19:53 2015 +0300 description: doveadm fs delete -R: More fixes to handling errors without crashing. diffstat: src/doveadm/doveadm-fs.c | 65 ++++++++++++++++++++++++++++++----------------- 1 files changed, 41 insertions(+), 24 deletions(-) diffs (97 lines): diff -r 457daf0bfbfa -r cf05939c3d09 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Wed Jun 10 20:29:35 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Thu Jun 11 15:19:53 2015 +0300 @@ -222,6 +222,9 @@ } struct fs_delete_ctx { + struct fs *fs; + const char *path_prefix; + unsigned int files_count; struct fs_file **files; }; @@ -250,6 +253,36 @@ return ret; } +static int doveadm_fs_delete_recursive_fname(struct fs_delete_ctx *ctx, + const char *fname) +{ + unsigned int i; + int ret; + + for (i = 0; i < ctx->files_count; i++) { + if (ctx->files[i] != NULL) + continue; + + ctx->files[i] = fs_file_init(ctx->fs, + t_strdup_printf("%s%s", ctx->path_prefix, fname), + FS_OPEN_MODE_READONLY | FS_OPEN_FLAG_ASYNC | + FS_OPEN_FLAG_ASYNC_NOQUEUE); + fname = NULL; + break; + } + if ((ret = cmd_fs_delete_ctx_run(ctx)) < 0) + return -1; + if (fname != NULL) { + if (ret > 0 && fs_wait_async(ctx->fs) < 0) { + i_error("fs_wait_async() failed: %s", fs_last_error(ctx->fs)); + doveadm_exit_code = EX_TEMPFAIL; + return -1;; + } + return doveadm_fs_delete_recursive_fname(ctx, fname); + } + return 0; +} + static void cmd_fs_delete_dir_recursive(struct fs *fs, unsigned int async_count, const char *path_prefix) @@ -262,6 +295,8 @@ int ret; memset(&ctx, 0, sizeof(ctx)); + ctx.fs = fs; + ctx.path_prefix = path_prefix; ctx.files_count = I_MAX(async_count, 1); ctx.files = t_new(struct fs_file *, ctx.files_count); @@ -304,31 +339,13 @@ doveadm_exit_code = EX_TEMPFAIL; } - array_foreach(&fnames, fnamep) T_BEGIN { - fname = *fnamep; - retry: - for (i = 0; i < ctx.files_count; i++) { - if (ctx.files[i] != NULL) - continue; - - ctx.files[i] = fs_file_init(fs, - t_strdup_printf("%s%s", path_prefix, fname), - FS_OPEN_MODE_READONLY | FS_OPEN_FLAG_ASYNC | - FS_OPEN_FLAG_ASYNC_NOQUEUE); - fname = NULL; + array_foreach(&fnames, fnamep) { + T_BEGIN { + ret = doveadm_fs_delete_recursive_fname(&ctx, *fnamep); + } T_END; + if (ret < 0) break; - } - if ((ret = cmd_fs_delete_ctx_run(&ctx)) < 0) - break; - if (fname != NULL) { - if (ret > 0 && fs_wait_async(fs) < 0) { - i_error("fs_wait_async() failed: %s", fs_last_error(fs)); - doveadm_exit_code = EX_TEMPFAIL; - break; - } - goto retry; - } - } T_END; + } while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(&ctx) > 0) { if (fs_wait_async(fs) < 0) { i_error("fs_wait_async() failed: %s", fs_last_error(fs)); From dovecot at dovecot.org Thu Jun 11 12:31:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Jun 2015 12:31:15 +0000 Subject: dovecot-2.2: cassandra: Don't access freed memory when doing an ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/247c6ae2eae4 changeset: 18839:247c6ae2eae4 user: Timo Sirainen date: Thu Jun 11 15:28:05 2015 +0300 description: cassandra: Don't access freed memory when doing an assert-check. diffstat: src/lib-sql/driver-cassandra.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (13 lines): diff -r cf05939c3d09 -r 247c6ae2eae4 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Jun 11 15:19:53 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Thu Jun 11 15:28:05 2015 +0300 @@ -941,8 +941,8 @@ driver_cassandra_try_commit_s(ctx, error_r); i_assert(ctx->refcount == 1); + i_assert((*error_r != NULL) == ctx->failed); driver_cassandra_transaction_unref(ctx); - i_assert((*error_r != NULL) == ctx->failed); return *error_r == NULL ? 0 : -1; } From dovecot at dovecot.org Thu Jun 11 12:31:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Jun 2015 12:31:15 +0000 Subject: dovecot-2.2: cassandra: Minor code cleanup to make sure we don't... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/34943b67e01f changeset: 18840:34943b67e01f user: Timo Sirainen date: Thu Jun 11 15:28:46 2015 +0300 description: cassandra: Minor code cleanup to make sure we don't try to access freed transaction memory. diffstat: src/lib-sql/driver-cassandra.c | 17 ++++++++++------- 1 files changed, 10 insertions(+), 7 deletions(-) diffs (70 lines): diff -r 247c6ae2eae4 -r 34943b67e01f src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Jun 11 15:28:05 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Thu Jun 11 15:28:46 2015 +0300 @@ -769,8 +769,11 @@ } static void -driver_cassandra_transaction_unref(struct cassandra_transaction_context *ctx) +driver_cassandra_transaction_unref(struct cassandra_transaction_context **_ctx) { + struct cassandra_transaction_context *ctx = *_ctx; + + *_ctx = NULL; i_assert(ctx->refcount > 0); if (--ctx->refcount > 0) return; @@ -790,7 +793,7 @@ } else { ctx->begin_succeeded = TRUE; } - driver_cassandra_transaction_unref(ctx); + driver_cassandra_transaction_unref(&ctx); } static void @@ -801,7 +804,7 @@ ctx->callback(sql_result_get_error(result), ctx->context); else ctx->callback(NULL, ctx->context); - driver_cassandra_transaction_unref(ctx); + driver_cassandra_transaction_unref(&ctx); } static void @@ -815,7 +818,7 @@ ctx->failed = TRUE; ctx->error = sql_result_get_error(result); } - driver_cassandra_transaction_unref(ctx); + driver_cassandra_transaction_unref(&ctx); } static void @@ -830,7 +833,7 @@ if (ctx->failed || _ctx->head == NULL) { callback(ctx->failed ? ctx->error : NULL, context); - driver_cassandra_transaction_unref(ctx); + driver_cassandra_transaction_unref(&ctx); } else if (_ctx->head->next == NULL) { /* just a single query, send it */ sql_query(_ctx->db, _ctx->head->query, @@ -942,7 +945,7 @@ i_assert(ctx->refcount == 1); i_assert((*error_r != NULL) == ctx->failed); - driver_cassandra_transaction_unref(ctx); + driver_cassandra_transaction_unref(&ctx); return *error_r == NULL ? 0 : -1; } @@ -953,7 +956,7 @@ (struct cassandra_transaction_context *)_ctx; i_assert(ctx->refcount == 1); - driver_cassandra_transaction_unref(ctx); + driver_cassandra_transaction_unref(&ctx); } static void From dovecot at dovecot.org Thu Jun 11 13:21:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Jun 2015 13:21:35 +0000 Subject: dovecot-2.2: cassandra: Error handling cleanup. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/34fad121ec7b changeset: 18841:34fad121ec7b user: Timo Sirainen date: Thu Jun 11 16:19:05 2015 +0300 description: cassandra: Error handling cleanup. Probably doesn't fix any actual bugs. diffstat: src/lib-sql/driver-cassandra.c | 51 +++++++++++++++++++++-------------------- 1 files changed, 26 insertions(+), 25 deletions(-) diffs (114 lines): diff -r 34943b67e01f -r 34fad121ec7b src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Thu Jun 11 15:28:46 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Thu Jun 11 16:19:05 2015 +0300 @@ -74,7 +74,7 @@ void *context; pool_t query_pool; - const char *error; + char *error; unsigned int begin_succeeded:1; unsigned int begin_failed:1; @@ -779,17 +779,30 @@ return; pool_unref(&ctx->query_pool); + i_free(ctx->error); i_free(ctx); } static void +transaction_set_failed(struct cassandra_transaction_context *ctx, + const char *error) +{ + if (ctx->failed) { + i_assert(ctx->error != NULL); + } else { + i_assert(ctx->error == NULL); + ctx->failed = TRUE; + ctx->error = i_strdup(error); + } +} + +static void transaction_begin_callback(struct sql_result *result, struct cassandra_transaction_context *ctx) { if (sql_result_next_row(result) < 0) { ctx->begin_failed = TRUE; - ctx->failed = TRUE; - ctx->error = sql_result_get_error(result); + transaction_set_failed(ctx, sql_result_get_error(result)); } else { ctx->begin_succeeded = TRUE; } @@ -814,10 +827,8 @@ struct cassandra_transaction_context *ctx = (struct cassandra_transaction_context *)query->trans; - if (sql_result_next_row(result) < 0) { - ctx->failed = TRUE; - ctx->error = sql_result_get_error(result); - } + if (sql_result_next_row(result) < 0) + transaction_set_failed(ctx, sql_result_get_error(result)); driver_cassandra_transaction_unref(&ctx); } @@ -856,9 +867,8 @@ commit_multi_fail(struct cassandra_transaction_context *ctx, struct sql_result *result, const char *query) { - ctx->failed = TRUE; - ctx->error = t_strdup_printf("%s (query: %s)", - sql_result_get_error(result), query); + transaction_set_failed(ctx, t_strdup_printf( + "%s (query: %s)", sql_result_get_error(result), query)); sql_result_unref(result); } @@ -895,8 +905,7 @@ } static void -driver_cassandra_try_commit_s(struct cassandra_transaction_context *ctx, - const char **error_r) +driver_cassandra_try_commit_s(struct cassandra_transaction_context *ctx) { struct sql_transaction_context *_ctx = &ctx->ctx; struct cassandra_db *db = (struct cassandra_db *)_ctx->db; @@ -916,17 +925,10 @@ driver_cassandra_sync_deinit(db); } - if (ctx->failed) { - i_assert(ctx->error != NULL); - *error_r = ctx->error; - } else { - if (sql_result_next_row(result) < 0) { - ctx->failed = TRUE; - *error_r = sql_result_get_error(result); - i_assert(*error_r != NULL); - } + if (!ctx->failed) { + if (sql_result_next_row(result) < 0) + transaction_set_failed(ctx, sql_result_get_error(result)); } - *error_r = t_strdup(*error_r); if (result != NULL) sql_result_unref(result); } @@ -938,10 +940,9 @@ struct cassandra_transaction_context *ctx = (struct cassandra_transaction_context *)_ctx; - *error_r = NULL; - if (_ctx->head != NULL) - driver_cassandra_try_commit_s(ctx, error_r); + driver_cassandra_try_commit_s(ctx); + *error_r = ctx->error; i_assert(ctx->refcount == 1); i_assert((*error_r != NULL) == ctx->failed); From dovecot at dovecot.org Thu Jun 11 16:10:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Jun 2015 16:10:43 +0000 Subject: dovecot-2.2: doveadm fetch: Added "refcount" field. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ffecdb676c53 changeset: 18842:ffecdb676c53 user: Timo Sirainen date: Thu Jun 11 19:08:13 2015 +0300 description: doveadm fetch: Added "refcount" field. diffstat: src/doveadm/doveadm-mail-fetch.c | 13 ++++++++++++- 1 files changed, 12 insertions(+), 1 deletions(-) diffs (30 lines): diff -r 34fad121ec7b -r ffecdb676c53 src/doveadm/doveadm-mail-fetch.c --- a/src/doveadm/doveadm-mail-fetch.c Thu Jun 11 16:19:05 2015 +0300 +++ b/src/doveadm/doveadm-mail-fetch.c Thu Jun 11 19:08:13 2015 +0300 @@ -376,6 +376,16 @@ return 0; } +static int fetch_refcount(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_REFCOUNT, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + static const struct fetch_field fetch_fields[] = { { "user", 0, fetch_user }, { "mailbox", 0, fetch_mailbox }, @@ -401,7 +411,8 @@ { "imap.body", MAIL_FETCH_IMAP_BODY, fetch_imap_body }, { "imap.bodystructure", MAIL_FETCH_IMAP_BODYSTRUCTURE, fetch_imap_bodystructure }, { "pop3.uidl", MAIL_FETCH_UIDL_BACKEND, fetch_pop3_uidl }, - { "pop3.order", MAIL_FETCH_POP3_ORDER, fetch_pop3_order } + { "pop3.order", MAIL_FETCH_POP3_ORDER, fetch_pop3_order }, + { "refcount", MAIL_FETCH_REFCOUNT, fetch_refcount } }; static const struct fetch_field *fetch_field_find(const char *name) From dovecot at dovecot.org Thu Jun 11 16:24:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Jun 2015 16:24:00 +0000 Subject: dovecot-2.2: lib-index: Avoid writing tail offset update to tran... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/03b3b97b84aa changeset: 18843:03b3b97b84aa user: Timo Sirainen date: Thu Jun 11 19:21:16 2015 +0300 description: lib-index: Avoid writing tail offset update to transaction log if it's not necessary. This should avoid extra writes that do nothing but update the tail offset. mdbox's map requires this behavior, so disable it for it. For others it might cause the next sync to read more data unnecessarily, but it should be worth the extra cost of write most of the times. diffstat: src/lib-index/mail-index-sync.c | 14 +++++++++++--- src/lib-index/mail-index.h | 5 ++++- src/lib-storage/index/dbox-multi/mdbox-map.c | 3 ++- 3 files changed, 17 insertions(+), 5 deletions(-) diffs (66 lines): diff -r ffecdb676c53 -r 03b3b97b84aa src/lib-index/mail-index-sync.c --- a/src/lib-index/mail-index-sync.c Thu Jun 11 19:08:13 2015 +0300 +++ b/src/lib-index/mail-index-sync.c Thu Jun 11 19:21:16 2015 +0300 @@ -26,6 +26,7 @@ uint32_t last_tail_seq, last_tail_offset; unsigned int no_warning:1; + unsigned int seen_nonexternal_transactions:1; }; static void mail_index_sync_add_expunge(struct mail_index_sync_ctx *ctx) @@ -173,6 +174,7 @@ mail_transaction_log_view_get_prev_pos(ctx->view->log_view, &seq, &offset); + ctx->seen_nonexternal_transactions = TRUE; ctx->last_tail_seq = seq; ctx->last_tail_offset = offset + ctx->hdr->size + sizeof(*ctx->hdr); } @@ -763,9 +765,15 @@ mail_transaction_log_set_mailbox_sync_pos(ctx->index->log, seq, offset); /* If tail offset has changed, make sure it gets written to - transaction log. do this only if we're required to changes. */ - if (ctx->last_tail_seq != seq || - ctx->last_tail_offset < offset) { + transaction log. do this only if we're required to make changes. + + avoid writing a new tail offset if all the transactions were + external, because that wouldn't change effective the tail offset. + except e.g. mdbox map requires this to happen, so do it + optionally. */ + if ((ctx->last_tail_seq != seq || ctx->last_tail_offset < offset) && + (ctx->seen_nonexternal_transactions || + (ctx->flags & MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET) != 0)) { ctx->ext_trans->log_updates = TRUE; ctx->ext_trans->tail_offset_changed = TRUE; } diff -r ffecdb676c53 -r 03b3b97b84aa src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Thu Jun 11 19:08:13 2015 +0300 +++ b/src/lib-index/mail-index.h Thu Jun 11 19:21:16 2015 +0300 @@ -167,7 +167,10 @@ /* Same as MAIL_INDEX_SYNC_FLAG_DELETING_INDEX, but finish index deletion only once and fail the rest (= avoid race conditions when multiple processes try to mark the index deleted) */ - MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX = 0x40 + MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX = 0x40, + /* Update header's tail_offset to head_offset, even if it's the only + thing we do and there's no strict need for it. */ + MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET = 0x80 }; enum mail_index_view_sync_flags { diff -r ffecdb676c53 -r 03b3b97b84aa src/lib-storage/index/dbox-multi/mdbox-map.c --- a/src/lib-storage/index/dbox-multi/mdbox-map.c Thu Jun 11 19:08:13 2015 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Thu Jun 11 19:21:16 2015 +0300 @@ -497,7 +497,8 @@ /* use syncing to lock the transaction log, so that we always see log's head_offset = tail_offset */ ret = mail_index_sync_begin(atomic->map->index, &atomic->sync_ctx, - &atomic->sync_view, &atomic->sync_trans, 0); + &atomic->sync_view, &atomic->sync_trans, + MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET); if (mail_index_reset_fscked(atomic->map->index)) mdbox_storage_set_corrupted(atomic->map->storage); if (ret <= 0) { From dovecot at dovecot.org Fri Jun 12 10:04:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Jun 2015 10:04:41 +0000 Subject: dovecot-2.2: lib-fts: Don't crash if fts_icu_*() are used after ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a01dc6ef392d changeset: 18844:a01dc6ef392d user: Timo Sirainen date: Fri Jun 12 13:02:08 2015 +0300 description: lib-fts: Don't crash if fts_icu_*() are used after fts_icu_deinit() diffstat: src/lib-fts/fts-icu.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 03b3b97b84aa -r a01dc6ef392d src/lib-fts/fts-icu.c --- a/src/lib-fts/fts-icu.c Thu Jun 11 19:21:16 2015 +0300 +++ b/src/lib-fts/fts-icu.c Fri Jun 12 13:02:08 2015 +0300 @@ -158,7 +158,9 @@ void fts_icu_deinit(void) { - if (icu_csm != NULL) + if (icu_csm != NULL) { ucasemap_close(icu_csm); + icu_csm = NULL; + } u_cleanup(); } From dovecot at dovecot.org Fri Jun 12 12:17:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Jun 2015 12:17:25 +0000 Subject: dovecot-2.2: doveadm mailbox delete -r: Delete all mailboxes whe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0e947c4dbb10 changeset: 18845:0e947c4dbb10 user: Timo Sirainen date: Fri Jun 12 15:14:52 2015 +0300 description: doveadm mailbox delete -r: Delete all mailboxes when empty mailbox name is given. diffstat: src/doveadm/doveadm-mail-mailbox.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r a01dc6ef392d -r 0e947c4dbb10 src/doveadm/doveadm-mail-mailbox.c --- a/src/doveadm/doveadm-mail-mailbox.c Fri Jun 12 13:02:08 2015 +0300 +++ b/src/doveadm/doveadm-mail-mailbox.c Fri Jun 12 15:14:52 2015 +0300 @@ -306,7 +306,8 @@ const char *pattern, *child_name; ns = mail_namespace_find(user->namespaces, name); - pattern = t_strdup_printf("%s%c*", name, mail_namespace_get_sep(ns)); + pattern = name[0] == '\0' ? "*" : + t_strdup_printf("%s%c*", name, mail_namespace_get_sep(ns)); iter = mailbox_list_iter_init(ns->list, pattern, MAILBOX_LIST_ITER_RETURN_NO_FLAGS); while ((info = mailbox_list_iter_next(iter)) != NULL) { @@ -337,7 +338,8 @@ doveadm_mail_failed_error(_ctx, MAIL_ERROR_TEMP); ret = -1; } - array_append(&recursive_mailboxes, namep, 1); + if ((*namep)[0] != '\0') + array_append(&recursive_mailboxes, namep, 1); } array_sort(&recursive_mailboxes, i_strcmp_reverse_p); mailboxes = &recursive_mailboxes; From dovecot at dovecot.org Sat Jun 13 11:39:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Jun 2015 11:39:16 +0000 Subject: dovecot-2.2: config: Added hook_config_parser_end for plugins. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b9601cc46ace changeset: 18846:b9601cc46ace user: Timo Sirainen date: Sat Jun 13 14:36:43 2015 +0300 description: config: Added hook_config_parser_end for plugins. diffstat: src/config/config-parser-private.h | 2 ++ src/config/config-parser.c | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diffs (45 lines): diff -r 0e947c4dbb10 -r b9601cc46ace src/config/config-parser-private.h --- a/src/config/config-parser-private.h Fri Jun 12 15:14:52 2015 +0300 +++ b/src/config/config-parser-private.h Sat Jun 13 14:36:43 2015 +0300 @@ -59,6 +59,8 @@ }; extern void (*hook_config_parser_begin)(struct config_parser_context *ctx); +extern int (*hook_config_parser_end)(struct config_parser_context *ctx, + const char **error_r); int config_apply_line(struct config_parser_context *ctx, const char *key, const char *line, const char *section_name) ATTR_NULL(4); diff -r 0e947c4dbb10 -r b9601cc46ace src/config/config-parser.c --- a/src/config/config-parser.c Fri Jun 12 15:14:52 2015 +0300 +++ b/src/config/config-parser.c Sat Jun 13 14:36:43 2015 +0300 @@ -40,6 +40,8 @@ struct config_filter_context *config_filter; struct module *modules; void (*hook_config_parser_begin)(struct config_parser_context *ctx); +int (*hook_config_parser_end)(struct config_parser_context *ctx, + const char **error_r); static const char *info_type_name_find(const struct setting_parser_info *info) { @@ -697,13 +699,18 @@ { struct config_filter_context *new_filter; const char *error; - int ret; + int ret = 0; + + if (hook_config_parser_end != NULL) + ret = hook_config_parser_end(ctx, error_r); new_filter = config_filter_init(ctx->pool); array_append_zero(&ctx->all_parsers); config_filter_add_all(new_filter, array_idx(&ctx->all_parsers, 0)); - if (ctx->hide_errors) + if (ret < 0) + ; + else if (ctx->hide_errors) ret = 0; else if ((ret = config_all_parsers_check(ctx, new_filter, &error)) < 0) { *error_r = t_strdup_printf("Error in configuration file %s: %s", From dovecot at dovecot.org Sat Jun 13 12:12:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Jun 2015 12:12:55 +0000 Subject: dovecot-2.2: lib-fts: Removed unnecessary data stack frames from... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d766dbdf871c changeset: 18847:d766dbdf871c user: Timo Sirainen date: Sat Jun 13 15:10:23 2015 +0300 description: lib-fts: Removed unnecessary data stack frames from test-fts-filter. They are already added by lib-test/test-common.c diffstat: src/lib-fts/test-fts-filter.c | 82 +++++++++++++++++++----------------------- 1 files changed, 37 insertions(+), 45 deletions(-) diffs (114 lines): diff -r b9601cc46ace -r d766dbdf871c src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sat Jun 13 14:36:43 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Sat Jun 13 15:10:23 2015 +0300 @@ -345,15 +345,13 @@ test_begin("fts filter normalizer Swedish short text"); - T_BEGIN { - test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); - for (i = 0; i < N_ELEMENTS(input); i++) { - token = input[i]; - test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); - test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); - } - fts_filter_unref(&norm); - } T_END; + test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); + for (i = 0; i < N_ELEMENTS(input); i++) { + token = input[i]; + test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); + test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); + } + fts_filter_unref(&norm); test_assert(norm == NULL); test_end(); } @@ -380,15 +378,13 @@ test_begin("fts filter normalizer Swedish short text using default ID"); - T_BEGIN { - test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, NULL, &norm, &error) == 0); - for (i = 0; i < N_ELEMENTS(input); i++) { - token = input[i]; - test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); - test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); - } - fts_filter_unref(&norm); - } T_END; + test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, NULL, &norm, &error) == 0); + for (i = 0; i < N_ELEMENTS(input); i++) { + token = input[i]; + test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); + test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); + } + fts_filter_unref(&norm); test_assert(norm == NULL); test_end(); } @@ -419,25 +415,23 @@ test_begin("fts filter normalizer French UDHR"); - T_BEGIN { - udhr_path = t_strconcat(UDHRDIR, UDHR_FRA_NAME, NULL); - test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); - input = fopen(udhr_path, "r"); - test_assert(input != NULL); - sha512_init(&ctx); - while (NULL != fgets(buf, sizeof(buf), input)) { - tokens = buf; - if (fts_filter_filter(norm, &tokens, &error) != 1){ - break; - } - sha512_loop(&ctx, tokens, strlen(tokens)); + udhr_path = t_strconcat(UDHRDIR, UDHR_FRA_NAME, NULL); + test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); + input = fopen(udhr_path, "r"); + test_assert(input != NULL); + sha512_init(&ctx); + while (NULL != fgets(buf, sizeof(buf), input)) { + tokens = buf; + if (fts_filter_filter(norm, &tokens, &error) != 1){ + break; } - fclose(input); - sha512_result(&ctx, sha512_digest); - test_assert(memcmp(sha512_digest, correct_digest, - sizeof(sha512_digest)) == 0); - fts_filter_unref(&norm); - } T_END; + sha512_loop(&ctx, tokens, strlen(tokens)); + } + fclose(input); + sha512_result(&ctx, sha512_digest); + test_assert(memcmp(sha512_digest, correct_digest, + sizeof(sha512_digest)) == 0); + fts_filter_unref(&norm); test_assert(norm == NULL); test_end(); } @@ -615,15 +609,13 @@ test_begin("fts filter english possessive"); - T_BEGIN { - test_assert(fts_filter_create(fts_filter_english_possessive, NULL, NULL, NULL, &norm, &error) == 0); - for (i = 0; i < N_ELEMENTS(input); i++) { - token = input[i]; - test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); - test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); - } - fts_filter_unref(&norm); - } T_END; + test_assert(fts_filter_create(fts_filter_english_possessive, NULL, NULL, NULL, &norm, &error) == 0); + for (i = 0; i < N_ELEMENTS(input); i++) { + token = input[i]; + test_assert_idx(fts_filter_filter(norm, &token, &error) == 1, i); + test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); + } + fts_filter_unref(&norm); test_assert(norm == NULL); test_end(); } From dovecot at dovecot.org Sat Jun 13 13:48:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Jun 2015 13:48:46 +0000 Subject: dovecot-2.2: cassandra: Commit failures returned an already free... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/47fa4a6c6b9a changeset: 18848:47fa4a6c6b9a user: Timo Sirainen date: Sat Jun 13 16:46:06 2015 +0300 description: cassandra: Commit failures returned an already freed error string. diffstat: src/lib-sql/driver-cassandra.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d766dbdf871c -r 47fa4a6c6b9a src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Sat Jun 13 15:10:23 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Sat Jun 13 16:46:06 2015 +0300 @@ -942,7 +942,7 @@ if (_ctx->head != NULL) driver_cassandra_try_commit_s(ctx); - *error_r = ctx->error; + *error_r = t_strdup(ctx->error); i_assert(ctx->refcount == 1); i_assert((*error_r != NULL) == ctx->failed); From dovecot at dovecot.org Sat Jun 13 14:08:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Jun 2015 14:08:21 +0000 Subject: dovecot-2.2: lib-index: Added mail_index_ext_register_resize_def... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9c90eba19aee changeset: 18849:9c90eba19aee user: Timo Sirainen date: Sat Jun 13 17:05:49 2015 +0300 description: lib-index: Added mail_index_ext_register_resize_defaults() diffstat: src/lib-index/mail-index.c | 14 ++++++++++++++ src/lib-index/mail-index.h | 6 ++++++ 2 files changed, 20 insertions(+), 0 deletions(-) diffs (40 lines): diff -r 47fa4a6c6b9a -r 9c90eba19aee src/lib-index/mail-index.c --- a/src/lib-index/mail-index.c Sat Jun 13 16:46:06 2015 +0300 +++ b/src/lib-index/mail-index.c Sat Jun 13 17:05:49 2015 +0300 @@ -159,6 +159,20 @@ return rext.index_idx; } +void mail_index_ext_register_resize_defaults(struct mail_index *index, + uint32_t ext_id, + uint32_t default_hdr_size, + uint16_t default_record_size, + uint16_t default_record_align) +{ + struct mail_index_registered_ext *rext; + + rext = array_idx_modifiable(&index->extensions, ext_id); + rext->hdr_size = default_hdr_size; + rext->record_size = default_record_size; + rext->record_align = default_record_align; +} + bool mail_index_ext_lookup(struct mail_index *index, const char *name, uint32_t *ext_id_r) { diff -r 47fa4a6c6b9a -r 9c90eba19aee src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Sat Jun 13 16:46:06 2015 +0300 +++ b/src/lib-index/mail-index.h Sat Jun 13 17:05:49 2015 +0300 @@ -562,6 +562,12 @@ uint32_t default_hdr_size, uint16_t default_record_size, uint16_t default_record_align); +/* Change an already registered extension's default sizes. */ +void mail_index_ext_register_resize_defaults(struct mail_index *index, + uint32_t ext_id, + uint32_t default_hdr_size, + uint16_t default_record_size, + uint16_t default_record_align); /* Returns TRUE and sets ext_id_r if extension with given name is registered. */ bool mail_index_ext_lookup(struct mail_index *index, const char *name, uint32_t *ext_id_r); From dovecot at dovecot.org Mon Jun 15 11:40:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:40:23 +0000 Subject: dovecot-2.2: lib: array - permit array_swap with uninitialised a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2ab0c5855548 changeset: 18850:2ab0c5855548 user: Phil Carmody date: Mon Jun 15 14:25:07 2015 +0300 description: lib: array - permit array_swap with uninitialised arrays Before initialisation, all fields are blank, so the element_size field is not the actual size, and the i_assert would fire. Signed-off-by: Phil Carmody diffstat: src/lib/array.h | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 9c90eba19aee -r 2ab0c5855548 src/lib/array.h --- a/src/lib/array.h Sat Jun 13 17:05:49 2015 +0300 +++ b/src/lib/array.h Mon Jun 15 14:25:07 2015 +0300 @@ -278,15 +278,18 @@ /* Exchange ownership of two arrays, which should have been allocated from the same pool/context. Useful for updating an array with a - replacement. */ + replacement. Can also do it with uninitialised arrays (which will + have .element_size == 0). */ static inline void array_swap_i(struct array *array1, struct array *array2) { buffer_t *buffer = array1->buffer; + size_t elsize = array1->element_size; - i_assert(array1->element_size == array2->element_size); array1->buffer = array2->buffer; + array1->element_size = array2->element_size; array2->buffer = buffer; + array2->element_size = elsize; } #define array_swap(array1, array2) \ array_swap_i(&(array1)->arr + ARRAY_TYPES_CHECK(array1, array2), \ From dovecot at dovecot.org Mon Jun 15 11:40:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:40:23 +0000 Subject: dovecot-2.2: lib-test: flush test-assert failure messages Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b5608c537daa changeset: 18851:b5608c537daa user: Phil Carmody date: Mon Jun 15 14:31:19 2015 +0300 description: lib-test: flush test-assert failure messages Diagnostic output to stderr may overtake these stdout logs if the streams are combined by the shell or on the console. fflush() might help realign them. Signed-off-by: Phil Carmody diffstat: src/lib-test/test-common.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (26 lines): diff -r 2ab0c5855548 -r b5608c537daa src/lib-test/test-common.c --- a/src/lib-test/test-common.c Mon Jun 15 14:25:07 2015 +0300 +++ b/src/lib-test/test-common.c Mon Jun 15 14:31:19 2015 +0300 @@ -165,12 +165,14 @@ void test_assert_failed(const char *code, const char *file, unsigned int line) { printf("%s:%u: Assert failed: %s\n", file, line, code); + fflush(stdout); test_success = FALSE; } void test_assert_failed_idx(const char *code, const char *file, unsigned int line, long long i) { printf("%s:%u: Assert(#%lld) failed: %s\n", file, line, i, code); + fflush(stdout); test_success = FALSE; } @@ -250,6 +252,7 @@ if (reason != NULL && *reason != '\0') printf(": %s", reason); putchar('\n'); + fflush(stdout); total_count++; } From dovecot at dovecot.org Mon Jun 15 11:40:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:40:23 +0000 Subject: dovecot-2.2: lib-test: let tests invoke i_warning/i_error behaviour Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3275eee3613b changeset: 18852:3275eee3613b user: Phil Carmody date: Mon Jun 15 14:31:19 2015 +0300 description: lib-test: let tests invoke i_warning/i_error behaviour Error-handling paths should be testable too. Permit a test case to register that a known number of warnings/errors are to be expected, and to verify that those warnings did occur afterwards. Too many messages will fail the test exactly as it did in the past, they're unexpected messages. However, too few messages will also cause the test case to fail. Signed-off-by: Phil Carmody diffstat: src/lib-test/test-common.c | 18 ++++++++++++++++++ src/lib-test/test-common.h | 3 +++ 2 files changed, 21 insertions(+), 0 deletions(-) diffs (55 lines): diff -r b5608c537daa -r 3275eee3613b src/lib-test/test-common.c --- a/src/lib-test/test-common.c Mon Jun 15 14:31:19 2015 +0300 +++ b/src/lib-test/test-common.c Mon Jun 15 14:31:19 2015 +0300 @@ -19,6 +19,7 @@ static bool test_success; static unsigned int failure_count; static unsigned int total_count; +static unsigned int expected_errors; struct test_istream { struct istream_private istream; @@ -256,6 +257,19 @@ total_count++; } +void +test_expect_errors(unsigned int expected) +{ + i_assert(expected_errors == 0); + expected_errors = expected; +} +void +test_expect_no_more_errors(void) +{ + test_assert(expected_errors == 0); + expected_errors = 0; +} + static void ATTR_FORMAT(2, 0) test_error_handler(const struct failure_context *ctx, const char *format, va_list args) @@ -270,6 +284,10 @@ return; } #endif + if (expected_errors > 0) { + expected_errors--; + return; + } test_success = FALSE; } diff -r b5608c537daa -r 3275eee3613b src/lib-test/test-common.h --- a/src/lib-test/test-common.h Mon Jun 15 14:31:19 2015 +0300 +++ b/src/lib-test/test-common.h Mon Jun 15 14:31:19 2015 +0300 @@ -20,6 +20,9 @@ void test_assert_failed(const char *code, const char *file, unsigned int line); void test_assert_failed_idx(const char *code, const char *file, unsigned int line, long long i); bool test_has_failed(void); +/* If you're testing nasty cases which you want to warn, surround the noisy op with these */ +void test_expect_errors(unsigned int expected); +void test_expect_no_more_errors(void); void test_end(void); void test_out(const char *name, bool success); From dovecot at dovecot.org Mon Jun 15 11:40:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:40:24 +0000 Subject: dovecot-2.2: lib-test: test_expect_error_string() to match a sin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8739d201898c changeset: 18853:8739d201898c user: Phil Carmody date: Mon Jun 15 14:31:19 2015 +0300 description: lib-test: test_expect_error_string() to match a single known message This gives us very fine control over what is acceptable as an expected warning. Alas you have to do it for each message individually. Signed-off-by: Phil Carmody diffstat: src/lib-test/test-common.c | 14 +++++++++++++- src/lib-test/test-common.h | 1 + 2 files changed, 14 insertions(+), 1 deletions(-) diffs (56 lines): diff -r 3275eee3613b -r 8739d201898c src/lib-test/test-common.c --- a/src/lib-test/test-common.c Mon Jun 15 14:31:19 2015 +0300 +++ b/src/lib-test/test-common.c Mon Jun 15 14:31:19 2015 +0300 @@ -20,6 +20,7 @@ static unsigned int failure_count; static unsigned int total_count; static unsigned int expected_errors; +static char *expected_error_str; struct test_istream { struct istream_private istream; @@ -258,6 +259,13 @@ } void +test_expect_error_string(const char *substr) +{ + i_assert(expected_errors == 0); + expected_errors = 1; + expected_error_str = i_strdup(substr); +} +void test_expect_errors(unsigned int expected) { i_assert(expected_errors == 0); @@ -266,7 +274,7 @@ void test_expect_no_more_errors(void) { - test_assert(expected_errors == 0); + test_assert(expected_errors == 0 && expected_error_str == NULL); expected_errors = 0; } @@ -285,6 +293,10 @@ } #endif if (expected_errors > 0) { + if (expected_error_str != NULL) { + test_assert(strstr(format, expected_error_str) != NULL); + i_free(expected_error_str); + } expected_errors--; return; } diff -r 3275eee3613b -r 8739d201898c src/lib-test/test-common.h --- a/src/lib-test/test-common.h Mon Jun 15 14:31:19 2015 +0300 +++ b/src/lib-test/test-common.h Mon Jun 15 14:31:19 2015 +0300 @@ -22,6 +22,7 @@ bool test_has_failed(void); /* If you're testing nasty cases which you want to warn, surround the noisy op with these */ void test_expect_errors(unsigned int expected); +void test_expect_error_string(const char *substr); /* expect just 1 message matching the printf format */ void test_expect_no_more_errors(void); void test_end(void); From dovecot at dovecot.org Mon Jun 15 11:40:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:40:24 +0000 Subject: dovecot-2.2: lib: test-failures - tests get/set handlers and the... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6d965c726810 changeset: 18854:6d965c726810 user: Phil Carmody date: Mon Jun 15 14:31:19 2015 +0300 description: lib: test-failures - tests get/set handlers and the various log levels Also indirectoy tests the new lib-test test_expect* family. Signed-off-by: Phil Carmody diffstat: src/lib/Makefile.am | 1 + src/lib/test-failures.c | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 4 files changed, 74 insertions(+), 0 deletions(-) diffs (108 lines): diff -r 8739d201898c -r 6d965c726810 src/lib/Makefile.am --- a/src/lib/Makefile.am Mon Jun 15 14:31:19 2015 +0300 +++ b/src/lib/Makefile.am Mon Jun 15 14:31:19 2015 +0300 @@ -294,6 +294,7 @@ test-buffer.c \ test-crc32.c \ test-data-stack.c \ + test-failures.c \ test-guid.c \ test-hash.c \ test-hash-format.c \ diff -r 8739d201898c -r 6d965c726810 src/lib/test-failures.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-failures.c Mon Jun 15 14:31:19 2015 +0300 @@ -0,0 +1,71 @@ +/* Copyright (c) 2001-2015 Dovecot authors, see the included COPYING file */ + +/* Unit tests for failure helpers */ + +#include "test-lib.h" +#include "failures.h" + +static int handlers_set_me; + +static void test_failures_handler(const struct failure_context *ctx, + const char *format ATTR_UNUSED, + va_list args ATTR_UNUSED) +{ + handlers_set_me = ctx->type; +} +static void test_get_set_handlers(void) +{ + failure_callback_t *handlers[4]; + test_begin("get_handlers"); + i_get_failure_handlers(handlers, handlers+1, handlers+2, handlers+3); + test_end(); + + test_begin("set_handlers"); + + i_set_debug_handler(&test_failures_handler); + i_debug("If you see this debug, something's gone wrong"); + test_assert(handlers_set_me == LOG_TYPE_DEBUG); + i_set_debug_handler(handlers[3]); + + i_set_info_handler(&test_failures_handler); + i_info("If you see this info, something's gone wrong"); + test_assert(handlers_set_me == LOG_TYPE_INFO); + i_set_info_handler(handlers[2]); + + i_set_error_handler(&test_failures_handler); + i_warning("If you see this warning, something's gone wrong"); + test_assert(handlers_set_me == LOG_TYPE_WARNING); + i_error("If you see this error, something's gone wrong"); + test_assert(handlers_set_me == LOG_TYPE_ERROR); + i_set_error_handler(handlers[1]); + + //i_set_fatal_handler(&test_failures_handler); + //i_fatal("If you see this fatal, something's gone wrong"); + //test_assert(handlers_set_me == LOG_TYPE_FATAL); + //i_set_fatal_handler(handlers[0]); + + test_end(); +} +static void test_expected(void) +{ + test_begin("expected messages"); + test_expect_errors(1); + i_warning("deliberate warning - be happy you're seeing this"); + test_expect_no_more_errors(); + test_end(); +} +static void test_expected_str(void) +{ + test_begin("expected strings in messages"); + test_expect_error_string("be happy"); + i_error("deliberate error - be happy you're seeing this"); + test_expect_no_more_errors(); + test_end(); +} + +void test_failures(void) +{ + test_get_set_handlers(); + test_expected(); + test_expected_str(); +} diff -r 8739d201898c -r 6d965c726810 src/lib/test-lib.c --- a/src/lib/test-lib.c Mon Jun 15 14:31:19 2015 +0300 +++ b/src/lib/test-lib.c Mon Jun 15 14:31:19 2015 +0300 @@ -14,6 +14,7 @@ test_buffer, test_crc32, test_data_stack, + test_failures, test_guid, test_hash, test_hash_format, diff -r 8739d201898c -r 6d965c726810 src/lib/test-lib.h --- a/src/lib/test-lib.h Mon Jun 15 14:31:19 2015 +0300 +++ b/src/lib/test-lib.h Mon Jun 15 14:31:19 2015 +0300 @@ -15,6 +15,7 @@ void test_crc32(void); void test_data_stack(void); enum fatal_test_state fatal_data_stack(int); +void test_failures(void); void test_guid(void); void test_hash(void); void test_hash_format(void); From dovecot at dovecot.org Mon Jun 15 11:40:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:40:24 +0000 Subject: dovecot-2.2: lib-fts: Fix comment in fts-filter.h Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf99661dca25 changeset: 18855:cf99661dca25 user: Teemu Huovila date: Mon Jun 15 14:35:25 2015 +0300 description: lib-fts: Fix comment in fts-filter.h diffstat: src/lib-fts/fts-filter.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 6d965c726810 -r cf99661dca25 src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Mon Jun 15 14:31:19 2015 +0300 +++ b/src/lib-fts/fts-filter.h Mon Jun 15 14:35:25 2015 +0300 @@ -29,7 +29,7 @@ */ extern const struct fts_filter *fts_filter_normalizer_icu; -/* Lowecases the input. Currently only ASCII data is lowercased. */ +/* Lowecases the input. Supports UTF8, if libicu is available. */ extern const struct fts_filter *fts_filter_lowercase; /* Removes <'s> suffix from words. */ From dovecot at dovecot.org Mon Jun 15 11:52:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:52:22 +0000 Subject: dovecot-2.2: fs-metawrap: Don't assert-crash when writing an emp... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/db8877fdd39f changeset: 18856:db8877fdd39f user: Timo Sirainen date: Mon Jun 15 14:49:46 2015 +0300 description: fs-metawrap: Don't assert-crash when writing an empty file. diffstat: src/lib-fs/fs-metawrap.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r cf99661dca25 -r db8877fdd39f src/lib-fs/fs-metawrap.c --- a/src/lib-fs/fs-metawrap.c Mon Jun 15 14:35:25 2015 +0300 +++ b/src/lib-fs/fs-metawrap.c Mon Jun 15 14:49:46 2015 +0300 @@ -457,7 +457,7 @@ /* fs_stat() after a write. we can do this quickly. */ if (fs_stat(file->super, st_r) < 0) return -1; - i_assert((uoff_t)st_r->st_size > file->metadata_write_size); + i_assert((uoff_t)st_r->st_size >= file->metadata_write_size); st_r->st_size -= file->metadata_write_size; return 0; } From dovecot at dovecot.org Mon Jun 15 11:54:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:54:20 +0000 Subject: dovecot-2.2: fs-metawrap: If written file unexpectedly shrinks, ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bd1dabfcfae3 changeset: 18857:bd1dabfcfae3 user: Timo Sirainen date: Mon Jun 15 14:51:46 2015 +0300 description: fs-metawrap: If written file unexpectedly shrinks, return error instead of assert-crash. This shouldn't be happening, but we can't fully control it so an error is better. diffstat: src/lib-fs/fs-metawrap.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (19 lines): diff -r db8877fdd39f -r bd1dabfcfae3 src/lib-fs/fs-metawrap.c --- a/src/lib-fs/fs-metawrap.c Mon Jun 15 14:49:46 2015 +0300 +++ b/src/lib-fs/fs-metawrap.c Mon Jun 15 14:51:46 2015 +0300 @@ -457,7 +457,14 @@ /* fs_stat() after a write. we can do this quickly. */ if (fs_stat(file->super, st_r) < 0) return -1; - i_assert((uoff_t)st_r->st_size >= file->metadata_write_size); + if ((uoff_t)st_r->st_size < file->metadata_write_size) { + fs_set_error(_file->fs, + "Just-written %s shrank unexpectedly " + "(%"PRIuUOFF_T" < %"PRIuUOFF_T")", + fs_file_path(_file), st_r->st_size, + file->metadata_write_size); + return -1; + } st_r->st_size -= file->metadata_write_size; return 0; } From dovecot at dovecot.org Mon Jun 15 11:58:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:58:37 +0000 Subject: dovecot-2.2: director: explicitly mark _disconnect() helper stat... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8bde19f080ad changeset: 18858:8bde19f080ad user: Phil Carmody date: Mon Jun 15 14:54:58 2015 +0300 description: director: explicitly mark _disconnect() helper static to match proto No functional change, compiler would not have exported the symbol, this just shuts up sparse. Signed-off-by: Phil Carmody diffstat: src/director/director-connection.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r bd1dabfcfae3 -r 8bde19f080ad src/director/director-connection.c --- a/src/director/director-connection.c Mon Jun 15 14:51:46 2015 +0300 +++ b/src/director/director-connection.c Mon Jun 15 14:54:58 2015 +0300 @@ -1868,8 +1868,8 @@ } } -void director_connection_disconnected(struct director_connection **_conn, - const char *reason) +static void director_connection_disconnected(struct director_connection **_conn, + const char *reason) { struct director_connection *conn = *_conn; struct director *dir = conn->dir; From dovecot at dovecot.org Mon Jun 15 11:58:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:58:37 +0000 Subject: dovecot-2.2: lib-fts: use NULL rather than 0 to shut up sparse Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d2a637585bfa changeset: 18859:d2a637585bfa user: Phil Carmody date: Mon Jun 15 14:55:01 2015 +0300 description: lib-fts: use NULL rather than 0 to shut up sparse Signed-off-by: Phil Carmody diffstat: src/lib-fts/fts-tokenizer.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 8bde19f080ad -r d2a637585bfa src/lib-fts/fts-tokenizer.c --- a/src/lib-fts/fts-tokenizer.c Mon Jun 15 14:54:58 2015 +0300 +++ b/src/lib-fts/fts-tokenizer.c Mon Jun 15 14:55:01 2015 +0300 @@ -89,7 +89,7 @@ settings = &empty_settings; if (tok_class->v->create(settings, &tok, error_r) < 0) { - *tokenizer_r = 0; + *tokenizer_r = NULL; return -1; } tok->refcount = 1; From dovecot at dovecot.org Mon Jun 15 11:58:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 11:58:38 +0000 Subject: dovecot-2.2: lib-stats: fix tristate-int vs. bool return value c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cac29b3a1e96 changeset: 18860:cac29b3a1e96 user: Phil Carmody date: Mon Jun 15 14:55:03 2015 +0300 description: lib-stats: fix tristate-int vs. bool return value confusion -1 was being silently converted to true. However, this was an error condition. The callers don't need anything more complex than a bool, so kill the int. Signed-off-by: Phil Carmody diffstat: src/lib-stats/stats.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r d2a637585bfa -r cac29b3a1e96 src/lib-stats/stats.c --- a/src/lib-stats/stats.c Mon Jun 15 14:55:01 2015 +0300 +++ b/src/lib-stats/stats.c Mon Jun 15 14:55:03 2015 +0300 @@ -195,12 +195,12 @@ p = memchr(data, '\0', size); if (p == NULL) { *error_r = "Expected name, but NUL is missing"; - return -1; + return FALSE; } item = stats_item_find_by_name(next_name); if (item == NULL) { *error_r = t_strdup_printf("Unknown stats name: '%s'", next_name); - return -1; + return FALSE; } size -= (p+1) - data; data = p+1; From dovecot at dovecot.org Mon Jun 15 21:19:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 21:19:16 +0000 Subject: dovecot-2.2: fs-posix: fs_copy() didn't work with prefix= parameter Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5450f9217b97 changeset: 18861:5450f9217b97 user: Timo Sirainen date: Tue Jun 16 00:16:42 2015 +0300 description: fs-posix: fs_copy() didn't work with prefix= parameter diffstat: src/lib-fs/fs-posix.c | 15 ++++++++------- 1 files changed, 8 insertions(+), 7 deletions(-) diffs (40 lines): diff -r cac29b3a1e96 -r 5450f9217b97 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Mon Jun 15 14:55:03 2015 +0300 +++ b/src/lib-fs/fs-posix.c Tue Jun 16 00:16:42 2015 +0300 @@ -624,28 +624,29 @@ static int fs_posix_copy(struct fs_file *_src, struct fs_file *_dest) { + struct posix_fs_file *src = (struct posix_fs_file *)_src; struct posix_fs_file *dest = (struct posix_fs_file *)_dest; struct posix_fs *fs = (struct posix_fs *)_src->fs; unsigned int try_count = 0; int ret; - ret = link(_src->path, _dest->path); + ret = link(src->full_path, dest->full_path); if (errno == EEXIST && dest->open_mode == FS_OPEN_MODE_REPLACE) { /* destination file already exists - replace it */ - if (unlink(_dest->path) < 0 && errno != ENOENT) - i_error("unlink(%s) failed: %m", _dest->path); - ret = link(_src->path, _dest->path); + if (unlink(dest->full_path) < 0 && errno != ENOENT) + i_error("unlink(%s) failed: %m", dest->full_path); + ret = link(src->full_path, dest->full_path); } while (ret < 0 && errno == ENOENT && try_count <= MAX_MKDIR_RETRY_COUNT) { - if (fs_posix_mkdir_parents(fs, _dest->path) < 0) + if (fs_posix_mkdir_parents(fs, dest->full_path) < 0) return -1; - ret = link(_src->path, _dest->path); + ret = link(src->full_path, dest->full_path); try_count++; } if (ret < 0) { fs_set_error(_src->fs, "link(%s, %s) failed: %m", - _src->path, _dest->path); + src->full_path, dest->full_path); return -1; } return 0; From dovecot at dovecot.org Mon Jun 15 21:31:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Jun 2015 21:31:45 +0000 Subject: dovecot-2.2: fts: Avoid expanding mail_search_args multiple time... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d255f8627d95 changeset: 18862:d255f8627d95 user: Timo Sirainen date: Tue Jun 16 00:29:07 2015 +0300 description: fts: Avoid expanding mail_search_args multiple times (for optimization & crash-avoidance) Primarily this fixes the crash: index-search-result.c: line 132 (index_search_result_update_flags): assertion failed: (result->search_args->args == &search_arg) It could be triggered by: a search return (update) body body seen b store 1 +flags \seen c store 1 -flags \seen diffstat: src/lib-storage/mail-search.h | 3 +++ src/plugins/fts/fts-search-args.c | 7 +++++++ 2 files changed, 10 insertions(+), 0 deletions(-) diffs (30 lines): diff -r 5450f9217b97 -r d255f8627d95 src/lib-storage/mail-search.h --- a/src/lib-storage/mail-search.h Tue Jun 16 00:16:42 2015 +0300 +++ b/src/lib-storage/mail-search.h Tue Jun 16 00:29:07 2015 +0300 @@ -115,6 +115,9 @@ /* Stop mail_search_next() when finding a non-matching mail. (Could be useful when wanting to find only the oldest mails.) */ unsigned int stop_on_nonmatch:1; + /* fts plugin has already expanded the search args - no need to do + it again. */ + unsigned int fts_expanded:1; }; #define ARG_SET_RESULT(arg, res) \ diff -r 5450f9217b97 -r d255f8627d95 src/plugins/fts/fts-search-args.c --- a/src/plugins/fts/fts-search-args.c Tue Jun 16 00:16:42 2015 +0300 +++ b/src/plugins/fts/fts-search-args.c Tue Jun 16 00:29:07 2015 +0300 @@ -185,6 +185,13 @@ { struct mail_search_arg *args_dup, *orig_args = args->args; + /* don't keep re-expanding every time the search args are used. + this is especially important to avoid an assert-crash in + index_search_result_update_flags(). */ + if (args->fts_expanded) + return 0; + args->fts_expanded = TRUE; + /* duplicate the args, so if expansion fails we haven't changed anything */ args_dup = mail_search_arg_dup(args->pool, args->args); From dovecot at dovecot.org Tue Jun 16 11:16:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Jun 2015 11:16:55 +0000 Subject: dovecot-2.2: doveadm: Added doveadm_killed_signo() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a58b3e70fc8a changeset: 18863:a58b3e70fc8a user: Timo Sirainen date: Tue Jun 16 14:12:33 2015 +0300 description: doveadm: Added doveadm_killed_signo() diffstat: src/doveadm/doveadm-mail.c | 5 +++++ src/doveadm/doveadm-mail.h | 1 + 2 files changed, 6 insertions(+), 0 deletions(-) diffs (26 lines): diff -r d255f8627d95 -r a58b3e70fc8a src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Tue Jun 16 00:29:07 2015 +0300 +++ b/src/doveadm/doveadm-mail.c Tue Jun 16 14:12:33 2015 +0300 @@ -45,6 +45,11 @@ return killed_signo != 0; } +int doveadm_killed_signo(void) +{ + return killed_signo; +} + void doveadm_mail_failed_error(struct doveadm_mail_cmd_context *ctx, enum mail_error error) { diff -r d255f8627d95 -r a58b3e70fc8a src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Tue Jun 16 00:29:07 2015 +0300 +++ b/src/doveadm/doveadm-mail.h Tue Jun 16 14:12:33 2015 +0300 @@ -112,6 +112,7 @@ extern char doveadm_mail_cmd_hide; bool doveadm_is_killed(void); +int doveadm_killed_signo(void); bool doveadm_mail_try_run(const char *cmd_name, int argc, char *argv[]); void doveadm_mail_register_cmd(const struct doveadm_mail_cmd *cmd); From dovecot at dovecot.org Tue Jun 16 11:16:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Jun 2015 11:16:55 +0000 Subject: dovecot-2.2: dsync: If we stop because of a signal, log a warnin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/80c1be850fdc changeset: 18864:80c1be850fdc user: Timo Sirainen date: Tue Jun 16 14:12:59 2015 +0300 description: dsync: If we stop because of a signal, log a warning about it. diffstat: src/doveadm/doveadm-dsync.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (18 lines): diff -r a58b3e70fc8a -r 80c1be850fdc src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Tue Jun 16 14:12:33 2015 +0300 +++ b/src/doveadm/doveadm-dsync.c Tue Jun 16 14:12:59 2015 +0300 @@ -383,9 +383,12 @@ changed1 = changed2 = TRUE; while (brain1_running || brain2_running) { if (dsync_brain_has_failed(brain) || - dsync_brain_has_failed(brain2) || - doveadm_is_killed()) + dsync_brain_has_failed(brain2)) break; + if (doveadm_is_killed()) { + i_warning("Killed with signal %d", doveadm_killed_signo()); + break; + } i_assert(changed1 || changed2); brain1_running = dsync_brain_run(brain, &changed1); From dovecot at dovecot.org Tue Jun 16 13:43:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Jun 2015 13:43:09 +0000 Subject: dovecot-2.2: lib: Added i_stream_create_failure_at() to inject E... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/99827acc1888 changeset: 18865:99827acc1888 user: Timo Sirainen date: Tue Jun 16 16:21:56 2015 +0300 description: lib: Added i_stream_create_failure_at() to inject EIO at given offset in istream. diffstat: src/lib/Makefile.am | 3 + src/lib/istream-failure-at.c | 88 +++++++++++++++++++++++++++++++++++++++ src/lib/istream-failure-at.h | 10 ++++ src/lib/test-istream-failure-at.c | 46 ++++++++++++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 6 files changed, 149 insertions(+), 0 deletions(-) diffs (205 lines): diff -r 80c1be850fdc -r 99827acc1888 src/lib/Makefile.am --- a/src/lib/Makefile.am Tue Jun 16 14:12:59 2015 +0300 +++ b/src/lib/Makefile.am Tue Jun 16 16:21:56 2015 +0300 @@ -65,6 +65,7 @@ istream-concat.c \ istream-crlf.c \ istream-data.c \ + istream-failure-at.c \ istream-file.c \ istream-hash.c \ istream-jsonstr.c \ @@ -203,6 +204,7 @@ istream-chain.h \ istream-concat.h \ istream-crlf.h \ + istream-failure-at.h \ istream-file-private.h \ istream-hash.h \ istream-jsonstr.h \ @@ -306,6 +308,7 @@ test-istream-base64-encoder.c \ test-istream-concat.c \ test-istream-crlf.c \ + test-istream-failure-at.c \ test-istream-seekable.c \ test-istream-tee.c \ test-istream-unix.c \ diff -r 80c1be850fdc -r 99827acc1888 src/lib/istream-failure-at.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-failure-at.c Tue Jun 16 16:21:56 2015 +0300 @@ -0,0 +1,88 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream-private.h" +#include "istream-failure-at.h" + +struct failure_at_istream { + struct istream_private istream; + char *error_string; + uoff_t failure_offset; +}; + +static void i_stream_failure_at_destroy(struct iostream_private *stream) +{ + struct failure_at_istream *fstream = + (struct failure_at_istream *)stream; + + i_free(fstream->error_string); +} + +static ssize_t +i_stream_failure_at_read(struct istream_private *stream) +{ + struct failure_at_istream *fstream = (struct failure_at_istream *)stream; + uoff_t new_offset; + ssize_t ret; + + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + + ret = i_stream_read_copy_from_parent(&stream->istream); + new_offset = stream->istream.v_offset + (stream->pos - stream->skip); + if (ret >= 0 && new_offset >= fstream->failure_offset) { + if (stream->istream.v_offset >= fstream->failure_offset) { + /* we already passed the wanted failure offset, + return error immediately. */ + stream->pos = stream->skip; + stream->istream.stream_errno = errno = EIO; + io_stream_set_error(&stream->iostream, "%s", + fstream->error_string); + ret = -1; + } else { + /* return data up to the wanted failure offset and + on the next read() call return failure */ + size_t new_pos = fstream->failure_offset - + stream->istream.v_offset + stream->skip; + i_assert(new_pos >= stream->skip && + stream->pos >= new_pos); + ret -= stream->pos - new_pos; + stream->pos = new_pos; + } + } else if (ret < 0 && stream->istream.stream_errno == 0 && + fstream->failure_offset == (uoff_t)-1) { + /* failure at EOF */ + stream->istream.stream_errno = errno = EIO; + io_stream_set_error(&stream->iostream, "%s", + fstream->error_string); + } + return ret; +} + +struct istream * +i_stream_create_failure_at(struct istream *input, uoff_t failure_offset, + const char *error_string) +{ + struct failure_at_istream *fstream; + + fstream = i_new(struct failure_at_istream, 1); + fstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + fstream->istream.stream_size_passthrough = TRUE; + + fstream->istream.read = i_stream_failure_at_read; + fstream->istream.iostream.destroy = i_stream_failure_at_destroy; + + fstream->istream.istream.blocking = input->blocking; + fstream->istream.istream.seekable = input->seekable; + + fstream->error_string = i_strdup(error_string); + fstream->failure_offset = failure_offset; + return i_stream_create(&fstream->istream, input, + i_stream_get_fd(input)); +} + +struct istream * +i_stream_create_failure_at_eof(struct istream *input, const char *error_string) +{ + return i_stream_create_failure_at(input, (uoff_t)-1, error_string); +} diff -r 80c1be850fdc -r 99827acc1888 src/lib/istream-failure-at.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-failure-at.h Tue Jun 16 16:21:56 2015 +0300 @@ -0,0 +1,10 @@ +#ifndef ISTREAM_FAILURE_AT_H +#define ISTREAM_FAILURE_AT_H + +struct istream * +i_stream_create_failure_at(struct istream *input, uoff_t failure_offset, + const char *error_string); +struct istream * +i_stream_create_failure_at_eof(struct istream *input, const char *error_string); + +#endif diff -r 80c1be850fdc -r 99827acc1888 src/lib/test-istream-failure-at.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-istream-failure-at.c Tue Jun 16 16:21:56 2015 +0300 @@ -0,0 +1,46 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "istream.h" +#include "istream-failure-at.h" + +#define TEST_DATA_LENGTH 128 +#define TEST_ERRMSG "test-istream-failure-at error triggered" + +void test_istream_failure_at(void) +{ + struct istream *input, *data_input; + unsigned char test_data[TEST_DATA_LENGTH]; + unsigned int i; + ssize_t ret; + + test_begin("istream failure at"); + for (i = 0; i < sizeof(test_data); i++) + test_data[i] = i; + data_input = i_stream_create_from_data(test_data, sizeof(test_data)); + for (i = 0; i < TEST_DATA_LENGTH; i++) { + i_stream_seek(data_input, 0); + input = i_stream_create_failure_at(data_input, i, TEST_ERRMSG); + while ((ret = i_stream_read(input)) > 0) + i_stream_skip(input, ret); + test_assert_idx(ret == -1 && input->v_offset == i && + input->stream_errno == EIO && + strcmp(i_stream_get_error(input), TEST_ERRMSG) == 0, i); + i_stream_destroy(&input); + } + /* shouldn't fail */ + i_stream_seek(data_input, 0); + input = i_stream_create_failure_at(data_input, TEST_DATA_LENGTH, TEST_ERRMSG); + while ((ret = i_stream_read(input)) > 0) + i_stream_skip(input, ret); + test_assert(ret == -1 && input->stream_errno == 0); + /* fail at EOF */ + i_stream_seek(data_input, 0); + input = i_stream_create_failure_at_eof(data_input, TEST_ERRMSG); + while ((ret = i_stream_read(input)) > 0) + i_stream_skip(input, ret); + test_assert_idx(ret == -1 && input->v_offset == TEST_DATA_LENGTH && + input->stream_errno == EIO && + strcmp(i_stream_get_error(input), TEST_ERRMSG) == 0, i); + test_end(); +} diff -r 80c1be850fdc -r 99827acc1888 src/lib/test-lib.c --- a/src/lib/test-lib.c Tue Jun 16 14:12:59 2015 +0300 +++ b/src/lib/test-lib.c Tue Jun 16 16:21:56 2015 +0300 @@ -26,6 +26,7 @@ test_istream_base64_encoder, test_istream_concat, test_istream_crlf, + test_istream_failure_at, test_istream_seekable, test_istream_tee, test_istream_unix, diff -r 80c1be850fdc -r 99827acc1888 src/lib/test-lib.h --- a/src/lib/test-lib.h Tue Jun 16 14:12:59 2015 +0300 +++ b/src/lib/test-lib.h Tue Jun 16 16:21:56 2015 +0300 @@ -27,6 +27,7 @@ void test_istream_base64_encoder(void); void test_istream_concat(void); void test_istream_crlf(void); +void test_istream_failure_at(void); void test_istream_seekable(void); void test_istream_tee(void); void test_istream_unix(void); From dovecot at dovecot.org Tue Jun 16 13:43:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Jun 2015 13:43:20 +0000 Subject: dovecot-2.2: lib: Added o_stream_create_failure_at() to inject E... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c7288c054b1 changeset: 18866:1c7288c054b1 user: Timo Sirainen date: Tue Jun 16 16:22:18 2015 +0300 description: lib: Added o_stream_create_failure_at() to inject EIO at given offset in ostream diffstat: src/lib/Makefile.am | 3 + src/lib/ostream-failure-at.c | 108 ++++++++++++++++++++++++++++++++++++++ src/lib/ostream-failure-at.h | 10 +++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + src/lib/test-ostream-failure-at.c | 52 ++++++++++++++++++ 6 files changed, 175 insertions(+), 0 deletions(-) diffs (231 lines): diff -r 99827acc1888 -r 1c7288c054b1 src/lib/Makefile.am --- a/src/lib/Makefile.am Tue Jun 16 16:21:56 2015 +0300 +++ b/src/lib/Makefile.am Tue Jun 16 16:22:18 2015 +0300 @@ -109,6 +109,7 @@ numpack.c \ ostream.c \ ostream-buffer.c \ + ostream-failure-at.c \ ostream-file.c \ ostream-hash.c \ ostream-rawlog.c \ @@ -237,6 +238,7 @@ nfs-workarounds.h \ numpack.h \ ostream.h \ + ostream-failure-at.h \ ostream-hash.h \ ostream-private.h \ ostream-rawlog.h \ @@ -318,6 +320,7 @@ test-mempool-alloconly.c \ test-net.c \ test-numpack.c \ + test-ostream-failure-at.c \ test-ostream-file.c \ test-primes.c \ test-printf-format-fix.c \ diff -r 99827acc1888 -r 1c7288c054b1 src/lib/ostream-failure-at.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/ostream-failure-at.c Tue Jun 16 16:22:18 2015 +0300 @@ -0,0 +1,108 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "ostream-private.h" +#include "ostream-failure-at.h" + +struct failure_at_ostream { + struct ostream_private ostream; + char *error_string; + uoff_t failure_offset; + bool failed; +}; + +static void o_stream_failure_at_destroy(struct iostream_private *stream) +{ + struct failure_at_ostream *fstream = + (struct failure_at_ostream *)stream; + + i_free(fstream->error_string); +} + +static ssize_t +o_stream_failure_at_sendv(struct ostream_private *stream, + const struct const_iovec *iov, unsigned int iov_count) +{ + struct failure_at_ostream *fstream = + (struct failure_at_ostream *)stream; + unsigned int i; + struct const_iovec *iov_dup; + unsigned int iov_dup_count; + uoff_t bytes_until_failure; + ssize_t ret; + + if (fstream->failure_offset <= stream->ostream.offset) { + io_stream_set_error(&stream->iostream, "%s", + fstream->error_string); + stream->ostream.stream_errno = errno = EIO; + fstream->failed = TRUE; + return -1; + } + bytes_until_failure = fstream->failure_offset - stream->ostream.offset; + + iov_dup = i_new(struct const_iovec, iov_count); + iov_dup_count = iov_count; + for (i = 0; i < iov_count; i++) { + iov_dup[i] = iov[i]; + if (iov_dup[i].iov_len >= bytes_until_failure) { + iov_dup[i].iov_len = bytes_until_failure; + iov_dup_count = i+1; + break; + } + } + ret = o_stream_sendv(stream->parent, iov_dup, iov_dup_count); + i_free(iov_dup); + + if (ret < 0) { + o_stream_copy_error_from_parent(stream); + return -1; + } + stream->ostream.offset += ret; + return ret; +} + +static int +o_stream_failure_at_flush(struct ostream_private *stream) +{ + struct failure_at_ostream *fstream = + (struct failure_at_ostream *)stream; + + if (fstream->failed) { + io_stream_set_error(&stream->iostream, "%s", + fstream->error_string); + stream->ostream.stream_errno = errno = EIO; + return -1; + } + return o_stream_flush(stream->parent); +} + +struct ostream * +o_stream_create_failure_at(struct ostream *output, uoff_t failure_offset, + const char *error_string) +{ + struct failure_at_ostream *fstream; + + fstream = i_new(struct failure_at_ostream, 1); + fstream->ostream.sendv = o_stream_failure_at_sendv; + fstream->ostream.flush = o_stream_failure_at_flush; + fstream->ostream.iostream.destroy = o_stream_failure_at_destroy; + fstream->failure_offset = failure_offset; + fstream->error_string = i_strdup(error_string); + return o_stream_create(&fstream->ostream, output, + o_stream_get_fd(output)); +} + +struct ostream * +o_stream_create_failure_at_flush(struct ostream *output, const char *error_string) +{ + struct failure_at_ostream *fstream; + + fstream = i_new(struct failure_at_ostream, 1); + fstream->ostream.flush = o_stream_failure_at_flush; + fstream->ostream.iostream.destroy = o_stream_failure_at_destroy; + fstream->error_string = i_strdup(error_string); + fstream->failed = TRUE; + return o_stream_create(&fstream->ostream, output, + o_stream_get_fd(output)); +} diff -r 99827acc1888 -r 1c7288c054b1 src/lib/ostream-failure-at.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/ostream-failure-at.h Tue Jun 16 16:22:18 2015 +0300 @@ -0,0 +1,10 @@ +#ifndef OSTREAM_FAILURE_AT_H +#define OSTREAM_FAILURE_AT_H + +struct ostream * +o_stream_create_failure_at(struct ostream *output, uoff_t failure_offset, + const char *error_string); +struct ostream * +o_stream_create_failure_at_flush(struct ostream *output, const char *error_string); + +#endif diff -r 99827acc1888 -r 1c7288c054b1 src/lib/test-lib.c --- a/src/lib/test-lib.c Tue Jun 16 16:21:56 2015 +0300 +++ b/src/lib/test-lib.c Tue Jun 16 16:22:18 2015 +0300 @@ -36,6 +36,7 @@ test_mempool_alloconly, test_net, test_numpack, + test_ostream_failure_at, test_ostream_file, test_primes, test_printf_format_fix, diff -r 99827acc1888 -r 1c7288c054b1 src/lib/test-lib.h --- a/src/lib/test-lib.h Tue Jun 16 16:21:56 2015 +0300 +++ b/src/lib/test-lib.h Tue Jun 16 16:22:18 2015 +0300 @@ -38,6 +38,7 @@ enum fatal_test_state fatal_mempool(int); void test_net(void); void test_numpack(void); +void test_ostream_failure_at(void); void test_ostream_file(void); void test_primes(void); void test_printf_format_fix(void); diff -r 99827acc1888 -r 1c7288c054b1 src/lib/test-ostream-failure-at.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-ostream-failure-at.c Tue Jun 16 16:22:18 2015 +0300 @@ -0,0 +1,52 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "buffer.h" +#include "ostream.h" +#include "ostream-failure-at.h" + +#define TEST_DATA_LENGTH 128 +#define TEST_ERRMSG "test-ostream-failure-at error triggered" + +void test_ostream_failure_at(void) +{ + unsigned char test_data[TEST_DATA_LENGTH]; + struct ostream *output, *buf_output; + buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), 256); + unsigned int i; + + test_begin("ostream failure at"); + for (i = 0; i < sizeof(test_data); i++) + test_data[i] = i; + for (i = 0; i < TEST_DATA_LENGTH; i++) { + buf_output = o_stream_create_buffer(buf); + output = o_stream_create_failure_at(buf_output, i, TEST_ERRMSG); + if (i > 0) + test_assert(o_stream_send(output, test_data, sizeof(test_data)) == i); + test_assert_idx(o_stream_send(output, test_data, sizeof(test_data)) == -1 && + output->offset == i && + output->stream_errno == EIO && + strcmp(o_stream_get_error(output), TEST_ERRMSG) == 0, i); + o_stream_destroy(&output); + o_stream_destroy(&buf_output); + } + /* shouldn't fail */ + buf_output = o_stream_create_buffer(buf); + output = o_stream_create_failure_at(buf_output, TEST_DATA_LENGTH, TEST_ERRMSG); + test_assert(o_stream_send(output, test_data, sizeof(test_data)) == TEST_DATA_LENGTH); + test_assert(o_stream_flush(output) > 0 && + output->offset == TEST_DATA_LENGTH && + output->stream_errno == 0); + o_stream_destroy(&output); + o_stream_destroy(&buf_output); + + /* fail at flush */ + buf_output = o_stream_create_buffer(buf); + output = o_stream_create_failure_at_flush(buf_output, TEST_ERRMSG); + test_assert(o_stream_send(output, test_data, sizeof(test_data)) == TEST_DATA_LENGTH); + test_assert(o_stream_flush(output) < 0 && output->stream_errno == EIO && + strcmp(o_stream_get_error(output), TEST_ERRMSG) == 0); + o_stream_destroy(&output); + o_stream_destroy(&buf_output); + test_end(); +} From dovecot at dovecot.org Tue Jun 16 13:43:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Jun 2015 13:43:20 +0000 Subject: dovecot-2.2: lib-fs: Added "randomfail" driver. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c6ed48c7f2a0 changeset: 18867:c6ed48c7f2a0 user: Timo Sirainen date: Tue Jun 16 16:40:29 2015 +0300 description: lib-fs: Added "randomfail" driver. Using this in front of fs drivers allows randomly injecting failures. For example: mail_attachment_fs = randomfail:all=10,read=30,read-range=2000-3000:sis posix This means that all FS operations have a 10% chance of failing, except reads have a 30% chance of failing. If the read fails, it'll fail somewhere between offsets 2000-3000 (the default is 0, so it'll fail at the start of file). The supported operations are: wait metadata prefetch read write lock exists stat copy rename delete iter. "all" applies to all of them. The supported ranges are: read-range, write-range, iter-range. diffstat: src/lib-fs/Makefile.am | 1 + src/lib-fs/fs-api-private.h | 1 + src/lib-fs/fs-api.c | 1 + src/lib-fs/fs-randomfail.c | 551 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 554 insertions(+), 0 deletions(-) diffs (truncated from 588 to 300 lines): diff -r 1c7288c054b1 -r c6ed48c7f2a0 src/lib-fs/Makefile.am --- a/src/lib-fs/Makefile.am Tue Jun 16 16:22:18 2015 +0300 +++ b/src/lib-fs/Makefile.am Tue Jun 16 16:40:29 2015 +0300 @@ -8,6 +8,7 @@ libfs_la_SOURCES = \ fs-api.c \ fs-metawrap.c \ + fs-randomfail.c \ fs-posix.c \ fs-sis.c \ fs-sis-common.c \ diff -r 1c7288c054b1 -r c6ed48c7f2a0 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Tue Jun 16 16:22:18 2015 +0300 +++ b/src/lib-fs/fs-api-private.h Tue Jun 16 16:40:29 2015 +0300 @@ -135,6 +135,7 @@ }; extern const struct fs fs_class_posix; +extern const struct fs fs_class_randomfail; extern const struct fs fs_class_metawrap; extern const struct fs fs_class_sis; extern const struct fs fs_class_sis_queue; diff -r 1c7288c054b1 -r c6ed48c7f2a0 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Tue Jun 16 16:22:18 2015 +0300 +++ b/src/lib-fs/fs-api.c Tue Jun 16 16:40:29 2015 +0300 @@ -64,6 +64,7 @@ { i_array_init(&fs_classes, 8); fs_class_register(&fs_class_posix); + fs_class_register(&fs_class_randomfail); fs_class_register(&fs_class_metawrap); fs_class_register(&fs_class_sis); fs_class_register(&fs_class_sis_queue); diff -r 1c7288c054b1 -r c6ed48c7f2a0 src/lib-fs/fs-randomfail.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fs/fs-randomfail.c Tue Jun 16 16:40:29 2015 +0300 @@ -0,0 +1,551 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "istream-private.h" +#include "istream-concat.h" +#include "istream-failure-at.h" +#include "ostream-failure-at.h" +#include "ostream.h" +#include "fs-api-private.h" + +#include + +#define RANDOMFAIL_ERROR "Random failure injection" + +enum fs_op { + FS_OP_WAIT, + FS_OP_METADATA, + FS_OP_PREFETCH, + FS_OP_READ, + FS_OP_WRITE, + FS_OP_LOCK, + FS_OP_EXISTS, + FS_OP_STAT, + FS_OP_COPY, + FS_OP_RENAME, + FS_OP_DELETE, + FS_OP_ITER, + + FS_OP_COUNT +}; +static const char *fs_op_names[FS_OP_COUNT] = { + "wait", "metadata", "prefetch", "read", "write", "lock", "exists", + "stat", "copy", "rename", "delete", "iter" +}; + +struct randomfail_fs { + struct fs fs; + unsigned int op_probability[FS_OP_COUNT]; + uoff_t range_start[FS_OP_COUNT], range_end[FS_OP_COUNT]; +}; + +struct randomfail_fs_file { + struct fs_file file; + struct fs_file *super, *super_read; + struct istream *input; + + struct ostream *super_output; +}; + +struct randomfail_fs_iter { + struct fs_iter iter; + struct fs_iter *super; + unsigned int fail_pos; +}; + +static struct fs *fs_randomfail_alloc(void) +{ + struct randomfail_fs *fs; + + fs = i_new(struct randomfail_fs, 1); + fs->fs = fs_class_randomfail; + return &fs->fs; +} + +static bool fs_op_find(const char *str, enum fs_op *op_r) +{ + enum fs_op op; + + for (op = 0; op < FS_OP_COUNT; op++) { + if (strcmp(fs_op_names[op], str) == 0) { + *op_r = op; + return TRUE; + } + } + return FALSE; +} + +static bool +fs_randomfail_add_probability(struct randomfail_fs *fs, + const char *key, const char *value, + const char **error_r) +{ + unsigned int num; + enum fs_op op; + bool invalid_value = FALSE; + + if (str_to_uint(value, &num) < 0 || num > 100) + invalid_value = TRUE; + if (fs_op_find(key, &op)) { + if (invalid_value) { + *error_r = "Invalid probability value"; + return -1; + } + fs->op_probability[op] = num; + return 1; + } + if (strcmp(key, "all") == 0) { + if (invalid_value) { + *error_r = "Invalid probability value"; + return -1; + } + for (op = 0; op < FS_OP_COUNT; op++) + fs->op_probability[op] = num; + return 1; + } + return 0; +} + +static int +fs_randomfail_add_probability_range(struct randomfail_fs *fs, + const char *key, const char *value, + const char **error_r) +{ + enum fs_op op; + const char *p; + uoff_t num1, num2; + + if (strcmp(key, "read-range") == 0) + op = FS_OP_READ; + else if (strcmp(key, "write-range") == 0) + op = FS_OP_WRITE; + else if (strcmp(key, "iter-range") == 0) + op = FS_OP_ITER; + else + return 0; + + p = strchr(value, '-'); + if (p == NULL) { + if (str_to_uoff(value, &num1) < 0) { + *error_r = "Invalid range value"; + return -1; + } + num2 = num1; + } else if (str_to_uoff(t_strdup_until(value, p), &num1) < 0 || + str_to_uoff(p+1, &num2) < 0 || num1 > num2) { + *error_r = "Invalid range values"; + return -1; + } + fs->range_start[op] = num1; + fs->range_end[op] = num2; + return 1; +} + +static int fs_randomfail_parse_params(struct randomfail_fs *fs, + const char *params, const char **error_r) +{ + const char *const *tmp; + int ret; + + for (tmp = t_strsplit_spaces(params, ","); *tmp != NULL; tmp++) { + const char *key = *tmp; + const char *value = strchr(key, '='); + + if (value == NULL) { + *error_r = "Missing '='"; + return -1; + } + key = t_strdup_until(key, value++); + if ((ret = fs_randomfail_add_probability(fs, key, value, error_r)) != 0) { + if (ret < 0) + return -1; + continue; + } + if ((ret = fs_randomfail_add_probability_range(fs, key, value, error_r)) != 0) { + if (ret < 0) + return -1; + continue; + } + *error_r = t_strdup_printf("Unknown key '%s'", key); + return -1; + } + return 0; +} + +static int +fs_randomfail_init(struct fs *_fs, const char *args, + const struct fs_settings *set) +{ + struct randomfail_fs *fs = (struct randomfail_fs *)_fs; + const char *p, *parent_name, *parent_args, *error; + + p = strchr(args, ':'); + if (p == NULL) { + fs_set_error(_fs, "Randomfail parameters missing"); + return -1; + } + if (fs_randomfail_parse_params(fs, t_strdup_until(args, p++), &error) < 0) { + fs_set_error(_fs, "Invalid randomfail parameters: %s", error); + return -1; + } + args = p; + + if (*args == '\0') { + fs_set_error(_fs, "Parent filesystem not given as parameter"); + return -1; + } + + parent_args = strchr(args, ':'); + if (parent_args == NULL) { + parent_name = args; + parent_args = ""; + } else { + parent_name = t_strdup_until(args, parent_args); + parent_args++; + } + if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) { + fs_set_error(_fs, "%s: %s", parent_name, error); + return -1; + } + return 0; +} + +static void fs_randomfail_deinit(struct fs *_fs) +{ + struct randomfail_fs *fs = (struct randomfail_fs *)_fs; + + if (_fs->parent != NULL) + fs_deinit(&_fs->parent); + i_free(fs); +} + +static enum fs_properties fs_randomfail_get_properties(struct fs *_fs) +{ + return fs_get_properties(_fs->parent); +} + +static struct fs_file * +fs_randomfail_file_init(struct fs *_fs, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct randomfail_fs_file *file; + + file = i_new(struct randomfail_fs_file, 1); + file->file.fs = _fs; + file->file.path = i_strdup(path); + file->super = fs_file_init(_fs->parent, path, mode | flags); + return &file->file; +} + +static void fs_randomfail_file_deinit(struct fs_file *_file) +{ + struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + + fs_file_deinit(&file->super); + i_free(file->file.path); + i_free(file); +} + +static void fs_randomfail_file_close(struct fs_file *_file) +{ + struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + + fs_file_close(file->super); +} + +static const char *fs_randomfail_file_get_path(struct fs_file *_file) +{ + struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + + return fs_file_path(file->super); +} + From dovecot at dovecot.org Tue Jun 16 13:58:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Jun 2015 13:58:45 +0000 Subject: dovecot-2.2: lib: Compiler warning fix for test-ostream-failure-at. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8076318a7b0e changeset: 18868:8076318a7b0e user: Timo Sirainen date: Tue Jun 16 16:56:08 2015 +0300 description: lib: Compiler warning fix for test-ostream-failure-at. diffstat: src/lib/test-ostream-failure-at.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c6ed48c7f2a0 -r 8076318a7b0e src/lib/test-ostream-failure-at.c --- a/src/lib/test-ostream-failure-at.c Tue Jun 16 16:40:29 2015 +0300 +++ b/src/lib/test-ostream-failure-at.c Tue Jun 16 16:56:08 2015 +0300 @@ -22,7 +22,7 @@ buf_output = o_stream_create_buffer(buf); output = o_stream_create_failure_at(buf_output, i, TEST_ERRMSG); if (i > 0) - test_assert(o_stream_send(output, test_data, sizeof(test_data)) == i); + test_assert(o_stream_send(output, test_data, sizeof(test_data)) == (int)i); test_assert_idx(o_stream_send(output, test_data, sizeof(test_data)) == -1 && output->offset == i && output->stream_errno == EIO && From dovecot at dovecot.org Wed Jun 17 08:16:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 08:16:35 +0000 Subject: dovecot-2.2: lib: Added unix_client_connect_msecs setting to con... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/697c6f04baca changeset: 18869:697c6f04baca user: Timo Sirainen date: Tue Jun 16 17:19:12 2015 +0300 description: lib: Added unix_client_connect_msecs setting to connection API. diffstat: src/lib/connection.c | 4 +++- src/lib/connection.h | 3 +++ 2 files changed, 6 insertions(+), 1 deletions(-) diffs (28 lines): diff -r 8076318a7b0e -r 697c6f04baca src/lib/connection.c --- a/src/lib/connection.c Tue Jun 16 16:56:08 2015 +0300 +++ b/src/lib/connection.c Tue Jun 16 17:19:12 2015 +0300 @@ -266,8 +266,10 @@ if (conn->port != 0) fd = net_connect_ip(&conn->ip, conn->port, NULL); + else if (conn->list->set.unix_client_connect_msecs == 0) + fd = net_connect_unix(conn->name); else - fd = net_connect_unix(conn->name); + fd = net_connect_unix_with_retries(conn->name, conn->list->set.unix_client_connect_msecs); if (fd == -1) return -1; conn->fd_in = conn->fd_out = fd; diff -r 8076318a7b0e -r 697c6f04baca src/lib/connection.h --- a/src/lib/connection.h Tue Jun 16 16:56:08 2015 +0300 +++ b/src/lib/connection.h Tue Jun 16 17:19:12 2015 +0300 @@ -63,6 +63,9 @@ to make the functionality identical with inet sockets, which may simplify the calling code. */ bool delayed_unix_client_connected_callback; + /* If connect() to UNIX socket fails with EAGAIN, retry for this many + milliseconds before giving up (0 = try once) */ + unsigned int unix_client_connect_msecs; }; struct connection { From dovecot at dovecot.org Wed Jun 17 08:33:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 08:33:37 +0000 Subject: dovecot-2.2: auth: Added allow_real_nets setting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9be3fbad33c5 changeset: 18870:9be3fbad33c5 user: Timo Sirainen date: Wed Jun 17 11:31:02 2015 +0300 description: auth: Added allow_real_nets setting. The difference to allow_nets is that it matches against the connection's "real IP" rather than what the connection told was the original client's IP address (%{rip} vs %{real_rip}) diffstat: src/auth/auth-request.c | 27 ++++++++++++++++----------- 1 files changed, 16 insertions(+), 11 deletions(-) diffs (70 lines): diff -r 697c6f04baca -r 9be3fbad33c5 src/auth/auth-request.c --- a/src/auth/auth-request.c Tue Jun 16 17:19:12 2015 +0300 +++ b/src/auth/auth-request.c Wed Jun 17 11:31:02 2015 +0300 @@ -1305,8 +1305,10 @@ return TRUE; } -static void auth_request_validate_networks(struct auth_request *request, - const char *networks) +static void +auth_request_validate_networks(struct auth_request *request, + const char *name, const char *networks, + const struct ip_addr *remote_ip) { const char *const *net; struct ip_addr net_ip; @@ -1315,20 +1317,20 @@ for (net = t_strsplit_spaces(networks, ", "); *net != NULL; net++) { auth_request_log_debug(request, AUTH_SUBSYS_DB, - "allow_nets: Matching for network %s", *net); + "%s: Matching for network %s", name, *net); - if (strcmp(*net, "local") == 0 && request->remote_ip.family == 0) { + if (strcmp(*net, "local") == 0 && remote_ip->family == 0) { found = TRUE; break; } if (net_parse_range(*net, &net_ip, &bits) < 0) { auth_request_log_info(request, AUTH_SUBSYS_DB, - "allow_nets: Invalid network '%s'", *net); + "%s: Invalid network '%s'", name, *net); } - if (request->remote_ip.family != 0 && - net_is_in_network(&request->remote_ip, &net_ip, bits)) { + if (remote_ip->family != 0 && + net_is_in_network(remote_ip, &net_ip, bits)) { found = TRUE; break; } @@ -1336,12 +1338,13 @@ if (found) ; - else if (request->remote_ip.family == 0) { + else if (remote_ip->family == 0) { auth_request_log_info(request, AUTH_SUBSYS_DB, - "allow_nets check failed: Remote IP not known and 'local' missing"); + "%s check failed: Remote IP not known and 'local' missing", name); } else if (!found) { auth_request_log_info(request, AUTH_SUBSYS_DB, - "allow_nets check failed: IP not in allowed networks"); + "%s check failed: IP %s not in allowed networks", + name, net_ip2addr(remote_ip)); } request->failed = !found; } @@ -1465,7 +1468,9 @@ } else if (strcmp(name, "login_user") == 0) { request->requested_login_user = p_strdup(request->pool, value); } else if (strcmp(name, "allow_nets") == 0) { - auth_request_validate_networks(request, value); + auth_request_validate_networks(request, name, value, &request->remote_ip); + } else if (strcmp(name, "allow_real_nets") == 0) { + auth_request_validate_networks(request, name, value, &request->real_remote_ip); } else if (strncmp(name, "userdb_", 7) == 0) { /* for prefetch userdb */ request->userdb_prefetch_set = TRUE; From dovecot at dovecot.org Wed Jun 17 08:45:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 08:45:41 +0000 Subject: dovecot-2.2: lib: test-data-stack - ensure t_push() and t_pop() ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/69b231fdf5d7 changeset: 18871:69b231fdf5d7 user: Phil Carmody date: Wed Jun 17 11:42:30 2015 +0300 description: lib: test-data-stack - ensure t_push() and t_pop() are balanced in fatal tests If the t_pop() unexpectedly succeeds, we won't want to do another one upon entering the function again. Signed-off-by: Phil Carmody diffstat: src/lib/test-data-stack.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (35 lines): diff -r 9be3fbad33c5 -r 69b231fdf5d7 src/lib/test-data-stack.c --- a/src/lib/test-data-stack.c Wed Jun 17 11:31:02 2015 +0300 +++ b/src/lib/test-data-stack.c Wed Jun 17 11:42:30 2015 +0300 @@ -158,13 +158,13 @@ /* Presume that we need to clean up from the prior test: undo the evil write, then we will be able to t_pop cleanly, and finally we can end the test stanza. */ - if (things_are_messed_up || undo_ptr == NULL || t_id == 999999999) + if (things_are_messed_up || undo_ptr == NULL) return FATAL_TEST_ABORT; /* abort, things are messed up with t_pop */ *undo_ptr = undo_data; undo_ptr = NULL; /* t_pop musn't abort, that would cause recursion */ things_are_messed_up = TRUE; - if (t_pop() != t_id) + if (t_id != 999999999 && t_pop() != t_id) return FATAL_TEST_ABORT; /* abort, things are messed up with us */ things_are_messed_up = FALSE; t_id = 999999999; @@ -206,6 +206,7 @@ *undo_ptr = '*'; /* t_pop will now fail */ (void)t_pop(); + t_id = 999999999; /* We're FUBAR, mustn't pop next entry */ return FATAL_TEST_FAILURE; } @@ -218,6 +219,7 @@ *undo_ptr = '*'; /* t_pop will now fail */ (void)t_pop(); + t_id = 999999999; /* We're FUBAR, mustn't pop next entry */ return FATAL_TEST_FAILURE; } From dovecot at dovecot.org Wed Jun 17 08:45:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 08:45:56 +0000 Subject: dovecot-2.2: lib: test-data-stack - simplify #if-ing out of DEBU... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e7c896eff939 changeset: 18872:e7c896eff939 user: Phil Carmody date: Wed Jun 17 11:42:53 2015 +0300 description: lib: test-data-stack - simplify #if-ing out of DEBUG-only fatal test With no canaries, nothing can be tested, so just reduce this to a trivial 1-line return on the non-DEBUG case. Signed-off-by: Phil Carmody diffstat: src/lib/test-data-stack.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (32 lines): diff -r 69b231fdf5d7 -r e7c896eff939 src/lib/test-data-stack.c --- a/src/lib/test-data-stack.c Wed Jun 17 11:42:30 2015 +0300 +++ b/src/lib/test-data-stack.c Wed Jun 17 11:42:53 2015 +0300 @@ -148,6 +148,7 @@ enum fatal_test_state fatal_data_stack(int stage) { +#ifdef DEBUG /* If we abort, then we'll be left with a dangling t_push() keep a record of our temporary stack id, so we can clean up. */ static unsigned int t_id = 999999999; @@ -172,7 +173,6 @@ } switch(stage) { -#ifdef DEBUG case 0: { unsigned char *p; test_begin("fatal data-stack underrun"); @@ -223,9 +223,11 @@ return FATAL_TEST_FAILURE; } -#endif default: things_are_messed_up = TRUE; return FATAL_TEST_FINISHED; } +#else + return stage == 0 ? FATAL_TEST_FINISHED : FATAL_TEST_ABORT; +#endif } From dovecot at dovecot.org Wed Jun 17 09:24:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 09:24:55 +0000 Subject: dovecot-2.2: lib-fs: Call o_stream_nfinish() automatically for a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/21c0ffec1d2b changeset: 18873:21c0ffec1d2b user: Timo Sirainen date: Wed Jun 17 12:21:52 2015 +0300 description: lib-fs: Call o_stream_nfinish() automatically for all backends in fs_write_stream_finish() Otherwise each backend needs to do it internally. fs-metawrap for example was missing this and causing assert-crashes. diffstat: src/lib-fs/fs-api.c | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diffs (28 lines): diff -r e7c896eff939 -r 21c0ffec1d2b src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Wed Jun 17 11:42:53 2015 +0300 +++ b/src/lib-fs/fs-api.c Wed Jun 17 12:21:52 2015 +0300 @@ -557,14 +557,23 @@ int fs_write_stream_finish(struct fs_file *file, struct ostream **output) { int ret; + bool success = TRUE; i_assert(*output == file->output || *output == NULL); *output = NULL; if (file->output != NULL) o_stream_uncork(file->output); + if (file->output != NULL) { + if (o_stream_nfinish(file->output) < 0) { + fs_set_error(file->fs, "write(%s) failed: %s", + o_stream_get_name(file->output), + o_stream_get_error(file->output)); + success = FALSE; + } + } T_BEGIN { - ret = file->fs->v.write_stream_finish(file, TRUE); + ret = file->fs->v.write_stream_finish(file, success); } T_END; if (ret != 0) file->metadata_changed = FALSE; From dovecot at dovecot.org Wed Jun 17 09:24:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 09:24:56 +0000 Subject: dovecot-2.2: lib-fs: Removed redundant o_stream_nfinish() calls Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/343602625064 changeset: 18874:343602625064 user: Timo Sirainen date: Wed Jun 17 12:22:17 2015 +0300 description: lib-fs: Removed redundant o_stream_nfinish() calls diffstat: src/lib-fs/fs-posix.c | 10 ---------- src/lib-fs/fs-randomfail.c | 6 ------ src/lib-fs/fs-sis.c | 2 -- 3 files changed, 0 insertions(+), 18 deletions(-) diffs (62 lines): diff -r 21c0ffec1d2b -r 343602625064 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Wed Jun 17 12:21:52 2015 +0300 +++ b/src/lib-fs/fs-posix.c Wed Jun 17 12:22:17 2015 +0300 @@ -47,7 +47,6 @@ bool seek_to_beginning; bool success; - bool open_failed; }; struct posix_fs_lock { @@ -475,7 +474,6 @@ } else if (file->fd == -1 && fs_posix_open(file) < 0) { _file->output = o_stream_create_error_str(errno, "%s", fs_file_last_error(_file)); - file->open_failed = TRUE; } else { _file->output = o_stream_create_fd_file(file->fd, (uoff_t)-1, FALSE); @@ -488,14 +486,6 @@ struct posix_fs_file *file = (struct posix_fs_file *)_file; int ret = success ? 0 : -1; - if (file->open_failed) - ret = -1; - else if (o_stream_nfinish(_file->output) < 0) { - fs_set_error(_file->fs, "write(%s) failed: %s", - o_stream_get_name(_file->output), - o_stream_get_error(_file->output)); - ret = -1; - } o_stream_destroy(&_file->output); switch (file->open_mode) { diff -r 21c0ffec1d2b -r 343602625064 src/lib-fs/fs-randomfail.c --- a/src/lib-fs/fs-randomfail.c Wed Jun 17 12:21:52 2015 +0300 +++ b/src/lib-fs/fs-randomfail.c Wed Jun 17 12:22:17 2015 +0300 @@ -382,12 +382,6 @@ struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; if (_file->output != NULL) { - if (o_stream_nfinish(_file->output) < 0) { - fs_set_error(_file->fs, "write(%s) failed: %s", - o_stream_get_name(_file->output), - o_stream_get_error(_file->output)); - success = FALSE; - } if (_file->output == file->super_output) _file->output = NULL; else diff -r 21c0ffec1d2b -r 343602625064 src/lib-fs/fs-sis.c --- a/src/lib-fs/fs-sis.c Wed Jun 17 12:21:52 2015 +0300 +++ b/src/lib-fs/fs-sis.c Wed Jun 17 12:22:17 2015 +0300 @@ -375,8 +375,6 @@ { struct sis_fs_file *file = (struct sis_fs_file *)_file; - if (o_stream_nfinish(_file->output) < 0) - success = FALSE; if (!success) { if (file->super != NULL) { fs_write_stream_abort(file->super, &file->fs_output); From dovecot at dovecot.org Wed Jun 17 10:15:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 10:15:24 +0000 Subject: dovecot-2.2: fts-tika: Fixed crashes when indexing larger attach... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3ae8cf3f8320 changeset: 18875:3ae8cf3f8320 user: Timo Sirainen date: Wed Jun 17 13:12:37 2015 +0300 description: fts-tika: Fixed crashes when indexing larger attachments with fts-solr. We were mixing ioloops without switching back to the original ioloop in the middle. Also io_remove() at deinit caused another timeout to be added, so this needs to be done while original ioloop is active or we'll just leak the just-added timeout in io_loop_destroy() and crash later. diffstat: src/plugins/fts/fts-parser-tika.c | 29 +++++++++++++++++++---------- 1 files changed, 19 insertions(+), 10 deletions(-) diffs (65 lines): diff -r 343602625064 -r 3ae8cf3f8320 src/plugins/fts/fts-parser-tika.c --- a/src/plugins/fts/fts-parser-tika.c Wed Jun 17 12:22:17 2015 +0300 +++ b/src/plugins/fts/fts-parser-tika.c Wed Jun 17 13:12:37 2015 +0300 @@ -148,6 +148,7 @@ struct message_block *block) { struct tika_fts_parser *parser = (struct tika_fts_parser *)_parser; + struct ioloop *prev_ioloop = current_ioloop; const unsigned char *data; size_t size; ssize_t ret; @@ -174,10 +175,11 @@ return; i_assert(parser->payload != NULL); } - /* continue returning data from Tika */ + /* continue returning data from Tika. we'll create a new ioloop just + for reading this one payload. */ while ((ret = i_stream_read_data(parser->payload, &data, &size, 0)) == 0) { if (parser->failed) - return; + break; /* wait for more input from Tika */ if (parser->ioloop == NULL) { parser->ioloop = io_loop_create(); @@ -188,7 +190,12 @@ } io_loop_run(current_ioloop); } - if (size > 0) { + /* switch back to original ioloop. */ + io_loop_set_current(prev_ioloop); + + if (parser->failed) + ; + else if (size > 0) { i_assert(ret > 0); block->data = data; block->size = size; @@ -210,16 +217,18 @@ struct tika_fts_parser *parser = (struct tika_fts_parser *)_parser; int ret = parser->failed ? -1 : 0; + /* remove io before unrefing payload - otherwise lib-http adds another + timeout to ioloop unnecessarily */ + if (parser->payload != NULL) + i_stream_unref(&parser->payload); + if (parser->io != NULL) + io_remove(&parser->io); + if (parser->http_req != NULL) + http_client_request_abort(&parser->http_req); if (parser->ioloop != NULL) { - io_remove(&parser->io); + io_loop_set_current(parser->ioloop); io_loop_destroy(&parser->ioloop); } - if (parser->payload != NULL) - i_stream_unref(&parser->payload); - /* FIXME: kludgy, http_req should be NULL here if we don't want to - free it. requires lib-http changes. */ - if (parser->http_req != NULL) - http_client_request_abort(&parser->http_req); i_free(parser); return ret; } From dovecot at dovecot.org Wed Jun 17 10:58:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 10:58:14 +0000 Subject: dovecot-2.2: fts-solr: Removed assert to fix indexing multiple m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fec87fb9b071 changeset: 18876:fec87fb9b071 user: Timo Sirainen date: Wed Jun 17 13:55:37 2015 +0300 description: fts-solr: Removed assert to fix indexing multiple mailboxes. fts plugin may want to do last-uid lookup for a new mailbox while we still have posting open for the previous one. diffstat: src/plugins/fts-solr/solr-connection.c | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (12 lines): diff -r 3ae8cf3f8320 -r fec87fb9b071 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Wed Jun 17 13:12:37 2015 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Wed Jun 17 13:55:37 2015 +0300 @@ -409,8 +409,6 @@ const char *url; int parse_ret; - i_assert(!conn->posting); - memset(&solr_lookup_context, 0, sizeof(solr_lookup_context)); solr_lookup_context.result_pool = pool; hash_table_create(&solr_lookup_context.mailboxes, default_pool, 0, From dovecot at dovecot.org Wed Jun 17 11:28:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 11:28:31 +0000 Subject: dovecot-2.2: fts-solr: Avoid sending too large queries to Solr d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1e6f38e8d5d5 changeset: 18877:1e6f38e8d5d5 user: Timo Sirainen date: Wed Jun 17 14:04:42 2015 +0300 description: fts-solr: Avoid sending too large queries to Solr due to listing wanted mailboxes. This is mainly a problem if there are a lot of mailboxes and "All Mails" virtual mailbox. For now hardcoded to send max 10 mailboxes in the query, afterwards it just returns matches in all the mailboxes and we'll filter out the unwanted mailboxes. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 26 +++++++++++++++++++------- 1 files changed, 19 insertions(+), 7 deletions(-) diffs (65 lines): diff -r fec87fb9b071 -r 1e6f38e8d5d5 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Wed Jun 17 13:55:37 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Wed Jun 17 14:04:42 2015 +0300 @@ -26,6 +26,8 @@ header fields as long as they're smaller than this */ #define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024 +#define SOLR_QUERY_MAX_MAILBOX_COUNT 10 + struct solr_fts_backend { struct fts_backend backend; struct solr_connection *solr_conn; @@ -831,6 +833,7 @@ struct mailbox *box; const char *box_guid; unsigned int i, len; + bool search_all_mailboxes; /* use a separate filter query for selecting the mailbox. it shouldn't affect the score and there could be some caching benefits too. */ @@ -841,19 +844,26 @@ str_append(str, "%22%22"); hash_table_create(&mailboxes, default_pool, 0, str_hash, strcmp); - str_append(str, "%2B("); + for (i = 0; boxes[i] != NULL; i++) ; + search_all_mailboxes = i > SOLR_QUERY_MAX_MAILBOX_COUNT; + if (!search_all_mailboxes) + str_append(str, "%2B("); len = str_len(str); + for (i = 0; boxes[i] != NULL; i++) { if (fts_mailbox_get_guid(boxes[i], &box_guid) < 0) continue; - if (str_len(str) != len) - str_append(str, "+OR+"); - str_printfa(str, "box:%s", box_guid); + if (!search_all_mailboxes) { + if (str_len(str) != len) + str_append(str, "+OR+"); + str_printfa(str, "box:%s", box_guid); + } hash_table_insert(mailboxes, t_strdup_noconst(box_guid), boxes[i]); } - str_append_c(str, ')'); + if (!search_all_mailboxes) + str_append_c(str, ')'); if (solr_connection_select(backend->solr_conn, str_c(str), result->pool, &solr_results) < 0) { @@ -865,8 +875,10 @@ for (i = 0; solr_results[i] != NULL; i++) { box = hash_table_lookup(mailboxes, solr_results[i]->box_id); if (box == NULL) { - i_warning("fts_solr: Lookup returned unexpected mailbox " - "with guid=%s", solr_results[i]->box_id); + if (!search_all_mailboxes) { + i_warning("fts_solr: Lookup returned unexpected mailbox " + "with guid=%s", solr_results[i]->box_id); + } continue; } fts_result = array_append_space(&fts_results); From dovecot at dovecot.org Wed Jun 17 11:28:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 11:28:31 +0000 Subject: dovecot-2.2: virtual: Added more asserts. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/91d8f998bf83 changeset: 18878:91d8f998bf83 user: Timo Sirainen date: Wed Jun 17 14:23:38 2015 +0300 description: virtual: Added more asserts. diffstat: src/plugins/virtual/virtual-storage.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (19 lines): diff -r 1e6f38e8d5d5 -r 91d8f998bf83 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Wed Jun 17 14:04:42 2015 +0300 +++ b/src/plugins/virtual/virtual-storage.c Wed Jun 17 14:23:38 2015 +0300 @@ -628,6 +628,7 @@ while (seq_range_array_iter_nth(&iter, n++, &uid)) { while (i < count && uids[i].real_uid < uid) i++; if (i < count && uids[i].real_uid == uid) { + i_assert(uids[i].virtual_uid > 0); seq_range_array_add(virtual_uids_r, uids[i].virtual_uid); i++; @@ -667,6 +668,7 @@ array_append(virtual_uids_r, &zero, 1); } else { + i_assert(uids[i].virtual_uid > 0); array_append(virtual_uids_r, &uids[i].virtual_uid, 1); i++; } From dovecot at dovecot.org Wed Jun 17 11:28:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 11:28:31 +0000 Subject: dovecot-2.2: virtual: Added more asserts when DEBUG is enabled. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac8fc3db0ef6 changeset: 18879:ac8fc3db0ef6 user: Timo Sirainen date: Wed Jun 17 14:24:06 2015 +0300 description: virtual: Added more asserts when DEBUG is enabled. diffstat: src/plugins/virtual/virtual-sync.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (18 lines): diff -r 91d8f998bf83 -r ac8fc3db0ef6 src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Wed Jun 17 14:23:38 2015 +0300 +++ b/src/plugins/virtual/virtual-sync.c Wed Jun 17 14:24:06 2015 +0300 @@ -1583,6 +1583,14 @@ virtual_sync_new_backend_boxes(ctx); } ret = virtual_sync_backend_add_new(ctx); +#ifdef DEBUG + for (i = 0; i < count; i++) { + const struct virtual_backend_uidmap *uidmap; + + array_foreach(&bboxes[i]->uids, uidmap) + i_assert(uidmap->virtual_uid > 0); + } +#endif array_free(&ctx->all_adds); if (array_is_created(&ctx->all_mails)) array_free(&ctx->all_mails); From dovecot at dovecot.org Wed Jun 17 11:28:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 11:28:31 +0000 Subject: dovecot-2.2: virtual: virtual_uids weren't always set to all mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d609454bdf64 changeset: 18880:d609454bdf64 user: Timo Sirainen date: Wed Jun 17 14:25:48 2015 +0300 description: virtual: virtual_uids weren't always set to all mails, causing missing mails and crashes in search. diffstat: src/plugins/virtual/virtual-sync.c | 16 +++++++--------- 1 files changed, 7 insertions(+), 9 deletions(-) diffs (40 lines): diff -r ac8fc3db0ef6 -r d609454bdf64 src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Wed Jun 17 14:24:06 2015 +0300 +++ b/src/plugins/virtual/virtual-sync.c Wed Jun 17 14:25:48 2015 +0300 @@ -1228,8 +1228,6 @@ if (bbox == NULL || bbox->mailbox_id != vrec->mailbox_id) { /* add the rest of the newly seen messages */ - i_assert(j == uidmap_count || - bbox->search_result != NULL); for (; j < uidmap_count; j++) { add_rec.rec.real_uid = uidmap[j].real_uid; array_append(&ctx->all_adds, &add_rec, 1); @@ -1247,11 +1245,6 @@ add_rec.rec.mailbox_id = bbox->mailbox_id; bbox->sync_seen = TRUE; } - if (bbox->search_result == NULL) { - /* mailbox is completely unchanged since last sync */ - j = uidmap_count; - continue; - } mail_index_lookup_uid(ctx->sync_view, vseq, &vuid); /* if virtual record doesn't exist in uidmap, it's expunged */ @@ -1268,8 +1261,13 @@ else { /* exists - update uidmap and flags */ uidmap[j++].virtual_uid = vuid; - virtual_sync_external_flags(ctx, bbox, vseq, - vrec->real_uid); + if (bbox->search_result == NULL) { + /* mailbox is completely unchanged since last + sync - no need to sync flags */ + } else { + virtual_sync_external_flags(ctx, bbox, vseq, + vrec->real_uid); + } } } i_free(vmails); From dovecot at dovecot.org Wed Jun 17 13:40:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 17 Jun 2015 13:40:14 +0000 Subject: dovecot-2.2: fs-posix: Hide temporary files from fs_iter_*() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7f7b77feb9ce changeset: 18881:7f7b77feb9ce user: Timo Sirainen date: Wed Jun 17 16:37:37 2015 +0300 description: fs-posix: Hide temporary files from fs_iter_*() Those could exist just because another process is just using them to create new files to the directory. Although they could also be older files caused by earlier crashes. It would probably be a good idea to have fs_iter_*() delete the old temporary files automatically. diffstat: src/lib-fs/fs-posix.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (37 lines): diff -r d609454bdf64 -r 7f7b77feb9ce src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Wed Jun 17 14:25:48 2015 +0300 +++ b/src/lib-fs/fs-posix.c Wed Jun 17 16:37:37 2015 +0300 @@ -32,6 +32,7 @@ struct posix_fs { struct fs fs; char *temp_file_prefix, *root_path, *path_prefix; + unsigned int temp_file_prefix_len; enum fs_posix_lock_method lock_method; mode_t mode, dir_mode; }; @@ -79,6 +80,7 @@ fs->temp_file_prefix = set->temp_file_prefix != NULL ? i_strdup(set->temp_file_prefix) : i_strdup("temp.dovecot."); + fs->temp_file_prefix_len = strlen(fs->temp_file_prefix); fs->root_path = i_strdup(set->root_path); fs->fs.set.temp_file_prefix = fs->temp_file_prefix; fs->fs.set.root_path = fs->root_path; @@ -731,6 +733,7 @@ static const char *fs_posix_iter_next(struct fs_iter *_iter) { struct posix_fs_iter *iter = (struct posix_fs_iter *)_iter; + struct posix_fs *fs = (struct posix_fs *)_iter->fs; struct dirent *d; if (iter->dir == NULL) @@ -741,6 +744,9 @@ if (strcmp(d->d_name, ".") == 0 || strcmp(d->d_name, "..") == 0) continue; + if (strncmp(d->d_name, fs->temp_file_prefix, + fs->temp_file_prefix_len) == 0) + continue; #ifdef HAVE_DIRENT_D_TYPE switch (d->d_type) { case DT_UNKNOWN: From dovecot at dovecot.org Thu Jun 18 08:12:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Jun 2015 08:12:44 +0000 Subject: dovecot-2.2: lib-http: http_client_request_send() failure return... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eddc7a17dd46 changeset: 18882:eddc7a17dd46 user: Timo Sirainen date: Thu Jun 18 11:10:05 2015 +0300 description: lib-http: http_client_request_send() failure returned already-freed error string. diffstat: src/lib-http/http-client-request.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (20 lines): diff -r 7f7b77feb9ce -r eddc7a17dd46 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Wed Jun 17 16:37:37 2015 +0300 +++ b/src/lib-http/http-client-request.c Thu Jun 18 11:10:05 2015 +0300 @@ -832,11 +832,16 @@ int http_client_request_send(struct http_client_request *req, const char **error_r) { + char *errstr = NULL; int ret; T_BEGIN { ret = http_client_request_send_real(req, error_r); + if (ret < 0) + errstr = i_strdup(*error_r); } T_END; + *error_r = t_strdup(errstr); + i_free(errstr); return ret; } From dovecot at dovecot.org Tue Jun 23 09:05:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Jun 2015 09:05:01 +0000 Subject: dovecot-2.2: imapc: Don't automatically login to IMAP server for... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/771254b34a42 changeset: 18883:771254b34a42 user: Timo Sirainen date: Tue Jun 23 11:02:20 2015 +0200 description: imapc: Don't automatically login to IMAP server for list=no namespaces. diffstat: src/lib-storage/index/imapc/imapc-storage.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r eddc7a17dd46 -r 771254b34a42 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Thu Jun 18 11:10:05 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.c Tue Jun 23 11:02:20 2015 +0200 @@ -292,8 +292,10 @@ client->client = imapc_client_init(&set); imapc_client_register_untagged(client->client, imapc_storage_client_untagged_cb, client); - /* start logging in immediately */ - imapc_client_login(client->client, imapc_storage_client_login, client); + if ((ns->flags & NAMESPACE_FLAG_LIST_PREFIX) != 0) { + /* start logging in immediately */ + imapc_client_login(client->client, imapc_storage_client_login, client); + } *client_r = client; return 0; From dovecot at dovecot.org Tue Jun 23 09:15:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Jun 2015 09:15:25 +0000 Subject: dovecot-2.2: imapc: If login is aborted at deinit, don't log an ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f06008be0d1d changeset: 18884:f06008be0d1d user: Timo Sirainen date: Tue Jun 23 11:12:38 2015 +0200 description: imapc: If login is aborted at deinit, don't log an error. diffstat: src/lib-storage/index/imapc/imapc-list.c | 2 ++ src/lib-storage/index/imapc/imapc-storage.c | 8 ++++++++ src/lib-storage/index/imapc/imapc-storage.h | 1 + 3 files changed, 11 insertions(+), 0 deletions(-) diffs (48 lines): diff -r 771254b34a42 -r f06008be0d1d src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Tue Jun 23 11:02:20 2015 +0200 +++ b/src/lib-storage/index/imapc/imapc-list.c Tue Jun 23 11:12:38 2015 +0200 @@ -87,6 +87,8 @@ { struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + list->client->destroying = TRUE; + /* make sure all pending commands are aborted before anything is deinitialized */ if (list->client != NULL) { diff -r 771254b34a42 -r f06008be0d1d src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Tue Jun 23 11:02:20 2015 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Tue Jun 23 11:12:38 2015 +0200 @@ -207,6 +207,12 @@ if (reply->state == IMAPC_COMMAND_STATE_OK) return; + if (client->destroying && + reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + /* user's work was finished before imapc login finished - + it's not an error */ + return; + } i_error("imapc: Authentication failed: %s", reply->text_full); client->auth_failed = TRUE; @@ -350,6 +356,8 @@ { struct imapc_storage *storage = (struct imapc_storage *)_storage; + storage->client->destroying = TRUE; + /* make sure all pending commands are aborted before anything is deinitialized */ imapc_client_disconnect(storage->client->client); diff -r 771254b34a42 -r f06008be0d1d src/lib-storage/index/imapc/imapc-storage.h --- a/src/lib-storage/index/imapc/imapc-storage.h Tue Jun 23 11:02:20 2015 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.h Tue Jun 23 11:12:38 2015 +0200 @@ -52,6 +52,7 @@ ARRAY(struct imapc_storage_event_callback) untagged_callbacks; unsigned int auth_failed:1; + unsigned int destroying:1; }; struct imapc_storage { From dovecot at dovecot.org Tue Jun 23 09:34:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Jun 2015 09:34:29 +0000 Subject: dovecot-2.2: imapc: Don't return mailbox list entries that would... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/563b93302831 changeset: 18885:563b93302831 user: Timo Sirainen date: Tue Jun 23 11:30:27 2015 +0200 description: imapc: Don't return mailbox list entries that would result in name="". So basically don't return namespace prefix if imapc_list_prefix="". diffstat: src/lib-storage/index/imapc/imapc-list.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (27 lines): diff -r f06008be0d1d -r 563b93302831 src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Tue Jun 23 11:12:38 2015 +0200 +++ b/src/lib-storage/index/imapc/imapc-list.c Tue Jun 23 11:30:27 2015 +0200 @@ -679,6 +679,8 @@ { struct imapc_mailbox_list_iterate_context *ctx = (struct imapc_mailbox_list_iterate_context *)_ctx; + struct imapc_mailbox_list *list = + (struct imapc_mailbox_list *)_ctx->list; struct mailbox_node *node; const char *vname; @@ -694,6 +696,14 @@ return NULL; } while ((node->flags & MAILBOX_MATCHED) == 0); + if (ctx->info.ns->prefix > 0 && + strncmp(vname, ctx->info.ns->prefix, ctx->info.ns->prefix_len-1) == 0 && + vname[ctx->info.ns->prefix_len] == '\0' && + list->set->imapc_list_prefix[0] == '\0') { + /* don't return "" name */ + return imapc_list_iter_next(_ctx); + } + ctx->info.vname = vname; ctx->info.flags = node->flags; if ((_ctx->list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { From dovecot at dovecot.org Tue Jun 23 09:34:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Jun 2015 09:34:30 +0000 Subject: dovecot-2.2: lib-storage: Set MAILBOX_SELECT/NONEXISTENT for nam... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e6cb25855abf changeset: 18886:e6cb25855abf user: Timo Sirainen date: Tue Jun 23 11:31:21 2015 +0200 description: lib-storage: Set MAILBOX_SELECT/NONEXISTENT for namespace prefix even if no flags are wanted. This fixes doveadm commands attempting to access such nonexistent mailbox prefixes. diffstat: src/lib-storage/list/mailbox-list-iter.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r 563b93302831 -r e6cb25855abf src/lib-storage/list/mailbox-list-iter.c --- a/src/lib-storage/list/mailbox-list-iter.c Tue Jun 23 11:30:27 2015 +0200 +++ b/src/lib-storage/list/mailbox-list-iter.c Tue Jun 23 11:31:21 2015 +0200 @@ -501,8 +501,7 @@ } } - if ((ctx->ctx.flags & MAILBOX_LIST_ITER_RETURN_NO_FLAGS) == 0 && - (ctx->ns_info.flags & MAILBOX_SELECT) == 0) { + if ((ctx->ns_info.flags & MAILBOX_SELECT) == 0) { /* see if namespace prefix is selectable */ box = mailbox_alloc(ns->list, ctx->ns_info.vname, 0); if (mailbox_exists(box, TRUE, &existence) == 0 && From dovecot at dovecot.org Wed Jun 24 05:57:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Jun 2015 05:57:23 +0000 Subject: dovecot-2.2: imapc: Fixed check to see if prefix!="" in previous... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/459cced0a15d changeset: 18887:459cced0a15d user: Timo Sirainen date: Wed Jun 24 07:54:35 2015 +0200 description: imapc: Fixed check to see if prefix!="" in previous commit. diffstat: src/lib-storage/index/imapc/imapc-list.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e6cb25855abf -r 459cced0a15d src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Tue Jun 23 11:31:21 2015 +0200 +++ b/src/lib-storage/index/imapc/imapc-list.c Wed Jun 24 07:54:35 2015 +0200 @@ -696,7 +696,7 @@ return NULL; } while ((node->flags & MAILBOX_MATCHED) == 0); - if (ctx->info.ns->prefix > 0 && + if (ctx->info.ns->prefix_len > 0 && strncmp(vname, ctx->info.ns->prefix, ctx->info.ns->prefix_len-1) == 0 && vname[ctx->info.ns->prefix_len] == '\0' && list->set->imapc_list_prefix[0] == '\0') { From dovecot at dovecot.org Mon Jun 29 08:46:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Jun 2015 08:46:52 +0000 Subject: dovecot-2.2: indexer: Fixed crash at deinit if there were still ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5945ba000a45 changeset: 18888:5945ba000a45 user: Timo Sirainen date: Mon Jun 29 11:44:00 2015 +0300 description: indexer: Fixed crash at deinit if there were still queued requests. diffstat: src/indexer/indexer.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 459cced0a15d -r 5945ba000a45 src/indexer/indexer.c --- a/src/indexer/indexer.c Wed Jun 24 07:54:35 2015 +0200 +++ b/src/indexer/indexer.c Mon Jun 29 11:44:00 2015 +0300 @@ -59,6 +59,11 @@ struct worker_connection *conn; struct indexer_request *request; + if (worker_pool == NULL) { + /* deinitializing */ + return; + } + while ((request = indexer_queue_request_peek(queue)) != NULL) { conn = worker_pool_find_username_connection(worker_pool, request->username); From dovecot at dovecot.org Mon Jun 29 09:08:18 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Jun 2015 09:08:18 +0000 Subject: dovecot-2.2: indexer: Reverted 5945ba000a45 - it didn't really help Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1458276c0a07 changeset: 18889:1458276c0a07 user: Timo Sirainen date: Mon Jun 29 12:00:05 2015 +0300 description: indexer: Reverted 5945ba000a45 - it didn't really help diffstat: src/indexer/indexer.c | 5 ----- 1 files changed, 0 insertions(+), 5 deletions(-) diffs (15 lines): diff -r 5945ba000a45 -r 1458276c0a07 src/indexer/indexer.c --- a/src/indexer/indexer.c Mon Jun 29 11:44:00 2015 +0300 +++ b/src/indexer/indexer.c Mon Jun 29 12:00:05 2015 +0300 @@ -59,11 +59,6 @@ struct worker_connection *conn; struct indexer_request *request; - if (worker_pool == NULL) { - /* deinitializing */ - return; - } - while ((request = indexer_queue_request_peek(queue)) != NULL) { conn = worker_pool_find_username_connection(worker_pool, request->username); From dovecot at dovecot.org Mon Jun 29 09:08:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Jun 2015 09:08:19 +0000 Subject: dovecot-2.2: indexer: Properly fix crashing at deinit when there... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/85822087fe3d changeset: 18890:85822087fe3d user: Timo Sirainen date: Mon Jun 29 12:05:25 2015 +0300 description: indexer: Properly fix crashing at deinit when there are pending requests. diffstat: src/indexer/indexer-queue.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diffs (19 lines): diff -r 1458276c0a07 -r 85822087fe3d src/indexer/indexer-queue.c --- a/src/indexer/indexer-queue.c Mon Jun 29 12:00:05 2015 +0300 +++ b/src/indexer/indexer-queue.c Mon Jun 29 12:05:25 2015 +0300 @@ -235,6 +235,15 @@ void indexer_queue_cancel_all(struct indexer_queue *queue) { struct indexer_request *request; + struct hash_iterate_context *iter; + + /* remove all reindex-markers so when the current requests finish + (or are cancelled) we don't try to retry them (especially during + deinit where it crashes) */ + iter = hash_table_iterate_init(queue->requests); + while (hash_table_iterate(iter, queue->requests, &request, &request)) + request->reindex_head = request->reindex_tail = FALSE; + hash_table_iterate_deinit(&iter); while ((request = indexer_queue_request_peek(queue)) != NULL) { indexer_queue_request_remove(queue); From dovecot at dovecot.org Mon Jun 29 10:08:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Jun 2015 10:08:50 +0000 Subject: dovecot-2.2: lib: Fixed read buffer overflow in wildcard_match*() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/740935acc0f8 changeset: 18891:740935acc0f8 user: Timo Sirainen date: Mon Jun 29 13:06:03 2015 +0300 description: lib: Fixed read buffer overflow in wildcard_match*() Patch by Hanno B?ck. Note that input to wildard_match*() is always coming only from trusted sources, like config file or doveadm commands. diffstat: src/lib/wildcard-match.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (18 lines): diff -r 85822087fe3d -r 740935acc0f8 src/lib/wildcard-match.c --- a/src/lib/wildcard-match.c Mon Jun 29 12:05:25 2015 +0300 +++ b/src/lib/wildcard-match.c Mon Jun 29 13:06:03 2015 +0300 @@ -35,10 +35,10 @@ return ma[0] == '\0' ? MATCH : NOMATCH; } /* find the end of each string */ - while (*(++mask)); - mask--; - while (*(++data)); - data--; + while (*(mask++)); + mask-=2; + while (*(data++)); + data-=2; while (data >= na) { /* If the mask runs out of chars before the string, fall back on From dovecot at dovecot.org Mon Jun 29 10:39:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Jun 2015 10:39:28 +0000 Subject: dovecot-2.2: lib: Fixed unit test with big-endian CPUs. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b268e186d029 changeset: 18892:b268e186d029 user: Timo Sirainen date: Mon Jun 29 13:36:38 2015 +0300 description: lib: Fixed unit test with big-endian CPUs. Patch by Michal Hlavinka / Redhat diffstat: src/lib/test-net.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 740935acc0f8 -r b268e186d029 src/lib/test-net.c --- a/src/lib/test-net.c Mon Jun 29 13:06:03 2015 +0300 +++ b/src/lib/test-net.c Mon Jun 29 13:36:38 2015 +0300 @@ -56,7 +56,7 @@ test_begin("net_ip2addr()"); test_assert(net_addr2ip("127.0.0.1", &ip) == 0 && ip.family == AF_INET && - ip.u.ip4.s_addr == (127 | (1 << 24))); + ntohl(ip.u.ip4.s_addr) == (0x7f000001)); #ifdef HAVE_IPV6 test_assert(net_addr2ip("::5", &ip) == 0 && ip.family == AF_INET6 && From dovecot at dovecot.org Mon Jun 29 17:28:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Jun 2015 17:28:05 +0000 Subject: dovecot-2.2: imap/pop3-login: If auth failure reason already beg... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/86b75d360241 changeset: 18893:86b75d360241 user: Timo Sirainen date: Mon Jun 29 20:25:15 2015 +0300 description: imap/pop3-login: If auth failure reason already begins with [resp-code], don't prefix it with another one. diffstat: src/imap-login/client-authenticate.c | 8 ++++++-- src/pop3-login/client.c | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diffs (33 lines): diff -r b268e186d029 -r 86b75d360241 src/imap-login/client-authenticate.c --- a/src/imap-login/client-authenticate.c Mon Jun 29 13:36:38 2015 +0300 +++ b/src/imap-login/client-authenticate.c Mon Jun 29 20:25:15 2015 +0300 @@ -81,8 +81,12 @@ client_send_reply(client, IMAP_CMD_REPLY_BAD, text); break; case CLIENT_AUTH_RESULT_AUTHFAILED_REASON: - client_send_reply_code(client, IMAP_CMD_REPLY_NO, - "ALERT", text); + if (text[0] == '[') + client_send_reply(client, IMAP_CMD_REPLY_NO, text); + else { + client_send_reply_code(client, IMAP_CMD_REPLY_NO, + "ALERT", text); + } break; case CLIENT_AUTH_RESULT_AUTHZFAILED: client_send_reply_code(client, IMAP_CMD_REPLY_NO, diff -r b268e186d029 -r 86b75d360241 src/pop3-login/client.c --- a/src/pop3-login/client.c Mon Jun 29 13:36:38 2015 +0300 +++ b/src/pop3-login/client.c Mon Jun 29 20:25:15 2015 +0300 @@ -243,7 +243,10 @@ prefix = "-ERR [SYS/TEMP]"; break; case POP3_CMD_REPLY_AUTH_ERROR: - prefix = "-ERR [AUTH]"; + if (text[0] == '[') + prefix = "-ERR"; + else + prefix = "-ERR [AUTH]"; break; case POP3_CMD_REPLY_ERROR: break; From dovecot at dovecot.org Tue Jun 30 08:29:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Jun 2015 08:29:44 +0000 Subject: dovecot-2.2: master: When sending SIGTERM/SIGKILL to processes, ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5631687b52ef changeset: 18894:5631687b52ef user: Timo Sirainen date: Tue Jun 30 11:26:47 2015 +0300 description: master: When sending SIGTERM/SIGKILL to processes, log which services they were sent to. diffstat: src/master/service.c | 12 ++++++++++-- 1 files changed, 10 insertions(+), 2 deletions(-) diffs (40 lines): diff -r 86b75d360241 -r 5631687b52ef src/master/service.c --- a/src/master/service.c Mon Jun 29 20:25:15 2015 +0300 +++ b/src/master/service.c Tue Jun 30 11:26:47 2015 +0300 @@ -16,7 +16,7 @@ #include #include -#define SERVICE_DIE_TIMEOUT_MSECS (1000*60) +#define SERVICE_DIE_TIMEOUT_MSECS (1000*6) #define SERVICE_LOGIN_NOTIFY_MIN_INTERVAL_SECS 2 HASH_TABLE_TYPE(pid_process) service_pids; @@ -508,6 +508,7 @@ void service_signal(struct service *service, int signo) { struct service_process *process = service->processes; + unsigned int count = 0; for (; process != NULL; process = process->next) { i_assert(process->service == service); @@ -518,11 +519,18 @@ continue; } - if (kill(process->pid, signo) < 0 && errno != ESRCH) { + if (kill(process->pid, signo) == 0) + count++; + else if (errno != ESRCH) { service_error(service, "kill(%s, %d) failed: %m", dec2str(process->pid), signo); } } + if (count > 0) { + i_warning("Sent %s to %u %s processes", + signo == SIGTERM ? "SIGTERM" : "SIGKILL", + count, service->set->name); + } } static void service_login_notify_send(struct service *service) From dovecot at dovecot.org Tue Jun 30 11:01:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Jun 2015 11:01:07 +0000 Subject: dovecot-2.2: lib: istream_file.stat() fails, set stream_errno an... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d4422b0560fe changeset: 18895:d4422b0560fe user: Timo Sirainen date: Tue Jun 30 12:17:35 2015 +0300 description: lib: istream_file.stat() fails, set stream_errno and error string. For now we'll also keep logging the error since everybody isn't using i_stream_get_error(). diffstat: src/lib/istream-file.c | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 5631687b52ef -r d4422b0560fe src/lib/istream-file.c --- a/src/lib/istream-file.c Tue Jun 30 11:26:47 2015 +0300 +++ b/src/lib/istream-file.c Tue Jun 30 12:17:35 2015 +0300 @@ -161,12 +161,18 @@ /* return defaults */ } else if (stream->fd != -1) { if (fstat(stream->fd, &stream->statbuf) < 0) { - i_error("file_istream.fstat(%s) failed: %m", name); + stream->istream.stream_errno = errno; + io_stream_set_error(&stream->iostream, + "file_istream.fstat(%s) failed: %m", name); + i_error("%s", i_stream_get_error(&stream->istream)); return -1; } } else { if (stat(name, &stream->statbuf) < 0) { - i_error("file_istream.stat(%s) failed: %m", name); + stream->istream.stream_errno = errno; + io_stream_set_error(&stream->iostream, + "file_istream.stat(%s) failed: %m", name); + i_error("%s", i_stream_get_error(&stream->istream)); return -1; } } From dovecot at dovecot.org Tue Jun 30 11:01:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Jun 2015 11:01:07 +0000 Subject: dovecot-2.2: istream filters: If parent's i_stream_stat() fails,... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac64c307d179 changeset: 18896:ac64c307d179 user: Timo Sirainen date: Tue Jun 30 12:18:08 2015 +0300 description: istream filters: If parent's i_stream_stat() fails, copy the stream_errno. This doesn't actually change any functionality yet, since most i_stream_stat() callers aren't using i_stream_get_error(). diffstat: src/lib-compression/istream-bzlib.c | 4 +++- src/lib-compression/istream-lz4.c | 4 +++- src/lib-compression/istream-lzma.c | 4 +++- src/lib-compression/istream-zlib.c | 4 +++- src/lib-fs/istream-metawrap.c | 4 +++- src/lib-mail/istream-header-filter.c | 4 +++- src/lib-storage/index/mbox/istream-raw-mbox.c | 4 +++- src/lib/istream-limit.c | 4 +++- src/lib/istream-sized.c | 4 +++- src/lib/istream.c | 8 ++++++-- src/plugins/mail-filter/istream-ext-filter.c | 4 +++- 11 files changed, 36 insertions(+), 12 deletions(-) diffs (177 lines): diff -r d4422b0560fe -r ac64c307d179 src/lib-compression/istream-bzlib.c --- a/src/lib-compression/istream-bzlib.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib-compression/istream-bzlib.c Tue Jun 30 12:18:08 2015 +0300 @@ -266,8 +266,10 @@ const struct stat *st; size_t size; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; /* when exact=FALSE always return the parent stat's size, even if we diff -r d4422b0560fe -r ac64c307d179 src/lib-compression/istream-lz4.c --- a/src/lib-compression/istream-lz4.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib-compression/istream-lz4.c Tue Jun 30 12:18:08 2015 +0300 @@ -249,8 +249,10 @@ const struct stat *st; size_t size; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; /* when exact=FALSE always return the parent stat's size, even if we diff -r d4422b0560fe -r ac64c307d179 src/lib-compression/istream-lzma.c --- a/src/lib-compression/istream-lzma.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib-compression/istream-lzma.c Tue Jun 30 12:18:08 2015 +0300 @@ -275,8 +275,10 @@ const struct stat *st; size_t size; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; /* when exact=FALSE always return the parent stat's size, even if we diff -r d4422b0560fe -r ac64c307d179 src/lib-compression/istream-zlib.c --- a/src/lib-compression/istream-zlib.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib-compression/istream-zlib.c Tue Jun 30 12:18:08 2015 +0300 @@ -417,8 +417,10 @@ const struct stat *st; size_t size; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; /* when exact=FALSE always return the parent stat's size, even if we diff -r d4422b0560fe -r ac64c307d179 src/lib-fs/istream-metawrap.c --- a/src/lib-fs/istream-metawrap.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib-fs/istream-metawrap.c Tue Jun 30 12:18:08 2015 +0300 @@ -92,8 +92,10 @@ const struct stat *st; int ret; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; if (mstream->in_metadata) { diff -r d4422b0560fe -r ac64c307d179 src/lib-mail/istream-header-filter.c --- a/src/lib-mail/istream-header-filter.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib-mail/istream-header-filter.c Tue Jun 30 12:18:08 2015 +0300 @@ -505,8 +505,10 @@ const struct stat *st; uoff_t old_offset; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; if (stream->statbuf.st_size == -1 || !exact) return 0; diff -r d4422b0560fe -r ac64c307d179 src/lib-storage/index/mbox/istream-raw-mbox.c --- a/src/lib-storage/index/mbox/istream-raw-mbox.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib-storage/index/mbox/istream-raw-mbox.c Tue Jun 30 12:18:08 2015 +0300 @@ -417,8 +417,10 @@ const struct stat *st; struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; stream->statbuf.st_size = diff -r d4422b0560fe -r ac64c307d179 src/lib/istream-limit.c --- a/src/lib/istream-limit.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib/istream-limit.c Tue Jun 30 12:18:08 2015 +0300 @@ -77,8 +77,10 @@ struct limit_istream *lstream = (struct limit_istream *) stream; const struct stat *st; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; if (lstream->v_size != (uoff_t)-1) diff -r d4422b0560fe -r ac64c307d179 src/lib/istream-sized.c --- a/src/lib/istream-sized.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib/istream-sized.c Tue Jun 30 12:18:08 2015 +0300 @@ -98,8 +98,10 @@ /* parent stream may be base64-decoder. don't waste time decoding the entire stream, since we already know what the size is supposed to be. */ - if (i_stream_stat(stream->parent, FALSE, &st) < 0) + if (i_stream_stat(stream->parent, FALSE, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; stream->statbuf.st_size = sstream->size; diff -r d4422b0560fe -r ac64c307d179 src/lib/istream.c --- a/src/lib/istream.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/lib/istream.c Tue Jun 30 12:18:08 2015 +0300 @@ -767,8 +767,10 @@ if (stream->parent == NULL) return stream->istream.stream_errno == 0 ? 0 : -1; - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; if (exact && !stream->stream_size_passthrough) { /* exact size is not known, even if parent returned something */ @@ -781,8 +783,10 @@ i_stream_default_get_size(struct istream_private *stream, bool exact, uoff_t *size_r) { - if (stream->stat(stream, exact) < 0) + if (stream->stat(stream, exact) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } if (stream->statbuf.st_size == -1) return 0; diff -r d4422b0560fe -r ac64c307d179 src/plugins/mail-filter/istream-ext-filter.c --- a/src/plugins/mail-filter/istream-ext-filter.c Tue Jun 30 12:17:35 2015 +0300 +++ b/src/plugins/mail-filter/istream-ext-filter.c Tue Jun 30 12:18:08 2015 +0300 @@ -135,8 +135,10 @@ i_assert(!exact); - if (i_stream_stat(stream->parent, exact, &st) < 0) + if (i_stream_stat(stream->parent, exact, &st) < 0) { + stream->istream.stream_errno = stream->parent->stream_errno; return -1; + } stream->statbuf = *st; return 0; } From dovecot at dovecot.org Tue Jun 30 11:01:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Jun 2015 11:01:08 +0000 Subject: dovecot-2.2: doveadm: Added doveadm_username to specify the user... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3bf5df6f63b2 changeset: 18897:3bf5df6f63b2 user: Timo Sirainen date: Tue Jun 30 13:58:15 2015 +0300 description: doveadm: Added doveadm_username to specify the username for authentication. The default is still "doveadm" and the server side doesn't currently support anything except "doveadm". diffstat: src/doveadm/doveadm-settings.c | 2 ++ src/doveadm/doveadm-settings.h | 1 + src/doveadm/server-connection.c | 2 +- 3 files changed, 4 insertions(+), 1 deletions(-) diffs (42 lines): diff -r ac64c307d179 -r 3bf5df6f63b2 src/doveadm/doveadm-settings.c --- a/src/doveadm/doveadm-settings.c Tue Jun 30 12:18:08 2015 +0300 +++ b/src/doveadm/doveadm-settings.c Tue Jun 30 13:58:15 2015 +0300 @@ -61,6 +61,7 @@ DEF(SET_UINT, doveadm_worker_count), DEF(SET_UINT, doveadm_port), { SET_ALIAS, "doveadm_proxy_port", 0, NULL }, + DEF(SET_STR, doveadm_username), DEF(SET_STR, doveadm_password), DEF(SET_STR, doveadm_allowed_commands), DEF(SET_STR, dsync_alt_char), @@ -83,6 +84,7 @@ .doveadm_socket_path = "doveadm-server", .doveadm_worker_count = 0, .doveadm_port = 0, + .doveadm_username = "doveadm", .doveadm_password = "", .doveadm_allowed_commands = "", .dsync_alt_char = "_", diff -r ac64c307d179 -r 3bf5df6f63b2 src/doveadm/doveadm-settings.h --- a/src/doveadm/doveadm-settings.h Tue Jun 30 12:18:08 2015 +0300 +++ b/src/doveadm/doveadm-settings.h Tue Jun 30 13:58:15 2015 +0300 @@ -10,6 +10,7 @@ const char *doveadm_socket_path; unsigned int doveadm_worker_count; unsigned int doveadm_port; + const char *doveadm_username; const char *doveadm_password; const char *doveadm_allowed_commands; const char *dsync_alt_char; diff -r ac64c307d179 -r 3bf5df6f63b2 src/doveadm/server-connection.c --- a/src/doveadm/server-connection.c Tue Jun 30 12:18:08 2015 +0300 +++ b/src/doveadm/server-connection.c Tue Jun 30 13:58:15 2015 +0300 @@ -243,7 +243,7 @@ } str_append_c(plain, '\0'); - str_append(plain, "doveadm"); + str_append(plain, conn->set->doveadm_username); str_append_c(plain, '\0'); str_append(plain, conn->set->doveadm_password); From dovecot at dovecot.org Tue Jun 30 11:29:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Jun 2015 11:29:21 +0000 Subject: dovecot-2.2: istream-zlib: Don't overwrite parent istream's erro... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7a6452869981 changeset: 18898:7a6452869981 user: Timo Sirainen date: Tue Jun 30 14:26:00 2015 +0300 description: istream-zlib: Don't overwrite parent istream's error on gz header/trailer read errors. diffstat: src/lib-compression/istream-zlib.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r 3bf5df6f63b2 -r 7a6452869981 src/lib-compression/istream-zlib.c --- a/src/lib-compression/istream-zlib.c Tue Jun 30 13:58:15 2015 +0300 +++ b/src/lib-compression/istream-zlib.c Tue Jun 30 14:26:00 2015 +0300 @@ -75,7 +75,8 @@ ret = i_stream_read_data(stream->parent, &data, &size, zstream->prev_size); if (size == zstream->prev_size) { - if (ret == -1) { + stream->istream.stream_errno = stream->parent->stream_errno; + if (ret == -1 && stream->istream.stream_errno == 0) { zlib_read_error(zstream, "missing gz header"); stream->istream.stream_errno = EINVAL; } @@ -141,7 +142,8 @@ ret = i_stream_read_data(stream->parent, &data, &size, GZ_TRAILER_SIZE-1); if (size == zstream->prev_size) { - if (ret == -1) { + stream->istream.stream_errno = stream->parent->stream_errno; + if (ret == -1 && stream->istream.stream_errno == 0) { zlib_read_error(zstream, "missing gz trailer"); stream->istream.stream_errno = EINVAL; }