dovecot-2.2: lib-fts: tr29 cleanup - token can never be empty by...

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.2/rev/c9b2d4e228f5
changeset: 18770:c9b2d4e228f5
user:      Timo Sirainen <tss at iki.fi>
date:      Mon Jun 01 21:10:11 2015 +0300
description:
lib-fts: tr29 cleanup - token can never be empty by the time it's being returned.

diffstat:

 src/lib-fts/fts-tokenizer-generic.c |  17 ++++++++++-------
 1 files changed, 10 insertions(+), 7 deletions(-)

diffs (53 lines):

diff -r 3ff93cabcac3 -r c9b2d4e228f5 src/lib-fts/fts-tokenizer-generic.c
--- a/src/lib-fts/fts-tokenizer-generic.c	Mon Jun 01 21:08:27 2015 +0300
+++ b/src/lib-fts/fts-tokenizer-generic.c	Mon Jun 01 21:10:11 2015 +0300
@@ -547,7 +547,7 @@
 
 	return FALSE;
 }
-static bool
+static void
 fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok,
                                          const char **token_r)
 {
@@ -561,13 +561,16 @@
 		i_assert(len > 0);
 		len--;
 	}
+	/* we're skipping all non-text at the beginning of the word,
+	   so by this point we must have something here - even if we just
+	   deleted the last character */
+	i_assert(len > 0);
 
 	tok->prev_prev_letter = LETTER_TYPE_NONE;
 	tok->prev_letter = LETTER_TYPE_NONE;
 
-	*token_r = len == 0 ? "" : fts_uni_strndup(data, len);
+	*token_r = fts_uni_strndup(data, len);
 	buffer_set_used_size(tok->token, 0);
-	return len > 0;
 }
 
 struct letter_fn {
@@ -644,8 +647,8 @@
 			tok_append_truncated(tok, data + start_skip,
 					     char_start_i - start_skip);
 			*skip_r = i + 1;
-			if (fts_tokenizer_generic_tr29_current_token(tok, token_r))
-				return 1;
+			fts_tokenizer_generic_tr29_current_token(tok, token_r);
+			return 1;
 		}
 	}
 	i_assert(i >= start_skip && size >= start_skip);
@@ -655,8 +658,8 @@
 	if (size == 0 && tok->token->used > 0) {
 		/* return the last token */
 		*skip_r = 0;
-		if (fts_tokenizer_generic_tr29_current_token(tok, token_r))
-			return 1;
+		fts_tokenizer_generic_tr29_current_token(tok, token_r);
+		return 1;
 	}
 	return 0;
 }