dovecot-2.2: fts-solr: fts_solr=use_libfts send data to Solr via...

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.2/rev/9974c748f72a
changeset: 18474:9974c748f72a
user:      Timo Sirainen <tss at iki.fi>
date:      Fri Apr 24 14:25:06 2015 +0300
description:
fts-solr: fts_solr=use_libfts send data to Solr via space-separated tokens.
In this case Solr should be configured to not do any kind of filtering and
use only WhitespaceTokenizerFactory.

diffstat:

 src/plugins/fts-solr/fts-backend-solr.c |  20 ++++++++++++++++++--
 src/plugins/fts-solr/fts-solr-plugin.c  |  10 ++++++++++
 src/plugins/fts-solr/fts-solr-plugin.h  |   1 +
 3 files changed, 29 insertions(+), 2 deletions(-)

diffs (107 lines):

diff -r 066db31d4dea -r 9974c748f72a src/plugins/fts-solr/fts-backend-solr.c
--- a/src/plugins/fts-solr/fts-backend-solr.c	Fri Apr 24 14:08:51 2015 +0300
+++ b/src/plugins/fts-solr/fts-backend-solr.c	Fri Apr 24 14:25:06 2015 +0300
@@ -50,6 +50,7 @@
 
 	uint32_t last_indexed_uid;
 
+	unsigned int tokenized_input:1;
 	unsigned int last_indexed_uid_set:1;
 	unsigned int body_open:1;
 	unsigned int documents_added:1;
@@ -165,6 +166,11 @@
 		*error_r = "Invalid fts_solr setting";
 		return -1;
 	}
+	if (fuser->set.use_libfts) {
+		/* change our flags so we get proper input */
+		_backend->flags &= ~FTS_BACKEND_FLAG_FUZZY_SEARCH;
+		_backend->flags |= FTS_BACKEND_FLAG_TOKENIZED_INPUT;
+	}
 	return solr_connection_init(fuser->set.url, fuser->set.debug,
 				    &backend->solr_conn, error_r);
 }
@@ -249,6 +255,8 @@
 
 	ctx = i_new(struct solr_fts_backend_update_context, 1);
 	ctx->ctx.backend = _backend;
+	ctx->tokenized_input =
+		(_backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0;
 	i_array_init(&ctx->fields, 16);
 	return &ctx->ctx;
 }
@@ -547,13 +555,21 @@
 			size -= len;
 		}
 		xml_encode_data(ctx->cmd, data, size);
+		if (ctx->tokenized_input)
+			str_append_c(ctx->cmd, ' ');
 	} else {
-		if (!ctx->truncate_header)
+		if (!ctx->truncate_header) {
 			xml_encode_data(ctx->cur_value, data, size);
+			if (ctx->tokenized_input)
+				str_append_c(ctx->cur_value, ' ');
+		}
 		if (ctx->cur_value2 != NULL &&
 		    (!ctx->truncate_header ||
-		     str_len(ctx->cur_value2) < SOLR_HEADER_LINE_MAX_TRUNC_SIZE))
+		     str_len(ctx->cur_value2) < SOLR_HEADER_LINE_MAX_TRUNC_SIZE)) {
 			xml_encode_data(ctx->cur_value2, data, size);
+			if (ctx->tokenized_input)
+				str_append_c(ctx->cur_value2, ' ');
+		}
 	}
 
 	if (str_len(ctx->cmd) >= SOLR_CMDBUF_FLUSH_SIZE) {
diff -r 066db31d4dea -r 9974c748f72a src/plugins/fts-solr/fts-solr-plugin.c
--- a/src/plugins/fts-solr/fts-solr-plugin.c	Fri Apr 24 14:08:51 2015 +0300
+++ b/src/plugins/fts-solr/fts-solr-plugin.c	Fri Apr 24 14:25:06 2015 +0300
@@ -6,6 +6,7 @@
 #include "mail-user.h"
 #include "mail-storage-hooks.h"
 #include "solr-connection.h"
+#include "fts-user.h"
 #include "fts-solr-plugin.h"
 
 #include <stdlib.h>
@@ -30,6 +31,8 @@
 			set->url = p_strdup(user->pool, *tmp + 4);
 		} else if (strcmp(*tmp, "debug") == 0) {
 			set->debug = TRUE;
+		} else if (strcmp(*tmp, "use_libfts") == 0) {
+			set->use_libfts = TRUE;
 		} else if (strcmp(*tmp, "break-imap-search") == 0) {
 			/* for backwards compatibility */
 		} else if (strcmp(*tmp, "default_ns=") == 0) {
@@ -50,12 +53,19 @@
 static void fts_solr_mail_user_create(struct mail_user *user, const char *env)
 {
 	struct fts_solr_user *fuser;
+	const char *error;
 
 	fuser = p_new(user->pool, struct fts_solr_user, 1);
 	if (fts_solr_plugin_init_settings(user, &fuser->set, env) < 0) {
 		/* invalid settings, disabling */
 		return;
 	}
+	if (fuser->set.use_libfts) {
+		if (fts_mail_user_create(user, &error) < 0) {
+			i_error("fts-solr: %s", error);
+			return;
+		}
+	}
 
 	MODULE_CONTEXT_SET(user, fts_solr_user_module, fuser);
 }
diff -r 066db31d4dea -r 9974c748f72a src/plugins/fts-solr/fts-solr-plugin.h
--- a/src/plugins/fts-solr/fts-solr-plugin.h	Fri Apr 24 14:08:51 2015 +0300
+++ b/src/plugins/fts-solr/fts-solr-plugin.h	Fri Apr 24 14:25:06 2015 +0300
@@ -10,6 +10,7 @@
 
 struct fts_solr_settings {
 	const char *url, *default_ns_prefix;
+	bool use_libfts;
 	bool debug;
 };