dovecot-1.2: fts-solr: Send data to Solr in larger packets. Comm...

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-1.2/rev/3fbfdda3e5d3
changeset: 8001:3fbfdda3e5d3
user:      Timo Sirainen <tss at iki.fi>
date:      Sun Jul 13 15:05:52 2008 +0300
description:
fts-solr: Send data to Solr in larger packets. Commit after expunges.

diffstat:

2 files changed, 15 insertions(+), 14 deletions(-)
src/plugins/fts-solr/fts-backend-solr.c |   28 ++++++++++++++--------------
src/plugins/fts-solr/solr-connection.c  |    1 +

diffs (87 lines):

diff -r 61be6a4722c5 -r 3fbfdda3e5d3 src/plugins/fts-solr/fts-backend-solr.c
--- a/src/plugins/fts-solr/fts-backend-solr.c	Sun Jul 13 14:44:03 2008 +0300
+++ b/src/plugins/fts-solr/fts-backend-solr.c	Sun Jul 13 15:05:52 2008 +0300
@@ -9,6 +9,8 @@
 
 #include <stdlib.h>
 #include <curl/curl.h>
+
+#define SOLR_CMDBUF_SIZE (1024*64)
 
 struct solr_fts_backend_build_context {
 	struct fts_backend_build_context ctx;
@@ -119,7 +121,7 @@ fts_backend_solr_build_init(struct fts_b
 	ctx = i_new(struct solr_fts_backend_build_context, 1);
 	ctx->ctx.backend = backend;
 	ctx->post = solr_connection_post_begin(solr_conn);
-	ctx->cmd = str_new(default_pool, 256);
+	ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
 
 	mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
 	ctx->uid_validity = status.uidvalidity;
@@ -141,7 +143,6 @@ fts_backend_solr_build_more(struct fts_b
 	/* body comes first, then headers */
 	if (ctx->prev_uid != uid) {
 		/* uid changed */
-		str_truncate(cmd, 0);
 		if (ctx->prev_uid == 0)
 			str_append(cmd, "<add>");
 		else
@@ -171,20 +172,18 @@ fts_backend_solr_build_more(struct fts_b
 		} else {
 			str_append(cmd, "<field name=\"body\">");
 		}
+	} else if (headers && !ctx->headers) {
+		str_append(cmd, "</field><field name=\"hdr\">");
+	} else {
+		i_assert(!(!headers && ctx->headers));
+	}
+
+	xml_encode_data(cmd, data, size);
+	if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) {
 		solr_connection_post_more(ctx->post, str_data(cmd),
 					  str_len(cmd));
-	} else if (headers && !ctx->headers) {
 		str_truncate(cmd, 0);
-		str_append(cmd, "</field><field name=\"hdr\">");
-		solr_connection_post_more(ctx->post, str_data(cmd),
-					  str_len(cmd));
-	} else {
-		i_assert(!(!headers && ctx->headers));
-	}
-
-	str_truncate(cmd, 0);
-	xml_encode_data(cmd, data, size);
-	solr_connection_post_more(ctx->post, str_data(cmd), str_len(cmd));
+	}
 	return 0;
 }
 
@@ -196,7 +195,6 @@ fts_backend_solr_build_deinit(struct fts
 	int ret = 0;
 
 	if (ctx->prev_uid != 0) {
-		str_truncate(ctx->cmd, 0);
 		str_append(ctx->cmd, "</field></doc></add>");
 		solr_connection_post_more(ctx->post, str_data(ctx->cmd),
 					  str_len(ctx->cmd));
@@ -241,6 +239,8 @@ fts_backend_solr_expunge_finish(struct f
 				struct mailbox *box ATTR_UNUSED,
 				bool committed ATTR_UNUSED)
 {
+	solr_connection_post(solr_conn,
+		"<commit waitFlush=\"false\" waitSearcher=\"false\"/>");
 }
 
 static int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED)
diff -r 61be6a4722c5 -r 3fbfdda3e5d3 src/plugins/fts-solr/solr-connection.c
--- a/src/plugins/fts-solr/solr-connection.c	Sun Jul 13 14:44:03 2008 +0300
+++ b/src/plugins/fts-solr/solr-connection.c	Sun Jul 13 15:05:52 2008 +0300
@@ -319,6 +319,7 @@ int solr_connection_select(struct solr_c
 
 	i_free_and_null(conn->http_failure);
 	conn->xml_failed = FALSE;
+	XML_ParserReset(conn->xml_parser, "UTF-8");
 	XML_SetElementHandler(conn->xml_parser,
 			      solr_lookup_xml_start, solr_lookup_xml_end);
 	XML_SetCharacterDataHandler(conn->xml_parser, solr_lookup_xml_data);