dovecot-2.2: fts-solr: Flush Solr indexing HTTP requests every 1...

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.2/rev/58f65e129b4d
changeset: 18973:58f65e129b4d
user:      Timo Sirainen <tss at iki.fi>
date:      Tue Aug 18 23:20:35 2015 +0300
description:
fts-solr: Flush Solr indexing HTTP requests every 1000 mails.
This avoids the HTTP request from becoming too huge.

diffstat:

 src/plugins/fts-solr/fts-backend-solr.c |  16 +++++++++++-----
 1 files changed, 11 insertions(+), 5 deletions(-)

diffs (68 lines):

diff -r ba4ac54e79a7 -r 58f65e129b4d src/plugins/fts-solr/fts-backend-solr.c
--- a/src/plugins/fts-solr/fts-backend-solr.c	Tue Aug 18 23:20:07 2015 +0300
+++ b/src/plugins/fts-solr/fts-backend-solr.c	Tue Aug 18 23:20:35 2015 +0300
@@ -27,6 +27,8 @@
 #define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024
 
 #define SOLR_QUERY_MAX_MAILBOX_COUNT 10
+/* How often to flush indexing request to Solr before beginning a new one. */
+#define SOLR_MAIL_FLUSH_INTERVAL 1000
 
 struct solr_fts_backend {
 	struct fts_backend backend;
@@ -51,6 +53,7 @@
 	ARRAY(struct solr_fts_field) fields;
 
 	uint32_t last_indexed_uid;
+	unsigned int mails_since_flush;
 
 	unsigned int tokenized_input:1;
 	unsigned int last_indexed_uid_set:1;
@@ -330,13 +333,14 @@
 }
 
 static int
-fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx)
+fts_backed_solr_build_flush(struct solr_fts_backend_update_context *ctx)
 {
 	if (ctx->post == NULL)
 		return 0;
 
 	fts_backend_solr_doc_close(ctx);
 	str_append(ctx->cmd, "</add>");
+	ctx->mails_since_flush = 0;
 
 	solr_connection_post_more(ctx->post, str_data(ctx->cmd),
 				  str_len(ctx->cmd));
@@ -367,7 +371,7 @@
 	const char *str;
 	int ret = _ctx->failed ? -1 : 0;
 
-	if (fts_backed_solr_build_commit(ctx) < 0)
+	if (fts_backed_solr_build_flush(ctx) < 0)
 		ret = -1;
 
 	if (ctx->documents_added || ctx->expunges) {
@@ -405,7 +409,7 @@
 	if (ctx->prev_uid != 0) {
 		/* flush solr between mailboxes, so we don't wrongly update
 		   last_uid before we know it has succeeded */
-		if (fts_backed_solr_build_commit(ctx) < 0)
+		if (fts_backed_solr_build_flush(ctx) < 0)
 			_ctx->failed = TRUE;
 		else if (!_ctx->failed)
 			fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid);
@@ -466,9 +470,11 @@
 	struct solr_fts_backend *backend =
 		(struct solr_fts_backend *)ctx->ctx.backend;
 
+	if (ctx->mails_since_flush++ >= SOLR_MAIL_FLUSH_INTERVAL) {
+		if (fts_backed_solr_build_flush(ctx) < 0)
+			ctx->ctx.failed = TRUE;
+	}
 	if (ctx->post == NULL) {
-		i_assert(ctx->prev_uid == 0);
-
 		if (ctx->cmd == NULL)
 			ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
 		ctx->post = solr_connection_post_begin(backend->solr_conn);