dovecot-2.2: fts-solr: Flush Solr indexing HTTP requests every 1...
dovecot at dovecot.org
dovecot at dovecot.org
Tue Aug 18 20:21:25 UTC 2015
details: http://hg.dovecot.org/dovecot-2.2/rev/58f65e129b4d
changeset: 18973:58f65e129b4d
user: Timo Sirainen <tss at iki.fi>
date: Tue Aug 18 23:20:35 2015 +0300
description:
fts-solr: Flush Solr indexing HTTP requests every 1000 mails.
This avoids the HTTP request from becoming too huge.
diffstat:
src/plugins/fts-solr/fts-backend-solr.c | 16 +++++++++++-----
1 files changed, 11 insertions(+), 5 deletions(-)
diffs (68 lines):
diff -r ba4ac54e79a7 -r 58f65e129b4d src/plugins/fts-solr/fts-backend-solr.c
--- a/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:20:07 2015 +0300
+++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:20:35 2015 +0300
@@ -27,6 +27,8 @@
#define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024
#define SOLR_QUERY_MAX_MAILBOX_COUNT 10
+/* How often to flush indexing request to Solr before beginning a new one. */
+#define SOLR_MAIL_FLUSH_INTERVAL 1000
struct solr_fts_backend {
struct fts_backend backend;
@@ -51,6 +53,7 @@
ARRAY(struct solr_fts_field) fields;
uint32_t last_indexed_uid;
+ unsigned int mails_since_flush;
unsigned int tokenized_input:1;
unsigned int last_indexed_uid_set:1;
@@ -330,13 +333,14 @@
}
static int
-fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx)
+fts_backed_solr_build_flush(struct solr_fts_backend_update_context *ctx)
{
if (ctx->post == NULL)
return 0;
fts_backend_solr_doc_close(ctx);
str_append(ctx->cmd, "</add>");
+ ctx->mails_since_flush = 0;
solr_connection_post_more(ctx->post, str_data(ctx->cmd),
str_len(ctx->cmd));
@@ -367,7 +371,7 @@
const char *str;
int ret = _ctx->failed ? -1 : 0;
- if (fts_backed_solr_build_commit(ctx) < 0)
+ if (fts_backed_solr_build_flush(ctx) < 0)
ret = -1;
if (ctx->documents_added || ctx->expunges) {
@@ -405,7 +409,7 @@
if (ctx->prev_uid != 0) {
/* flush solr between mailboxes, so we don't wrongly update
last_uid before we know it has succeeded */
- if (fts_backed_solr_build_commit(ctx) < 0)
+ if (fts_backed_solr_build_flush(ctx) < 0)
_ctx->failed = TRUE;
else if (!_ctx->failed)
fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid);
@@ -466,9 +470,11 @@
struct solr_fts_backend *backend =
(struct solr_fts_backend *)ctx->ctx.backend;
+ if (ctx->mails_since_flush++ >= SOLR_MAIL_FLUSH_INTERVAL) {
+ if (fts_backed_solr_build_flush(ctx) < 0)
+ ctx->ctx.failed = TRUE;
+ }
if (ctx->post == NULL) {
- i_assert(ctx->prev_uid == 0);
-
if (ctx->cmd == NULL)
ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
ctx->post = solr_connection_post_begin(backend->solr_conn);
More information about the dovecot-cvs
mailing list