dovecot-1.1: fts-solr: Send data to Solr in larger packets. Comm...
dovecot at dovecot.org
dovecot at dovecot.org
Sun Jul 13 15:37:51 EEST 2008
details: http://hg.dovecot.org/dovecot-1.1/rev/a0b9325d4967
changeset: 7750:a0b9325d4967
user: Timo Sirainen <tss at iki.fi>
date: Sun Jul 13 15:05:52 2008 +0300
description:
fts-solr: Send data to Solr in larger packets. Commit after expunges.
diffstat:
2 files changed, 15 insertions(+), 14 deletions(-)
src/plugins/fts-solr/fts-backend-solr.c | 28 ++++++++++++++--------------
src/plugins/fts-solr/solr-connection.c | 1 +
diffs (87 lines):
diff -r b25da07179c6 -r a0b9325d4967 src/plugins/fts-solr/fts-backend-solr.c
--- a/src/plugins/fts-solr/fts-backend-solr.c Sun Jul 13 14:44:03 2008 +0300
+++ b/src/plugins/fts-solr/fts-backend-solr.c Sun Jul 13 15:05:52 2008 +0300
@@ -9,6 +9,8 @@
#include <stdlib.h>
#include <curl/curl.h>
+
+#define SOLR_CMDBUF_SIZE (1024*64)
struct solr_fts_backend_build_context {
struct fts_backend_build_context ctx;
@@ -119,7 +121,7 @@ fts_backend_solr_build_init(struct fts_b
ctx = i_new(struct solr_fts_backend_build_context, 1);
ctx->ctx.backend = backend;
ctx->post = solr_connection_post_begin(solr_conn);
- ctx->cmd = str_new(default_pool, 256);
+ ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
ctx->uid_validity = status.uidvalidity;
@@ -141,7 +143,6 @@ fts_backend_solr_build_more(struct fts_b
/* body comes first, then headers */
if (ctx->prev_uid != uid) {
/* uid changed */
- str_truncate(cmd, 0);
if (ctx->prev_uid == 0)
str_append(cmd, "<add>");
else
@@ -171,20 +172,18 @@ fts_backend_solr_build_more(struct fts_b
} else {
str_append(cmd, "<field name=\"body\">");
}
+ } else if (headers && !ctx->headers) {
+ str_append(cmd, "</field><field name=\"hdr\">");
+ } else {
+ i_assert(!(!headers && ctx->headers));
+ }
+
+ xml_encode_data(cmd, data, size);
+ if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) {
solr_connection_post_more(ctx->post, str_data(cmd),
str_len(cmd));
- } else if (headers && !ctx->headers) {
str_truncate(cmd, 0);
- str_append(cmd, "</field><field name=\"hdr\">");
- solr_connection_post_more(ctx->post, str_data(cmd),
- str_len(cmd));
- } else {
- i_assert(!(!headers && ctx->headers));
- }
-
- str_truncate(cmd, 0);
- xml_encode_data(cmd, data, size);
- solr_connection_post_more(ctx->post, str_data(cmd), str_len(cmd));
+ }
return 0;
}
@@ -196,7 +195,6 @@ fts_backend_solr_build_deinit(struct fts
int ret = 0;
if (ctx->prev_uid != 0) {
- str_truncate(ctx->cmd, 0);
str_append(ctx->cmd, "</field></doc></add>");
solr_connection_post_more(ctx->post, str_data(ctx->cmd),
str_len(ctx->cmd));
@@ -241,6 +239,8 @@ fts_backend_solr_expunge_finish(struct f
struct mailbox *box ATTR_UNUSED,
bool committed ATTR_UNUSED)
{
+ solr_connection_post(solr_conn,
+ "<commit waitFlush=\"false\" waitSearcher=\"false\"/>");
}
static int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED)
diff -r b25da07179c6 -r a0b9325d4967 src/plugins/fts-solr/solr-connection.c
--- a/src/plugins/fts-solr/solr-connection.c Sun Jul 13 14:44:03 2008 +0300
+++ b/src/plugins/fts-solr/solr-connection.c Sun Jul 13 15:05:52 2008 +0300
@@ -319,6 +319,7 @@ int solr_connection_select(struct solr_c
i_free_and_null(conn->http_failure);
conn->xml_failed = FALSE;
+ XML_ParserReset(conn->xml_parser, "UTF-8");
XML_SetElementHandler(conn->xml_parser,
solr_lookup_xml_start, solr_lookup_xml_end);
XML_SetCharacterDataHandler(conn->xml_parser, solr_lookup_xml_data);
More information about the dovecot-cvs
mailing list