dovecot-2.2: indexer: Improved handling multiple indexing reques...

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.2/rev/2f5799572a2f
changeset: 18824:2f5799572a2f
user:      Timo Sirainen <tss at iki.fi>
date:      Wed Jun 03 22:30:05 2015 +0300
description:
indexer: Improved handling multiple indexing requests to the same mailbox.
If a request arrives for a mailbox that we were already indexes, the
previous code simply sent the indexing request to the existing worker
process. This could have caused a lot of requests to be buffered to the same
mailbox if the indexing took a long time, which could have taken a while to
process even though they weren't really doing anything indexing work.

The new code instead just keeps track in memory that when the earlier
indexing is finished, it's done again once to finish any pending changes.

diffstat:

 src/indexer/indexer-queue.c |  40 +++++++++++++++++++++++++++++++++++++---
 src/indexer/indexer-queue.h |  13 +++++++++++++
 src/indexer/indexer.c       |   1 +
 3 files changed, 51 insertions(+), 3 deletions(-)

diffs (124 lines):

diff -r a5d5d2dd6aea -r 2f5799572a2f src/indexer/indexer-queue.c
--- a/src/indexer/indexer-queue.c	Wed Jun 03 22:26:05 2015 +0300
+++ b/src/indexer/indexer-queue.c	Wed Jun 03 22:30:05 2015 +0300
@@ -98,6 +98,14 @@
 		if (request->max_recent_msgs > max_recent_msgs)
 			request->max_recent_msgs = max_recent_msgs;
 		request_add_context(request, context);
+		if (request->working) {
+			/* we're already indexing this mailbox. */
+			if (append)
+				request->reindex_tail = TRUE;
+			else
+				request->reindex_head = TRUE;
+			return request;
+		}
 		if (append) {
 			/* keep the request in its old position */
 			return request;
@@ -156,8 +164,6 @@
 	i_assert(request != NULL);
 
 	DLLIST2_REMOVE(&queue->head, &queue->tail, request);
-	hash_table_remove(queue->requests, request);
-	indexer_refresh_proctitle();
 }
 
 static void indexer_queue_request_status_int(struct indexer_queue *queue,
@@ -167,7 +173,7 @@
 	void *const *contextp;
 	unsigned int i;
 
-	for (i = 0; i < array_count(&request->contexts); i++) {
+	for (i = 0; i < request->working_context_idx; i++) {
 		contextp = array_idx(&request->contexts, i);
 		queue->callback(percentage, *contextp);
 	}
@@ -182,6 +188,14 @@
 	indexer_queue_request_status_int(queue, request, percentage);
 }
 
+void indexer_queue_request_work(struct indexer_request *request)
+{
+	request->working = TRUE;
+	request->working_context_idx =
+		!array_is_created(&request->contexts) ? 0 :
+		array_count(&request->contexts);
+}
+
 void indexer_queue_request_finish(struct indexer_queue *queue,
 				  struct indexer_request **_request,
 				  bool success)
@@ -191,11 +205,31 @@
 	*_request = NULL;
 
 	indexer_queue_request_status_int(queue, request, success ? 100 : -1);
+
+	if (request->reindex_head || request->reindex_tail) {
+		i_assert(request->working);
+		request->working = FALSE;
+		request->reindex_head = FALSE;
+		request->reindex_tail = FALSE;
+		if (request->working_context_idx > 0) {
+			array_delete(&request->contexts, 0,
+				     request->working_context_idx);
+		}
+		if (request->reindex_head)
+			DLLIST2_PREPEND(&queue->head, &queue->tail, request);
+		else
+			DLLIST2_APPEND(&queue->head, &queue->tail, request);
+		return;
+	}
+
+	hash_table_remove(queue->requests, request);
 	if (array_is_created(&request->contexts))
 		array_free(&request->contexts);
 	i_free(request->username);
 	i_free(request->mailbox);
 	i_free(request);
+
+	indexer_refresh_proctitle();
 }
 
 void indexer_queue_cancel_all(struct indexer_queue *queue)
diff -r a5d5d2dd6aea -r 2f5799572a2f src/indexer/indexer-queue.h
--- a/src/indexer/indexer-queue.h	Wed Jun 03 22:26:05 2015 +0300
+++ b/src/indexer/indexer-queue.h	Wed Jun 03 22:30:05 2015 +0300
@@ -14,6 +14,17 @@
 	unsigned int index:1;
 	/* optimize this mailbox */
 	unsigned int optimize:1;
+	/* currently indexing this mailbox */
+	unsigned int working:1;
+	/* after indexing is finished, add this request back to the queue and
+	   reindex it (i.e. a new indexing request came while we were
+	   working.) */
+	unsigned int reindex_head:1;
+	unsigned int reindex_tail:1;
+
+	/* when working finished, call this number of contexts and leave the
+	   rest to the reindexing. */
+	unsigned int working_context_idx;
 
 	ARRAY(void *) contexts;
 };
@@ -45,6 +56,8 @@
 void indexer_queue_request_status(struct indexer_queue *queue,
 				  struct indexer_request *request,
 				  int percentage);
+/* Start working on a request */
+void indexer_queue_request_work(struct indexer_request *request);
 /* Finish the request and free its memory. */
 void indexer_queue_request_finish(struct indexer_queue *queue,
 				  struct indexer_request **request,
diff -r a5d5d2dd6aea -r 2f5799572a2f src/indexer/indexer.c
--- a/src/indexer/indexer.c	Wed Jun 03 22:26:05 2015 +0300
+++ b/src/indexer/indexer.c	Wed Jun 03 22:30:05 2015 +0300
@@ -50,6 +50,7 @@
 	wrequest->conn = conn;
 	wrequest->request = request;
 
+	indexer_queue_request_work(request);
 	worker_connection_request(conn, request, wrequest);
 }