dovecot-2.2: lib-imap-client: Implemented support for GMail [THR...

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.2/rev/6c2ea1d6ab58
changeset: 18214:6c2ea1d6ab58
user:      Timo Sirainen <tss at iki.fi>
date:      Tue Feb 03 10:15:38 2015 +0200
description:
lib-imap-client: Implemented support for GMail [THROTTLED] resp-text-code.
If we receive it, start throttling future commands by waiting exponentially
longer until we no longer receive [THROTTLED]. Max wait time is currently
16 seconds.

diffstat:

 src/lib-imap-client/imapc-connection.c |  59 ++++++++++++++++++++++++++++++++++
 1 files changed, 59 insertions(+), 0 deletions(-)

diffs (125 lines):

diff -r 5205f3bd1a27 -r 6c2ea1d6ab58 src/lib-imap-client/imapc-connection.c
--- a/src/lib-imap-client/imapc-connection.c	Mon Feb 02 23:48:30 2015 +0200
+++ b/src/lib-imap-client/imapc-connection.c	Tue Feb 03 10:15:38 2015 +0200
@@ -8,6 +8,7 @@
 #include "base64.h"
 #include "write-full.h"
 #include "str.h"
+#include "time-util.h"
 #include "dns-lookup.h"
 #include "dsasl-client.h"
 #include "iostream-rawlog.h"
@@ -24,6 +25,10 @@
 #define IMAPC_COMMAND_STATE_AUTHENTICATE_CONTINUE 10000
 #define IMAPC_MAX_INLINE_LITERAL_SIZE (1024*32)
 
+/* max seconds to wait after receiving [THROTTLED] as
+   2^IMAPC_THROTTLE_COUNTER_MAX_EXP */
+#define IMAPC_THROTTLE_COUNTER_MAX_EXP 4
+
 enum imapc_input_state {
 	IMAPC_INPUT_STATE_NONE = 0,
 	IMAPC_INPUT_STATE_PLUS,
@@ -116,6 +121,11 @@
 	struct imapc_connection_literal literal;
 	ARRAY(struct imapc_arg_file) literal_files;
 
+	unsigned int throttle_counter;
+	bool throttle_pending;
+	struct timeval throttle_end_timeval;
+	struct timeout *to_throttle;
+
 	unsigned int idling:1;
 	unsigned int idle_stopping:1;
 	unsigned int idle_plus_waiting:1;
@@ -190,6 +200,8 @@
 		conn->io = io_loop_move_io(&conn->io);
 	if (conn->to != NULL)
 		conn->to = io_loop_move_timeout(&conn->to);
+	if (conn->to_throttle != NULL)
+		conn->to_throttle = io_loop_move_timeout(&conn->to_throttle);
 	if (conn->output != NULL)
 		o_stream_switch_ioloop(conn->output);
 	if (conn->dns_lookup != NULL)
@@ -373,6 +385,8 @@
 		timeout_remove(&conn->to);
 	if (conn->to_output != NULL)
 		timeout_remove(&conn->to_output);
+	if (conn->to_throttle != NULL)
+		timeout_remove(&conn->to_throttle);
 	if (conn->parser != NULL)
 		imap_parser_unref(&conn->parser);
 	if (conn->io != NULL)
@@ -611,6 +625,15 @@
 			conn->selecting_box = NULL;
 		}
 	}
+	if (strcasecmp(key, "THROTTLED") == 0 && !conn->throttle_pending) {
+		/* GMail throttling - start slowing down commands. */
+		conn->throttle_end_timeval = ioloop_timeval;
+		timeval_add_msecs(&conn->throttle_end_timeval,
+				  (1U << conn->throttle_counter) * 1000);
+		conn->throttle_pending = TRUE;
+		if (conn->throttle_counter < IMAPC_THROTTLE_COUNTER_MAX_EXP)
+			conn->throttle_counter++;
+	}
 	return 0;
 }
 
@@ -1157,6 +1180,13 @@
 	} else {
 		reply.text_without_resp = reply.text_full;
 	}
+	if (!conn->throttle_pending &&
+	    timeval_cmp(&ioloop_timeval, &conn->throttle_end_timeval) >= 0) {
+		/* tagged reply without [THROTTLED] and it was received after
+		   the throttling ended. we can completely reset the throttling
+		   state now */
+		conn->throttle_counter = 0;
+	}
 
 	/* find the command. it's either the first command in send queue
 	   (literal failed) or somewhere in wait list. */
@@ -1740,6 +1770,32 @@
 	}
 }
 
+static bool imapc_connection_is_throttled(struct imapc_connection *conn)
+{
+	if (conn->to_throttle != NULL)
+		timeout_remove(&conn->to_throttle);
+
+	if (conn->throttle_counter == 0) {
+		/* we haven't received [THROTTLED] recently */
+		return FALSE;
+	}
+	if (array_count(&conn->cmd_wait_list) > 0) {
+		/* wait until we have received the existing commands' tagged
+		   replies to see if we're still throttled */
+		return TRUE;
+	}
+	if (timeval_cmp(&ioloop_timeval, &conn->throttle_end_timeval) > 0) {
+		/* we reached the throttle timeout - send the next command */
+		conn->throttle_pending = FALSE;
+		return FALSE;
+	}
+
+	/* we're still being throttled - wait for it to end */
+	conn->to_throttle = timeout_add_absolute(&conn->throttle_end_timeval,
+						 imapc_command_send_more, conn);
+	return TRUE;
+}
+
 static void imapc_command_send_more(struct imapc_connection *conn)
 {
 	struct imapc_command *const *cmds, *cmd;
@@ -1748,6 +1804,9 @@
 	unsigned int count, seek_pos, start_pos, end_pos, size;
 	int ret;
 
+	if (imapc_connection_is_throttled(conn))
+		return;
+
 	cmds = array_get(&conn->cmd_send_queue, &count);
 	if (count == 0)
 		return;