dovecot-2.2: Added imap-hibernate process for gathering IDLEing ...
dovecot at dovecot.org
dovecot at dovecot.org
Mon Aug 24 11:29:32 UTC 2015
details: http://hg.dovecot.org/dovecot-2.2/rev/64c73e6bd397
changeset: 18996:64c73e6bd397
user: Timo Sirainen <tss at iki.fi>
date: Mon Aug 24 14:13:02 2015 +0300
description:
Added imap-hibernate process for gathering IDLEing imap processes.
imap_hibernate_timeout setting controls how quickly the connection is moved
from imap process to imap-hibernate process.
Some IMAP extensions like NOTIFY, SEARCH=CONTEXT and COMPRESS aren't
supported yet.
There's also a new X-STATE command, which can be used to export the current
IMAP connection state to a string and later on imported to get back to the
original state (a quick resync feature for IMAP clients). However, this
command is disabled for now due to the current code being unoptimized for
untrusted input.
diffstat:
.hgignore | 1 +
configure.ac | 1 +
src/Makefile.am | 1 +
src/imap-hibernate/Makefile.am | 24 +
src/imap-hibernate/imap-client.c | 530 ++++++++++++++++
src/imap-hibernate/imap-client.h | 35 +
src/imap-hibernate/imap-hibernate-client.c | 262 +++++++
src/imap-hibernate/imap-hibernate-client.h | 9 +
src/imap-hibernate/imap-hibernate-settings.c | 48 +
src/imap-hibernate/imap-master-connection.c | 121 +++
src/imap-hibernate/imap-master-connection.h | 21 +
src/imap-hibernate/main.c | 56 +
src/imap/Makefile.am | 8 +-
src/imap/cmd-idle.c | 49 +-
src/imap/cmd-x-state.c | 68 ++
src/imap/imap-client-hibernate.c | 225 ++++++
src/imap/imap-client.c | 7 +-
src/imap/imap-client.h | 19 +
src/imap/imap-commands.c | 1 +
src/imap/imap-commands.h | 1 +
src/imap/imap-common.h | 7 +
src/imap/imap-master-client.c | 321 +++++++++
src/imap/imap-master-client.h | 9 +
src/imap/imap-settings.c | 8 +-
src/imap/imap-settings.h | 1 +
src/imap/imap-state.c | 889 +++++++++++++++++++++++++++
src/imap/imap-state.h | 30 +
src/imap/main.c | 25 +-
src/lib-storage/mail-user.c | 2 +
src/plugins/imap-zlib/imap-zlib-plugin.c | 18 +
30 files changed, 2780 insertions(+), 17 deletions(-)
diffs (truncated from 3226 to 300 lines):
diff -r 4da6efa3fd91 -r 64c73e6bd397 .hgignore
--- a/.hgignore Mon Aug 24 14:14:59 2015 +0300
+++ b/.hgignore Mon Aug 24 14:13:02 2015 +0300
@@ -68,6 +68,7 @@
src/dns/dns-client
src/doveadm/doveadm
src/doveadm/doveadm-server
+src/imap-hibernate/imap-hibernate
src/imap-login/imap-login
src/imap-urlauth/imap-urlauth
src/imap-urlauth/imap-urlauth-login
diff -r 4da6efa3fd91 -r 64c73e6bd397 configure.ac
--- a/configure.ac Mon Aug 24 14:14:59 2015 +0300
+++ b/configure.ac Mon Aug 24 14:13:02 2015 +0300
@@ -2931,6 +2931,7 @@
src/indexer/Makefile
src/ipc/Makefile
src/imap/Makefile
+src/imap-hibernate/Makefile
src/imap-login/Makefile
src/imap-urlauth/Makefile
src/login-common/Makefile
diff -r 4da6efa3fd91 -r 64c73e6bd397 src/Makefile.am
--- a/src/Makefile.am Mon Aug 24 14:14:59 2015 +0300
+++ b/src/Makefile.am Mon Aug 24 14:13:02 2015 +0300
@@ -37,6 +37,7 @@
ipc \
master \
login-common \
+ imap-hibernate \
imap-login \
imap \
imap-urlauth \
diff -r 4da6efa3fd91 -r 64c73e6bd397 src/imap-hibernate/Makefile.am
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/imap-hibernate/Makefile.am Mon Aug 24 14:13:02 2015 +0300
@@ -0,0 +1,24 @@
+pkglibexecdir = $(libexecdir)/dovecot
+
+pkglibexec_PROGRAMS = imap-hibernate
+
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-settings \
+ -I$(top_srcdir)/src/lib-master \
+ -I$(top_srcdir)/src/lib-imap
+
+imap_hibernate_LDADD = $(LIBDOVECOT)
+imap_hibernate_DEPENDENCIES = $(LIBDOVECOT_DEPS)
+
+imap_hibernate_SOURCES = \
+ imap-client.c \
+ imap-hibernate-client.c \
+ imap-hibernate-settings.c \
+ imap-master-connection.c \
+ main.c
+
+noinst_HEADERS = \
+ imap-client.h \
+ imap-hibernate-client.h \
+ imap-master-connection.h
diff -r 4da6efa3fd91 -r 64c73e6bd397 src/imap-hibernate/imap-client.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/imap-hibernate/imap-client.c Mon Aug 24 14:13:02 2015 +0300
@@ -0,0 +1,530 @@
+/* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ioloop.h"
+#include "fd-set-nonblock.h"
+#include "fdpass.h"
+#include "hostpid.h"
+#include "connection.h"
+#include "iostream.h"
+#include "istream.h"
+#include "ostream.h"
+#include "llist.h"
+#include "base64.h"
+#include "str.h"
+#include "strescape.h"
+#include "var-expand.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "imap-keepalive.h"
+#include "imap-master-connection.h"
+#include "imap-client.h"
+
+#include <unistd.h>
+
+#define IMAP_MASTER_SOCKET_NAME "imap-master"
+
+/* we only need enough for "DONE\r\nIDLE\r\n" */
+#define IMAP_MAX_INBUF 12
+
+enum imap_client_input_state {
+ IMAP_CLIENT_INPUT_STATE_UNKNOWN,
+ IMAP_CLIENT_INPUT_STATE_BAD,
+ IMAP_CLIENT_INPUT_STATE_DONE_LF,
+ IMAP_CLIENT_INPUT_STATE_DONE_CRLF,
+ IMAP_CLIENT_INPUT_STATE_DONEIDLE
+};
+
+struct imap_client_notify {
+ int fd;
+ struct io *io;
+};
+
+struct imap_client {
+ struct imap_client *prev, *next;
+ pool_t pool;
+ struct imap_client_state state;
+ ARRAY(struct imap_client_notify) notifys;
+
+ int fd;
+ struct io *io;
+ struct istream *input;
+ struct timeout *to_keepalive;
+ struct imap_master_connection *master_conn;
+ struct ioloop_context *ioloop_ctx;
+ const char *log_prefix;
+ unsigned int imap_still_here_text_pos;
+ unsigned int next_read_threshold;
+ bool bad_done, idle_done;
+};
+
+static struct imap_client *imap_clients;
+static const char imap_still_here_text[] = "* OK Still here\r\n";
+
+static void imap_client_stop(struct imap_client *client);
+void imap_client_destroy(struct imap_client **_client, const char *reason);
+static void imap_client_add_idle_keepalive_timeout(struct imap_client *client);
+
+static void imap_client_disconnected(struct imap_client **_client)
+{
+ struct imap_client *client = *_client;
+ const char *reason;
+
+ reason = io_stream_get_disconnect_reason(client->input, NULL);
+ imap_client_destroy(_client, reason);
+}
+
+static void
+imap_client_parse_userdb_fields(struct imap_client *client,
+ const char **auth_user_r)
+{
+ const char *const *field;
+ unsigned int i;
+
+ *auth_user_r = NULL;
+
+ if (client->state.userdb_fields == NULL)
+ return;
+
+ field = t_strsplit_tabescaped(client->state.userdb_fields);
+ for (i = 0; field[i] != NULL; i++) {
+ if (strncmp(field[i], "auth_user=", 10) == 0)
+ *auth_user_r = field[i] + 10;
+ }
+}
+
+static void
+imap_client_move_back_send_callback(void *context, struct ostream *output)
+{
+ struct imap_client *client = context;
+ const struct imap_client_state *state = &client->state;
+ string_t *str = t_str_new(256);
+ const unsigned char *input_data;
+ size_t input_size;
+ ssize_t ret;
+
+ str_append_tabescaped(str, state->username);
+ if (state->session_id != NULL) {
+ str_append(str, "\tsession=");
+ str_append_tabescaped(str, state->session_id);
+ }
+ if (state->local_ip.family != 0)
+ str_printfa(str, "\tlip=%s", net_ip2addr(&state->local_ip));
+ if (state->remote_ip.family != 0)
+ str_printfa(str, "\trip=%s", net_ip2addr(&state->remote_ip));
+ if (state->userdb_fields != NULL) {
+ str_append(str, "\tuserdb_fields=");
+ str_append_tabescaped(str, state->userdb_fields);
+ }
+ if (state->peer_ip.family != 0)
+ str_printfa(str, "\tpeer_ip=%s", net_ip2addr(&state->peer_ip));
+ if (state->peer_port != 0)
+ str_printfa(str, "\tpeer_port=%u", state->peer_port);
+ if (state->state_size > 0) {
+ str_append(str, "\tstate=");
+ base64_encode(state->state, state->state_size, str);
+ }
+ input_data = i_stream_get_data(client->input, &input_size);
+ if (input_size > 0) {
+ str_append(str, "\tclient_input=");
+ base64_encode(input_data, input_size, str);
+ }
+ if (client->imap_still_here_text_pos != 0) {
+ str_append(str, "\tclient_output=");
+ base64_encode(imap_still_here_text + client->imap_still_here_text_pos,
+ sizeof(imap_still_here_text)-1 - client->imap_still_here_text_pos,
+ str);
+ }
+ if (client->idle_done) {
+ if (client->bad_done)
+ str_append(str, "\tbad-done");
+ } else if (client->state.idle_cmd) {
+ /* IDLE continues after sending changes */
+ str_append(str, "\tidle-continue");
+ }
+ str_append_c(str, '\n');
+
+ /* send the fd first */
+ ret = fd_send(o_stream_get_fd(output), client->fd, str_data(str), 1);
+ if (ret < 0) {
+ i_error("fd_send(%s) failed: %m",
+ o_stream_get_name(output));
+ imap_client_destroy(&client, "Failed to recreate imap process");
+ return;
+ }
+ i_assert(ret > 0);
+ o_stream_nsend(output, str_data(str) + 1, str_len(str) - 1);
+}
+
+static void
+imap_client_move_back_read_callback(void *context, const char *line)
+{
+ struct imap_client *client = context;
+
+ if (line[0] != '+') {
+ /* failed - FIXME: retry later? */
+ imap_client_destroy(&client, t_strdup_printf(
+ "Failed to recreate imap process: %s", line+1));
+ } else {
+ imap_client_destroy(&client, NULL);
+ }
+}
+
+static void imap_client_move_back(struct imap_client *client)
+{
+ const struct master_service_settings *master_set;
+ const char *path;
+
+ imap_client_stop(client);
+
+ master_set = master_service_settings_get(master_service);
+ path = t_strconcat(master_set->base_dir,
+ "/"IMAP_MASTER_SOCKET_NAME, NULL);
+ if (imap_master_connection_init(path,
+ imap_client_move_back_send_callback,
+ imap_client_move_back_read_callback,
+ client, &client->master_conn) < 0) {
+ /* failed to connect to the imap-master socket */
+ imap_client_destroy(&client, "Failed to connect to master socket");
+ }
+}
+
+static enum imap_client_input_state
+imap_client_input_parse(const unsigned char *data, size_t size)
+{
+ enum imap_client_input_state state = IMAP_CLIENT_INPUT_STATE_DONE_LF;
+
+ /* skip over DONE[\r]\n */
+ if (i_memcasecmp(data, "DONE", I_MIN(size, 4)) != 0)
+ return IMAP_CLIENT_INPUT_STATE_BAD;
+ if (size <= 4)
+ return IMAP_CLIENT_INPUT_STATE_UNKNOWN;
+ data += 4; size -= 4;
+
+ if (data[0] == '\r') {
+ state = IMAP_CLIENT_INPUT_STATE_DONE_CRLF;
+ data++; size--;
+ }
+ if (size == 0)
+ return IMAP_CLIENT_INPUT_STATE_UNKNOWN;
+ if (data[0] != '\n')
+ return IMAP_CLIENT_INPUT_STATE_BAD;
+ data++; size--;
+
+ /* skip over IDLE[\r]\n - checking this assumes that the DONE and IDLE
+ are sent in the same IP packet, otherwise we'll unnecessarily
+ recreate the imap process and immediately resume IDLE there. if this
+ becomes an issue we could add a small delay to the imap process
+ creation and wait for the IDLE command during it. */
+ if (size <= 4 || i_memcasecmp(data, "IDLE", 4) != 0)
+ return state;
+ data += 4; size -= 4;
+
+ if (data[0] == '\r') {
+ data++; size--;
+ }
+ return size == 1 && data[0] == '\n' ?
+ IMAP_CLIENT_INPUT_STATE_DONEIDLE : state;
+}
+
+static void imap_client_input_idle_cmd(struct imap_client *client)
+{
+ const unsigned char *data;
+ size_t size;
+ int ret;
More information about the dovecot-cvs
mailing list