dovecot-2.2: login proxy: Added delayed disconnection of clients...
dovecot at dovecot.org
dovecot at dovecot.org
Mon Sep 7 15:51:35 UTC 2015
details: http://hg.dovecot.org/dovecot-2.2/rev/2eeef1619161
changeset: 19107:2eeef1619161
user: Timo Sirainen <tss at iki.fi>
date: Mon Sep 07 18:50:24 2015 +0300
description:
login proxy: Added delayed disconnection of clients on server mass-disconnect.
login_proxy_max_disconnect_delay setting (default 0 = disabled) controls for
how long time period the disconnections are spread to. The idea behind this
is to avoid load spikes due to client reconnections when a backend server
dies or is restarted.
diffstat:
src/login-common/login-proxy-state.h | 14 +++
src/login-common/login-proxy.c | 134 +++++++++++++++++++++++++++++++++-
src/login-common/login-settings.c | 2 +
src/login-common/login-settings.h | 1 +
4 files changed, 144 insertions(+), 7 deletions(-)
diffs (truncated from 326 to 300 lines):
diff -r 757a566a0d1c -r 2eeef1619161 src/login-common/login-proxy-state.h
--- a/src/login-common/login-proxy-state.h Mon Sep 07 17:10:19 2015 +0300
+++ b/src/login-common/login-proxy-state.h Mon Sep 07 18:50:24 2015 +0300
@@ -7,8 +7,22 @@
struct ip_addr ip;
in_port_t port;
+ /* These are used to spread client-visible disconnects over longer
+ periods of time to avoid reconnect spikes when a server dies.
+
+ If num_disconnects_since_ts=0 when server disconnects us, it's
+ increased and disconnect_timestamp is updated. Afterwards it's
+ increased for each new disconnection. num_disconnects_since_ts gets
+ reset back to zero whenever a) last_success gets updated or b)
+ num_delayed_client_disconnects drops to 0. */
+ struct timeval disconnect_timestamp;
+ unsigned int num_disconnects_since_ts;
+ unsigned int num_delayed_client_disconnects;
+
/* these are tracking connect()s, not necessarily logins: */
unsigned int num_waiting_connections;
+ /* number of connections we're proxying now (post-login) */
+ unsigned int num_proxying_connections;
struct timeval last_failure;
struct timeval last_success;
};
diff -r 757a566a0d1c -r 2eeef1619161 src/login-common/login-proxy.c
--- a/src/login-common/login-proxy.c Mon Sep 07 17:10:19 2015 +0300
+++ b/src/login-common/login-proxy.c Mon Sep 07 18:50:24 2015 +0300
@@ -16,6 +16,8 @@
#include "login-proxy-state.h"
#include "login-proxy.h"
+#include <stdlib.h>
+
#define MAX_PROXY_INPUT_SIZE 4096
#define OUTBUF_THRESHOLD 1024
#define LOGIN_PROXY_DIE_IDLE_SECS 2
@@ -24,6 +26,7 @@
#define KILLED_BY_ADMIN_REASON "Killed by admin"
#define PROXY_IMMEDIATE_FAILURE_SECS 30
#define PROXY_CONNECT_RETRY_MSECS 1000
+#define PROXY_DISCONNECT_INTERVAL_MSECS 100
struct login_proxy {
struct login_proxy *prev, *next;
@@ -53,21 +56,27 @@
unsigned int connected:1;
unsigned int destroying:1;
unsigned int disconnecting:1;
+ unsigned int delayed_disconnect:1;
unsigned int num_waiting_connections_updated:1;
};
static struct login_proxy_state *proxy_state;
static struct login_proxy *login_proxies = NULL;
static struct login_proxy *login_proxies_pending = NULL;
+static struct login_proxy *login_proxies_disconnecting = NULL;
static struct ipc_server *login_proxy_ipc_server;
static int login_proxy_connect(struct login_proxy *proxy);
static void login_proxy_disconnect(struct login_proxy *proxy);
static void login_proxy_ipc_cmd(struct ipc_cmd *cmd, const char *line);
+static void login_proxy_free_final(struct login_proxy *proxy);
static void
login_proxy_free_reason(struct login_proxy **_proxy, const char *reason)
ATTR_NULL(2);
+static void
+login_proxy_free_delayed(struct login_proxy **_proxy, const char *reason)
+ ATTR_NULL(2);
static void login_proxy_free_errno(struct login_proxy **proxy,
int err, bool server)
@@ -77,7 +86,10 @@
reason = err == 0 || err == EPIPE ?
t_strdup_printf("Disconnected by %s", who) :
t_strdup_printf("Disconnected by %s: %s", who, strerror(errno));
- login_proxy_free_reason(proxy, reason);
+ if (server)
+ login_proxy_free_delayed(proxy, reason);
+ else
+ login_proxy_free_reason(proxy, reason);
}
static void server_input(struct login_proxy *proxy)
@@ -136,6 +148,16 @@
}
}
+static void proxy_client_disconnected_input(struct login_proxy *proxy)
+{
+ unsigned char buf[OUTBUF_THRESHOLD];
+
+ /* we're already disconnected from server. either wait for
+ disconnection timeout or for client to disconnect itself. */
+ if (net_receive(proxy->client_fd, buf, sizeof(buf)) < 0)
+ login_proxy_free_final(proxy);
+}
+
static int server_output(struct login_proxy *proxy)
{
proxy->last_io = ioloop_time;
@@ -283,6 +305,8 @@
proxy->state_rec->last_success = ioloop_timeval;
i_assert(proxy->state_rec->num_waiting_connections > 0);
proxy->state_rec->num_waiting_connections--;
+ proxy->state_rec->num_proxying_connections++;
+ proxy->state_rec->num_disconnects_since_ts = 0;
if ((proxy->ssl_flags & PROXY_SSL_FLAG_YES) != 0 &&
(proxy->ssl_flags & PROXY_SSL_FLAG_STARTTLS) == 0) {
@@ -408,6 +432,10 @@
i_assert(proxy->state_rec->num_waiting_connections > 0);
proxy->state_rec->num_waiting_connections--;
}
+ if (proxy->connected) {
+ i_assert(proxy->state_rec->num_proxying_connections > 0);
+ proxy->state_rec->num_proxying_connections--;
+ }
if (proxy->server_io != NULL)
io_remove(&proxy->server_io);
@@ -421,6 +449,17 @@
static void login_proxy_free_final(struct login_proxy *proxy)
{
+ if (proxy->delayed_disconnect) {
+ DLLIST_REMOVE(&login_proxies_disconnecting, proxy);
+
+ i_assert(proxy->state_rec->num_delayed_client_disconnects > 0);
+ if (--proxy->state_rec->num_delayed_client_disconnects == 0)
+ proxy->state_rec->num_disconnects_since_ts = 0;
+ timeout_remove(&proxy->to);
+ }
+
+ if (proxy->client_io != NULL)
+ io_remove(&proxy->client_io);
if (proxy->client_output != NULL)
o_stream_destroy(&proxy->client_output);
if (proxy->client_fd != -1)
@@ -431,12 +470,69 @@
i_free(proxy);
}
+static unsigned int login_proxy_delay_disconnect(struct login_proxy *proxy)
+{
+ struct login_proxy_record *rec = proxy->state_rec;
+ const unsigned int max_delay =
+ proxy->client->set->login_proxy_max_disconnect_delay;
+ struct timeval disconnect_time_offset;
+ unsigned int max_disconnects_per_sec, delay_msecs_since_ts, max_conns;
+ int delay_msecs;
+
+ if (rec->num_disconnects_since_ts == 0) {
+ rec->disconnect_timestamp = ioloop_timeval;
+ /* start from a slightly random timestamp. this way all proxy
+ processes will disconnect at slightly different times to
+ spread the load. */
+ timeval_add_msecs(&rec->disconnect_timestamp,
+ rand() % PROXY_DISCONNECT_INTERVAL_MSECS);
+ }
+ rec->num_disconnects_since_ts++;
+ if (proxy->to != NULL) {
+ /* we were already lazily disconnecting this */
+ return 0;
+ }
+ if (max_delay == 0) {
+ /* delaying is disabled */
+ return 0;
+ }
+ max_conns = rec->num_proxying_connections + rec->num_disconnects_since_ts;
+ max_disconnects_per_sec = (max_conns + max_delay-1) / max_delay;
+ if (rec->num_disconnects_since_ts <= max_disconnects_per_sec &&
+ rec->num_delayed_client_disconnects == 0) {
+ /* wait delaying until we have 1 second's worth of clients
+ disconnected */
+ return 0;
+ }
+
+ /* see at which time we should be disconnecting the client.
+ do it in 100ms intervals so the timeouts are triggered together. */
+ disconnect_time_offset = rec->disconnect_timestamp;
+ delay_msecs_since_ts = PROXY_DISCONNECT_INTERVAL_MSECS *
+ (max_delay * rec->num_disconnects_since_ts *
+ (1000/PROXY_DISCONNECT_INTERVAL_MSECS) / max_conns);
+ timeval_add_msecs(&disconnect_time_offset, delay_msecs_since_ts);
+ delay_msecs = timeval_diff_msecs(&disconnect_time_offset, &ioloop_timeval);
+ if (delay_msecs <= 0) {
+ /* we already reached the time */
+ return 0;
+ }
+
+ rec->num_delayed_client_disconnects++;
+ proxy->delayed_disconnect = TRUE;
+ proxy->to = timeout_add(delay_msecs, login_proxy_free_final, proxy);
+ DLLIST_PREPEND(&login_proxies_disconnecting, proxy);
+ return delay_msecs;
+}
+
static void ATTR_NULL(2)
-login_proxy_free_reason(struct login_proxy **_proxy, const char *reason)
+login_proxy_free_full(struct login_proxy **_proxy, const char *reason,
+ bool delayed)
{
struct login_proxy *proxy = *_proxy;
struct client *client = proxy->client;
const char *ipstr;
+ unsigned int delay_ms = 0;
*_proxy = NULL;
@@ -444,18 +540,23 @@
return;
proxy->destroying = TRUE;
+ /* we'll disconnect server side in any case. */
login_proxy_disconnect(proxy);
if (proxy->client_fd != -1) {
/* detached proxy */
DLLIST_REMOVE(&login_proxies, proxy);
+ if (delayed)
+ delay_ms = login_proxy_delay_disconnect(proxy);
+
ipstr = net_ip2addr(&proxy->client->ip);
client_log(proxy->client, t_strdup_printf(
- "proxy(%s): disconnecting %s%s",
+ "proxy(%s): disconnecting %s%s%s",
proxy->client->virtual_user,
ipstr != NULL ? ipstr : "",
- reason == NULL ? "" : t_strdup_printf(" (%s)", reason)));
+ reason == NULL ? "" : t_strdup_printf(" (%s)", reason),
+ delay_ms == 0 ? "" : t_strdup_printf(" - disconnecting client in %ums", delay_ms)));
if (proxy->client_io != NULL)
io_remove(&proxy->client_io);
@@ -469,12 +570,29 @@
if (proxy->callback != NULL)
proxy->callback(proxy->client);
}
- login_proxy_free_final(proxy);
+ if (delay_ms == 0)
+ login_proxy_free_final(proxy);
+ else {
+ proxy->client_io = io_add(proxy->client_fd, IO_READ,
+ proxy_client_disconnected_input, proxy);
+ }
client->login_proxy = NULL;
client_unref(&client);
}
+static void ATTR_NULL(2)
+login_proxy_free_reason(struct login_proxy **_proxy, const char *reason)
+{
+ login_proxy_free_full(_proxy, reason, FALSE);
+}
+
+static void ATTR_NULL(2)
+login_proxy_free_delayed(struct login_proxy **_proxy, const char *reason)
+{
+ login_proxy_free_full(_proxy, reason, TRUE);
+}
+
void login_proxy_free(struct login_proxy **_proxy)
{
login_proxy_free_reason(_proxy, NULL);
@@ -680,7 +798,7 @@
next = proxy->next;
if (strcmp(proxy->client->virtual_user, args[0]) == 0) {
- login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON);
+ login_proxy_free_delayed(&proxy, KILLED_BY_ADMIN_REASON);
count++;
}
}
@@ -726,7 +844,7 @@
if (director_username_hash(proxy->client) == hash &&
!net_ip_compare(&proxy->ip, &except_ip)) {
- login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON);
+ login_proxy_free_delayed(&proxy, KILLED_BY_ADMIN_REASON);
count++;
}
}
@@ -799,6 +917,8 @@
proxy = login_proxies;
login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON);
}
+ while (login_proxies_disconnecting != NULL)
+ login_proxy_free_final(login_proxies_disconnecting);
if (login_proxy_ipc_server != NULL)
ipc_server_deinit(&login_proxy_ipc_server);
login_proxy_state_deinit(&proxy_state);
diff -r 757a566a0d1c -r 2eeef1619161 src/login-common/login-settings.c
--- a/src/login-common/login-settings.c Mon Sep 07 17:10:19 2015 +0300
+++ b/src/login-common/login-settings.c Mon Sep 07 18:50:24 2015 +0300
@@ -26,6 +26,7 @@
More information about the dovecot-cvs
mailing list