dovecot-2.0: lib-sql: Use generic sql connection pooling code fo...
dovecot at dovecot.org
dovecot at dovecot.org
Tue May 4 17:57:48 EEST 2010
details: http://hg.dovecot.org/dovecot-2.0/rev/1c8cc349ef55
changeset: 11261:1c8cc349ef55
user: Timo Sirainen <tss at iki.fi>
date: Tue May 04 17:55:23 2010 +0300
description:
lib-sql: Use generic sql connection pooling code for mysql/pgsql.
It's possible to give multiple host settings to do load balancing / HA.
If one host is down, another one is tried. All queries are automatically
retried in another host if they fail in first one.
Since PostgreSQL support async queries, Dovecot can create multiple
connections to the database as needed, so it can do lookups in parallel. The
number of connections can be changed with maxconns=n in connect_query, the
default is 5.
diffstat:
doc/example-config/dovecot-sql.conf.ext | 7 +-
src/lib-sql/Makefile.am | 1 +
src/lib-sql/driver-mysql.c | 321 ++++-----------
src/lib-sql/driver-pgsql.c | 660 +++++++++++++-------------------
src/lib-sql/driver-sqlite.c | 20 +-
src/lib-sql/driver-sqlpool.c | 756 +++++++++++++++++++++++++++++++++++++
src/lib-sql/sql-api-private.h | 73 +++-
src/lib-sql/sql-api.c | 98 ++++-
src/lib-sql/sql-api.h | 6 +-
9 files changed, 1296 insertions(+), 646 deletions(-)
diffs (truncated from 2763 to 300 lines):
diff -r 669e8266927e -r 1c8cc349ef55 doc/example-config/dovecot-sql.conf.ext
--- a/doc/example-config/dovecot-sql.conf.ext Tue May 04 16:12:00 2010 +0300
+++ b/doc/example-config/dovecot-sql.conf.ext Tue May 04 17:55:23 2010 +0300
@@ -30,9 +30,14 @@
# Database connection string. This is driver-specific setting.
#
+# HA / round-robin load-balancing is supported by giving multiple host
+# settings, like: host=sql1.host.org host=sql2.host.org
+#
# pgsql:
# For available options, see the PostgreSQL documention for the
# PQconnectdb function of libpq.
+# Use maxconns=n (default 5) to change how many connections Dovecot can
+# create to pgsql.
#
# mysql:
# Basic options emulate PostgreSQL option names:
@@ -50,8 +55,6 @@
# You can connect to UNIX sockets by using host: host=/var/run/mysql.sock
# Note that currently you can't use spaces in parameters.
#
-# MySQL supports multiple host parameters for load balancing / HA.
-#
# sqlite:
# The path to the database file.
#
diff -r 669e8266927e -r 1c8cc349ef55 src/lib-sql/Makefile.am
--- a/src/lib-sql/Makefile.am Tue May 04 16:12:00 2010 +0300
+++ b/src/lib-sql/Makefile.am Tue May 04 17:55:23 2010 +0300
@@ -38,6 +38,7 @@
driver_sources = \
driver-mysql.c \
driver-pgsql.c \
+ driver-sqlpool.c \
driver-sqlite.c
endif
diff -r 669e8266927e -r 1c8cc349ef55 src/lib-sql/driver-mysql.c
--- a/src/lib-sql/driver-mysql.c Tue May 04 16:12:00 2010 +0300
+++ b/src/lib-sql/driver-mysql.c Tue May 04 17:55:23 2010 +0300
@@ -12,49 +12,23 @@
#include <mysql.h>
#include <errmsg.h>
-/* Abort connect() if it can't connect within this time. */
-#define MYSQL_CONNECT_FAILURE_TIMEOUT 10
-
-/* Minimum delay between reconnecting to same server */
-#define CONNECT_MIN_DELAY 1
-/* Maximum time to avoiding reconnecting to same server */
-#define CONNECT_MAX_DELAY (60*30)
-/* If no servers are connected but a query is requested, try reconnecting to
- next server which has been disconnected longer than this (with a single
- server setup this is really the "max delay" and the CONNECT_MAX_DELAY
- is never used). */
-#define CONNECT_RESET_DELAY 15
-
struct mysql_db {
struct sql_db api;
pool_t pool;
- const char *user, *password, *dbname, *unix_socket;
+ const char *user, *password, *dbname, *host, *unix_socket;
const char *ssl_cert, *ssl_key, *ssl_ca, *ssl_ca_path, *ssl_cipher;
const char *option_file, *option_group;
unsigned int port, client_flags;
- ARRAY_DEFINE(connections, struct mysql_connection);
+ MYSQL *mysql;
unsigned int next_query_connection;
-};
-struct mysql_connection {
- struct mysql_db *db;
-
- MYSQL *mysql;
- const char *host;
-
- unsigned int connect_delay;
- unsigned int connect_failure_count;
-
- time_t last_connect;
- unsigned int connected:1;
unsigned int ssl_set:1;
};
struct mysql_result {
struct sql_result api;
- struct mysql_connection *conn;
MYSQL_RES *result;
MYSQL_ROW row;
@@ -67,137 +41,83 @@
struct sql_transaction_context ctx;
pool_t query_pool;
- struct mysql_query_list *head, *tail;
-
const char *error;
unsigned int failed:1;
};
-struct mysql_query_list {
- struct mysql_query_list *next;
- const char *query;
- unsigned int *affected_rows;
-};
-
extern const struct sql_db driver_mysql_db;
extern const struct sql_result driver_mysql_result;
extern const struct sql_result driver_mysql_error_result;
-static bool driver_mysql_connect(struct mysql_connection *conn)
+static int driver_mysql_connect(struct sql_db *_db)
{
- struct mysql_db *db = conn->db;
+ struct mysql_db *db = (struct mysql_db *)_db;
const char *unix_socket, *host;
unsigned long client_flags = db->client_flags;
- time_t now;
bool failed;
- if (conn->connected)
- return TRUE;
+ i_assert(db->api.state == SQL_DB_STATE_DISCONNECTED);
- /* don't try reconnecting more than once a second */
- now = time(NULL);
- if (conn->last_connect + (time_t)conn->connect_delay > now)
- return FALSE;
- conn->last_connect = now;
-
- if (*conn->host == '/') {
- unix_socket = conn->host;
+ if (*db->host == '/') {
+ unix_socket = db->host;
host = NULL;
} else {
unix_socket = NULL;
- host = conn->host;
+ host = db->host;
}
if (db->option_file != NULL) {
- mysql_options(conn->mysql, MYSQL_READ_DEFAULT_FILE,
+ mysql_options(db->mysql, MYSQL_READ_DEFAULT_FILE,
db->option_file);
}
- mysql_options(conn->mysql, MYSQL_READ_DEFAULT_GROUP,
+ mysql_options(db->mysql, MYSQL_READ_DEFAULT_GROUP,
db->option_group != NULL ? db->option_group : "client");
- if (!conn->ssl_set && (db->ssl_ca != NULL || db->ssl_ca_path != NULL)) {
+ if (!db->ssl_set && (db->ssl_ca != NULL || db->ssl_ca_path != NULL)) {
#ifdef HAVE_MYSQL_SSL
- mysql_ssl_set(conn->mysql, db->ssl_key, db->ssl_cert,
+ mysql_ssl_set(db->mysql, db->ssl_key, db->ssl_cert,
db->ssl_ca, db->ssl_ca_path
#ifdef HAVE_MYSQL_SSL_CIPHER
, db->ssl_cipher
#endif
);
- conn->ssl_set = TRUE;
+ db->ssl_set = TRUE;
#else
i_fatal("mysql: SSL support not compiled in "
"(remove ssl_ca and ssl_ca_path settings)");
#endif
}
- alarm(MYSQL_CONNECT_FAILURE_TIMEOUT);
+ alarm(SQL_CONNECT_TIMEOUT_SECS);
#ifdef CLIENT_MULTI_RESULTS
client_flags |= CLIENT_MULTI_RESULTS;
#endif
/* CLIENT_MULTI_RESULTS allows the use of stored procedures */
- failed = mysql_real_connect(conn->mysql, host, db->user, db->password,
+ failed = mysql_real_connect(db->mysql, host, db->user, db->password,
db->dbname, db->port, unix_socket,
client_flags) == NULL;
alarm(0);
if (failed) {
- if (conn->connect_failure_count > 0) {
- /* increase delay between reconnections to this
- server */
- conn->connect_delay *= 5;
- if (conn->connect_delay > CONNECT_MAX_DELAY)
- conn->connect_delay = CONNECT_MAX_DELAY;
- }
- conn->connect_failure_count++;
-
+ sql_db_set_state(&db->api, SQL_DB_STATE_DISCONNECTED);
i_error("mysql: Connect failed to %s (%s): %s - "
"waiting for %u seconds before retry",
host != NULL ? host : unix_socket, db->dbname,
- mysql_error(conn->mysql), conn->connect_delay);
- return FALSE;
+ mysql_error(db->mysql), db->api.connect_delay);
+ return -1;
} else {
i_info("mysql: Connected to %s%s (%s)",
host != NULL ? host : unix_socket,
- conn->ssl_set ? " using SSL" : "", db->dbname);
+ db->ssl_set ? " using SSL" : "", db->dbname);
- conn->connect_failure_count = 0;
- conn->connect_delay = CONNECT_MIN_DELAY;
- conn->connected = TRUE;
- return TRUE;
+ sql_db_set_state(&db->api, SQL_DB_STATE_IDLE);
+ return 1;
}
}
-static int driver_mysql_connect_all(struct sql_db *_db)
+static void driver_mysql_disconnect(struct sql_db *_db ATTR_UNUSED)
{
- struct mysql_db *db = (struct mysql_db *)_db;
- struct mysql_connection *conn;
- int ret = -1;
-
- array_foreach_modifiable(&db->connections, conn) {
- if (driver_mysql_connect(conn))
- ret = 1;
- }
- return ret;
-}
-
-static void driver_mysql_connection_add(struct mysql_db *db, const char *host)
-{
- struct mysql_connection *conn;
-
- conn = array_append_space(&db->connections);
- conn->db = db;
- conn->host = p_strdup(db->pool, host);
- conn->mysql = mysql_init(NULL);
- if (conn->mysql == NULL)
- i_fatal("mysql_init() failed");
-
- conn->connect_delay = CONNECT_MIN_DELAY;
-}
-
-static void driver_mysql_connection_free(struct mysql_connection *conn)
-{
- mysql_close(conn->mysql);
}
static void driver_mysql_parse_connect_string(struct mysql_db *db,
@@ -221,7 +141,7 @@
field = NULL;
if (strcmp(name, "host") == 0 ||
strcmp(name, "hostaddr") == 0)
- driver_mysql_connection_add(db, value);
+ field = &db->host;
else if (strcmp(name, "user") == 0)
field = &db->user;
else if (strcmp(name, "password") == 0)
@@ -253,8 +173,12 @@
*field = p_strdup(db->pool, value);
}
- if (array_count(&db->connections) == 0)
+ if (db->host == NULL)
i_fatal("mysql: No hosts given in connect string");
+
+ db->mysql = mysql_init(NULL);
+ if (db->mysql == NULL)
+ i_fatal("mysql_init() failed");
}
static struct sql_db *driver_mysql_init_v(const char *connect_string)
@@ -266,7 +190,6 @@
db = p_new(pool, struct mysql_db, 1);
db->pool = pool;
db->api = driver_mysql_db;
- p_array_init(&db->connections, db->pool, 6);
T_BEGIN {
driver_mysql_parse_connect_string(db, connect_string);
@@ -277,88 +200,26 @@
static void driver_mysql_deinit_v(struct sql_db *_db)
{
struct mysql_db *db = (struct mysql_db *)_db;
- struct mysql_connection *conn;
- array_foreach_modifiable(&db->connections, conn)
- (void)driver_mysql_connection_free(conn);
-
+ mysql_close(db->mysql);
array_free(&_db->module_contexts);
pool_unref(&db->pool);
}
-static enum sql_db_flags
-driver_mysql_get_flags(struct sql_db *db ATTR_UNUSED)
+static int driver_mysql_do_query(struct mysql_db *db, const char *query)
{
More information about the dovecot-cvs
mailing list