dovecot-2.2: lib-http: http-client: Fixed request scheduling and...
dovecot at dovecot.org
dovecot at dovecot.org
Sun Sep 15 03:38:25 EEST 2013
details: http://hg.dovecot.org/dovecot-2.2/rev/bdc5d6dcfc53
changeset: 16733:bdc5d6dcfc53
user: Stephan Bosch <stephan at rename-it.nl>
date: Sun Sep 15 03:31:28 2013 +0300
description:
lib-http: http-client: Fixed request scheduling and connection management.
diffstat:
src/lib-http/http-client-connection.c | 49 ++--
src/lib-http/http-client-host.c | 159 +++++++++-------
src/lib-http/http-client-peer.c | 317 ++++++++++++++++++++++++++-------
src/lib-http/http-client-private.h | 38 ++-
src/lib-http/http-client-request.c | 2 +-
src/lib-http/http-client.c | 5 +
src/lib-http/http-client.h | 4 +-
7 files changed, 393 insertions(+), 181 deletions(-)
diffs (truncated from 1003 to 300 lines):
diff -r 0bcbc30b81b7 -r bdc5d6dcfc53 src/lib-http/http-client-connection.c
--- a/src/lib-http/http-client-connection.c Sun Sep 15 03:29:03 2013 +0300
+++ b/src/lib-http/http-client-connection.c Sun Sep 15 03:31:28 2013 +0300
@@ -167,8 +167,7 @@
http_client_connection_unref(&conn);
}
-static void
-http_client_connection_check_idle(struct http_client_connection *conn)
+void http_client_connection_check_idle(struct http_client_connection *conn)
{
unsigned int timeout, count;
@@ -254,7 +253,7 @@
}
}
-bool http_client_connection_next_request(struct http_client_connection *conn)
+int http_client_connection_next_request(struct http_client_connection *conn)
{
struct http_client_request *req = NULL;
const char *error;
@@ -262,17 +261,15 @@
if (!http_client_connection_is_ready(conn)) {
http_client_connection_debug(conn, "Not ready for next request");
- return FALSE;
+ return 0;
}
/* claim request, but no urgent request can be second in line */
have_pending_requests = array_count(&conn->request_wait_list) > 0 ||
conn->pending_request != NULL;
req = http_client_peer_claim_request(conn->peer, have_pending_requests);
- if (req == NULL) {
- http_client_connection_check_idle(conn);
- return FALSE;
- }
+ if (req == NULL)
+ return 0;
if (conn->to_idle != NULL)
timeout_remove(&conn->to_idle);
@@ -300,7 +297,7 @@
http_client_connection_abort_temp_error(&conn,
HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
t_strdup_printf("Failed to send request: %s", error));
- return FALSE;
+ return -1;
}
/* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21;
@@ -321,7 +318,7 @@
http_client_connection_continue_timeout, conn);
}
- return TRUE;
+ return 1;
}
static void http_client_connection_destroy(struct connection *_conn)
@@ -595,7 +592,7 @@
conn->output_locked = FALSE;
conn->peer->no_payload_sync = TRUE;
http_client_request_retry(req, response->status, response->reason);
- return;
+
} else if (response->status / 100 == 3 && response->status != 304 &&
response->location != NULL) {
/* redirect */
@@ -648,9 +645,13 @@
}
if (finished > 0) {
+ /* connection still alive after (at least one) request;
+ we can pipeline -> mark for subsequent connections */
+ conn->peer->allows_pipelining = TRUE;
+
/* room for new requests */
- http_client_peer_handle_requests(conn->peer);
- http_client_connection_check_idle(conn);
+ if (http_client_connection_is_ready(conn))
+ http_client_peer_trigger_request_handler(conn->peer);
}
}
@@ -688,8 +689,8 @@
}
if (!conn->output_locked) {
/* room for new requests */
- http_client_peer_handle_requests(conn->peer);
- http_client_connection_check_idle(conn);
+ if (http_client_connection_is_ready(conn))
+ http_client_peer_trigger_request_handler(conn->peer);
}
}
}
@@ -701,27 +702,28 @@
{
struct stat st;
+ /* connected */
conn->connected = TRUE;
- conn->connect_succeeded = TRUE;
if (conn->to_connect != NULL &&
(conn->ssl_iostream == NULL ||
ssl_iostream_is_handshaked(conn->ssl_iostream)))
timeout_remove(&conn->to_connect);
+ /* indicate connection success */
+ conn->connect_succeeded = TRUE;
http_client_peer_connection_success(conn->peer);
+ /* start raw log */
if (conn->client->set.rawlog_dir != NULL &&
stat(conn->client->set.rawlog_dir, &st) == 0) {
iostream_rawlog_create(conn->client->set.rawlog_dir,
&conn->conn.input, &conn->conn.output);
}
+ /* start protocol I/O */
conn->http_parser = http_response_parser_init(conn->conn.input);
o_stream_set_flush_callback(conn->conn.output,
http_client_connection_output, conn);
-
- /* we never pipeline before the first response */
- (void)http_client_connection_next_request(conn);
}
static int
@@ -872,23 +874,24 @@
{
struct http_client_connection *conn;
static unsigned int id = 0;
+ const struct http_client_peer_addr *addr = &peer->addr;
conn = i_new(struct http_client_connection, 1);
conn->refcount = 1;
conn->client = peer->client;
+ conn->id = id++;
conn->peer = peer;
i_array_init(&conn->request_wait_list, 16);
connection_init_client_ip
- (peer->client->conn_list, &conn->conn, &peer->addr.ip, peer->addr.port);
+ (peer->client->conn_list, &conn->conn, &addr->ip, addr->port);
http_client_connection_connect(conn);
- conn->id = id++;
array_append(&peer->conns, &conn, 1);
http_client_connection_debug(conn,
- "Connection created (%d parallel connections exist)",
- array_count(&peer->conns));
+ "Connection created (%d parallel connections exist)%s",
+ array_count(&peer->conns), (conn->to_input == NULL ? "" : " [broken]"));
return conn;
}
diff -r 0bcbc30b81b7 -r bdc5d6dcfc53 src/lib-http/http-client-host.c
--- a/src/lib-http/http-client-host.c Sun Sep 15 03:29:03 2013 +0300
+++ b/src/lib-http/http-client-host.c Sun Sep 15 03:31:28 2013 +0300
@@ -46,13 +46,13 @@
static struct http_client_host_port *
http_client_host_port_find(struct http_client_host *host,
- unsigned int port, const char *https_name)
+ in_port_t port, const char *https_name)
{
struct http_client_host_port *hport;
array_foreach_modifiable(&host->ports, hport) {
- if (hport->port == port &&
- null_strcmp(hport->https_name, https_name) == 0)
+ if (hport->addr.port == port &&
+ null_strcmp(hport->addr.https_name, https_name) == 0)
return hport;
}
@@ -61,7 +61,7 @@
static struct http_client_host_port *
http_client_host_port_init(struct http_client_host *host,
- unsigned int port, const char *https_name)
+ in_port_t port, const char *https_name)
{
struct http_client_host_port *hport;
@@ -69,8 +69,8 @@
if (hport == NULL) {
hport = array_append_space(&host->ports);
hport->host = host;
- hport->port = port;
- hport->https_name = i_strdup(https_name);
+ hport->addr.port = port;
+ hport->addr.https_name = i_strdup(https_name);
hport->ips_connect_idx = 0;
i_array_init(&hport->request_queue, 16);
}
@@ -94,7 +94,9 @@
{
http_client_host_port_error
(hport, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted");
- i_free(hport->https_name);
+ i_free(hport->addr.https_name);
+ if (array_is_created(&hport->pending_peers))
+ array_free(&hport->pending_peers);
array_free(&hport->request_queue);
}
@@ -136,20 +138,25 @@
if (hport->to_connect != NULL)
timeout_remove(&hport->to_connect);
- if (http_client_hport_is_last_connect_ip(hport))
+ if (http_client_hport_is_last_connect_ip(hport)) {
+ /* no more IPs to try */
return;
+ }
/* if our our previous connection attempt takes longer than the
- soft_connect_timeout we start a connection attempt to the next IP in
+ soft_connect_timeout, we start a connection attempt to the next IP in
parallel */
http_client_host_debug(host, "Connection to %s:%u%s is taking a long time; "
"starting parallel connection attempt to next IP",
- net_ip2addr(&host->ips[hport->ips_connect_idx]), hport->port,
- hport->https_name == NULL ? "" :
- t_strdup_printf(" (SSL=%s)", hport->https_name));
+ net_ip2addr(&hport->addr.ip), hport->addr.port,
+ hport->addr.https_name == NULL ? "" :
+ t_strdup_printf(" (SSL=%s)", hport->addr.https_name));
+ /* next IP */
hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count;
+
+ /* setup connection to new peer (can start new soft timeout) */
http_client_host_port_connection_setup(hport);
}
@@ -158,59 +165,41 @@
{
struct http_client_host *host = hport->host;
struct http_client_peer *peer = NULL;
- struct http_client_peer_addr addr;
- unsigned int msecs;
+ const struct http_client_peer_addr *addr = &hport->addr;
+ unsigned int num_requests = array_count(&hport->request_queue);
- addr.ip = host->ips[hport->ips_connect_idx];
- addr.port = hport->port;
- addr.https_name = hport->https_name;
+ if (num_requests == 0)
+ return;
- http_client_host_debug(host, "Setting up connection to %s:%u%s",
- net_ip2addr(&addr.ip), addr.port, addr.https_name == NULL ? "" :
- t_strdup_printf(" (SSL=%s)", addr.https_name));
+ /* update our peer address */
+ hport->addr.ip = host->ips[hport->ips_connect_idx];
- peer = http_client_peer_get(host->client, &addr);
+ http_client_host_debug(host, "Setting up connection to %s:%u%s "
+ "(%u requests pending)", net_ip2addr(&addr->ip), addr->port,
+ addr->https_name == NULL ? "" :
+ t_strdup_printf(" (SSL=%s)", addr->https_name), num_requests);
+
+ /* create/get peer */
+ peer = http_client_peer_get(host->client, addr);
http_client_peer_add_host(peer, host);
- if (http_client_peer_handle_requests(peer))
- hport->pending_connection_count++;
- /* start soft connect time-out (but only if we have another IP left) */
- msecs = host->client->set.soft_connect_timeout_msecs;
- if (!http_client_hport_is_last_connect_ip(hport) && msecs > 0 &&
- hport->to_connect == NULL) {
- hport->to_connect =
- timeout_add(msecs, http_client_host_port_soft_connect_timeout, hport);
- }
-}
+ /* handle requests; creates new connections when needed/possible */
+ http_client_peer_trigger_request_handler(peer);
-static void
-http_client_host_drop_pending_connections(struct http_client_host_port *hport,
- const struct http_client_peer_addr *addr)
-{
- struct http_client_peer *peer;
- struct http_client_connection *const *conns, *conn;
- unsigned int i, count;
+ if (!http_client_peer_is_connected(peer)) {
+ unsigned int msecs;
- for (peer = hport->host->client->peers_list; peer != NULL; peer = peer->next) {
- if (http_client_peer_addr_cmp(&peer->addr, addr) == 0) {
- /* don't drop any connections to the successfully
- connected peer, even if some of the connections
- are pending. they may be intended for urgent
More information about the dovecot-cvs
mailing list