dovecot-2.2: http: Implemented delayed requests scheduling.
dovecot at dovecot.org
dovecot at dovecot.org
Fri Nov 22 22:13:13 EET 2013
details: http://hg.dovecot.org/dovecot-2.2/rev/d2f2423615d7
changeset: 17005:d2f2423615d7
user: Stephan Bosch <stephan at rename-it.nl>
date: Fri Nov 22 22:12:08 2013 +0200
description:
http: Implemented delayed requests scheduling.
Requests can now be (re)submitted with a delay. The request is not sent
until the delay time expires. This facilitates handling the Retry-After
header in responses. This can either be performed automatically if the
indicated delay is not too long or explicitly by the code using lib-http.
diffstat:
src/lib-http/http-client-connection.c | 27 +++++++-
src/lib-http/http-client-private.h | 10 ++-
src/lib-http/http-client-queue.c | 101 +++++++++++++++++++++++++++++++++-
src/lib-http/http-client-request.c | 46 +++++++++++---
src/lib-http/http-client.c | 1 +
src/lib-http/http-client.h | 11 +++
6 files changed, 174 insertions(+), 22 deletions(-)
diffs (truncated from 362 to 300 lines):
diff -r c215131c2c18 -r d2f2423615d7 src/lib-http/http-client-connection.c
--- a/src/lib-http/http-client-connection.c Fri Nov 22 22:08:44 2013 +0200
+++ b/src/lib-http/http-client-connection.c Fri Nov 22 22:12:08 2013 +0200
@@ -595,19 +595,36 @@
conn->close_indicated = response.connection_close;
if (!aborted) {
+ bool handled = FALSE;
+
+ /* failed Expect: */
if (response.status == 417 && req->payload_sync) {
/* drop Expect: continue */
req->payload_sync = FALSE;
conn->output_locked = FALSE;
conn->peer->no_payload_sync = TRUE;
- http_client_request_retry_response(req, &response);
-
+ if (http_client_request_try_retry(req))
+ handled = TRUE;
+ /* redirection */
} else if (!req->client->set.no_auto_redirect &&
response.status / 100 == 3 && response.status != 304 &&
response.location != NULL) {
- /* redirect */
- http_client_request_redirect(req, response.status, response.location);
- } else {
+ /* redirect (possibly after delay) */
+ if (http_client_request_delay_from_response(req, &response) >= 0) {
+ http_client_request_redirect
+ (req, response.status, response.location);
+ handled = TRUE;
+ }
+ /* service unavailable */
+ } else if (response.status == 503) {
+ /* automatically retry after delay if indicated */
+ if ( response.retry_after != (time_t)-1 &&
+ http_client_request_delay_from_response(req, &response) > 0 &&
+ http_client_request_try_retry(req))
+ handled = TRUE;
+ }
+
+ if (!handled) {
/* response for application */
if (!http_client_connection_return_response(conn, req, &response))
return;
diff -r c215131c2c18 -r d2f2423615d7 src/lib-http/http-client-private.h
--- a/src/lib-http/http-client-private.h Fri Nov 22 22:08:44 2013 +0200
+++ b/src/lib-http/http-client-private.h Fri Nov 22 22:12:08 2013 +0200
@@ -70,6 +70,8 @@
uoff_t payload_size, payload_offset;
struct ostream *payload_output;
+ struct timeval release_time;
+
unsigned int attempts;
unsigned int redirects;
@@ -182,9 +184,9 @@
ARRAY_TYPE(http_client_peer) pending_peers;
/* requests pending in queue to be picked up by connections */
- ARRAY_TYPE(http_client_request) request_queue;
+ ARRAY_TYPE(http_client_request) request_queue, delayed_request_queue;
- struct timeout *to_connect;
+ struct timeout *to_connect, *to_delayed;
};
struct http_client_host {
@@ -229,6 +231,8 @@
void http_client_request_ref(struct http_client_request *req);
void http_client_request_unref(struct http_client_request **_req);
+int http_client_request_delay_from_response(struct http_client_request *req,
+ const struct http_response *response);
enum http_response_payload_type
http_client_request_get_payload_type(struct http_client_request *req);
int http_client_request_send(struct http_client_request *req,
@@ -243,8 +247,6 @@
void http_client_request_resubmit(struct http_client_request *req);
void http_client_request_retry(struct http_client_request *req,
unsigned int status, const char *error);
-void http_client_request_retry_response(struct http_client_request *req,
- struct http_response *response);
void http_client_request_send_error(struct http_client_request *req,
unsigned int status, const char *error);
void http_client_request_error_delayed(struct http_client_request **_req);
diff -r c215131c2c18 -r d2f2423615d7 src/lib-http/http-client-queue.c
--- a/src/lib-http/http-client-queue.c Fri Nov 22 22:08:44 2013 +0200
+++ b/src/lib-http/http-client-queue.c Fri Nov 22 22:12:08 2013 +0200
@@ -5,10 +5,12 @@
#include "str.h"
#include "hash.h"
#include "array.h"
+#include "bsearch-insert-pos.h"
#include "llist.h"
#include "ioloop.h"
#include "istream.h"
#include "ostream.h"
+#include "time-util.h"
#include "dns-lookup.h"
#include "http-response-parser.h"
@@ -41,6 +43,10 @@
* Queue
*/
+static void
+http_client_queue_set_delay_timer(struct http_client_queue *queue,
+ struct timeval time);
+
static struct http_client_queue *
http_client_queue_find(struct http_client_host *host,
const struct http_client_peer_addr *addr)
@@ -92,6 +98,7 @@
queue->name = name;
queue->ips_connect_idx = 0;
i_array_init(&queue->request_queue, 16);
+ i_array_init(&queue->delayed_request_queue, 4);
array_append(&host->queues, &queue, 1);
}
@@ -106,8 +113,11 @@
if (array_is_created(&queue->pending_peers))
array_free(&queue->pending_peers);
array_free(&queue->request_queue);
+ array_free(&queue->delayed_request_queue);
if (queue->to_connect != NULL)
timeout_remove(&queue->to_connect);
+ if (queue->to_delayed != NULL)
+ timeout_remove(&queue->to_delayed);
i_free(queue->name);
i_free(queue);
}
@@ -122,6 +132,12 @@
http_client_request_error(*req, status, error);
}
array_clear(&queue->request_queue);
+
+ /* abort all delayed requests */
+ array_foreach_modifiable(&queue->delayed_request_queue, req) {
+ http_client_request_error(*req, status, error);
+ }
+ array_clear(&queue->delayed_request_queue);
}
void
@@ -309,10 +325,11 @@
return TRUE;
}
-void http_client_queue_submit_request(struct http_client_queue *queue,
+static void http_client_queue_submit_now(struct http_client_queue *queue,
struct http_client_request *req)
{
- req->queue = queue;
+ req->release_time.tv_sec = 0;
+ req->release_time.tv_usec = 0;
if (req->urgent)
array_insert(&queue->request_queue, 0, &req, 1);
@@ -320,6 +337,84 @@
array_append(&queue->request_queue, &req, 1);
}
+static void
+http_client_queue_delay_timeout(struct http_client_queue *queue)
+{
+ struct http_client_request *const *reqs;
+ unsigned int count, i, finished;
+
+ io_loop_time_refresh();
+
+ finished = 0;
+ reqs = array_get(&queue->delayed_request_queue, &count);
+ for (i = 0; i < count; i++) {
+ if (timeval_cmp(&reqs[i]->release_time, &ioloop_timeval) > 0) {
+ break;
+ }
+
+ http_client_queue_debug(queue,
+ "Activated delayed request %s%s",
+ http_client_request_label(reqs[i]),
+ (reqs[i]->urgent ? " (urgent)" : ""));
+ http_client_queue_submit_now(queue, reqs[i]);
+ finished++;
+ }
+ i_assert(finished > 0);
+ if (i < count) {
+ http_client_queue_set_delay_timer(queue, reqs[i]->release_time);
+ }
+ array_delete(&queue->delayed_request_queue, 0, finished);
+
+ http_client_queue_connection_setup(queue);
+}
+
+static void
+http_client_queue_set_delay_timer(struct http_client_queue *queue,
+ struct timeval time)
+{
+ int usecs = timeval_diff_usecs(&time, &ioloop_timeval);
+ int msecs;
+
+ /* round up to nearest microsecond */
+ msecs = (usecs + 999) / 1000;
+
+ /* set timer */
+ if (queue->to_delayed != NULL)
+ timeout_remove(&queue->to_delayed);
+ queue->to_delayed = timeout_add
+ (msecs, http_client_queue_delay_timeout, queue);
+}
+
+static int
+http_client_queue_delayed_cmp(struct http_client_request *const *req1,
+ struct http_client_request *const *req2)
+{
+ return timeval_cmp(&(*req1)->release_time, &(*req2)->release_time);
+}
+
+void http_client_queue_submit_request(struct http_client_queue *queue,
+ struct http_client_request *req)
+{
+ unsigned int insert_idx;
+
+ req->queue = queue;
+
+ if (req->release_time.tv_sec > 0) {
+ io_loop_time_refresh();
+
+ if (timeval_cmp(&req->release_time, &ioloop_timeval) > 0) {
+ (void)array_bsearch_insert_pos(&queue->delayed_request_queue,
+ &req, http_client_queue_delayed_cmp, &insert_idx);
+ array_insert(&queue->delayed_request_queue, insert_idx, &req, 1);
+ if (insert_idx == 0)
+ http_client_queue_set_delay_timer(queue, req->release_time);
+ return;
+ }
+ }
+
+ http_client_queue_submit_now(queue, req);
+}
+
struct http_client_request *
http_client_queue_claim_request(struct http_client_queue *queue,
const struct http_client_peer_addr *addr, bool no_urgent)
@@ -372,4 +467,6 @@
{
if (queue->to_connect != NULL)
queue->to_connect = io_loop_move_timeout(&queue->to_connect);
+ if (queue->to_delayed != NULL)
+ queue->to_delayed = io_loop_move_timeout(&queue->to_delayed);
}
diff -r c215131c2c18 -r d2f2423615d7 src/lib-http/http-client-request.c
--- a/src/lib-http/http-client-request.c Fri Nov 22 22:08:44 2013 +0200
+++ b/src/lib-http/http-client-request.c Fri Nov 22 22:12:08 2013 +0200
@@ -269,6 +269,40 @@
req->payload_sync = TRUE;
}
+void http_client_request_delay_until(struct http_client_request *req,
+ time_t time)
+{
+ req->release_time.tv_sec = time;
+ req->release_time.tv_usec = 0;
+}
+
+void http_client_request_delay(struct http_client_request *req,
+ time_t seconds)
+{
+ req->release_time = ioloop_timeval;
+ req->release_time.tv_sec += seconds;
+}
+
+int http_client_request_delay_from_response(struct http_client_request *req,
+ const struct http_response *response)
+{
+ time_t retry_after = response->retry_after;
+ unsigned int max;
+
+ if (retry_after == (time_t)-1)
+ return 0; /* no delay */
+ if (retry_after < ioloop_time)
+ return 0; /* delay already expired */
+ max = (req->client->set.max_auto_retry_delay == 0 ?
+ req->client->set.request_timeout_msecs / 1000 :
+ req->client->set.max_auto_retry_delay);
+ if ((retry_after - ioloop_time) > max)
+ return -1; /* delay too long */
+ req->release_time.tv_sec = retry_after;
+ req->release_time.tv_usec = 0;
+ return 1; /* valid delay */
+}
+
enum http_request_state
http_client_request_get_state(struct http_client_request *req)
{
@@ -902,7 +936,7 @@
req->conn = NULL;
req->peer = NULL;
More information about the dovecot-cvs
mailing list