dovecot-2.2: lib-storage: Added caching for mail_get_binary_size()
dovecot at dovecot.org
dovecot at dovecot.org
Sat Aug 11 07:47:40 EEST 2012
details: http://hg.dovecot.org/dovecot-2.2/rev/f75d5a2eddcb
changeset: 14862:f75d5a2eddcb
user: Timo Sirainen <tss at iki.fi>
date: Sat Aug 11 07:47:29 2012 +0300
description:
lib-storage: Added caching for mail_get_binary_size()
diffstat:
src/lib-mail/Makefile.am | 2 +
src/lib-mail/message-binary-part.c | 40 +++
src/lib-mail/message-binary-part.h | 26 ++
src/lib-storage/index/index-mail-binary.c | 356 ++++++++++++++++++++++-------
src/lib-storage/index/index-mail.c | 6 +-
src/lib-storage/index/index-mail.h | 2 +
src/lib-storage/mail.c | 28 +-
7 files changed, 364 insertions(+), 96 deletions(-)
diffs (truncated from 689 to 300 lines):
diff -r b45d968adff8 -r f75d5a2eddcb src/lib-mail/Makefile.am
--- a/src/lib-mail/Makefile.am Sat Aug 11 07:46:42 2012 +0300
+++ b/src/lib-mail/Makefile.am Sat Aug 11 07:47:29 2012 +0300
@@ -16,6 +16,7 @@
mail-user-hash.c \
mbox-from.c \
message-address.c \
+ message-binary-part.c \
message-date.c \
message-decoder.c \
message-header-decode.c \
@@ -43,6 +44,7 @@
mbox-from.h \
mail-types.h \
message-address.h \
+ message-binary-part.h \
message-date.h \
message-decoder.h \
message-header-decode.h \
diff -r b45d968adff8 -r f75d5a2eddcb src/lib-mail/message-binary-part.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-mail/message-binary-part.c Sat Aug 11 07:47:29 2012 +0300
@@ -0,0 +1,40 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "numpack.h"
+#include "message-binary-part.h"
+
+void message_binary_part_serialize(const struct message_binary_part *parts,
+ buffer_t *dest)
+{
+ const struct message_binary_part *part;
+
+ for (part = parts; part != NULL; part = part->next) {
+ numpack_encode(dest, part->physical_pos);
+ numpack_encode(dest, part->binary_hdr_size);
+ numpack_encode(dest, part->binary_body_size);
+ }
+}
+
+int message_binary_part_deserialize(pool_t pool, const void *data, size_t size,
+ struct message_binary_part **parts_r)
+{
+ const uint8_t *p = data, *end = p + size;
+ uint64_t n1, n2, n3;
+ struct message_binary_part *part = NULL, *prev_part = NULL;
+
+ while (p != end) {
+ part = p_new(pool, struct message_binary_part, 1);
+ part->next = prev_part;
+ prev_part = part;
+ if (numpack_decode(&p, end, &n1) < 0 ||
+ numpack_decode(&p, end, &n2) < 0 ||
+ numpack_decode(&p, end, &n3) < 0)
+ return -1;
+ part->physical_pos = n1;
+ part->binary_hdr_size = n2;
+ part->binary_body_size = n3;
+ }
+ *parts_r = part;
+ return 0;
+}
diff -r b45d968adff8 -r f75d5a2eddcb src/lib-mail/message-binary-part.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-mail/message-binary-part.h Sat Aug 11 07:47:29 2012 +0300
@@ -0,0 +1,26 @@
+#ifndef MESSAGE_BINARY_PART_H
+#define MESSAGE_BINARY_PART_H
+
+struct message_binary_part {
+ struct message_binary_part *next;
+
+ /* Absolute position from beginning of message. This can be used to
+ match the part to struct message_part. */
+ uoff_t physical_pos;
+ /* Decoded binary header/body size. The binary header size may differ
+ from message_part's, because Content-Transfer-Encoding is changed to
+ "binary". */
+ uoff_t binary_hdr_size;
+ uoff_t binary_body_size;
+};
+
+/* Serialize message binary_part. */
+void message_binary_part_serialize(const struct message_binary_part *parts,
+ buffer_t *dest);
+
+/* Generate struct message_binary_part from serialized data. Returns 0 if ok,
+ -1 if any problems are detected. */
+int message_binary_part_deserialize(pool_t pool, const void *data, size_t size,
+ struct message_binary_part **parts_r);
+
+#endif
diff -r b45d968adff8 -r f75d5a2eddcb src/lib-storage/index/index-mail-binary.c
--- a/src/lib-storage/index/index-mail-binary.c Sat Aug 11 07:46:42 2012 +0300
+++ b/src/lib-storage/index/index-mail-binary.c Sat Aug 11 07:47:29 2012 +0300
@@ -10,24 +10,36 @@
#include "istream-qp.h"
#include "istream-header-filter.h"
#include "ostream.h"
+#include "message-binary-part.h"
#include "message-parser.h"
#include "message-decoder.h"
#include "mail-user.h"
+#include "index-storage.h"
#include "index-mail.h"
#define MAIL_BINARY_CACHE_EXPIRE_MSECS (60*1000)
+#define IS_CONVERTED_CTE(cte) \
+ ((cte) == MESSAGE_CTE_QP || (cte) == MESSAGE_CTE_BASE64)
+
+struct binary_block {
+ struct istream *input;
+ struct message_binary_part bin_part;
+ bool converted, converted_hdr;
+};
+
struct binary_ctx {
struct mail *mail;
struct istream *input;
bool has_nuls, converted;
- ARRAY_DEFINE(streams, struct istream *);
+ ARRAY_DEFINE(blocks, struct binary_block);
uoff_t copy_start_offset;
};
static void binary_copy_to(struct binary_ctx *ctx, uoff_t end_offset)
{
+ struct binary_block *block;
struct istream *linput, *cinput;
uoff_t orig_offset, size;
@@ -43,7 +55,9 @@
linput = i_stream_create_limit(ctx->input, size);
cinput = i_stream_create_crlf(linput);
i_stream_unref(&linput);
- array_append(&ctx->streams, &cinput, 1);
+
+ block = array_append_space(&ctx->blocks);
+ block->input = cinput;
i_stream_seek(ctx->input, orig_offset);
}
@@ -72,7 +86,8 @@
struct message_header_line *hdr;
struct message_part *child;
struct message_size hdr_size;
- struct istream *linput, *cinput;
+ struct istream *linput;
+ struct binary_block *block;
enum message_cte cte;
int ret;
@@ -109,17 +124,21 @@
i_stream_seek(ctx->input, part->physical_pos);
if (!include_hdr) {
/* body only */
- } else if (cte == MESSAGE_CTE_QP || cte == MESSAGE_CTE_BASE64) {
+ } else if (IS_CONVERTED_CTE(cte)) {
/* write header with modified content-type */
if (ctx->copy_start_offset != 0)
binary_copy_to(ctx, part->physical_pos);
+ block = array_append_space(&ctx->blocks);
+ block->bin_part.physical_pos = part->physical_pos;
+ block->converted = TRUE;
+ block->converted_hdr = TRUE;
+
linput = i_stream_create_limit(ctx->input, (uoff_t)-1);
- cinput = i_stream_create_header_filter(linput,
+ block->input = i_stream_create_header_filter(linput,
HEADER_FILTER_EXCLUDE | HEADER_FILTER_HIDE_BODY,
filter_headers, N_ELEMENTS(filter_headers),
binary_cte_filter_callback, ctx);
i_stream_unref(&linput);
- array_append(&ctx->streams, &cinput, 1);
} else {
/* copy everything as-is until the end of this header */
binary_copy_to(ctx, part->physical_pos +
@@ -141,6 +160,9 @@
}
/* single part - write decoded data */
+ block = array_append_space(&ctx->blocks);
+ block->bin_part.physical_pos = part->physical_pos;
+
i_stream_seek(ctx->input, part->physical_pos +
part->header_size.physical_size);
linput = i_stream_create_limit(ctx->input, part->body_size.physical_size);
@@ -152,20 +174,18 @@
/* no conversion necessary */
if ((part->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0)
ctx->has_nuls = TRUE;
- cinput = linput;
- i_stream_ref(cinput);
+ block->input = i_stream_create_crlf(linput);
break;
case MESSAGE_CTE_QP:
- cinput = i_stream_create_qp_decoder(linput);
- ctx->converted = TRUE;
+ block->input = i_stream_create_qp_decoder(linput);
+ ctx->converted = block->converted = TRUE;
break;
case MESSAGE_CTE_BASE64:
- cinput = i_stream_create_base64_decoder(linput);
- ctx->converted = TRUE;
+ block->input = i_stream_create_base64_decoder(linput);
+ ctx->converted = block->converted = TRUE;
break;
}
i_stream_unref(&linput);
- array_append(&ctx->streams, &cinput, 1);
ctx->copy_start_offset = part->physical_pos +
part->header_size.physical_size +
@@ -200,14 +220,225 @@
static void binary_streams_free(struct binary_ctx *ctx)
{
- struct istream **input;
+ struct binary_block *block;
- array_foreach_modifiable(&ctx->streams, input) {
- if (*input != NULL)
- i_stream_unref(input);
+ array_foreach_modifiable(&ctx->blocks, block)
+ i_stream_unref(&block->input);
+}
+
+static void
+binary_parts_update(struct binary_ctx *ctx, const struct message_part *part,
+ struct message_binary_part **msg_bin_parts)
+{
+ struct index_mail *mail = (struct index_mail *)ctx->mail;
+ struct binary_block *blocks;
+ struct message_binary_part bin_part;
+ unsigned int i, count;
+ uoff_t size;
+ bool found;
+
+ blocks = array_get_modifiable(&ctx->blocks, &count);
+ for (; part != NULL; part = part->next) {
+ binary_parts_update(ctx, part->children, msg_bin_parts);
+
+ memset(&bin_part, 0, sizeof(bin_part));
+ /* default to unchanged header */
+ bin_part.binary_hdr_size = part->header_size.virtual_size;
+ bin_part.physical_pos = part->physical_pos;
+ found = FALSE;
+ for (i = 0; i < count; i++) {
+ if (blocks[i].bin_part.physical_pos != part->physical_pos ||
+ !blocks[i].converted)
+ continue;
+
+ size = blocks[i].input->v_offset;
+ if (blocks[i].converted_hdr)
+ bin_part.binary_hdr_size = size;
+ else
+ bin_part.binary_body_size = size;
+ found = TRUE;
+ }
+ if (found) {
+ bin_part.next = *msg_bin_parts;
+ *msg_bin_parts = p_new(mail->data_pool,
+ struct message_binary_part, 1);
+ **msg_bin_parts = bin_part;
+ }
}
}
+static void binary_parts_cache(struct binary_ctx *ctx)
+{
+ struct index_mail *mail = (struct index_mail *)ctx->mail;
+ buffer_t *buf;
+
+ buf = buffer_create_dynamic(pool_datastack_create(), 128);
+ message_binary_part_serialize(mail->data.bin_parts, buf);
+ index_mail_cache_add(mail, MAIL_CACHE_BINARY_PARTS,
+ buf->data, buf->used);
+}
+
+static struct istream **blocks_get_streams(struct binary_ctx *ctx)
+{
+ struct istream **streams;
+ const struct binary_block *blocks;
+ unsigned int i, count;
+
+ blocks = array_get(&ctx->blocks, &count);
+ streams = t_new(struct istream *, count+1);
+ for (i = 0; i < count; i++)
+ streams[i] = blocks[i].input;
+ return streams;
+}
+
+static int
+index_mail_read_binary_to_cache(struct mail *_mail,
+ const struct message_part *part,
+ bool include_hdr, bool *binary_r,
+ bool *converted_r)
More information about the dovecot-cvs
mailing list