dovecot-2.1: Backported LZ4 compression support from v2.2.
dovecot at dovecot.org
dovecot at dovecot.org
Wed Feb 5 00:48:20 EET 2014
details: http://hg.dovecot.org/dovecot-2.1/rev/8e6b8d53c02b
changeset: 15011:8e6b8d53c02b
user: Timo Sirainen <tss at iki.fi>
date: Wed Feb 05 00:47:55 2014 +0200
description:
Backported LZ4 compression support from v2.2.
diffstat:
configure.in | 25 +++
src/plugins/zlib/Makefile.am | 8 +-
src/plugins/zlib/iostream-lz4.h | 30 +++
src/plugins/zlib/istream-lz4.c | 315 ++++++++++++++++++++++++++++++++++++++++
src/plugins/zlib/istream-zlib.h | 1 +
src/plugins/zlib/ostream-lz4.c | 184 +++++++++++++++++++++++
src/plugins/zlib/ostream-zlib.h | 1 +
src/plugins/zlib/zlib-plugin.c | 18 ++
8 files changed, 581 insertions(+), 1 deletions(-)
diffs (truncated from 681 to 300 lines):
diff -r a8408943ded7 -r 8e6b8d53c02b configure.in
--- a/configure.in Mon Feb 03 11:38:51 2014 -0500
+++ b/configure.in Wed Feb 05 00:47:55 2014 +0200
@@ -169,6 +169,11 @@
TEST_WITH(bzlib, $withval),
want_bzlib=auto)
+AC_ARG_WITH(lz4,
+AS_HELP_STRING([--with-lz4], [Build with LZ4 compression support]),
+ TEST_WITH(lz4, $withval),
+ want_lz4=auto)
+
AC_ARG_WITH(libcap,
AS_HELP_STRING([--with-libcap], [Build with libcap support (Dropping capabilities).]),
TEST_WITH(libcap, $withval),
@@ -2598,6 +2603,26 @@
])
fi
AM_CONDITIONAL(BUILD_BZLIB, test "$have_bzlib" = "yes")
+
+if test "$want_lz4" != "no"; then
+ AC_CHECK_HEADER(lz4.h, [
+ AC_CHECK_LIB(lz4, LZ4_compress, [
+ have_lz4=yes
+ have_zlib_plugin=yes
+ AC_DEFINE(HAVE_LZ4,, Define if you have lz4 library)
+ ], [
+ if test "$want_lz4" = "yes"; then
+ AC_ERROR([Can't build with lz4 support: liblz4 not found])
+ fi
+ ])
+ ], [
+ if test "$want_lz4" = "yes"; then
+ AC_ERROR([Can't build with lz4 support: lz4.h not found])
+ fi
+ ])
+fi
+AM_CONDITIONAL(BUILD_LZ4, test "$have_lz4" = "yes")
+
AM_CONDITIONAL(BUILD_ZLIB_PLUGIN, test "$have_zlib_plugin" = "yes")
RPCGEN=${RPCGEN-rpcgen}
diff -r a8408943ded7 -r 8e6b8d53c02b src/plugins/zlib/Makefile.am
--- a/src/plugins/zlib/Makefile.am Mon Feb 03 11:38:51 2014 -0500
+++ b/src/plugins/zlib/Makefile.am Wed Feb 05 00:47:55 2014 +0200
@@ -23,19 +23,25 @@
if BUILD_BZLIB
BZLIB_LIB = -lbz2
endif
+if BUILD_LZ4
+LZ4_LIB = -llz4
+endif
lib20_zlib_plugin_la_LIBADD = \
- $(ZLIB_LIB) $(BZLIB_LIB)
+ $(ZLIB_LIB) $(BZLIB_LIB) $(LZ4_LIB)
lib20_zlib_plugin_la_SOURCES = \
istream-bzlib.c \
+ istream-lz4.c \
istream-zlib.c \
ostream-zlib.c \
ostream-bzlib.c \
+ ostream-lz4.c \
zlib-plugin.c
noinst_HEADERS = \
istream-zlib.h \
+ iostream-lz4.h \
ostream-zlib.h \
zlib-plugin.h
diff -r a8408943ded7 -r 8e6b8d53c02b src/plugins/zlib/iostream-lz4.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/zlib/iostream-lz4.h Wed Feb 05 00:47:55 2014 +0200
@@ -0,0 +1,30 @@
+#ifndef IOSTREAM_LZ4_H
+#define IOSTREAM_LZ4_H
+
+/*
+ Dovecot's LZ4 compressed files contain:
+
+ IOSTREAM_LZ4_HEADER
+ n x (4 byte big-endian: compressed chunk length, compressed chunk)
+*/
+
+#define IOSTREAM_LZ4_MAGIC "Dovecot-LZ4\x0d\x2a\x9b\xc5"
+#define IOSTREAM_LZ4_MAGIC_LEN (sizeof(IOSTREAM_LZ4_MAGIC)-1)
+
+struct iostream_lz4_header {
+ unsigned char magic[IOSTREAM_LZ4_MAGIC_LEN];
+ /* OSTREAM_LZ4_CHUNK_SIZE in big-endian */
+ unsigned char max_uncompressed_chunk_size[4];
+};
+
+/* How large chunks we're buffering into memory before compressing them */
+#define OSTREAM_LZ4_CHUNK_SIZE (1024*64)
+/* How large chunks we allow in input data before returning a failure.
+ This must be at least OSTREAM_LZ4_CHUNK_SIZE, but for future compatibility
+ should be somewhat higher (but not too high to avoid wasting memory for
+ corrupted files). */
+#define ISTREAM_LZ4_CHUNK_SIZE (1024*1024)
+
+#define IOSTREAM_LZ4_CHUNK_PREFIX_LEN 4 /* big-endian size of chunk */
+
+#endif
diff -r a8408943ded7 -r 8e6b8d53c02b src/plugins/zlib/istream-lz4.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/zlib/istream-lz4.c Wed Feb 05 00:47:55 2014 +0200
@@ -0,0 +1,315 @@
+/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+
+#ifdef HAVE_LZ4
+
+#include "buffer.h"
+#include "istream-private.h"
+#include "istream-zlib.h"
+#include "iostream-lz4.h"
+#include <lz4.h>
+
+struct lz4_istream {
+ struct istream_private istream;
+
+ uoff_t stream_size;
+ struct stat last_parent_statbuf;
+
+ buffer_t *chunk_buf;
+ uint32_t chunk_size, chunk_left, max_uncompressed_chunk_size;
+
+ unsigned int log_errors:1;
+ unsigned int marked:1;
+ unsigned int header_read:1;
+};
+
+static void i_stream_lz4_close(struct iostream_private *stream)
+{
+ struct lz4_istream *zstream = (struct lz4_istream *)stream;
+
+ if (zstream->chunk_buf != NULL)
+ buffer_free(&zstream->chunk_buf);
+}
+
+static void lz4_read_error(struct lz4_istream *zstream, const char *error)
+{
+ if (!zstream->log_errors)
+ return;
+ i_error("lz4.read(%s): %s at %"PRIuUOFF_T,
+ i_stream_get_name(&zstream->istream.istream), error,
+ zstream->istream.abs_start_offset +
+ zstream->istream.istream.v_offset);
+}
+
+static int i_stream_lz4_read_header(struct lz4_istream *zstream)
+{
+ const struct iostream_lz4_header *hdr;
+ const unsigned char *data;
+ size_t size;
+ int ret;
+
+ ret = i_stream_read_data(zstream->istream.parent, &data, &size,
+ sizeof(*hdr)-1);
+ if (ret < 0) {
+ zstream->istream.istream.stream_errno =
+ zstream->istream.parent->stream_errno;
+ return ret;
+ }
+ if (ret == 0 && !zstream->istream.istream.eof)
+ return 0;
+ hdr = (const void *)data;
+ if (ret == 0 || memcmp(hdr->magic, IOSTREAM_LZ4_MAGIC,
+ IOSTREAM_LZ4_MAGIC_LEN) != 0) {
+ lz4_read_error(zstream, "wrong magic in header (not lz4 file?)");
+ zstream->istream.istream.stream_errno = EINVAL;
+ return -1;
+ }
+ zstream->max_uncompressed_chunk_size =
+ ((uint32_t)hdr->max_uncompressed_chunk_size[0] << 24) |
+ (hdr->max_uncompressed_chunk_size[1] << 16) |
+ (hdr->max_uncompressed_chunk_size[2] << 8) |
+ hdr->max_uncompressed_chunk_size[3];
+ if (zstream->max_uncompressed_chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
+ lz4_read_error(zstream, t_strdup_printf(
+ "lz4 max chunk size too large (%u > %u)",
+ zstream->max_uncompressed_chunk_size,
+ ISTREAM_LZ4_CHUNK_SIZE));
+ zstream->istream.istream.stream_errno = EINVAL;
+ return -1;
+ }
+ i_stream_skip(zstream->istream.parent, sizeof(*hdr));
+ return 1;
+}
+
+static ssize_t i_stream_lz4_read(struct istream_private *stream)
+{
+ struct lz4_istream *zstream = (struct lz4_istream *)stream;
+ const unsigned char *data;
+ size_t size, max_size;
+ int ret;
+
+ if (!zstream->header_read) {
+ if ((ret = i_stream_lz4_read_header(zstream)) <= 0)
+ return ret;
+ zstream->header_read = TRUE;
+ }
+
+ if (zstream->chunk_left == 0) {
+ ret = i_stream_read_data(stream->parent, &data, &size,
+ IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+ if (ret < 0) {
+ stream->istream.stream_errno =
+ stream->parent->stream_errno;
+ if (stream->istream.stream_errno == 0) {
+ stream->istream.eof = TRUE;
+ zstream->stream_size = stream->istream.v_offset +
+ stream->pos - stream->skip;
+ }
+ return ret;
+ }
+ if (ret == 0 && !stream->istream.eof)
+ return 0;
+ zstream->chunk_size = zstream->chunk_left =
+ ((uint32_t)data[0] << 24) |
+ (data[1] << 16) | (data[2] << 8) | data[3];
+ if (zstream->chunk_size == 0 ||
+ zstream->chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
+ lz4_read_error(zstream, t_strdup_printf(
+ "invalid lz4 chunk size: %u", zstream->chunk_size));
+ stream->istream.stream_errno = EINVAL;
+ return -1;
+ }
+ i_stream_skip(stream->parent, IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+ buffer_set_used_size(zstream->chunk_buf, 0);
+ }
+
+ /* read the whole compressed chunk into memory */
+ while (zstream->chunk_left > 0 &&
+ (ret = i_stream_read_data(zstream->istream.parent,
+ &data, &size, 0)) > 0) {
+ if (size > zstream->chunk_left)
+ size = zstream->chunk_left;
+ buffer_append(zstream->chunk_buf, data, size);
+ i_stream_skip(zstream->istream.parent, size);
+ zstream->chunk_left -= size;
+ }
+ if (zstream->chunk_left > 0) {
+ if (ret == -1 && zstream->istream.parent->stream_errno == 0) {
+ lz4_read_error(zstream, "truncated lz4 chunk");
+ stream->istream.stream_errno = EINVAL;
+ return -1;
+ }
+ zstream->istream.istream.stream_errno =
+ zstream->istream.parent->stream_errno;
+ return ret;
+ }
+ /* if we already have max_buffer_size amount of data, fail here */
+ i_stream_compress(stream);
+ if (stream->pos >= stream->max_buffer_size)
+ return -2;
+ /* allocate enough space for the old data and the new
+ decompressed chunk. we don't know the original compressed size,
+ so just allocate the max amount of memory. */
+ max_size = stream->pos + zstream->max_uncompressed_chunk_size;
+ if (stream->buffer_size < max_size) {
+ stream->w_buffer = i_realloc(stream->w_buffer,
+ stream->buffer_size, max_size);
+ stream->buffer_size = max_size;
+ stream->buffer = stream->w_buffer;
+ }
+ ret = LZ4_decompress_safe(zstream->chunk_buf->data,
+ (void *)(stream->w_buffer + stream->pos),
+ zstream->chunk_buf->used,
+ stream->buffer_size - stream->pos);
+ i_assert(ret <= (int)zstream->max_uncompressed_chunk_size);
+ if (ret < 0) {
+ lz4_read_error(zstream, "corrupted lz4 chunk");
+ stream->istream.stream_errno = EINVAL;
+ return -1;
+ }
+ i_assert(ret > 0);
+ stream->pos += ret;
+ i_assert(stream->pos <= stream->buffer_size);
+ return ret;
+}
+
+static void i_stream_lz4_reset(struct lz4_istream *zstream)
+{
+ struct istream_private *stream = &zstream->istream;
+
+ i_stream_seek(stream->parent, stream->parent_start_offset);
+ zstream->header_read = FALSE;
+ zstream->chunk_size = zstream->chunk_left = 0;
+
+ stream->parent_expected_offset = stream->parent_start_offset;
+ stream->skip = stream->pos = 0;
+ stream->istream.v_offset = 0;
+}
+
+static void
More information about the dovecot-cvs
mailing list