dovecot-2.0: Initial implementation of single instance attachmen...
dovecot at dovecot.org
dovecot at dovecot.org
Sat Oct 2 14:33:55 EEST 2010
details: http://hg.dovecot.org/dovecot-2.0/rev/21ceba48445d
changeset: 12234:21ceba48445d
user: Timo Sirainen <tss at iki.fi>
date: Wed Jul 21 21:08:18 2010 +0100
description:
Initial implementation of single instance attachment storage for dbox.
diffstat:
TODO | 8 +
configure.in | 3 +-
src/Makefile.am | 1 +
src/lib-dovecot/Makefile.am | 1 +
src/lib-fs/Makefile.am | 18 +
src/lib-fs/fs-api-private.h | 61 ++
src/lib-fs/fs-api.c | 198 +++++++++
src/lib-fs/fs-api.h | 101 ++++
src/lib-fs/fs-posix.c | 530 +++++++++++++++++++++++++
src/lib-fs/fs-sis.c | 418 +++++++++++++++++++
src/lib-fs/ostream-cmp.c | 118 +++++
src/lib-fs/ostream-cmp.h | 15 +
src/lib-storage/index/dbox-common/Makefile.am | 5 +
src/lib-storage/index/dbox-common/dbox-attachment.c | 520 ++++++++++++++++++++++++
src/lib-storage/index/dbox-common/dbox-attachment.h | 33 +
src/lib-storage/index/dbox-common/dbox-file.c | 4 +
src/lib-storage/index/dbox-common/dbox-save.c | 8 +
src/lib-storage/index/dbox-common/dbox-save.h | 26 +
src/lib-storage/index/dbox-common/dbox-settings.c | 41 +
src/lib-storage/index/dbox-common/dbox-settings.h | 11 +
src/lib-storage/index/dbox-common/dbox-storage.c | 40 +
src/lib-storage/index/dbox-common/dbox-storage.h | 12 +
src/lib-storage/index/dbox-multi/mdbox-purge.c | 102 ++++-
src/lib-storage/index/dbox-multi/mdbox-storage.c | 6 +-
src/lib-storage/index/dbox-single/sdbox-storage.c | 22 +-
src/lib-storage/index/dbox-single/sdbox-sync.c | 61 ++-
26 files changed, 2327 insertions(+), 36 deletions(-)
diffs (truncated from 2778 to 300 lines):
diff -r 584957cba877 -r 21ceba48445d TODO
--- a/TODO Wed Jul 21 15:19:19 2010 +0100
+++ b/TODO Wed Jul 21 21:08:18 2010 +0100
@@ -1,3 +1,11 @@
+ - dbox attachments:
+ - attachments can generate very long metadata lines. input stream reading
+ them probably has a limit..
+ - save attachments base64-decoded
+ - if attachment stream is too small/long, log an error
+ - if file is completely empty, maybe show it as spaces? this could be
+ useful for deleting viruses by truncating their files to zero bytes
+ - delayed deduplication daemon?
- NTLMv1 and LM should be disabled if disable_plaintext_auth=yes
- SEARCH SENT*/HEADER/etc. doesn't seem optimized when using with TEXT/BODY
diff -r 584957cba877 -r 21ceba48445d configure.in
--- a/configure.in Wed Jul 21 15:19:19 2010 +0100
+++ b/configure.in Wed Jul 21 21:08:18 2010 +0100
@@ -2461,7 +2461,7 @@
LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/libdovecot-login.la'
LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la'
else
- LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib/liblib.la'
+ LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib/liblib.la'
LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV)"
LIBDOVECOT_STORAGE_LAST='$(top_builddir)/src/lib-storage/list/libstorage_list.la $(top_builddir)/src/lib-storage/index/libstorage_index.la $(top_builddir)/src/lib-storage/libstorage.la $(top_builddir)/src/lib-index/libindex.la'
LIBDOVECOT_STORAGE_FIRST='$(top_builddir)/src/lib-storage/libstorage_service.la $(top_builddir)/src/lib-storage/register/libstorage_register.la'
@@ -2643,6 +2643,7 @@
src/lib-charset/Makefile
src/lib-dict/Makefile
src/lib-dns/Makefile
+src/lib-fs/Makefile
src/lib-imap/Makefile
src/lib-index/Makefile
src/lib-lda/Makefile
diff -r 584957cba877 -r 21ceba48445d src/Makefile.am
--- a/src/Makefile.am Wed Jul 21 15:19:19 2010 +0100
+++ b/src/Makefile.am Wed Jul 21 21:08:18 2010 +0100
@@ -3,6 +3,7 @@
lib-auth \
lib-charset \
lib-dns \
+ lib-fs \
lib-mail \
lib-imap \
lib-master \
diff -r 584957cba877 -r 21ceba48445d src/lib-dovecot/Makefile.am
--- a/src/lib-dovecot/Makefile.am Wed Jul 21 15:19:19 2010 +0100
+++ b/src/lib-dovecot/Makefile.am Wed Jul 21 21:08:18 2010 +0100
@@ -5,6 +5,7 @@
../lib-mail/libmail.la \
../lib-auth/libauth.la \
../lib-dns/libdns.la \
+ ../lib-fs/libfs.la \
../lib-charset/libcharset.la \
../lib-master/libmaster.la \
../lib/liblib.la
diff -r 584957cba877 -r 21ceba48445d src/lib-fs/Makefile.am
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-fs/Makefile.am Wed Jul 21 21:08:18 2010 +0100
@@ -0,0 +1,18 @@
+noinst_LTLIBRARIES = libfs.la
+
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib
+
+libfs_la_SOURCES = \
+ fs-api.c \
+ fs-posix.c \
+ fs-sis.c \
+ ostream-cmp.c
+
+headers = \
+ fs-api.h \
+ fs-api-private.h \
+ ostream-cmp.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
diff -r 584957cba877 -r 21ceba48445d src/lib-fs/fs-api-private.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-fs/fs-api-private.h Wed Jul 21 21:08:18 2010 +0100
@@ -0,0 +1,61 @@
+#ifndef FS_API_PRIVATE_H
+#define FS_API_PRIVATE_H
+
+#include "fs-api.h"
+
+struct fs_vfuncs {
+ struct fs *(*init)(const char *args, const struct fs_settings *set);
+ void (*deinit)(struct fs *fs);
+
+ int (*open)(struct fs *fs, const char *path, enum fs_open_mode mode,
+ enum fs_open_flags flags, struct fs_file **file_r);
+ void (*close)(struct fs_file *file);
+
+ ssize_t (*read)(struct fs_file *file, void *buf, size_t size);
+ struct istream *(*read_stream)(struct fs_file *file,
+ size_t max_buffer_size);
+
+ int (*write)(struct fs_file *file, const void *data, size_t size);
+ void (*write_stream)(struct fs_file *file);
+ int (*write_stream_finish)(struct fs_file *file, bool success);
+
+ int (*lock)(struct fs_file *file, unsigned int secs,
+ struct fs_lock **lock_r);
+ void (*unlock)(struct fs_lock *lock);
+ int (*fdatasync)(struct fs_file *file);
+
+ int (*exists)(struct fs *fs, const char *path);
+ int (*stat)(struct fs *fs, const char *path, struct stat *st_r);
+ int (*link)(struct fs *fs, const char *src, const char *dest);
+ int (*rename)(struct fs *fs, const char *src, const char *dest);
+ int (*unlink)(struct fs *fs, const char *path);
+ int (*rmdir)(struct fs *fs, const char *path);
+};
+
+struct fs {
+ const char *name;
+ struct fs_vfuncs v;
+
+ struct fs_settings set;
+ string_t *last_error;
+
+ unsigned int files_open_count;
+};
+
+struct fs_file {
+ struct fs *fs;
+ struct ostream *output;
+ char *path;
+};
+
+struct fs_lock {
+ struct fs_file *file;
+};
+
+extern struct fs fs_class_posix;
+extern struct fs fs_class_sis;
+
+void fs_set_error(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3);
+void fs_set_critical(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3);
+
+#endif
diff -r 584957cba877 -r 21ceba48445d src/lib-fs/fs-api.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-fs/fs-api.c Wed Jul 21 21:08:18 2010 +0100
@@ -0,0 +1,198 @@
+/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "fs-api-private.h"
+
+static struct fs *fs_classes[] = {
+ &fs_class_posix,
+ &fs_class_sis
+};
+
+static struct fs *
+fs_alloc(const struct fs *fs_class, const char *args,
+ const struct fs_settings *set)
+{
+ struct fs *fs;
+
+ fs = fs_class->v.init(args, set);
+ fs->last_error = str_new(default_pool, 64);
+ return fs;
+}
+
+struct fs *fs_init(const char *driver, const char *args,
+ const struct fs_settings *set)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(fs_classes); i++) {
+ if (strcmp(fs_classes[i]->name, driver) == 0)
+ return fs_alloc(fs_classes[i], args, set);
+ }
+ i_fatal("Unknown fs driver: %s", driver);
+}
+
+void fs_deinit(struct fs **_fs)
+{
+ struct fs *fs = *_fs;
+
+ *_fs = NULL;
+
+ if (fs->files_open_count > 0) {
+ i_panic("fs-%s: %u files still open",
+ fs->name, fs->files_open_count);
+ }
+
+ str_free(&fs->last_error);
+ fs->v.deinit(fs);
+}
+
+int fs_open(struct fs *fs, const char *path, int mode_flags,
+ struct fs_file **file_r)
+{
+ int ret;
+
+ T_BEGIN {
+ ret = fs->v.open(fs, path, mode_flags & FS_OPEN_MODE_MASK,
+ mode_flags & ~FS_OPEN_MODE_MASK, file_r);
+ } T_END;
+ if (ret == 0)
+ fs->files_open_count++;
+ return ret;
+}
+
+void fs_close(struct fs_file **_file)
+{
+ struct fs_file *file = *_file;
+
+ i_assert(file->fs->files_open_count > 0);
+
+ *_file = NULL;
+
+ file->fs->files_open_count--;
+ file->fs->v.close(file);
+}
+
+const char *fs_file_path(struct fs_file *file)
+{
+ return file->path;
+}
+
+const char *fs_last_error(struct fs *fs)
+{
+ if (str_len(fs->last_error) == 0)
+ return "BUG: Unknown fs error";
+ return str_c(fs->last_error);
+}
+
+const char *fs_file_last_error(struct fs_file *file)
+{
+ return fs_last_error(file->fs);
+}
+
+ssize_t fs_read(struct fs_file *file, void *buf, size_t size)
+{
+ return file->fs->v.read(file, buf, size);
+}
+
+struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size)
+{
+ return file->fs->v.read_stream(file, max_buffer_size);
+}
+
+int fs_write(struct fs_file *file, const void *data, size_t size)
+{
+ return file->fs->v.write(file, data, size);
+}
+
+struct ostream *fs_write_stream(struct fs_file *file)
+{
+ file->fs->v.write_stream(file);
+ i_assert(file->output != NULL);
+ return file->output;
+}
+
+int fs_write_stream_finish(struct fs_file *file, struct ostream **output)
+{
+ i_assert(*output == file->output);
+
+ *output = NULL;
+ return file->fs->v.write_stream_finish(file, TRUE);
+}
+
+void fs_write_stream_abort(struct fs_file *file, struct ostream **output)
+{
+ i_assert(*output == file->output);
+
+ *output = NULL;
+ (void)file->fs->v.write_stream_finish(file, FALSE);
+}
+
+int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r)
+{
+ return file->fs->v.lock(file, secs, lock_r);
+}
+
+void fs_unlock(struct fs_lock **_lock)
+{
+ struct fs_lock *lock = *_lock;
+
+ *_lock = NULL;
+ lock->file->fs->v.unlock(lock);
+}
+
+int fs_fdatasync(struct fs_file *file)
+{
+ return file->fs->v.fdatasync(file);
+}
+
+int fs_exists(struct fs *fs, const char *path)
+{
+ return fs->v.exists(fs, path);
+}
More information about the dovecot-cvs
mailing list