dovecot-2.2: lib-index: Cache record linking is now while writin...
dovecot at dovecot.org
dovecot at dovecot.org
Sat Nov 3 19:26:22 EET 2012
details: http://hg.dovecot.org/dovecot-2.2/rev/67afcb730109
changeset: 15371:67afcb730109
user: Timo Sirainen <tss at iki.fi>
date: Sat Nov 03 19:25:35 2012 +0200
description:
lib-index: Cache record linking is now while writing new records, not later with pwrite().
This should make performance somewhat better. It also means that now the
only overwrites to cache file are to the file header and the fields header.
This means that we no longer have to invalidate file_cache (with
mmap_disable=yes) except when reading the headers, which is always done
anyway.
It would be nice to move the overwritten fields in the headers to a cache
header in dovecot.index file, but this is a bit tricky currently. When
writing these headers the cache file needs to be locked to avoid race
conditions. To avoid deadlocks the transaction log would have to be locked
before the cache. The code to do all this would still be a bit complex, and
to keep backwards compatibility with old Dovecot versions it would still
have to do the overwriting until the backwards compatibility code is
disabled.
diffstat:
src/lib-index/mail-cache-fields.c | 9 +-
src/lib-index/mail-cache-private.h | 9 +-
src/lib-index/mail-cache-sync-update.c | 91 -----------------
src/lib-index/mail-cache-transaction.c | 176 ++++++++++++++++----------------
src/lib-index/mail-cache.c | 40 ++-----
5 files changed, 103 insertions(+), 222 deletions(-)
diffs (truncated from 516 to 300 lines):
diff -r 148dc0f6f990 -r 67afcb730109 src/lib-index/mail-cache-fields.c
--- a/src/lib-index/mail-cache-fields.c Sat Nov 03 19:13:24 2012 +0200
+++ b/src/lib-index/mail-cache-fields.c Sat Nov 03 19:25:35 2012 +0200
@@ -206,7 +206,6 @@
const void *data;
uint32_t offset = 0, next_offset;
unsigned int next_count = 0;
- bool invalidate = FALSE;
int ret;
if (MAIL_CACHE_IS_UNUSABLE(cache)) {
@@ -228,7 +227,6 @@
return -1;
}
offset = next_offset;
- invalidate = TRUE;
if (cache->mmap_base != NULL || cache->map_with_read) {
ret = mail_cache_map(cache, offset, sizeof(*field_hdr),
@@ -274,10 +272,9 @@
cache->need_compress_file_seq = cache->hdr->file_seq;
if (field_hdr_r != NULL) {
- if (cache->file_cache != NULL && invalidate) {
- /* if this isn't the first header in file and we hadn't
- read this before, we can't trust that the cached
- data is valid */
+ if (cache->file_cache != NULL) {
+ /* invalidate the cache fields area to make sure we
+ get the latest cache decisions/last_used fields */
file_cache_invalidate(cache->file_cache, offset,
field_hdr->size);
}
diff -r 148dc0f6f990 -r 67afcb730109 src/lib-index/mail-cache-private.h
--- a/src/lib-index/mail-cache-private.h Sat Nov 03 19:13:24 2012 +0200
+++ b/src/lib-index/mail-cache-private.h Sat Nov 03 19:25:35 2012 +0200
@@ -210,7 +210,7 @@
int mail_cache_write(struct mail_cache *cache, const void *data, size_t size,
uoff_t offset);
int mail_cache_append(struct mail_cache *cache, const void *data, size_t size,
- uint32_t *offset_r);
+ uint32_t *offset);
int mail_cache_header_fields_read(struct mail_cache *cache);
int mail_cache_header_fields_update(struct mail_cache *cache);
@@ -242,9 +242,6 @@
void mail_cache_file_close(struct mail_cache *cache);
int mail_cache_reopen(struct mail_cache *cache);
-/* Update new_offset's prev_offset field to old_offset. */
-int mail_cache_link(struct mail_cache *cache, uint32_t old_offset,
- uint32_t new_offset);
/* Mark record in given offset to be deleted. */
int mail_cache_delete(struct mail_cache *cache, uint32_t offset);
@@ -258,10 +255,6 @@
int mail_cache_expunge_handler(struct mail_index_sync_map_ctx *sync_ctx,
uint32_t seq, const void *data,
void **sync_context, void *context);
-int mail_cache_sync_handler(struct mail_index_sync_map_ctx *sync_ctx,
- uint32_t seq, void *old_data, const void *new_data,
- void **context);
-void mail_cache_sync_lost_handler(struct mail_index *index);
void mail_cache_set_syscall_error(struct mail_cache *cache,
const char *function);
diff -r 148dc0f6f990 -r 67afcb730109 src/lib-index/mail-cache-sync-update.c
--- a/src/lib-index/mail-cache-sync-update.c Sat Nov 03 19:13:24 2012 +0200
+++ b/src/lib-index/mail-cache-sync-update.c Sat Nov 03 19:25:35 2012 +0200
@@ -1,17 +1,12 @@
/* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */
#include "lib.h"
-#include "file-cache.h"
#include "mail-cache-private.h"
-#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
struct mail_cache_sync_context {
- uoff_t invalidate_highwater;
-
unsigned int locked:1;
unsigned int lock_failed:1;
- unsigned int nfs_read_cache_flushed:1;
};
static void mail_cache_handler_deinit(struct mail_index_sync_map_ctx *sync_ctx,
@@ -34,7 +29,6 @@
else {
*context = i_new(struct mail_cache_sync_context, 1);
ctx = *context;
- ctx->invalidate_highwater = (uoff_t)-1;
}
return ctx;
}
@@ -104,88 +98,3 @@
(void)mail_cache_delete(cache, *cache_offset);
return 0;
}
-
-int mail_cache_sync_handler(struct mail_index_sync_map_ctx *sync_ctx,
- uint32_t seq ATTR_UNUSED,
- void *old_data, const void *new_data,
- void **context)
-{
- struct mail_index_view *view = sync_ctx->view;
- struct mail_index *index = view->index;
- struct mail_cache *cache = index->cache;
- struct mail_cache_sync_context *ctx = *context;
- const uint32_t *old_cache_offset = old_data;
- const uint32_t *new_cache_offset = new_data;
- uint32_t cache_file_seq, cur_seq, tail_seq;
- uoff_t cur_offset, tail_offset;
- int ret;
-
- if (new_cache_offset == NULL) {
- mail_cache_handler_deinit(sync_ctx, ctx);
- *context = NULL;
- return 1;
- }
-
- ctx = mail_cache_handler_init(context);
- if (cache->file_cache != NULL && !MAIL_CACHE_IS_UNUSABLE(cache)) {
- /* flush read cache only once per sync */
- if (!ctx->nfs_read_cache_flushed &&
- (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) {
- ctx->nfs_read_cache_flushed = TRUE;
- mail_index_flush_read_cache(index,
- cache->filepath, cache->fd,
- cache->locked);
- }
- /* don't invalidate anything that's already been invalidated
- within this sync. */
- if (*new_cache_offset < ctx->invalidate_highwater) {
- file_cache_invalidate(cache->file_cache,
- *new_cache_offset,
- ctx->invalidate_highwater -
- *new_cache_offset);
- ctx->invalidate_highwater = *new_cache_offset;
- }
- }
-
- if (*old_cache_offset == 0 || *old_cache_offset == *new_cache_offset ||
- sync_ctx->type == MAIL_INDEX_SYNC_HANDLER_VIEW)
- return 1;
-
- mail_transaction_log_view_get_prev_pos(view->log_view,
- &cur_seq, &cur_offset);
- mail_transaction_log_get_mailbox_sync_pos(index->log,
- &tail_seq, &tail_offset);
- if (LOG_IS_BEFORE(cur_seq, cur_offset, tail_seq, tail_offset)) {
- /* already been linked */
- return 1;
- }
-
- /* we'll need to link the old and new cache records */
- ret = mail_cache_handler_lock(ctx, cache);
- if (ret <= 0)
- return ret < 0 ? -1 : 1;
-
- if (!get_cache_file_seq(view, &cache_file_seq))
- return 1;
-
- if (cache_file_seq != cache->hdr->file_seq) {
- /* cache has been compressed, don't modify it */
- return 1;
- }
-
- if (mail_cache_link(cache, *old_cache_offset, *new_cache_offset) < 0)
- return -1;
-
- return 1;
-}
-
-void mail_cache_sync_lost_handler(struct mail_index *index)
-{
- struct mail_cache *cache = index->cache;
-
- if (!MAIL_CACHE_IS_UNUSABLE(cache)) {
- mail_index_flush_read_cache(cache->index, cache->filepath,
- cache->fd, cache->locked);
- }
- file_cache_invalidate(cache->file_cache, 0, (uoff_t)-1);
-}
diff -r 148dc0f6f990 -r 67afcb730109 src/lib-index/mail-cache-transaction.c
--- a/src/lib-index/mail-cache-transaction.c Sat Nov 03 19:13:24 2012 +0200
+++ b/src/lib-index/mail-cache-transaction.c Sat Nov 03 19:25:35 2012 +0200
@@ -34,7 +34,7 @@
buffer_t *cache_data;
ARRAY(uint32_t) cache_data_seq;
- uint32_t prev_seq;
+ uint32_t prev_seq, min_seq;
size_t last_rec_pos;
uoff_t bytes_written;
@@ -47,8 +47,6 @@
&mail_index_module_register);
static int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx);
-static int mail_cache_link_locked(struct mail_cache *cache,
- uint32_t old_offset, uint32_t new_offset);
static void mail_index_transaction_cache_reset(struct mail_index_transaction *t)
{
@@ -281,7 +279,7 @@
struct mail_cache *cache = ctx->cache;
const struct mail_cache_record *rec = ctx->cache_data->data;
const uint32_t *seqs;
- uint32_t i, seq_count, old_offset;
+ uint32_t i, seq_count;
mail_index_ext_using_reset_id(ctx->trans, ctx->cache->ext_id,
ctx->cache_file_seq);
@@ -292,30 +290,7 @@
seqs = array_get(&ctx->cache_data_seq, &seq_count);
for (i = 0; i < seq_count; i++) {
mail_index_update_ext(ctx->trans, seqs[i], cache->ext_id,
- &write_offset, &old_offset);
- if (old_offset != 0) {
- /* we added records for this message multiple
- times in this same uncommitted transaction.
- only the new one will be written to
- transaction log, we need to do the linking
- ourself here. */
- if (old_offset > write_offset) {
- if (mail_cache_link_locked(cache, old_offset,
- write_offset) < 0)
- return -1;
- } else {
- /* if we're combining multiple transactions,
- make sure the one with the smallest offset
- is written into index. this is required for
- non-file-mmaped cache to work properly. */
- mail_index_update_ext(ctx->trans, seqs[i],
- cache->ext_id,
- &old_offset, NULL);
- if (mail_cache_link_locked(cache, write_offset,
- old_offset) < 0)
- return -1;
- }
- }
+ &write_offset, NULL);
write_offset += rec->size;
rec = CONST_PTR_OFFSET(rec, rec->size);
@@ -324,9 +299,70 @@
}
static int
+mail_cache_link_records(struct mail_cache_transaction_ctx *ctx,
+ uint32_t write_offset)
+{
+ struct mail_index_map *map;
+ struct mail_cache_record *rec;
+ const uint32_t *seqs, *prev_offsetp;
+ ARRAY_TYPE(uint32_t) seq_offsets;
+ uint32_t i, seq_count, reset_id, prev_offset, *offsetp;
+ const void *data;
+ bool expunged;
+
+ i_assert(ctx->min_seq != 0);
+
+ i_array_init(&seq_offsets, 64);
+ seqs = array_get(&ctx->cache_data_seq, &seq_count);
+ rec = buffer_get_modifiable_data(ctx->cache_data, NULL);
+ for (i = 0; i < seq_count; i++) {
+ offsetp = array_idx_modifiable(&seq_offsets,
+ seqs[i] - ctx->min_seq);
+ if (*offsetp != 0)
+ prev_offset = *offsetp;
+ else {
+ mail_index_lookup_ext_full(ctx->view->trans_view, seqs[i],
+ ctx->cache->ext_id, &map,
+ &data, &expunged);
+ prev_offsetp = data;
+
+ if (prev_offsetp == NULL || *prev_offsetp == 0)
+ prev_offset = 0;
+ else if (mail_index_ext_get_reset_id(ctx->view->trans_view, map,
+ ctx->cache->ext_id,
+ &reset_id) &&
+ reset_id == ctx->cache_file_seq)
+ prev_offset = *prev_offsetp;
+ else
+ prev_offset = 0;
+ if (prev_offset >= write_offset) {
+ mail_cache_set_corrupted(ctx->cache,
+ "Cache record offset points outside existing file");
+ array_free(&seq_offsets);
+ return -1;
+ }
+ }
+
+ if (prev_offset != 0) {
+ /* link this record to previous one */
+ rec->prev_offset = prev_offset;
+ ctx->cache->hdr_copy.continued_record_count++;
+ ctx->cache->hdr_modified = TRUE;
+ }
More information about the dovecot-cvs
mailing list