dovecot: NFS cache flushing APIs changed and backend implements ...
dovecot at dovecot.org
dovecot at dovecot.org
Sun Nov 18 08:41:04 EET 2007
details: http://hg.dovecot.org/dovecot/rev/dbab5e592577
changeset: 6829:dbab5e592577
user: Timo Sirainen <tss at iki.fi>
date: Sun Nov 18 08:40:59 2007 +0200
description:
NFS cache flushing APIs changed and backend implements it a bit differently.
Hopefully this works now more correctly.
diffstat:
20 files changed, 262 insertions(+), 233 deletions(-)
src/lib-index/mail-cache-compress.c | 8
src/lib-index/mail-cache-private.h | 1
src/lib-index/mail-cache-sync-update.c | 22 +-
src/lib-index/mail-cache.c | 60 ++-----
src/lib-index/mail-hash.c | 64 ++++---
src/lib-index/mail-index-lock.c | 28 +--
src/lib-index/mail-index-map.c | 4
src/lib-index/mail-index-private.h | 3
src/lib-index/mail-index.c | 21 --
src/lib-index/mail-transaction-log-file.c | 12 -
src/lib-index/mail-transaction-log.c | 8
src/lib-index/mailbox-list-index-sync.c | 6
src/lib-index/mailbox-list-index.c | 11 -
src/lib-storage/index/maildir/maildir-keywords.c | 2
src/lib-storage/index/maildir/maildir-uidlist.c | 14 +
src/lib-storage/index/mbox/mbox-lock.c | 12 -
src/lib-storage/mailbox-list.c | 1
src/lib/file-dotlock.c | 4
src/lib/nfs-workarounds.c | 186 ++++++++++++----------
src/lib/nfs-workarounds.h | 28 +--
diffs (truncated from 942 to 300 lines):
diff -r 8470d0845fc2 -r dbab5e592577 src/lib-index/mail-cache-compress.c
--- a/src/lib-index/mail-cache-compress.c Sun Nov 18 08:01:54 2007 +0200
+++ b/src/lib-index/mail-cache-compress.c Sun Nov 18 08:40:59 2007 +0200
@@ -354,9 +354,6 @@ static int mail_cache_compress_locked(st
unsigned int i, count;
int fd, ret;
- if (!MAIL_CACHE_IS_UNUSABLE(cache))
- mail_cache_flush_read_cache(cache, TRUE);
-
/* get the latest info on fields */
if (mail_cache_header_fields_read(cache) < 0)
return -1;
@@ -460,6 +457,11 @@ int mail_cache_compress(struct mail_cach
if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
/* we're using dotlocking, cache file creation itself creates
the dotlock file we need. */
+ if (!MAIL_CACHE_IS_UNUSABLE(cache)) {
+ mail_index_flush_read_cache(cache->index,
+ cache->filepath, cache->fd,
+ FALSE);
+ }
return mail_cache_compress_locked(cache, trans, &unlock);
}
diff -r 8470d0845fc2 -r dbab5e592577 src/lib-index/mail-cache-private.h
--- a/src/lib-index/mail-cache-private.h Sun Nov 18 08:01:54 2007 +0200
+++ b/src/lib-index/mail-cache-private.h Sun Nov 18 08:40:59 2007 +0200
@@ -220,7 +220,6 @@ int mail_cache_lock(struct mail_cache *c
/* Returns -1 if cache is / just got corrupted, 0 if ok. */
int mail_cache_unlock(struct mail_cache *cache);
-void mail_cache_flush_read_cache(struct mail_cache *cache, bool just_locked);
int mail_cache_write(struct mail_cache *cache, const void *data, size_t size,
uoff_t offset);
diff -r 8470d0845fc2 -r dbab5e592577 src/lib-index/mail-cache-sync-update.c
--- a/src/lib-index/mail-cache-sync-update.c Sun Nov 18 08:01:54 2007 +0200
+++ b/src/lib-index/mail-cache-sync-update.c Sun Nov 18 08:40:59 2007 +0200
@@ -11,7 +11,7 @@ struct mail_cache_sync_context {
unsigned int locked:1;
unsigned int lock_failed:1;
- unsigned int nfs_attr_cache_flushed:1;
+ unsigned int nfs_read_cache_flushed:1;
};
static void mail_cache_handler_deinit(struct mail_index_sync_map_ctx *sync_ctx,
@@ -127,10 +127,12 @@ int mail_cache_sync_handler(struct mail_
ctx = mail_cache_handler_init(context);
if (cache->file_cache != NULL && !MAIL_CACHE_IS_UNUSABLE(cache)) {
- /* flush attribute cache only once per sync */
- if (!ctx->nfs_attr_cache_flushed && cache->index->nfs_flush) {
- ctx->nfs_attr_cache_flushed = TRUE;
- mail_cache_flush_read_cache(cache, FALSE);
+ /* flush read cache only once per sync */
+ if (!ctx->nfs_read_cache_flushed && cache->index->nfs_flush) {
+ ctx->nfs_read_cache_flushed = TRUE;
+ mail_index_flush_read_cache(cache->index,
+ cache->filepath, cache->fd,
+ cache->locked);
}
/* don't invalidate anything that's already been invalidated
within this sync. */
@@ -177,7 +179,11 @@ int mail_cache_sync_handler(struct mail_
void mail_cache_sync_lost_handler(struct mail_index *index)
{
- if (!MAIL_CACHE_IS_UNUSABLE(index->cache))
- mail_cache_flush_read_cache(index->cache, FALSE);
- file_cache_invalidate(index->cache->file_cache, 0, (uoff_t)-1);
+ struct mail_cache *cache = index->cache;
+
+ if (!MAIL_CACHE_IS_UNUSABLE(cache)) {
+ mail_index_flush_read_cache(cache->index, cache->filepath,
+ cache->fd, cache->locked);
+ }
+ file_cache_invalidate(cache->file_cache, 0, (uoff_t)-1);
}
diff -r 8470d0845fc2 -r dbab5e592577 src/lib-index/mail-cache.c
--- a/src/lib-index/mail-cache.c Sun Nov 18 08:01:54 2007 +0200
+++ b/src/lib-index/mail-cache.c Sun Nov 18 08:40:59 2007 +0200
@@ -78,12 +78,6 @@ static void mail_cache_init_file_cache(s
if (cache->file_cache == NULL)
return;
- if (cache->index->nfs_flush) {
- nfs_flush_attr_cache_fd(cache->filepath, cache->fd);
- nfs_flush_read_cache(cache->filepath, cache->fd,
- F_UNLCK, FALSE);
- }
-
file_cache_set_fd(cache->file_cache, cache->fd);
if (fstat(cache->fd, &st) == 0)
@@ -114,22 +108,33 @@ static bool mail_cache_need_reopen(struc
return TRUE;
/* see if the file has changed */
- if (cache->index->nfs_flush)
- nfs_flush_attr_cache(cache->filepath);
+ if (cache->index->nfs_flush) {
+ i_assert(!cache->locked);
+ nfs_flush_attr_cache_unlocked(cache->filepath);
+ }
if (nfs_safe_stat(cache->filepath, &st) < 0) {
mail_cache_set_syscall_error(cache, "stat()");
return TRUE;
}
+
+ if (st.st_ino != cache->st_ino ||
+ !CMP_DEV_T(st.st_dev, cache->st_dev)) {
+ /* file changed */
+ return TRUE;
+ }
+
if (cache->index->nfs_flush) {
/* if the old file has been deleted, the new file may have
- the same inode as the old one, but we'll catch this by
- flushing attribute cache and seeing if it fails with
- ESTALE */
- if (!nfs_flush_attr_cache_fd(cache->filepath, cache->fd))
- return TRUE;
- }
- return st.st_ino != cache->st_ino ||
- !CMP_DEV_T(st.st_dev, cache->st_dev);
+ the same inode as the old one. we'll catch this here by
+ checking if fstat() fails with ESTALE */
+ if (fstat(cache->fd, &st) < 0) {
+ if (errno == ESTALE)
+ return TRUE;
+ mail_cache_set_syscall_error(cache, "fstat()");
+ return FALSE;
+ }
+ }
+ return FALSE;
}
int mail_cache_reopen(struct mail_cache *cache)
@@ -448,23 +453,6 @@ void mail_cache_free(struct mail_cache *
i_free(cache->fields);
i_free(cache->filepath);
i_free(cache);
-}
-
-void mail_cache_flush_read_cache(struct mail_cache *cache, bool just_locked)
-{
- if (!cache->index->nfs_flush)
- return;
-
- /* Assume flock() is independent of fcntl() locks. This isn't true
- with Linux 2.6 NFS, but with it there's no point in using flock() */
- if (cache->locked &&
- cache->index->lock_method == FILE_LOCK_METHOD_FCNTL) {
- nfs_flush_read_cache(cache->filepath, cache->fd,
- F_WRLCK, just_locked);
- } else {
- nfs_flush_read_cache(cache->filepath, cache->fd,
- F_UNLCK, FALSE);
- }
}
static int mail_cache_lock_file(struct mail_cache *cache)
@@ -490,10 +478,8 @@ static int mail_cache_lock_file(struct m
if (ret <= 0)
return ret;
- if (cache->index->nfs_flush &&
- cache->index->lock_method != FILE_LOCK_METHOD_FCNTL)
- nfs_flush_attr_cache_fd(cache->filepath, cache->fd);
- mail_cache_flush_read_cache(cache, TRUE);
+ mail_index_flush_read_cache(cache->index, cache->filepath, cache->fd,
+ TRUE);
return 1;
}
diff -r 8470d0845fc2 -r dbab5e592577 src/lib-index/mail-hash.c
--- a/src/lib-index/mail-hash.c Sun Nov 18 08:01:54 2007 +0200
+++ b/src/lib-index/mail-hash.c Sun Nov 18 08:40:59 2007 +0200
@@ -242,6 +242,8 @@ static int mail_hash_file_read(struct ma
{
int ret;
+ i_assert(hash->locked);
+
if (hash->mmap_base == NULL) {
if (file_size < size)
file_size = size;
@@ -269,9 +271,6 @@ static int mail_hash_file_read(struct ma
hash->mmap_size = size;
}
- if (hash->index->nfs_flush)
- nfs_flush_read_cache(hash->filepath, hash->fd, F_UNLCK, FALSE);
-
ret = pread_full(hash->fd, hash->mmap_base, size, 0);
if (ret < 0) {
mail_hash_set_syscall_error(hash, "pread_full()");
@@ -356,8 +355,7 @@ static int mail_hash_file_map(struct mai
struct stat st;
size_t size;
- if (hash->index->nfs_flush)
- nfs_flush_attr_cache_fd(hash->filepath, hash->fd);
+ i_assert(hash->locked);
if (fstat(hash->fd, &st) < 0) {
mail_hash_set_syscall_error(hash, "fstat()");
@@ -408,23 +406,29 @@ static int mail_hash_file_map(struct mai
static int mail_hash_file_lock(struct mail_hash *hash, int lock_type)
{
+ int ret;
+
i_assert(hash->fd != -1);
if (hash->index->lock_method != FILE_LOCK_METHOD_DOTLOCK) {
i_assert(hash->file_lock == NULL);
- return mail_index_lock_fd(hash->index, hash->filepath, hash->fd,
- lock_type, MAIL_HASH_TIMEOUT_SECS,
- &hash->file_lock);
+ ret = mail_index_lock_fd(hash->index, hash->filepath, hash->fd,
+ lock_type, MAIL_HASH_TIMEOUT_SECS,
+ &hash->file_lock);
} else {
i_assert(hash->dotlock == NULL);
- if (file_dotlock_create(&hash->dotlock_settings,
- hash->filepath, 0,
- &hash->dotlock) < 0) {
- mail_hash_set_syscall_error(hash, "open()");
- return -1;
- }
- return 0;
- }
+ ret = file_dotlock_create(&hash->dotlock_settings,
+ hash->filepath, 0, &hash->dotlock);
+ if (ret < 0) {
+ mail_hash_set_syscall_error(hash,
+ "file_dotlock_create()");
+ }
+ }
+ if (ret > 0) {
+ mail_index_flush_read_cache(hash->index, hash->filepath,
+ hash->fd, TRUE);
+ }
+ return ret;
}
static void mail_hash_file_unlock(struct mail_hash *hash)
@@ -653,20 +657,25 @@ static int mail_hash_reopen_if_needed(st
return mail_hash_reopen(hash);
if (hash->index->nfs_flush)
- nfs_flush_attr_cache(hash->filepath);
+ nfs_flush_attr_cache_unlocked(hash->filepath);
if (nfs_safe_stat(hash->filepath, &st) < 0) {
- if (errno == ENOENT)
- return mail_hash_reopen(hash);
-
+ if (errno != ENOENT) {
+ mail_hash_set_syscall_error(hash, "stat()");
+ return -1;
+ }
+ } else if (st.st_ino == hash->ino &&
+ CMP_DEV_T(st.st_dev, hash->dev)) {
+ /* the file looks the same */
+ if (!hash->index->nfs_flush)
+ return 0;
+
+ if (fstat(hash->fd, &st) == 0)
+ return 0;
if (errno != ESTALE) {
- mail_hash_set_syscall_error(hash, "stat()");
- return -1;
- }
- /* if ESTALE is returned, it most likely means it was rebuilt */
- } else {
- if (st.st_ino == hash->ino && CMP_DEV_T(st.st_dev, hash->dev))
- return 0;
+ mail_hash_set_syscall_error(hash, "fstat()");
+ return -1;
+ }
}
return mail_hash_reopen(hash);
}
@@ -894,6 +903,7 @@ int mail_hash_insert(struct mail_hash *h
{
uint32_t hash_key = key == NULL ? 0 : hash->key_hash_cb(key);
+ i_assert(hash->locked);
i_assert((key == NULL && hash->rec_hash_cb(value) == 0) ||
(key != NULL && hash_key != 0 &&
hash->rec_hash_cb(value) != 0));
diff -r 8470d0845fc2 -r dbab5e592577 src/lib-index/mail-index-lock.c
--- a/src/lib-index/mail-index-lock.c Sun Nov 18 08:01:54 2007 +0200
+++ b/src/lib-index/mail-index-lock.c Sun Nov 18 08:40:59 2007 +0200
@@ -120,19 +120,20 @@ static int mail_index_lock(struct mail_i
return 1;
More information about the dovecot-cvs
mailing list