[dovecot-cvs] dovecot: Initial commit for major index file code cleanup.
dovecot at dovecot.org
dovecot at dovecot.org
Mon Jun 11 14:51:14 EEST 2007
details: http://hg.dovecot.org/dovecot/rev/c2362f144f15
changeset: 5689:c2362f144f15
user: Timo Sirainen <tss at iki.fi>
date: Mon Jun 11 14:50:10 2007 +0300
description:
Initial commit for major index file code cleanup.
- dovecot.index file isn't anymore required to be updated when syncing.
- Getting the latest index file mapping is now done always by reading
dovecot.index and then reading the latest changes from dovecot.index.log.
- mmap()ing dovecot.index file is slower than reading it, so it's not
currently done unless the file is at 256kB. This may change though.
- Some things are still broken.
diffstat:
21 files changed, 2104 insertions(+), 2686 deletions(-)
src/lib-index/mail-index-fsck.c | 62 -
src/lib-index/mail-index-lock.c | 289 ------
src/lib-index/mail-index-map.c | 1125 +++++++++++++-------------
src/lib-index/mail-index-private.h | 90 --
src/lib-index/mail-index-sync-ext.c | 132 +--
src/lib-index/mail-index-sync-keywords.c | 34
src/lib-index/mail-index-sync-private.h | 31
src/lib-index/mail-index-sync-update.c | 749 ++++++++---------
src/lib-index/mail-index-sync.c | 453 ++++++----
src/lib-index/mail-index-view-private.h | 5
src/lib-index/mail-index-view-sync.c | 77 +
src/lib-index/mail-index-view.c | 52 -
src/lib-index/mail-index.c | 548 ++----------
src/lib-index/mail-index.h | 24
src/lib-index/mail-transaction-log-append.c | 104 +-
src/lib-index/mail-transaction-log-file.c | 529 ++++++------
src/lib-index/mail-transaction-log-private.h | 56 -
src/lib-index/mail-transaction-log-view.c | 14
src/lib-index/mail-transaction-log.c | 378 +++-----
src/lib-index/mail-transaction-log.h | 33
src/util/idxview.c | 5
diffs (truncated from 6478 to 300 lines):
diff -r 5a37076852d4 -r c2362f144f15 src/lib-index/mail-index-fsck.c
--- a/src/lib-index/mail-index-fsck.c Mon Jun 11 06:28:07 2007 +0300
+++ b/src/lib-index/mail-index-fsck.c Mon Jun 11 14:50:10 2007 +0300
@@ -23,43 +23,12 @@ static void mail_index_fsck_error(struct
map->hdr.field, hdr.field); \
}
-static void
-mail_index_fsck_locked(struct mail_index *index, struct mail_index_header *hdr)
-{
- uint32_t log_seq;
- uoff_t log_offset;
-
- mail_transaction_log_get_head(index->log, &log_seq, &log_offset);
-
- if (hdr->log_file_int_offset > hdr->log_file_ext_offset) {
- mail_index_fsck_error(index,
- "log_file_int_offset > log_file_ext_offset");
- hdr->log_file_int_offset = hdr->log_file_ext_offset;
- }
-
- if ((hdr->log_file_seq == log_seq &&
- hdr->log_file_ext_offset > log_offset) ||
- (hdr->log_file_seq != log_seq &&
- !mail_transaction_log_is_head_prev(index->log,
- hdr->log_file_seq,
- hdr->log_file_ext_offset))) {
- mail_index_fsck_error(index,
- "log file sync pos %u,%u -> %u, %"PRIuUOFF_T,
- hdr->log_file_seq, hdr->log_file_ext_offset,
- log_seq, log_offset);
- hdr->log_file_seq = log_seq;
- hdr->log_file_int_offset =
- hdr->log_file_ext_offset = log_offset;
- }
-}
-
static int
mail_index_fsck_map(struct mail_index *index, struct mail_index_map *map,
const char **error_r)
{
struct mail_index_header hdr;
const struct mail_index_record *rec;
- unsigned int records_count;
uint32_t i, last_uid;
*error_r = NULL;
@@ -70,13 +39,6 @@ mail_index_fsck_map(struct mail_index *i
if (hdr.uid_validity == 0 && hdr.next_uid != 1) {
*error_r = "uid_validity = 0 && next_uid != 1";
return 0;
- }
-
- if (!index->log_locked)
- records_count = map->hdr.messages_count;
- else {
- records_count = map->records_count;
- mail_index_fsck_locked(index, &hdr);
}
hdr.flags &= ~MAIL_INDEX_HDR_FLAG_FSCK;
@@ -149,30 +111,14 @@ int mail_index_fsck(struct mail_index *i
{
const char *error;
unsigned int lock_id;
- uint32_t file_seq;
- uoff_t file_offset;
int ret;
- bool lock_log;
-
- if (index->sync_update) {
- /* we're modifying index, don't do anything */
- return 1;
- }
i_warning("fscking index file %s", index->filepath);
- lock_log = !index->log_locked;
- if (lock_log) {
- if (mail_transaction_log_sync_lock(index->log, &file_seq,
- &file_offset) < 0)
- return -1;
- }
- if (mail_index_lock_exclusive(index, &lock_id) < 0) {
- mail_transaction_log_sync_unlock(index->log);
- return -1;
- }
+ // FIXME: should we be fscking a given map instead? anyway we probably
+ // want to rewrite the main index after fsck is finished.
error = NULL;
- ret = mail_index_map(index, TRUE);
+ ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD, &lock_id);
if (ret > 0) {
ret = mail_index_fsck_map(index, index->map, &error);
if (ret > 0) {
@@ -183,8 +129,6 @@ int mail_index_fsck(struct mail_index *i
}
mail_index_unlock(index, lock_id);
- if (lock_log)
- mail_transaction_log_sync_unlock(index->log);
if (error != NULL) {
mail_index_set_error(index, "Corrupted index file %s: %s",
diff -r 5a37076852d4 -r c2362f144f15 src/lib-index/mail-index-lock.c
--- a/src/lib-index/mail-index-lock.c Mon Jun 11 06:28:07 2007 +0300
+++ b/src/lib-index/mail-index-lock.c Mon Jun 11 14:50:10 2007 +0300
@@ -3,33 +3,22 @@
/*
Locking should never fail or timeout. Exclusive locks must be kept as short
time as possible. Shared locks can be long living, so if we can't get
- exclusive lock directly within 2 seconds, we'll replace the index file with
- a copy of it. That means the shared lock holders can keep using the old file
- while we're modifying the new file.
+ exclusive lock directly, we'll recreate the index. That means the shared
+ lock holders can keep using the old file.
lock_id is used to figure out if acquired lock is still valid. When index
file is reopened, the lock_id can become invalid. It doesn't matter however,
as no-one's going to modify the old file anymore.
- lock_id also tells if we're referring to shared or exclusive lock. This
- allows us to drop back to shared locking once all exclusive locks are
- dropped. Shared locks have even numbers, exclusive locks have odd numbers.
+ lock_id also tells us if we're referring to a shared or an exclusive lock.
+ This allows us to drop back to shared locking once all exclusive locks
+ are dropped. Shared locks have even numbers, exclusive locks have odd numbers.
The number is increased by two every time the lock is dropped or index file
is reopened.
*/
#include "lib.h"
-#include "buffer.h"
-#include "mmap-util.h"
-#include "write-full.h"
#include "mail-index-private.h"
-
-#include <stdio.h>
-#include <sys/stat.h>
-
-#ifdef HAVE_FLOCK
-# include <sys/file.h>
-#endif
int mail_index_lock_fd(struct mail_index *index, const char *path, int fd,
int lock_type, unsigned int timeout_secs,
@@ -45,10 +34,9 @@ int mail_index_lock_fd(struct mail_index
}
static int mail_index_lock(struct mail_index *index, int lock_type,
- unsigned int timeout_secs, int update_index,
- unsigned int *lock_id_r)
+ unsigned int timeout_secs, unsigned int *lock_id_r)
{
- int ret, ret2;
+ int ret;
i_assert(lock_type == F_RDLCK || lock_type == F_WRLCK);
@@ -64,20 +52,6 @@ static int mail_index_lock(struct mail_i
ret = 0;
}
- if (update_index && index->excl_lock_count == 0) {
- /* we wish to have the latest available index file. */
- i_assert(index->lock_type != F_WRLCK);
- if ((ret2 = mail_index_reopen_if_needed(index)) < 0)
- return -1;
- if (ret > 0 && ret2 == 0) {
- /* no new file and the old file is already locked */
- i_assert(lock_type == F_RDLCK);
- i_assert(index->lock_type == F_RDLCK);
- return 1;
- }
- ret = 0;
- }
-
if (ret > 0) {
/* file is already locked */
return 1;
@@ -90,10 +64,6 @@ static int mail_index_lock(struct mail_i
locks then, though */
if (lock_type == F_WRLCK)
return 0;
- if (update_index && index->lock_type == F_UNLCK) {
- if (mail_index_reopen_if_needed(index) < 0)
- return -1;
- }
index->shared_lock_count++;
index->lock_type = F_RDLCK;
@@ -147,13 +117,11 @@ static int mail_index_lock(struct mail_i
return 1;
}
-int mail_index_lock_shared(struct mail_index *index, bool update_index,
- unsigned int *lock_id_r)
+int mail_index_lock_shared(struct mail_index *index, unsigned int *lock_id_r)
{
int ret;
- ret = mail_index_lock(index, F_RDLCK, MAIL_INDEX_LOCK_SECS,
- update_index, lock_id_r);
+ ret = mail_index_lock(index, F_RDLCK, MAIL_INDEX_LOCK_SECS, lock_id_r);
if (ret > 0)
return 0;
if (ret < 0)
@@ -166,235 +134,14 @@ int mail_index_lock_shared(struct mail_i
return -1;
}
-static int mail_index_copy(struct mail_index *index, const char **path_r)
+int mail_index_try_lock_exclusive(struct mail_index *index,
+ unsigned int *lock_id_r)
{
- struct mail_index_map *map = index->map;
- unsigned int base_size;
- const char *path;
- int ret, fd;
-
- i_assert(!MAIL_INDEX_IS_IN_MEMORY(index));
-
- fd = mail_index_create_tmp_file(index, &path);
- if (fd == -1)
- return -1;
-
- /* write base header */
- base_size = I_MIN(map->hdr.base_header_size, sizeof(map->hdr));
- ret = write_full(fd, &map->hdr, base_size);
- if (ret == 0) {
- /* write extended headers */
- ret = write_full(fd, CONST_PTR_OFFSET(map->hdr_base, base_size),
- map->hdr.header_size - base_size);
- }
-
- if (ret < 0 || write_full(fd, map->records, map->records_count *
- map->hdr.record_size) < 0) {
- mail_index_file_set_syscall_error(index, path, "write_full()");
- (void)close(fd);
- (void)unlink(path);
- fd = -1;
- } else {
- *path_r = path;
- }
-
- return fd;
-}
-
-static int mail_index_lock_exclusive_copy(struct mail_index *index)
-{
- struct mail_index_map *map;
-
- i_assert(index->log_locked);
- i_assert(index->excl_lock_count == 0);
-
- map = mail_index_map_clone(index->map, index->map->hdr.record_size);
- mail_index_unmap(index, &index->map);
- index->map = map;
- index->hdr = &map->hdr;
-
- map->write_atomic = TRUE;
- map->write_to_disk = TRUE;
-
- index->excl_lock_count++;
- index->lock_type = F_WRLCK;
- return 0;
-}
-
-int mail_index_lock_exclusive(struct mail_index *index,
- unsigned int *lock_id_r)
-{
- int ret;
-
- /* exclusive transaction log lock protects exclusive locking
- for the main index file */
- i_assert(index->log_locked);
-
- /* if header size is smaller than what we have, we'll have to recreate
- the index to grow it. so don't even try regular locking. */
- if (index->map->hdr.base_header_size >= sizeof(*index->hdr) ||
- index->excl_lock_count > 0) {
- /* wait two seconds for exclusive lock */
- ret = mail_index_lock(index, F_WRLCK, 2, TRUE, lock_id_r);
- if (ret > 0)
- return 0;
- if (ret < 0)
- return -1;
- }
- if (mail_index_lock_exclusive_copy(index) < 0)
- return -1;
- *lock_id_r = index->lock_id + 1;
- return 0;
-}
-
-static int
-mail_index_copy_lock_finish(struct mail_index *index, const char *path)
-{
- int ret = 0;
-
- if (!index->fsync_disable) {
More information about the dovecot-cvs
mailing list