[dovecot-cvs] dovecot/src/lib-index mail-index-compress.c,1.15,1.16 mail-index-data.c,1.25,1.26 mail-index-data.h,1.9,1.10 mail-index-update.c,1.24,1.25 mail-index.c,1.60,1.61 mail-index.h,1.44,1.45 mail-modifylog.c,1.30,1.31 mail-modifylog.h,1.5,1.6 Message-Id: <20021023194925.583992382C@danu.procontrol.fi>
cras at procontrol.fi
cras at procontrol.fi
Wed Oct 23 23:49:25 EEST 2002
Update of /home/cvs/dovecot/src/lib-index
In directory danu:/tmp/cvs-serv22997/lib-index
Modified Files:
mail-index-compress.c mail-index-data.c mail-index-data.h
mail-index-update.c mail-index.c mail-index.h mail-modifylog.c
mail-modifylog.h mail-tree-redblack.c mail-tree.c mail-tree.h
Log Message:
Use fdatasync() instead of fsync() where possible. msync() all files first,
then fsync them instead of msync+fsync+msync+fsync+.. data file now keeps
track of when it's been changed, tree file didn't do mmap updates, plus
other related cleanups.
Index: mail-index-compress.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-compress.c,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- mail-index-compress.c 8 Oct 2002 23:26:08 -0000 1.15
+++ mail-index-compress.c 23 Oct 2002 19:49:23 -0000 1.16
@@ -18,7 +18,7 @@
if (index->mmap_full_length <= INDEX_FILE_MIN_SIZE)
return TRUE;
-
+
/* really truncate the file only when it's almost empty */
empty_space = index->mmap_full_length - index->mmap_used_length;
truncate_threshold =
@@ -46,6 +46,7 @@
{
MailIndexRecord *rec, *hole_rec, *end_rec;
unsigned int idx;
+ int tree_fd;
if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
return FALSE;
@@ -62,7 +63,7 @@
/* if we get interrupted, the whole index is probably corrupted.
so keep rebuild-flag on while doing this */
index->header->flags |= MAIL_INDEX_FLAG_REBUILD;
- if (!mail_index_fmsync(index, sizeof(MailIndexHeader)))
+ if (!mail_index_fmdatasync(index, sizeof(MailIndexHeader)))
return FALSE;
/* first actually compress the data */
@@ -96,7 +97,16 @@
index->header->first_hole_records = 0;
/* make sure the whole file is synced before removing rebuild-flag */
- if (!mail_index_fmsync(index, index->mmap_used_length))
+ if (!mail_tree_sync_file(index->tree, &tree_fd))
+ return FALSE;
+
+ if (fdatasync(tree_fd) < 0) {
+ index_file_set_syscall_error(index, index->tree->filepath,
+ "fdatasync()");
+ return FALSE;
+ }
+
+ if (!mail_index_fmdatasync(index, index->mmap_used_length))
return FALSE;
index->header->flags &= ~(MAIL_INDEX_FLAG_COMPRESS |
@@ -130,7 +140,7 @@
/* now we'll begin the actual moving. keep rebuild-flag on
while doing it. */
index->header->flags |= MAIL_INDEX_FLAG_REBUILD;
- if (!mail_index_fmsync(index, sizeof(MailIndexHeader)))
+ if (!mail_index_fmdatasync(index, sizeof(MailIndexHeader)))
return FALSE;
offset = sizeof(data_hdr);
@@ -195,8 +205,15 @@
failed = !mail_index_copy_data(index, fd, temppath);
- if (close(fd) < 0)
+ if (close(fd) < 0) {
index_file_set_syscall_error(index, temppath, "close()");
+ failed = TRUE;
+ }
+
+ if (fdatasync(fd) < 0) {
+ index_file_set_syscall_error(index, temppath, "fdatasync()");
+ failed = TRUE;
+ }
if (!failed) {
/* now, rename the temp file to new data file. but before that
@@ -226,7 +243,7 @@
}
/* make sure the whole file is synced before removing rebuild-flag */
- if (!mail_index_fmsync(index, index->mmap_used_length))
+ if (!mail_index_fmdatasync(index, index->mmap_used_length))
return FALSE;
index->header->flags &= ~(MAIL_INDEX_FLAG_COMPRESS_DATA |
Index: mail-index-data.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-data.c,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -d -r1.25 -r1.26
--- mail-index-data.c 17 Oct 2002 18:11:05 -0000 1.25
+++ mail-index-data.c 23 Oct 2002 19:49:23 -0000 1.26
@@ -40,6 +40,8 @@
unsigned int anon_mmap:1;
unsigned int dirty_mmap:1;
+ unsigned int modified:1;
+ unsigned int fsynced:1;
};
int index_data_set_corrupted(MailIndexData *data, const char *fmt, ...)
@@ -146,9 +148,11 @@
i_assert(!data->anon_mmap);
if (data->mmap_base != NULL) {
- if (data->mmap_used_length > 0 &&
+ if (data->modified &&
msync(data->mmap_base, data->mmap_used_length, MS_SYNC) < 0)
return index_data_set_syscall_error(data, "msync()");
+ data->modified = FALSE;
+ data->fsynced = FALSE;
if (munmap(data->mmap_base, data->mmap_full_length) < 0)
index_data_set_syscall_error(data, "munmap()");
@@ -359,6 +363,8 @@
return index_data_set_syscall_error(data, "write_full()");
}
+ data->modified = FALSE;
+ data->fsynced = FALSE;
return mmap_update(data, 0, 0);
}
@@ -371,9 +377,15 @@
if (msync(data->mmap_base, sizeof(MailIndexDataHeader), MS_SYNC) < 0)
return index_data_set_syscall_error(data, "msync()");
+ data->fsynced = FALSE;
return TRUE;
}
+void mail_index_data_mark_modified(MailIndexData *data)
+{
+ data->modified = TRUE;
+}
+
static int mail_index_data_grow(MailIndexData *data, size_t size)
{
void *base;
@@ -443,6 +455,7 @@
memcpy((char *) data->mmap_base + offset, buffer, size);
data->header->used_file_size += size;
+ data->modified = TRUE;
return offset;
}
@@ -462,20 +475,28 @@
if (data->header->deleted_space >= max_del_space)
data->index->set_flags |= MAIL_INDEX_FLAG_COMPRESS_DATA;
}
+
+ data->modified = TRUE;
return TRUE;
}
-int mail_index_data_sync_file(MailIndexData *data)
+int mail_index_data_sync_file(MailIndexData *data, int *fsync_fd)
{
+ *fsync_fd = -1;
+
if (data->anon_mmap)
return TRUE;
- if (data->mmap_base != NULL && data->mmap_used_length > 0) {
+ if (data->modified) {
if (msync(data->mmap_base, data->mmap_used_length, MS_SYNC) < 0)
return index_data_set_syscall_error(data, "msync()");
- if (fsync(data->fd) < 0)
- return index_data_set_syscall_error(data, "fsync()");
+ data->fsynced = FALSE;
+ }
+
+ if (!data->fsynced) {
+ data->fsynced = TRUE;
+ *fsync_fd = data->fd;
}
return TRUE;
Index: mail-index-data.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-data.h,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- mail-index-data.h 16 Oct 2002 22:54:18 -0000 1.9
+++ mail-index-data.h 23 Oct 2002 19:49:23 -0000 1.10
@@ -14,6 +14,9 @@
re-open it. */
int mail_index_data_mark_deleted(MailIndexData *data);
+/* Mark the file as being modified */
+void mail_index_data_mark_modified(MailIndexData *data);
+
/* Append new data at the end of the file. Returns the position in file
where the data begins, or 0 if error occured. */
uoff_t mail_index_data_append(MailIndexData *data, const void *buffer,
@@ -23,7 +26,7 @@
int mail_index_data_add_deleted_space(MailIndexData *data, size_t data_size);
/* Synchronize the data into disk */
-int mail_index_data_sync_file(MailIndexData *data);
+int mail_index_data_sync_file(MailIndexData *data, int *fsync_fd);
/* Looks up a field from data file. If field is 0, returns the first field
found. Returns NULL if not found or if error occured. */
Index: mail-index-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-update.c,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- mail-index-update.c 20 Oct 2002 01:12:27 -0000 1.24
+++ mail-index-update.c 23 Oct 2002 19:49:23 -0000 1.25
@@ -231,6 +231,8 @@
rec = mail_index_data_next(update->index->data,
update->rec, rec);
}
+
+ mail_index_data_mark_modified(update->index->data);
}
int mail_index_update_end(MailIndexUpdate *update)
Index: mail-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.60
retrieving revision 1.61
diff -u -d -r1.60 -r1.61
--- mail-index.c 20 Oct 2002 02:35:51 -0000 1.60
+++ mail-index.c 23 Oct 2002 19:49:23 -0000 1.61
@@ -177,12 +177,16 @@
static int mail_index_sync_file(MailIndex *index)
{
struct utimbuf ut;
- int failed;
+ unsigned int i;
+ int failed, fsync_fds[3];
if (index->anon_mmap)
return TRUE;
- if (!mail_index_data_sync_file(index->data))
+ for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++)
+ fsync_fds[i] = -1;
+
+ if (!mail_index_data_sync_file(index->data, &fsync_fds[0]))
return FALSE;
if (msync(index->mmap_base, index->mmap_used_length, MS_SYNC) < 0)
@@ -191,12 +195,12 @@
failed = FALSE;
if (index->tree != NULL) {
- if (!mail_tree_sync_file(index->tree))
+ if (!mail_tree_sync_file(index->tree, &fsync_fds[1]))
failed = TRUE;
}
if (index->modifylog != NULL) {
- if (!mail_modifylog_sync_file(index->modifylog))
+ if (!mail_modifylog_sync_file(index->modifylog, &fsync_fds[2]))
failed = TRUE;
}
@@ -206,20 +210,25 @@
if (utime(index->filepath, &ut) < 0)
return index_set_syscall_error(index, "utime()");
+ for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++) {
+ if (fsync_fds[i] != -1 && fdatasync(fsync_fds[i]) < 0)
+ index_set_error(index, "fdatasync(%d) failed: %m", i);
+ }
+
if (fsync(index->fd) < 0)
return index_set_syscall_error(index, "fsync()");
return !failed;
}
-int mail_index_fmsync(MailIndex *index, size_t size)
+int mail_index_fmdatasync(MailIndex *index, size_t size)
{
i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
if (!index->anon_mmap) {
if (msync(index->mmap_base, size, MS_SYNC) < 0)
return index_set_syscall_error(index, "msync()");
- if (fsync(index->fd) < 0)
+ if (fdatasync(index->fd) < 0)
return index_set_syscall_error(index, "fsync()");
}
@@ -370,7 +379,7 @@
when the lock is released, the FSCK flag will also be
removed. */
index->header->flags |= MAIL_INDEX_FLAG_FSCK;
- if (!mail_index_fmsync(index, sizeof(MailIndexHeader))) {
+ if (!mail_index_fmdatasync(index, sizeof(MailIndexHeader))) {
(void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK);
return FALSE;
}
Index: mail-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.44
retrieving revision 1.45
diff -u -d -r1.44 -r1.45
--- mail-index.h 20 Oct 2002 02:35:51 -0000 1.44
+++ mail-index.h 23 Oct 2002 19:49:23 -0000 1.45
@@ -395,7 +395,7 @@
int mail_index_mmap_update(MailIndex *index);
void mail_index_init_header(MailIndex *index, MailIndexHeader *hdr);
void mail_index_close(MailIndex *index);
-int mail_index_fmsync(MailIndex *index, size_t size);
+int mail_index_fmdatasync(MailIndex *index, size_t size);
int mail_index_verify_hole_range(MailIndex *index);
void mail_index_mark_flag_changes(MailIndex *index, MailIndexRecord *rec,
MailFlags old_flags, MailFlags new_flags);
Index: mail-modifylog.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-modifylog.c,v
retrieving revision 1.30
retrieving revision 1.31
diff -u -d -r1.30 -r1.31
--- mail-modifylog.c 23 Oct 2002 17:15:36 -0000 1.30
+++ mail-modifylog.c 23 Oct 2002 19:49:23 -0000 1.31
@@ -178,6 +178,7 @@
if (file->modified &&
msync(file->mmap_base, file->mmap_used_length, MS_SYNC) < 0)
return modifylog_set_syscall_error(file, "msync()");
+ file->modified = FALSE;
if (munmap(file->mmap_base, file->mmap_full_length) < 0)
modifylog_set_syscall_error(file, "munmap()");
@@ -578,22 +579,21 @@
i_free(log);
}
-int mail_modifylog_sync_file(MailModifyLog *log)
+int mail_modifylog_sync_file(MailModifyLog *log, int *fsync_fd)
{
ModifyLogFile *file = log->head;
+ *fsync_fd = -1;
+
if (!file->modified || file->anon_mmap)
return TRUE;
- if (file->mmap_base != NULL) {
- if (msync(file->mmap_base, file->mmap_used_length,
- MS_SYNC) < 0)
- return modifylog_set_syscall_error(file, "msync()");
- }
+ i_assert(file->mmap_base != NULL);
- if (fsync(file->fd) < 0)
- return modifylog_set_syscall_error(file, "fsync()");
+ if (msync(file->mmap_base, file->mmap_used_length, MS_SYNC) < 0)
+ return modifylog_set_syscall_error(file, "msync()");
+ *fsync_fd = file->fd;
file->modified = FALSE;
return TRUE;
}
Index: mail-modifylog.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-modifylog.h,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- mail-modifylog.h 23 Oct 2002 17:03:39 -0000 1.5
+++ mail-modifylog.h 23 Oct 2002 19:49:23 -0000 1.6
@@ -47,7 +47,7 @@
unsigned int uid, int external_change);
/* Synchronize the data into disk */
-int mail_modifylog_sync_file(MailModifyLog *log);
+int mail_modifylog_sync_file(MailModifyLog *log, int *fsync_fd);
/* Must be called when exclusive lock is dropped from index. */
void mail_modifylog_notify_lock_drop(MailModifyLog *log);
Index: mail-tree-redblack.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-tree-redblack.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- mail-tree-redblack.c 19 Oct 2002 23:09:56 -0000 1.13
+++ mail-tree-redblack.c 23 Oct 2002 19:49:23 -0000 1.14
@@ -630,6 +630,9 @@
i_assert(first_uid <= last_uid);
i_assert(tree->index->lock_type != MAIL_LOCK_UNLOCK);
+ if (!_mail_tree_mmap_update(tree, FALSE))
+ return (unsigned int)-1;
+
rb_check(tree);
if (seq_r != NULL)
@@ -684,6 +687,9 @@
i_assert(seq != 0);
i_assert(tree->index->lock_type != MAIL_LOCK_UNLOCK);
+ if (!_mail_tree_mmap_update(tree, FALSE))
+ return (unsigned int)-1;
+
rb_check(tree);
x = tree->header->root;
@@ -716,6 +722,9 @@
i_assert(uid != 0);
i_assert(tree->index->lock_type == MAIL_LOCK_EXCLUSIVE);
+ if (!_mail_tree_mmap_update(tree, FALSE))
+ return FALSE;
+
tree->modified = TRUE;
/* we'll always insert to right side of the tree */
@@ -770,6 +779,9 @@
i_assert(uid != 0);
i_assert(tree->index->lock_type == MAIL_LOCK_EXCLUSIVE);
+ if (!_mail_tree_mmap_update(tree, FALSE))
+ return FALSE;
+
rb_check(tree);
tree->modified = TRUE;
@@ -799,6 +811,9 @@
i_assert(uid != 0);
i_assert(tree->index->lock_type == MAIL_LOCK_EXCLUSIVE);
+
+ if (!_mail_tree_mmap_update(tree, FALSE))
+ return;
tree->modified = TRUE;
Index: mail-tree.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-tree.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mail-tree.c 20 Oct 2002 01:34:15 -0000 1.4
+++ mail-tree.c 23 Oct 2002 19:49:23 -0000 1.5
@@ -43,7 +43,7 @@
return FALSE;
}
-static int mmap_update(MailTree *tree, int forced)
+int _mail_tree_mmap_update(MailTree *tree, int forced)
{
if (!forced && tree->header != NULL &&
tree->mmap_full_length >= tree->header->used_file_size) {
@@ -58,6 +58,7 @@
if (tree->modified &&
msync(tree->mmap_base, tree->mmap_used_length, MS_SYNC) < 0)
return tree_set_syscall_error(tree, "msync()");
+ tree->modified = FALSE;
if (munmap(tree->mmap_base, tree->mmap_full_length) < 0)
tree_set_syscall_error(tree, "munmap()");
@@ -199,7 +200,7 @@
return FALSE;
do {
- if (!mmap_update(tree, TRUE))
+ if (!_mail_tree_mmap_update(tree, TRUE))
break;
if (tree->mmap_full_length == 0) {
@@ -271,7 +272,7 @@
return FALSE;
if (!mail_tree_init(tree) ||
- !mmap_update(tree, TRUE) ||
+ !_mail_tree_mmap_update(tree, TRUE) ||
!mmap_verify(tree)) {
tree->index->header->flags |= MAIL_INDEX_FLAG_REBUILD_TREE;
return FALSE;
@@ -292,19 +293,19 @@
return TRUE;
}
-int mail_tree_sync_file(MailTree *tree)
+int mail_tree_sync_file(MailTree *tree, int *fsync_fd)
{
+ *fsync_fd = -1;
+
if (!tree->modified || tree->anon_mmap)
return TRUE;
- if (tree->mmap_base != NULL) {
- if (msync(tree->mmap_base, tree->mmap_used_length, MS_SYNC) < 0)
- return tree_set_syscall_error(tree, "msync()");
- }
+ i_assert(tree->mmap_base != NULL);
- if (fsync(tree->fd) < 0)
- return tree_set_syscall_error(tree, "fsync()");
+ if (msync(tree->mmap_base, tree->mmap_used_length, MS_SYNC) < 0)
+ return tree_set_syscall_error(tree, "msync()");
+ *fsync_fd = tree->fd;
tree->modified = FALSE;
return TRUE;
}
@@ -343,7 +344,7 @@
return tree_set_syscall_error(tree, "file_set_size()");
}
- if (!mmap_update(tree, TRUE) || !mmap_verify(tree))
+ if (!_mail_tree_mmap_update(tree, TRUE) || !mmap_verify(tree))
return FALSE;
return TRUE;
Index: mail-tree.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-tree.h,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mail-tree.h 8 Oct 2002 23:26:08 -0000 1.2
+++ mail-tree.h 23 Oct 2002 19:49:23 -0000 1.3
@@ -51,7 +51,7 @@
void mail_tree_free(MailTree *tree);
int mail_tree_rebuild(MailTree *tree);
-int mail_tree_sync_file(MailTree *tree);
+int mail_tree_sync_file(MailTree *tree, int *fsync_fd);
/* Find first existing UID in range. Returns (unsigned int)-1 if not found. */
unsigned int mail_tree_lookup_uid_range(MailTree *tree, unsigned int *seq_r,
@@ -72,6 +72,7 @@
/* private: */
int _mail_tree_set_corrupted(MailTree *tree, const char *fmt, ...);
+int _mail_tree_mmap_update(MailTree *tree, int forced);
int _mail_tree_grow(MailTree *tree);
#endif
More information about the dovecot-cvs
mailing list