[dovecot-cvs] dovecot/src/lib-index mail-index-private.h, 1.36,
1.37 mail-index-sync.c, 1.39, 1.40 mail-index-view-sync.c,
1.29, 1.30 mail-index.c, 1.166,
1.167 mail-transaction-log-view.c, 1.34, 1.35
cras at dovecot.org
cras at dovecot.org
Sun Nov 14 03:23:24 EET 2004
- Previous message: [dovecot-cvs] dovecot/src/lib-index mail-index-lock.c, 1.36,
1.37 mail-index-private.h, 1.35, 1.36 mail-index-sync-update.c,
1.57, 1.58 mail-index-sync.c, 1.38,
1.39 mail-index-transaction-private.h, 1.17,
1.18 mail-index-transaction-view.c, 1.7,
1.8 mail-index-transaction.c, 1.34,
1.35 mail-index-view-private.h, 1.13,
1.14 mail-index-view-sync.c, 1.28, 1.29 mail-index-view.c,
1.27, 1.28 mail-index.c, 1.165, 1.166 mail-transaction-log.c,
1.73, 1.74
- Next message: [dovecot-cvs]
dovecot/src/master auth-process.c, 1.68, 1.69 common.h,
1.20, 1.21 login-process.c, 1.61, 1.62 mail-process.c, 1.62,
1.63 main.c, 1.57, 1.58 master-settings.c, 1.75, 1.76
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv32286
Modified Files:
mail-index-private.h mail-index-sync.c mail-index-view-sync.c
mail-index.c mail-transaction-log-view.c
Log Message:
Avoid re-reading index file with mmap_disabled=yes. Instead sync it by
reading transaction log file.
Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- mail-index-private.h 13 Nov 2004 23:08:07 -0000 1.36
+++ mail-index-private.h 14 Nov 2004 01:23:21 -0000 1.37
@@ -107,6 +107,11 @@
struct dotlock dotlock;
enum mail_index_lock_method lock_method;
+ /* These are typically same as map->hdr->log_file_*, but with
+ mmap_disable we may have synced more than index */
+ uint32_t sync_log_file_seq;
+ uoff_t sync_log_file_offset;
+
unsigned int last_grow_count;
char *error;
Index: mail-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.39
retrieving revision 1.40
diff -u -d -r1.39 -r1.40
--- mail-index-sync.c 13 Nov 2004 23:08:07 -0000 1.39
+++ mail-index-sync.c 14 Nov 2004 01:23:21 -0000 1.40
@@ -310,10 +310,29 @@
return -1;
}
- if (mail_index_map(index, FALSE) <= 0) {
- mail_transaction_log_sync_unlock(index->log);
- mail_index_unlock(index, lock_id);
- return -1;
+ if (index->mmap_disable) {
+ if (index->sync_log_file_seq != seq ||
+ index->sync_log_file_offset != offset) {
+ /* we may have synced our internal view more than what
+ is synced in index. re-read the whole index if our
+ sync seq/offset doesn't match what is in index's
+ header. */
+ if (mail_index_map(index, TRUE) <= 0) {
+ mail_transaction_log_sync_unlock(index->log);
+ mail_index_unlock(index, lock_id);
+ return -1;
+ }
+ } else {
+ /* the whole log file is synced already. */
+ i_assert(index->map->hdr.log_file_seq == seq);
+ i_assert(index->map->hdr.log_file_ext_offset == offset);
+ }
+ } else {
+ if (mail_index_map(index, FALSE) <= 0) {
+ mail_transaction_log_sync_unlock(index->log);
+ mail_index_unlock(index, lock_id);
+ return -1;
+ }
}
if (!mail_index_need_lock(index, sync_recent,
@@ -504,6 +523,7 @@
int mail_index_sync_commit(struct mail_index_sync_ctx *ctx)
{
+ struct mail_index *index = ctx->index;
const struct mail_index_header *hdr;
uint32_t seq, seq2;
uoff_t offset, offset2;
@@ -515,9 +535,9 @@
/* we have had the transaction log locked since the beginning of sync,
so only external changes could have been committed. write them to
the index here as well. */
- mail_transaction_log_get_head(ctx->index->log, &seq, &offset);
+ mail_transaction_log_get_head(index->log, &seq, &offset);
- hdr = ctx->index->hdr;
+ hdr = index->hdr;
if (ret == 0 && (hdr->log_file_seq != seq ||
hdr->log_file_int_offset != offset ||
hdr->log_file_ext_offset != offset)) {
@@ -529,12 +549,12 @@
ret = -1;
}
- if (ret == 0 && mail_cache_need_compress(ctx->index->cache)) {
- if (mail_cache_compress(ctx->index->cache, ctx->view) < 0)
+ if (ret == 0 && mail_cache_need_compress(index->cache)) {
+ if (mail_cache_compress(index->cache, ctx->view) < 0)
ret = -1;
else {
/* cache_offsets have changed, sync them */
- mail_transaction_log_get_head(ctx->index->log,
+ mail_transaction_log_get_head(index->log,
&seq2, &offset2);
if (mail_transaction_log_view_set(ctx->view->log_view,
seq, offset, seq2, offset2,
@@ -545,6 +565,9 @@
}
}
+ index->sync_log_file_seq = index->map->hdr.log_file_seq;
+ index->sync_log_file_offset = index->map->hdr.log_file_int_offset;
+
mail_index_sync_end(ctx);
return ret;
}
Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- mail-index-view-sync.c 13 Nov 2004 23:08:07 -0000 1.29
+++ mail-index-view-sync.c 14 Nov 2004 01:23:21 -0000 1.30
@@ -342,6 +342,9 @@
i_assert(view->syncing);
+ if (ctx->sync_map_update)
+ mail_index_sync_map_deinit(&ctx->sync_map_ctx);
+
if (view->log_syncs != NULL && !ctx->skipped_some)
buffer_set_used_size(view->log_syncs, 0);
Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.166
retrieving revision 1.167
diff -u -d -r1.166 -r1.167
--- mail-index.c 13 Nov 2004 23:08:07 -0000 1.166
+++ mail-index.c 14 Nov 2004 01:23:21 -0000 1.167
@@ -6,6 +6,7 @@
#include "read-full.h"
#include "write-full.h"
#include "mail-index-private.h"
+#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
#include "mail-cache.h"
@@ -432,6 +433,25 @@
return 1;
}
+static int mail_index_read_header(struct mail_index *index,
+ struct mail_index_header *hdr, size_t *pos_r)
+{
+ size_t pos;
+ int ret;
+
+ memset(hdr, 0, sizeof(*hdr));
+
+ ret = 1;
+ for (pos = 0; ret > 0 && pos < sizeof(*hdr); ) {
+ ret = pread(index->fd, PTR_OFFSET(hdr, pos),
+ sizeof(*hdr) - pos, pos);
+ if (ret > 0)
+ pos += ret;
+ }
+ *pos_r = pos;
+ return ret;
+}
+
static int mail_index_read_map(struct mail_index *index,
struct mail_index_map *map, int *retry_r)
{
@@ -443,15 +463,7 @@
i_assert(map->mmap_base == NULL);
*retry_r = FALSE;
- memset(&hdr, 0, sizeof(hdr));
-
- ret = 1;
- for (pos = 0; ret > 0 && pos < sizeof(hdr); ) {
- ret = pread(index->fd, PTR_OFFSET(&hdr, pos),
- sizeof(hdr) - pos, pos);
- if (ret > 0)
- pos += ret;
- }
+ ret = mail_index_read_header(index, &hdr, &pos);
if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
hdr.major_version != MAIL_INDEX_MAJOR_VERSION) {
@@ -523,14 +535,114 @@
memcpy(&map->hdr, &hdr, hdr.base_header_size);
}
map->hdr_base = map->hdr_copy_buf->data;
+
+ index->sync_log_file_seq = hdr.log_file_seq;
+ index->sync_log_file_offset = hdr.log_file_int_offset;
return 1;
}
+static int mail_index_sync_from_transactions(struct mail_index *index,
+ struct mail_index_map *map,
+ int sync_to_index)
+{
+ struct mail_index_view *view;
+ struct mail_transaction_log_view *log_view;
+ struct mail_index_sync_map_ctx sync_map_ctx;
+ struct mail_index_header hdr;
+ const struct mail_transaction_header *thdr;
+ const void *tdata;
+ uint32_t max_seq;
+ uoff_t max_offset;
+ size_t pos;
+ int ret, skipped;
+
+ if (sync_to_index) {
+ /* read the real log position where we are supposed to be
+ synced */
+ ret = mail_index_read_header(index, &hdr, &pos);
+ if (ret < 0 && errno != ESTALE) {
+ mail_index_set_syscall_error(index, "pread()");
+ return -1;
+ }
+ if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
+ return 0;
+
+ if (map->hdr.log_file_seq == hdr.log_file_seq &&
+ map->hdr.log_file_int_offset == hdr.log_file_int_offset) {
+ /* nothing to do */
+ return 1;
+ }
+
+ if (map->hdr.log_file_seq > hdr.log_file_seq ||
+ (map->hdr.log_file_seq == hdr.log_file_seq &&
+ map->hdr.log_file_int_offset > hdr.log_file_int_offset)) {
+ /* we went too far, have to re-read the file */
+ return 0;
+ }
+ if (map->hdr.log_file_ext_offset !=
+ map->hdr.log_file_int_offset ||
+ hdr.log_file_ext_offset != hdr.log_file_int_offset) {
+ /* too much trouble to get this right. */
+ return 0;
+ }
+ max_seq = hdr.log_file_seq;
+ max_offset = hdr.log_file_int_offset;
+ } else {
+ /* sync everything there is */
+ max_seq = (uint32_t)-1;
+ max_offset = (uoff_t)-1;
+ }
+
+ log_view = mail_transaction_log_view_open(index->log);
+ if (mail_transaction_log_view_set(log_view,
+ map->hdr.log_file_seq,
+ map->hdr.log_file_int_offset,
+ max_seq, max_offset,
+ MAIL_TRANSACTION_TYPE_MASK) < 0) {
+ mail_transaction_log_view_close(log_view);
+ return 0;
+ }
+
+ index->map = map;
+
+ view = mail_index_view_open(index);
+ mail_index_sync_map_init(&sync_map_ctx, view,
+ MAIL_INDEX_SYNC_HANDLER_VIEW);
+
+ while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,
+ &skipped)) > 0) {
+ if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {
+ ret = -1;
+ break;
+ }
+ }
+
+ mail_index_sync_map_deinit(&sync_map_ctx);
+ mail_index_view_close(view);
+ mail_transaction_log_view_close(log_view);
+
+ index->map = NULL;
+ return ret < 0 ? -1 : 1;
+}
+
static int mail_index_read_map_with_retry(struct mail_index *index,
- struct mail_index_map *map)
+ struct mail_index_map *map,
+ int sync_to_index)
{
int i, ret, retry;
+ if (map->hdr.indexid != 0) {
+ /* sync this as a view from transaction log. */
+ ret = mail_index_sync_from_transactions(index, map,
+ sync_to_index);
+ if (ret != 0)
+ return ret;
+
+ /* transaction log lost/broken, fallback to re-reading it */
+ /* FIXME: file cache need to be reset (except not really with
+ sync_to_index if we were just rewinding..) */
+ }
+
for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) {
ret = mail_index_read_map(index, map, &retry);
if (ret != 0 || !retry)
@@ -598,24 +710,27 @@
if (index->map != NULL && index->map->refcount > 1) {
/* this map is already used by some views and they may have
pointers into it. leave them and create a new mapping. */
+ if (!index->mmap_disable) {
+ map = NULL;
+ } else {
+ /* create a copy of the mapping instead so we don't
+ have to re-read it */
+ map = mail_index_map_to_memory(index->map,
+ index->map->hdr.record_size);
+ }
index->map->refcount--;
index->map = NULL;
+ } else {
+ map = index->map;
}
- map = index->map;
if (map == NULL) {
map = i_new(struct mail_index_map, 1);
map->refcount = 1;
map->hdr_copy_buf =
buffer_create_dynamic(default_pool, sizeof(map->hdr));
} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
- if (map->write_to_disk) {
- /* we have modified this mapping and it's waiting to
- be written to disk once we drop exclusive lock.
- mapping couldn't have changed, so do nothing. */
- return 1;
- }
- /* FIXME: we need to re-read header */
+ i_assert(!map->write_to_disk);
} else if (map->mmap_base != NULL) {
i_assert(map->buffer == NULL);
if (munmap(map->mmap_base, map->mmap_size) < 0)
@@ -628,8 +743,8 @@
if (!index->mmap_disable)
ret = mail_index_mmap(index, map);
- else
- ret = mail_index_read_map_with_retry(index, map);
+ else
+ ret = mail_index_read_map_with_retry(index, map, force);
if (ret <= 0) {
mail_index_unmap_forced(index, map);
return ret;
Index: mail-transaction-log-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.34
retrieving revision 1.35
diff -u -d -r1.34 -r1.35
--- mail-transaction-log-view.c 13 Nov 2004 22:27:57 -0000 1.34
+++ mail-transaction-log-view.c 14 Nov 2004 01:23:21 -0000 1.35
@@ -153,6 +153,12 @@
for (seq = min_file_seq+1; seq <= max_file_seq; seq++) {
file = file->next;
if (file == NULL || file->hdr.file_seq != seq) {
+ if (file == NULL && max_file_seq == (uint32_t)-1) {
+ /* we just wanted to sync everything */
+ max_file_seq = seq-1;
+ break;
+ }
+
mail_index_set_error(view->log->index,
"Lost transaction log file %s seq %u",
view->log->tail->filepath, seq);
- Previous message: [dovecot-cvs] dovecot/src/lib-index mail-index-lock.c, 1.36,
1.37 mail-index-private.h, 1.35, 1.36 mail-index-sync-update.c,
1.57, 1.58 mail-index-sync.c, 1.38,
1.39 mail-index-transaction-private.h, 1.17,
1.18 mail-index-transaction-view.c, 1.7,
1.8 mail-index-transaction.c, 1.34,
1.35 mail-index-view-private.h, 1.13,
1.14 mail-index-view-sync.c, 1.28, 1.29 mail-index-view.c,
1.27, 1.28 mail-index.c, 1.165, 1.166 mail-transaction-log.c,
1.73, 1.74
- Next message: [dovecot-cvs]
dovecot/src/master auth-process.c, 1.68, 1.69 common.h,
1.20, 1.21 login-process.c, 1.61, 1.62 mail-process.c, 1.62,
1.63 main.c, 1.57, 1.58 master-settings.c, 1.75, 1.76
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the dovecot-cvs
mailing list