[dovecot-cvs] dovecot/src/lib-index mail-cache.c,1.14,1.15
mail-index-file.c,1.3,1.4 mail-index.c,1.99,1.100
mail-index.h,1.92,1.93
cras at procontrol.fi
cras at procontrol.fi
Mon Oct 20 07:13:08 EEST 2003
Update of /home/cvs/dovecot/src/lib-index
In directory danu:/tmp/cvs-serv16105/lib-index
Modified Files:
mail-cache.c mail-index-file.c mail-index.c mail-index.h
Log Message:
We didn't notice cache compression immediately in other processes. Also some
other bugfixes.
Index: mail-cache.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- mail-cache.c 21 Sep 2003 16:21:37 -0000 1.14
+++ mail-cache.c 20 Oct 2003 03:13:06 -0000 1.15
@@ -42,6 +42,8 @@
struct mail_cache_header {
uint32_t indexid;
+ uint32_t sync_id;
+
uint32_t continued_record_count;
uint32_t used_file_size;
@@ -70,6 +72,7 @@
void *mmap_base;
size_t mmap_length;
uint32_t used_file_size;
+ uint32_t sync_id;
struct mail_cache_header *header;
@@ -249,6 +252,7 @@
if (cache->mmap_length < sizeof(struct mail_cache_header))
return mail_cache_set_corrupted(cache, "File too small");
cache->header = hdr = cache->mmap_base;
+ cache->sync_id = hdr->sync_id;
if (cache->header->indexid != cache->index->indexid) {
/* index id changed */
@@ -290,25 +294,33 @@
static int mmap_update_nocheck(struct mail_cache *cache,
size_t offset, size_t size)
{
- if (cache->header != NULL &&
- cache->header->indexid != cache->index->indexid) {
- /* indexid changed, most likely it was rebuilt.
- try reopening. */
+ struct stat st;
+
+ /* if sync id has changed, the file has to be reopened.
+ note that if main index isn't locked, it may change again */
+ if (cache->sync_id != cache->index->cache_sync_id &&
+ cache->mmap_base != NULL) {
if (!mail_cache_file_reopen(cache))
return -1;
-
- /* force mmap refresh */
- size = 0;
}
- if (size != 0 && offset < cache->mmap_length &&
- size <= cache->mmap_length - offset) {
+ if (offset < cache->mmap_length &&
+ size <= cache->mmap_length - offset &&
+ !cache->mmap_refresh) {
/* already mapped */
- if (!cache->mmap_refresh)
+ if (size != 0 || cache->anon_mmap)
return 1;
- cache->mmap_refresh = FALSE;
+ /* requesting the whole file - see if we need to
+ re-mmap */
+ if (fstat(cache->fd, &st) < 0) {
+ mail_cache_set_syscall_error(cache, "fstat()");
+ return -1;
+ }
+ if ((uoff_t)st.st_size == cache->mmap_length)
+ return 1;
}
+ cache->mmap_refresh = FALSE;
if (cache->anon_mmap)
return 1;
@@ -346,14 +358,28 @@
static int mmap_update(struct mail_cache *cache, size_t offset, size_t size)
{
- int ret;
+ int synced, ret;
- ret = mmap_update_nocheck(cache, offset, size);
- if (ret > 0)
- return TRUE;
- if (ret < 0)
- return FALSE;
- return mmap_verify_header(cache);
+ for (synced = FALSE;; synced = TRUE) {
+ ret = mmap_update_nocheck(cache, offset, size);
+ if (ret > 0)
+ return TRUE;
+ if (ret < 0)
+ return FALSE;
+
+ if (!mmap_verify_header(cache))
+ return FALSE;
+
+ /* see if cache file was rebuilt - do it only once to avoid
+ infinite looping */
+ if (cache->header->sync_id == cache->index->cache_sync_id ||
+ synced)
+ break;
+
+ if (!mail_cache_file_reopen(cache))
+ return FALSE;
+ }
+ return TRUE;
}
static int mail_cache_open_and_verify(struct mail_cache *cache, int silent)
@@ -475,6 +501,7 @@
memset(&hdr, 0, sizeof(hdr));
hdr.indexid = index->indexid;
+ hdr.sync_id = index->cache_sync_id;
hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
cache = i_new(struct mail_cache, 1);
@@ -688,6 +715,8 @@
/* update header */
hdr->indexid = cache->index->indexid;
+ hdr->sync_id = cache->sync_id = cache->index->cache_sync_id =
+ ++cache->index->header->cache_sync_id;
hdr->used_file_size = uint32_to_nbo(offset);
hdr->used_fields = used_fields;
hdr->field_usage_start = uint32_to_nbo(ioloop_time);
@@ -710,12 +739,19 @@
i_assert(cache->trans_ctx == NULL);
+ if (cache->anon_mmap)
+ return TRUE;
+
if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
return FALSE;
if (mail_cache_lock(cache, TRUE) <= 0)
return FALSE;
+#ifdef DEBUG
+ i_warning("Compressing cache file %s", cache->filepath);
+#endif
+
fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
MAIL_CACHE_LOCK_STALE_TIMEOUT, NULL, NULL);
if (fd == -1) {
@@ -759,8 +795,12 @@
struct mail_cache_header hdr;
int ret, fd;
+ i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE);
+
memset(&hdr, 0, sizeof(hdr));
hdr.indexid = cache->index->indexid;
+ hdr.sync_id = cache->sync_id = cache->index->cache_sync_id
+ =++cache->index->header->cache_sync_id;
hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
cache->used_file_size = sizeof(hdr);
@@ -840,9 +880,19 @@
}
if (ret > 0) {
- if (!mmap_verify_header(cache)) {
+ if (!mmap_update(cache, 0, 0)) {
(void)mail_cache_unlock(cache);
- ret = -1;
+ return -1;
+ }
+ if (cache->sync_id != cache->index->cache_sync_id) {
+ /* we have the cache file locked and sync_id still
+ doesn't match. it means we crashed between updating
+ cache file and updating sync_id in index header.
+ just update the sync_ids so they match. */
+ i_warning("Updating broken sync_id in cache file %s",
+ cache->filepath);
+ cache->sync_id = cache->header->sync_id =
+ cache->index->cache_sync_id;
}
}
return ret;
Index: mail-index-file.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-file.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mail-index-file.c 24 Aug 2003 06:32:08 -0000 1.3
+++ mail-index-file.c 20 Oct 2003 03:13:06 -0000 1.4
@@ -142,7 +142,7 @@
return FALSE;
}
- index->header->sync_id++;
+ index->header->master_sync_id++;
if (!mail_index_mmap_update(index))
return FALSE;
}
@@ -162,7 +162,7 @@
index->header->header_size = sizeof(struct mail_index_header);
index->header->used_file_size += diff;
- index->header->sync_id++;
+ index->header->master_sync_id++;
if (!mail_index_fmdatasync(index, index->mmap_used_length))
return FALSE;
@@ -201,7 +201,7 @@
if (ftruncate(index->fd, (off_t)index->mmap_full_length) < 0)
return index_set_syscall_error(index, "ftruncate()");
- index->header->sync_id++;
+ index->header->master_sync_id++;
}
return TRUE;
Index: mail-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.99
retrieving revision 1.100
diff -u -d -r1.99 -r1.100
--- mail-index.c 2 Sep 2003 22:33:33 -0000 1.99
+++ mail-index.c 20 Oct 2003 03:13:06 -0000 1.100
@@ -74,7 +74,9 @@
return FALSE;
}
- index->sync_id = hdr->sync_id;
+ index->master_sync_id = hdr->master_sync_id;
+ index->cache_sync_id = hdr->cache_sync_id;
+ index->log_sync_id = hdr->log_sync_id;
index->sync_stamp = hdr->sync_stamp;
index->sync_size = hdr->sync_size;
index->mmap_used_length = hdr->used_file_size;
@@ -89,6 +91,9 @@
if (index->mmap_base != NULL) {
index->header = (struct mail_index_header *) index->mmap_base;
+ index->cache_sync_id = index->header->cache_sync_id;
+ index->log_sync_id = index->header->log_sync_id;
+
if (index->mmap_invalidate) {
if (msync(index->mmap_base,
index->mmap_used_length,
@@ -99,7 +104,7 @@
}
/* make sure file size hasn't changed */
- if (index->header->sync_id == index->sync_id) {
+ if (index->header->master_sync_id == index->master_sync_id) {
index->mmap_used_length = index->header->used_file_size;
if (index->mmap_used_length > index->mmap_full_length) {
i_panic("Index file size was grown without "
@@ -647,7 +652,7 @@
/* file size changed, let others know about it too by changing
sync_id in header. */
- index->header->sync_id++;
+ index->header->master_sync_id++;
if (!mail_index_mmap_update(index))
return FALSE;
Index: mail-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.92
retrieving revision 1.93
diff -u -d -r1.92 -r1.93
--- mail-index.h 2 Sep 2003 22:33:33 -0000 1.92
+++ mail-index.h 20 Oct 2003 03:13:06 -0000 1.93
@@ -115,10 +115,12 @@
uint8_t compat_data[4];
uint32_t indexid;
-
uint32_t used_file_size;
- uint32_t sync_id; /* re-mmap() when changed, required only
- if file size is shrinked */
+
+ /* file needs to be reopened if sync_ids change. */
+ uint32_t master_sync_id;
+ uint32_t cache_sync_id;
+ uint32_t log_sync_id;
uint32_t flags;
@@ -281,7 +283,7 @@
char *mailbox_path; /* file/directory for mailbox location */
char *control_dir; /* destination for control files */
unsigned int indexid;
- unsigned int sync_id;
+ unsigned int master_sync_id, cache_sync_id, log_sync_id;
/* updated whenever exclusive lock is set/unset */
unsigned int excl_lock_counter;
@@ -358,11 +360,11 @@
members.. */
#define MAIL_INDEX_PRIVATE_FILL \
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
- 0, 0, 0, 0, { 0, 0, 0 }, 0, 0, 0, \
+ 0, 0, 0, 0, 0, 0, { 0, 0, 0 }, 0, \
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
- 0, 0, 0, 0, 0, 0, 0
+ 0, 0, 0, 0, 0, 0, 0, 0, 0
#endif
/* defaults - same as above but prefixed with mail_index_. */
More information about the dovecot-cvs
mailing list