[dovecot-cvs] dovecot/src/lib-index Makefile.am, 1.18,
1.19 mail-cache-compress.c, 1.2, 1.3 mail-cache-decisions.c,
NONE, 1.1 mail-cache-lookup.c, 1.6, 1.7 mail-cache-private.h,
1.3, 1.4 mail-cache-transaction.c, 1.6, 1.7 mail-cache.c, 1.32,
1.33 mail-cache.h, 1.11, 1.12 mail-index-sync-update.c, 1.30,
1.31 mail-index-sync.c, 1.24, 1.25 mail-index.h, 1.120, 1.121
cras at procontrol.fi
cras at procontrol.fi
Mon Jun 28 20:35:29 EEST 2004
Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv31698/lib-index
Modified Files:
Makefile.am mail-cache-compress.c mail-cache-lookup.c
mail-cache-private.h mail-cache-transaction.c mail-cache.c
mail-cache.h mail-index-sync-update.c mail-index-sync.c
mail-index.h
Added Files:
mail-cache-decisions.c
Log Message:
Added some smartness for deciding what to cache. Cache compression code compiles, but untested.
Index: Makefile.am
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/Makefile.am,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- Makefile.am 22 Jun 2004 07:36:33 -0000 1.18
+++ Makefile.am 28 Jun 2004 17:35:27 -0000 1.19
@@ -6,6 +6,8 @@
libindex_a_SOURCES = \
mail-cache.c \
+ mail-cache-compress.c \
+ mail-cache-decisions.c \
mail-cache-lookup.c \
mail-cache-transaction.c \
mail-index.c \
Index: mail-cache-compress.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-compress.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mail-cache-compress.c 20 Jun 2004 08:05:41 -0000 1.2
+++ mail-cache-compress.c 28 Jun 2004 17:35:27 -0000 1.3
@@ -1,9 +1,19 @@
+/* Copyright (C) 2003-2004 Timo Sirainen */
+
+#include "lib.h"
+#include "buffer.h"
+#include "byteorder.h"
+#include "ostream.h"
+#include "mail-cache-private.h"
+
+static unsigned char null4[4] = { 0, 0, 0, 0 };
+
static const struct mail_cache_record *
-mail_cache_compress_record(struct mail_cache *cache,
- struct mail_index_record *rec, int header_idx,
- uint32_t *size_r)
+mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
+ enum mail_cache_field orig_cached_fields,
+ int header_idx, uint32_t *size_r)
{
- enum mail_cache_field orig_cached_fields, cached_fields, field;
+ enum mail_cache_field cached_fields, field;
struct mail_cache_record cache_rec;
buffer_t *buffer;
const void *data;
@@ -15,14 +25,13 @@
buffer = buffer_create_dynamic(pool_datastack_create(),
4096, (size_t)-1);
- orig_cached_fields = mail_cache_get_fields(cache, rec);
cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK;
buffer_append(buffer, &cache_rec, sizeof(cache_rec));
for (i = 0, field = 1; i < 31; i++, field <<= 1) {
if ((cached_fields & field) == 0)
continue;
- if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) {
+ if (!mail_cache_lookup_field(view, seq, field, &data, &size)) {
cached_fields &= ~field;
continue;
}
@@ -44,14 +53,14 @@
for (i = 0; i <= header_idx; i++) {
field = mail_cache_header_fields[i];
- if (mail_cache_lookup_field(cache, rec, field,
+ if (mail_cache_lookup_field(view, seq, field,
&data, &size) && size > 1) {
size--; /* terminating \0 */
buffer_append(buffer, data, size);
nb_size += size;
}
}
- buffer_append(buffer, "", 1);
+ buffer_append(buffer, null4, 1);
nb_size++;
if ((nb_size & 3) != 0)
buffer_append(buffer, null4, 4 - (nb_size & 3));
@@ -71,36 +80,59 @@
return data;
}
-static int mail_cache_copy(struct mail_cache *cache, int fd)
+static int
+mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
{
-#if 0
- struct mail_cache_header *hdr;
+ struct mail_cache_view *cache_view;
+ struct mail_index_transaction *t;
+ const struct mail_index_header *idx_hdr;
const struct mail_cache_record *cache_rec;
- struct mail_index_record *rec;
- enum mail_cache_field used_fields;
- unsigned char *mmap_base;
+ struct mail_cache_header hdr;
+ struct ostream *output;
+ enum mail_cache_field keep_fields, temp_fields;
+ enum mail_cache_field cached_fields, new_fields;
const char *str;
- uint32_t new_file_size, offset, size, nb_size;
- int i, header_idx;
+ uint32_t size, nb_size, message_count, seq, first_new_seq;
+ uoff_t offset;
+ int i, header_idx, ret;
- /* pick some reasonably good file size */
- new_file_size = cache->used_file_size -
- nbo_to_uint32(cache->hdr->deleted_space);
- new_file_size = (new_file_size + 1023) & ~1023;
- if (new_file_size < MAIL_CACHE_INITIAL_SIZE)
- new_file_size = MAIL_CACHE_INITIAL_SIZE;
+ /* get sequence of first message which doesn't need it's temp fields
+ removed. */
+ if (mail_index_get_header(view, &idx_hdr) < 0)
+ return -1;
+ if (mail_index_lookup_uid_range(view, idx_hdr->day_first_uid[7],
+ (uint32_t)-1, &first_new_seq,
+ &message_count) < 0)
+ return -1;
+ if (first_new_seq == 0)
+ first_new_seq = message_count+1;
- if (file_set_size(fd, new_file_size) < 0)
- return mail_cache_set_syscall_error(cache, "file_set_size()");
+ cache_view = mail_cache_view_open(cache, view);
+ t = mail_index_transaction_begin(view, FALSE);
+ output = o_stream_create_file(fd, default_pool, 0, FALSE);
- mmap_base = mmap(NULL, new_file_size, PROT_READ | PROT_WRITE,
- MAP_SHARED, fd, 0);
- if (mmap_base == MAP_FAILED)
- return mail_cache_set_syscall_error(cache, "mmap()");
+ memset(&hdr, 0, sizeof(hdr));
+ hdr.indexid = cache->hdr->indexid;
+ hdr.file_seq = cache->hdr->file_seq + 1;
- /* skip file's header */
- hdr = (struct mail_cache_header *) mmap_base;
- offset = sizeof(*hdr);
+ memcpy(hdr.field_usage_decision_type,
+ cache->hdr->field_usage_decision_type,
+ sizeof(hdr.field_usage_decision_type));
+ memcpy(hdr.field_usage_last_used,
+ cache->hdr->field_usage_last_used,
+ sizeof(hdr.field_usage_last_used));
+
+ keep_fields = temp_fields = 0;
+ for (i = 0; i < 32; i++) {
+ if (cache->hdr->field_usage_decision_type[i] &
+ MAIL_CACHE_DECISION_YES)
+ keep_fields |= 1 << i;
+ else if (cache->hdr->field_usage_decision_type[i] &
+ MAIL_CACHE_DECISION_TEMP)
+ temp_fields |= 1 << i;
+ }
+
+ offset = sizeof(hdr);
/* merge all the header pieces into one. if some message doesn't have
all the required pieces, we'll just have to drop them all. */
@@ -113,136 +145,125 @@
if (str == NULL)
header_idx = -1;
else {
- hdr->header_offsets[0] = uint32_to_offset(offset);
+ hdr.header_offsets[0] = mail_cache_uint32_to_offset(offset);
header_idx = i;
size = strlen(str) + 1;
nb_size = uint32_to_nbo(size);
- memcpy(mmap_base + offset, &nb_size, sizeof(nb_size));
- offset += sizeof(nb_size);
- memcpy(mmap_base + offset, str, size);
- offset += (size + 3) & ~3;
+ o_stream_send(output, &nb_size, sizeof(nb_size));
+ o_stream_send(output, str, size);
+ if ((size & 3) != 0)
+ o_stream_send(output, null4, 4 - (size & 3));
}
- // FIXME: recreate index file with new cache_offsets
+ mail_index_reset_cache(t, hdr.file_seq);
- used_fields = 0;
- rec = cache->index->lookup(cache->index, 1);
- while (rec != NULL) {
- cache_rec = mail_cache_lookup(cache, rec, 0);
+ ret = 0;
+ for (seq = 1; seq <= message_count; seq++) {
+ cache_rec = mail_cache_lookup(cache_view, seq, 0);
if (cache_rec == NULL)
- rec->cache_offset = 0;
- else if (offset_to_uint32(cache_rec->next_offset) == 0) {
- /* just one unmodified block, copy it */
- size = nbo_to_uint32(cache_rec->size);
- i_assert(offset + size <= new_file_size);
+ continue;
- memcpy(mmap_base + offset, cache_rec, size);
- rec->cache_offset = uint32_to_offset(offset);
+ cached_fields = mail_cache_get_fields(cache_view, seq);
+ new_fields = cached_fields & keep_fields;
+ if ((cached_fields & temp_fields) != 0 &&
+ seq >= first_new_seq) {
+ /* new message, keep temp fields */
+ new_fields |= cached_fields & temp_fields;
+ }
- size = (size + 3) & ~3;
- offset += size;
+ if (keep_fields == cached_fields &&
+ mail_cache_offset_to_uint32(cache_rec->next_offset) == 0) {
+ /* just one unmodified block, save it */
+ size = nbo_to_uint32(cache_rec->size);
+ mail_index_update_cache(t, seq, output->offset);
+ o_stream_send(output, cache_rec, size);
+
+ if ((size & 3) != 0)
+ o_stream_send(output, null4, 4 - (size & 3));
} else {
- /* multiple blocks, sort them into buffer */
+ /* a) dropping fields
+ b) multiple blocks, sort them into buffer */
+ mail_index_update_cache(t, seq, output->offset);
+
t_push();
- cache_rec = mail_cache_compress_record(cache, rec,
+ cache_rec = mail_cache_compress_record(cache_view, seq,
+ keep_fields,
header_idx,
&size);
- i_assert(offset + size <= new_file_size);
- memcpy(mmap_base + offset, cache_rec, size);
- used_fields |= cache_rec->fields;
+ o_stream_send(output, cache_rec, size);
t_pop();
-
- rec->cache_offset = uint32_to_offset(offset);
- offset += size;
}
-
- rec = cache->index->next(cache->index, rec);
}
+ hdr.used_file_size = uint32_to_nbo(output->offset);
- /* update header */
- hdr->indexid = cache->index->indexid;
- hdr->file_seq = cache->index->hdr->cache_sync_id+1;
- hdr->used_file_size = uint32_to_nbo(offset);
- hdr->used_fields = used_fields;
- hdr->field_usage_start = uint32_to_nbo(ioloop_time);
-
- /* write everything to disk */
- if (msync(mmap_base, offset, MS_SYNC) < 0)
- return mail_cache_set_syscall_error(cache, "msync()");
+ o_stream_unref(output);
+ mail_cache_view_close(cache_view);
- if (munmap(mmap_base, new_file_size) < 0)
- return mail_cache_set_syscall_error(cache, "munmap()");
+ if (fdatasync(fd) < 0) {
+ mail_cache_set_syscall_error(cache, "fdatasync()");
+ (void)mail_index_transaction_rollback(t);
+ return -1;
+ }
- if (fdatasync(fd) < 0)
- return mail_cache_set_syscall_error(cache, "fdatasync()");
- return TRUE;
-#endif
+ return mail_index_transaction_commit(t, &seq, &offset);
}
-int mail_cache_compress(struct mail_cache *cache)
+int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view)
{
- int fd, ret = TRUE;
+ int fd, ret;
i_assert(cache->trans_ctx == NULL);
- if (cache->anon_mmap)
- return TRUE;
-
- if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- if (mail_cache_lock(cache, TRUE) <= 0)
- return FALSE;
+ if ((ret = mail_cache_lock(cache, TRUE)) <= 0)
+ return ret;
#ifdef DEBUG
i_warning("Compressing cache file %s", cache->filepath);
#endif
- fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
+ fd = file_dotlock_open(cache->filepath, NULL, NULL,
+ MAIL_CACHE_LOCK_TIMEOUT,
MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
if (fd == -1) {
mail_cache_set_syscall_error(cache, "file_dotlock_open()");
- return FALSE;
+ return -1;
}
- /* now we'll begin the actual moving. keep rebuild-flag on
- while doing it. */
- cache->index->hdr->flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
- if (!mail_index_fmdatasync(cache->index, cache->index->hdr_size))
- return FALSE;
-
- if (!mail_cache_copy(cache, fd)) {
- (void)file_dotlock_delete(cache->filepath, fd);
- ret = FALSE;
+ if (mail_cache_copy(cache, view, fd) < 0) {
+ (void)file_dotlock_delete(cache->filepath, NULL, fd);
+ ret = -1;
} else {
- mail_cache_file_close(cache);
- cache->fd = dup(fd);
-
- if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
+ if (file_dotlock_replace(cache->filepath, NULL,
+ -1, FALSE) < 0) {
mail_cache_set_syscall_error(cache,
"file_dotlock_replace()");
- ret = FALSE;
- }
+ (void)close(fd);
+ ret = -1;
+ } else {
+ mail_cache_file_close(cache);
+ cache->fd = fd;
- if (!mmap_update(cache, 0, 0))
- ret = FALSE;
+ if (mail_cache_mmap_update(cache, 0, 0) < 0)
+ ret = -1;
+ }
}
/* headers could have changed, reread them */
memset(cache->split_offsets, 0, sizeof(cache->split_offsets));
memset(cache->split_headers, 0, sizeof(cache->split_headers));
- if (ret) {
- cache->index->hdr->flags &=
- ~(MAIL_INDEX_HDR_FLAG_REBUILD |
- MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE);
- }
-
if (mail_cache_unlock(cache) < 0)
- ret = FALSE;
+ return -1;
+ if (ret == 0)
+ cache->need_compress = FALSE;
return ret;
}
+
+int mail_cache_need_compress(struct mail_cache *cache)
+{
+ return cache->need_compress;
+}
--- NEW FILE: mail-cache-decisions.c ---
/* Copyright (C) 2004 Timo Sirainen */
#include "lib.h"
#include "write-full.h"
#include "mail-cache-private.h"
#include <stddef.h>
static void
mail_cache_set_decision_type(struct mail_cache *cache, uint32_t idx,
enum mail_cache_decision_type type)
{
uint8_t value = type;
/* update the header without locking, we'll just write one byte and
it's very unlikely someone else tries to write different value for
it at the same time. even then it's just a wrong decision which
will be corrected sometimes later, not too bad.. */
if (pwrite_full(cache->fd, &value, 1,
offsetof(struct mail_cache_header,
field_usage_decision_type) + idx) < 0) {
mail_cache_set_syscall_error(cache, "pwrite_full()");
}
}
void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field field)
{
const struct mail_index_header *hdr;
unsigned int idx;
uint32_t uid;
idx = mail_cache_field_index(field);
if (view->cache->hdr->field_usage_decision_type[idx] !=
MAIL_CACHE_DECISION_TEMP) {
/* a) forced decision
b) not cached, mail_cache_mark_missing() will handle this
c) permanently cached already, okay. */
return;
}
/* see if we want to change decision from TEMP to YES */
if (mail_index_lookup_uid(view->view, seq, &uid) < 0 ||
mail_index_get_header(view->view, &hdr) < 0)
return;
if (uid < view->cache->field_usage_uid_highwater[idx] ||
uid < hdr->day_first_uid[7]) {
/* a) nonordered access within this session. if client doesn't
request messages in growing order, we assume it doesn't
have a permanent local cache.
b) accessing message older than one week. assume it's a
client with no local cache. if it was just a new client
generating the local cache for the first time, we'll
drop back to TEMP within few months. */
mail_cache_set_decision_type(view->cache, idx,
MAIL_CACHE_DECISION_YES);
} else {
view->cache->field_usage_uid_highwater[idx] = uid;
}
}
void mail_cache_mark_missing(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field field)
{
unsigned int idx;
uint32_t uid;
idx = mail_cache_field_index(field);
if (view->cache->hdr->field_usage_decision_type[idx] !=
MAIL_CACHE_DECISION_NO) {
/* a) forced decision
b) we're already caching it, so it just wasn't in cache */
return;
}
/* field used the first time */
mail_cache_set_decision_type(view->cache, idx,
MAIL_CACHE_DECISION_TEMP);
if (mail_index_lookup_uid(view->view, seq, &uid) == 0)
view->cache->field_usage_uid_highwater[idx] = uid;
}
Index: mail-cache-lookup.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-lookup.c,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- mail-cache-lookup.c 23 Jun 2004 13:43:36 -0000 1.6
+++ mail-cache-lookup.c 28 Jun 2004 17:35:27 -0000 1.7
@@ -210,7 +210,7 @@
static int cache_get_field(struct mail_cache *cache,
struct mail_cache_record *cache_rec,
enum mail_cache_field field,
- void **data_r, size_t *size_r)
+ const void **data_r, size_t *size_r)
{
unsigned char *buf;
unsigned int mask;
@@ -266,12 +266,14 @@
return FALSE;
}
-static int cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
- enum mail_cache_field field,
- void **data_r, size_t *size_r)
+int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
+ enum mail_cache_field field,
+ const void **data_r, size_t *size_r)
{
struct mail_cache_record *cache_rec;
+ mail_cache_handle_decisions(view, seq, field);
+
cache_rec = mail_cache_lookup(view, seq, field);
while (cache_rec != NULL) {
if ((cache_rec->fields & field) != 0) {
@@ -284,19 +286,6 @@
return FALSE;
}
-int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
- enum mail_cache_field field,
- const void **data_r, size_t *size_r)
-{
- void *data;
-
- if (!cache_lookup_field(view, seq, field, &data, size_r))
- return FALSE;
-
- *data_r = data;
- return TRUE;
-}
-
const char *
mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field field)
Index: mail-cache-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-private.h,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mail-cache-private.h 20 Jun 2004 09:13:14 -0000 1.3
+++ mail-cache-private.h 28 Jun 2004 17:35:27 -0000 1.4
@@ -27,6 +27,18 @@
#define CACHE_RECORD(cache, offset) \
((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset))
+enum mail_cache_decision_type {
+ /* Not needed currently */
+ MAIL_CACHE_DECISION_NO = 0x00,
+ /* Needed only for new mails. Drop when compressing. */
+ MAIL_CACHE_DECISION_TEMP = 0x01,
+ /* Needed. */
+ MAIL_CACHE_DECISION_YES = 0x02,
+
+ /* This decision has been forced manually, don't change it. */
+ MAIL_CACHE_DECISION_FORCED = 0x80
+};
+
struct mail_cache_header {
uint32_t indexid;
uint32_t file_seq;
@@ -36,10 +48,8 @@
uint32_t used_file_size;
uint32_t deleted_space;
- uint32_t used_fields; /* enum mail_cache_field */
-
- uint32_t field_usage_start; /* time_t */
- uint32_t field_usage_counts[32];
+ uint32_t field_usage_last_used[32]; /* time_t */
+ uint8_t field_usage_decision_type[32];
uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT];
};
@@ -68,10 +78,13 @@
enum mail_cache_field default_cache_fields;
enum mail_cache_field never_cache_fields;
+ uint32_t field_usage_uid_highwater[32];
+
struct mail_cache_transaction_ctx *trans_ctx;
unsigned int locks;
unsigned int mmap_refresh:1;
+ unsigned int need_compress:1;
unsigned int silent:1;
unsigned int disabled:1;
};
@@ -88,6 +101,7 @@
uint32_t mail_cache_uint32_to_offset(uint32_t offset);
uint32_t mail_cache_offset_to_uint32(uint32_t offset);
+unsigned int mail_cache_field_index(enum mail_cache_field field);
const char *
mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx);
@@ -111,6 +125,10 @@
int mail_cache_mmap_update(struct mail_cache *cache,
size_t offset, size_t size);
+void mail_cache_file_close(struct mail_cache *cache);
+
+void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq,
+ enum mail_cache_field field);
void mail_cache_set_syscall_error(struct mail_cache *cache,
const char *function);
Index: mail-cache-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-transaction.c,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- mail-cache-transaction.c 22 Jun 2004 07:36:33 -0000 1.6
+++ mail-cache-transaction.c 28 Jun 2004 17:35:27 -0000 1.7
@@ -145,7 +145,7 @@
COMPRESS_CONTINUED_PERCENTAGE &&
ctx->used_file_size >= COMPRESS_MIN_SIZE) {
/* too many continued rows, compress */
- //FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
+ cache->need_compress = TRUE;
}
cache->hdr->continued_record_count = uint32_to_nbo(cont);
@@ -302,8 +302,7 @@
if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
/* they're all used - compress the cache to get more */
- /* FIXME: ctx->cache->index->set_flags |=
- MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;*/
+ ctx->cache->need_compress = TRUE;
}
mail_cache_transaction_flush(ctx);
@@ -548,9 +547,8 @@
/* see if we've reached the max. deleted space in file */
max_del_space = ctx->used_file_size / 100 * COMPRESS_PERCENTAGE;
if (deleted_space >= max_del_space &&
- ctx->used_file_size >= COMPRESS_MIN_SIZE) {
- //FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
- }
+ ctx->used_file_size >= COMPRESS_MIN_SIZE)
+ cache->need_compress = TRUE;
cache->hdr->deleted_space = uint32_to_nbo(deleted_space);
return 0;
Index: mail-cache.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.32
retrieving revision 1.33
diff -u -d -r1.32 -r1.33
--- mail-cache.c 21 Jun 2004 14:20:08 -0000 1.32
+++ mail-cache.c 28 Jun 2004 17:35:27 -0000 1.33
@@ -1,7 +1,6 @@
/* Copyright (C) 2003-2004 Timo Sirainen */
#include "lib.h"
-#include "buffer.h"
#include "byteorder.h"
#include "file-lock.h"
#include "file-set-size.h"
@@ -9,7 +8,6 @@
#include "write-full.h"
#include "mail-cache-private.h"
-#include <stddef.h>
#include <unistd.h>
#include <sys/stat.h>
@@ -67,6 +65,17 @@
(((uint32_t)buf[0] & 0x7f) << 23);
}
+unsigned int mail_cache_field_index(enum mail_cache_field field)
+{
+ unsigned int i, num;
+
+ for (i = 0, num = 1; i < 32; i++, num <<= 1) {
+ if (field == num)
+ return i;
+ }
+ i_unreached();
+}
+
void mail_cache_set_syscall_error(struct mail_cache *cache,
const char *function)
{
@@ -99,7 +108,7 @@
va_end(va);
}
-static void mail_cache_file_close(struct mail_cache *cache)
+void mail_cache_file_close(struct mail_cache *cache)
{
if (cache->mmap_base != NULL) {
if (munmap(cache->mmap_base, cache->mmap_length) < 0)
@@ -539,9 +548,3 @@
{
i_free(view);
}
-
-void mail_cache_mark_missing(struct mail_cache_view *view,
- enum mail_cache_field fields)
-{
- // FIXME
-}
Index: mail-cache.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.h,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- mail-cache.h 23 Jun 2004 18:33:22 -0000 1.11
+++ mail-cache.h 28 Jun 2004 17:35:27 -0000 1.12
@@ -28,9 +28,9 @@
enum mail_cache_field {
/* fixed size fields */
MAIL_CACHE_INDEX_FLAGS = 0x00000001,
- MAIL_CACHE_SENT_DATE = 0x00000008,
- MAIL_CACHE_RECEIVED_DATE = 0x00000010,
- MAIL_CACHE_VIRTUAL_FULL_SIZE = 0x00000020,
+ MAIL_CACHE_SENT_DATE = 0x00000002,
+ MAIL_CACHE_RECEIVED_DATE = 0x00000004,
+ MAIL_CACHE_VIRTUAL_FULL_SIZE = 0x00000008,
/* variable sized field */
MAIL_CACHE_HEADERS1 = 0x40000000,
@@ -77,8 +77,10 @@
enum mail_cache_field default_cache_fields,
enum mail_cache_field never_cache_fields);
+/* Returns TRUE if cache should be compressed. */
+int mail_cache_need_compress(struct mail_cache *cache);
/* Compress cache file. */
-int mail_cache_compress(struct mail_cache *cache);
+int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view);
/* Reset the cache file, clearing all data. */
int mail_cache_reset(struct mail_cache *cache);
@@ -153,9 +155,9 @@
enum mail_cache_field field,
void *buffer, size_t buffer_size);
-/* Mark given fields as missing, ie. they should be cached when possible. */
-void mail_cache_mark_missing(struct mail_cache_view *view,
- enum mail_cache_field fields);
+/* Mark given field as missing, ie. it should be cached when possible. */
+void mail_cache_mark_missing(struct mail_cache_view *view, uint32_t uid,
+ enum mail_cache_field field);
/* Return record flags. */
enum mail_cache_record_flag
Index: mail-index-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.30
retrieving revision 1.31
diff -u -d -r1.30 -r1.31
--- mail-index-sync-update.c 24 Jun 2004 11:10:41 -0000 1.30
+++ mail-index-sync-update.c 28 Jun 2004 17:35:27 -0000 1.31
@@ -1,6 +1,7 @@
/* Copyright (C) 2004 Timo Sirainen */
#include "lib.h"
+#include "ioloop.h"
#include "buffer.h"
#include "file-set-size.h"
#include "mmap-util.h"
@@ -9,6 +10,8 @@
#include "mail-transaction-log.h"
#include "mail-transaction-util.h"
+#include <time.h>
+
static void
mail_index_header_update_counts(struct mail_index_header *hdr,
uint8_t old_flags, uint8_t new_flags)
@@ -306,6 +309,43 @@
map->write_to_disk = TRUE;
}
+static void
+mail_index_update_day_headers(struct mail_index_header *hdr, uint32_t uid)
+{
+ const int max_days =
+ sizeof(hdr->day_first_uid) / sizeof(hdr->day_first_uid[0]);
+ struct tm tm;
+ time_t stamp;
+ int i, days;
+
+ /* get beginning of today */
+ tm = *localtime(&ioloop_time);
+ tm.tm_hour = 0;
+ tm.tm_min = 0;
+ tm.tm_sec = 0;
+ stamp = mktime(&tm);
+ if (stamp == (time_t)-1)
+ i_panic("mktime(today) failed");
+
+ if ((time_t)hdr->day_stamp >= stamp)
+ return;
+
+ /* get number of days since last message */
+ days = (stamp - hdr->day_stamp) / (3600*24);
+ if (days > max_days)
+ days = max_days;
+
+ /* @UNSAFE: move days forward and fill the missing days with old
+ day_first_uid[0]. */
+ memcpy(hdr->day_first_uid + days,
+ hdr->day_first_uid, max_days - days);
+ for (i = 1; i < days; i++)
+ hdr->day_first_uid[i] = hdr->day_first_uid[0];
+
+ hdr->day_stamp = stamp;
+ hdr->day_first_uid[0] = uid;
+}
+
int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
{
struct mail_index *index = sync_ctx->index;
@@ -314,7 +354,7 @@
const struct mail_transaction_header *hdr;
const void *data;
unsigned int count, old_lock_id;
- uint32_t seq, i;
+ uint32_t seq, i, first_append_uid;
uoff_t offset;
int ret, had_dirty, skipped;
@@ -338,6 +378,7 @@
view->map = map;
view->map->refcount++;
+ first_append_uid = 0;
had_dirty = (map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
if (had_dirty)
map->hdr_copy.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
@@ -358,6 +399,11 @@
if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) {
const struct mail_transaction_append_header *append_hdr;
+ const struct mail_index_record *rec;
+
+ rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
+ if (first_append_uid == 0)
+ first_append_uid = rec->uid;
append_hdr = data;
if (append_hdr->record_size > map->hdr->record_size) {
@@ -367,7 +413,7 @@
mail_index_sync_replace_map(view, map);
}
count = (hdr->size - sizeof(*append_hdr)) /
- append_hdr->record_size;
+ append_hdr->record_size;
if (mail_index_grow(index, view->map, count) < 0) {
ret = -1;
break;
@@ -383,7 +429,6 @@
}
if (ret < 0) {
- /* */
mail_index_view_unlock(view);
return -1;
}
@@ -396,6 +441,9 @@
map->hdr_copy.log_file_seq = seq;
map->hdr_copy.log_file_offset = offset;
+ if (first_append_uid != 0)
+ mail_index_update_day_headers(&map->hdr_copy, first_append_uid);
+
if ((map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 &&
had_dirty) {
/* do we have dirty flags anymore? */
Index: mail-index-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- mail-index-sync.c 24 Jun 2004 11:10:41 -0000 1.24
+++ mail-index-sync.c 28 Jun 2004 17:35:27 -0000 1.25
@@ -332,8 +332,8 @@
int mail_index_sync_end(struct mail_index_sync_ctx *ctx)
{
const struct mail_index_header *hdr;
- uint32_t seq;
- uoff_t offset;
+ uint32_t seq, seq2;
+ uoff_t offset, offset2;
int ret = 0;
if (mail_transaction_log_view_is_corrupted(ctx->view->log_view))
@@ -348,8 +348,24 @@
hdr->log_file_seq, hdr->log_file_offset,
seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0)
ret = -1;
- if (mail_index_sync_update_index(ctx) < 0)
+ else if (mail_index_sync_update_index(ctx) < 0)
+ ret = -1;
+ }
+
+ if (ret == 0 && mail_cache_need_compress(ctx->index->cache)) {
+ if (mail_cache_compress(ctx->index->cache, ctx->view) < 0)
ret = -1;
+ else {
+ /* cache_offsets have changed, sync them */
+ mail_transaction_log_get_head(ctx->index->log,
+ &seq2, &offset2);
+ if (mail_transaction_log_view_set(ctx->view->log_view,
+ seq, offset, seq2, offset2,
+ MAIL_TRANSACTION_TYPE_MASK) < 0)
+ ret = -1;
+ else if (mail_index_sync_update_index(ctx) < 0)
+ ret = -1;
+ }
}
mail_index_unlock(ctx->index, ctx->lock_id);
Index: mail-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.120
retrieving revision 1.121
diff -u -d -r1.120 -r1.121
--- mail-index.h 24 Jun 2004 11:10:41 -0000 1.120
+++ mail-index.h 28 Jun 2004 17:35:27 -0000 1.121
@@ -6,7 +6,7 @@
#define MAIL_INDEX_MAJOR_VERSION 4
#define MAIL_INDEX_MINOR_VERSION 0
-#define MAIL_INDEX_HEADER_MIN_SIZE 88
+#define MAIL_INDEX_HEADER_MIN_SIZE 120
/* Number of keywords in mail_index_record. */
#define INDEX_KEYWORDS_COUNT (3*8)
@@ -93,11 +93,15 @@
uint32_t log_file_seq;
uint32_t log_file_offset;
- uint64_t sync_size;
uint32_t sync_stamp;
+ uint64_t sync_size;
uint32_t cache_file_seq;
uint32_t extra_records_hdr_offset;
+
+ /* daily first UIDs that have been added to index. */
+ uint32_t day_stamp;
+ uint32_t day_first_uid[8];
};
struct mail_index_record {
More information about the dovecot-cvs
mailing list