[dovecot-cvs] dovecot/src/lib-index mail-cache-compress.c, 1.9,
1.10 mail-cache-decisions.c, 1.3, 1.4 mail-cache-lookup.c,
1.11, 1.12 mail-cache-private.h, 1.7,
1.8 mail-cache-transaction.c, 1.11, 1.12 mail-cache.c, 1.38,
1.39 mail-cache.h, 1.17, 1.18
cras at dovecot.org
cras at dovecot.org
Thu Jul 8 23:26:18 EEST 2004
Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv1384/lib-index
Modified Files:
mail-cache-compress.c mail-cache-decisions.c
mail-cache-lookup.c mail-cache-private.h
mail-cache-transaction.c mail-cache.c mail-cache.h
Log Message:
Cache file fixes, API changes, etc. It's still in somewhat ugly state, but
getting better..
Index: mail-cache-compress.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-compress.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- mail-cache-compress.c 4 Jul 2004 21:56:12 -0000 1.9
+++ mail-cache-compress.c 8 Jul 2004 20:26:15 -0000 1.10
@@ -8,98 +8,76 @@
static unsigned char null4[4] = { 0, 0, 0, 0 };
-static const struct mail_cache_record *
-mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
- enum mail_cache_field orig_cached_fields,
- int header_idx, uint32_t *size_r)
+struct mail_cache_copy_context {
+ int new_msg;
+ char field_seen[32], keep_fields[32], temp_fields[32];
+ buffer_t *buffer, *header;
+};
+
+static int
+mail_cache_compress_callback(struct mail_cache_view *view __attr_unused__,
+ enum mail_cache_field field,
+ const void *data, size_t data_size, void *context)
{
- enum mail_cache_field cached_fields, field;
- struct mail_cache_record cache_rec;
- buffer_t *buffer;
- pool_t pool;
- const void *data;
- size_t size, pos;
- uint32_t *p, size32 = 0;
+ struct mail_cache_copy_context *ctx = context;
+ uint32_t size32;
int i;
- memset(&cache_rec, 0, sizeof(cache_rec));
- pool = pool_datastack_create();
- buffer = buffer_create_dynamic(pool, 4096, (size_t)-1);
-
- cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK;
- buffer_append(buffer, &cache_rec, sizeof(cache_rec));
- for (i = 0, field = 1; i < 31; i++, field <<= 1) {
- if ((cached_fields & field) == 0)
- continue;
-
- pos = buffer_get_used_size(buffer);
- if ((field & MAIL_CACHE_FIXED_MASK) == 0)
- buffer_append(buffer, &size32, sizeof(size32));
-
- if (!mail_cache_lookup_field(view, buffer, seq, field)) {
- cached_fields &= ~field;
- buffer_set_used_size(buffer, pos);
- continue;
- }
-
- if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
- p = buffer_get_space_unsafe(buffer, pos,
- sizeof(uint32_t));
- *p = (uint32_t)size;
- }
-
- if ((size & 3) != 0)
- buffer_append(buffer, null4, 4 - (size & 3));
+ if (ctx->new_msg) {
+ if (!ctx->temp_fields[field])
+ return 1;
+ } else {
+ if (!ctx->keep_fields[field])
+ return 1;
}
- /* now merge all the headers if we have them all */
- if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) {
- size32 = 0;
- pos = buffer_get_used_size(buffer);
- buffer_append(buffer, &size32, sizeof(size32));
+ if (ctx->field_seen[field]) {
+ /* drop duplicates */
+ return 1;
+ }
+ ctx->field_seen[field] = TRUE;
- for (i = 0; i <= header_idx; i++) {
- field = mail_cache_header_fields[i];
- if (mail_cache_lookup_field(view, buffer, seq, field)) {
- /* remove terminating \0 */
- buffer_set_used_size(buffer,
- buffer_get_used_size(buffer)-1);
+ for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
+ if (mail_cache_header_fields[i] == field) {
+ /* it's header - save it into header field */
+ size32 = buffer_get_used_size(ctx->header);
+ if (size32 > 0) {
+ /* remove old terminating \0 */
+ buffer_set_used_size(ctx->header, size32-1);
}
+ buffer_append(ctx->header, data, data_size);
+ return 1;
}
- buffer_append(buffer, null4, 1);
+ }
- size32 = buffer_get_used_size(buffer) - pos;
- if ((size32 & 3) != 0)
- buffer_append(buffer, null4, 4 - (size32 & 3));
- buffer_write(buffer, pos, &size32, sizeof(size32));
+ buffer_append(ctx->buffer, &field, sizeof(field));
- cached_fields |= MAIL_CACHE_HEADERS1;
+ if (mail_cache_field_sizes[field] == (unsigned int)-1) {
+ size32 = (uint32_t)data_size;
+ buffer_append(ctx->buffer, &size32, sizeof(size32));
}
- cache_rec.fields = cached_fields;
- cache_rec.size = buffer_get_used_size(buffer);
- buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec));
-
- data = buffer_get_data(buffer, &size);
- *size_r = size;
- return data;
+ buffer_append(ctx->buffer, data, data_size);
+ if ((data_size & 3) != 0)
+ buffer_append(ctx->buffer, null4, 4 - (data_size & 3));
+ return 1;
}
static int
mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
{
+ struct mail_cache_copy_context ctx;
struct mail_cache_view *cache_view;
struct mail_index_transaction *t;
const struct mail_index_header *idx_hdr;
- const struct mail_cache_record *cache_rec;
struct mail_cache_header hdr;
+ struct mail_cache_record cache_rec;
+ enum mail_cache_field field;
struct ostream *output;
- enum mail_cache_field keep_fields, temp_fields;
- enum mail_cache_field cached_fields, new_fields;
const char *str;
- uint32_t size, size32, message_count, seq, first_new_seq;
+ uint32_t size32, message_count, seq, first_new_seq, old_offset;
uoff_t offset;
- int i, header_idx, ret;
+ int i, ret, header_idx;
/* get sequence of first message which doesn't need it's temp fields
removed. */
@@ -133,15 +111,23 @@
memcpy(hdr.field_usage_last_used,
cache->hdr->field_usage_last_used,
sizeof(hdr.field_usage_last_used));
+ } else {
+ memcpy(hdr.field_usage_decision_type,
+ cache->default_field_usage_decision_type,
+ sizeof(hdr.field_usage_decision_type));
+ }
- keep_fields = temp_fields = 0;
- for (i = 0; i < 32; i++) {
- if (cache->hdr->field_usage_decision_type[i] &
- MAIL_CACHE_DECISION_YES)
- keep_fields |= 1 << i;
- else if (cache->hdr->field_usage_decision_type[i] &
- MAIL_CACHE_DECISION_TEMP)
- temp_fields |= 1 << i;
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.buffer = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
+ ctx.header = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
+
+ for (i = 0; i < 32; i++) {
+ if (hdr.field_usage_decision_type[i] & MAIL_CACHE_DECISION_YES)
+ ctx.keep_fields[i] = TRUE;
+ else if (hdr.field_usage_decision_type[i] &
+ MAIL_CACHE_DECISION_TEMP) {
+ ctx.temp_fields[i] = TRUE;
+ ctx.keep_fields[i] = TRUE;
}
}
@@ -173,45 +159,42 @@
ret = 0;
for (seq = 1; seq <= message_count; seq++) {
- cache_rec = mail_cache_lookup(cache_view, seq);
- if (cache_rec == NULL)
- continue;
+ ctx.new_msg = seq >= first_new_seq;
+ buffer_set_used_size(ctx.buffer, 0);
+ buffer_set_used_size(ctx.header, 0);
+ memset(ctx.field_seen, 0, sizeof(ctx.field_seen));
- cached_fields = mail_cache_get_fields(cache_view, seq);
- new_fields = cached_fields & keep_fields;
- if ((cached_fields & temp_fields) != 0 &&
- seq >= first_new_seq) {
- /* new message, keep temp fields */
- new_fields |= cached_fields & temp_fields;
- }
+ memset(&cache_rec, 0, sizeof(cache_rec));
+ buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
- if (keep_fields == cached_fields &&
- cache_rec->prev_offset == 0) {
- /* just one unmodified block, save it */
- mail_index_update_cache(t, seq, hdr.file_seq,
- output->offset, NULL);
- o_stream_send(output, cache_rec, cache_rec->size);
+ mail_cache_foreach(cache_view, seq,
+ mail_cache_compress_callback, &ctx);
- if ((cache_rec->size & 3) != 0) {
- o_stream_send(output, null4,
- 4 - (cache_rec->size & 3));
+ size32 = buffer_get_used_size(ctx.header);
+ if (size32 > 0 && ctx.field_seen[header_idx]) {
+ field = MAIL_CACHE_HEADERS1;
+ buffer_append(ctx.buffer, &field, sizeof(field));
+ buffer_append(ctx.buffer, &size32, sizeof(size32));
+ buffer_append(ctx.buffer,
+ buffer_get_data(ctx.header, NULL),
+ size32);
+ if ((size32 & 3) != 0) {
+ buffer_append(ctx.buffer, null4,
+ 4 - (size32 & 3));
}
- } else {
- /* a) dropping fields
- b) multiple blocks, sort them into buffer */
- mail_index_update_cache(t, seq, hdr.file_seq,
- output->offset, NULL);
-
- t_push();
- cache_rec = mail_cache_compress_record(cache_view, seq,
- keep_fields,
- header_idx,
- &size);
- o_stream_send(output, cache_rec, size);
- t_pop();
}
+
+ if (buffer_get_used_size(ctx.buffer) == sizeof(cache_rec))
+ continue;
+
+ mail_index_update_cache(t, seq, hdr.file_seq,
+ output->offset, &old_offset);
+ o_stream_send(output, buffer_get_data(ctx.buffer, NULL),
+ buffer_get_used_size(ctx.buffer));
}
hdr.used_file_size = output->offset;
+ buffer_free(ctx.buffer);
+ buffer_free(ctx.header);
o_stream_seek(output, 0);
o_stream_send(output, &hdr, sizeof(hdr));
@@ -263,7 +246,7 @@
return -1;
}
- // FIXME: check that cache file was just recreated
+ // FIXME: check that cache file wasn't just recreated
ret = 0;
if (mail_cache_copy(cache, view, fd) < 0) {
Index: mail-cache-decisions.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-decisions.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mail-cache-decisions.c 4 Jul 2004 11:50:49 -0000 1.3
+++ mail-cache-decisions.c 8 Jul 2004 20:26:15 -0000 1.4
@@ -73,7 +73,8 @@
#include <stddef.h>
static void
-mail_cache_set_decision_type(struct mail_cache *cache, uint32_t idx,
+mail_cache_set_decision_type(struct mail_cache *cache,
+ enum mail_cache_field field,
enum mail_cache_decision_type type)
{
uint8_t value = type;
@@ -84,20 +85,18 @@
will be corrected sometimes later, not too bad.. */
if (pwrite_full(cache->fd, &value, 1,
offsetof(struct mail_cache_header,
- field_usage_decision_type) + idx) < 0) {
+ field_usage_decision_type) + field) < 0) {
mail_cache_set_syscall_error(cache, "pwrite_full()");
}
}
-void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq,
- enum mail_cache_field field)
+void mail_cache_decision_lookup(struct mail_cache_view *view, uint32_t seq,
+ enum mail_cache_field field)
{
const struct mail_index_header *hdr;
- unsigned int idx;
uint32_t uid;
- idx = mail_cache_field_index(field);
- if (view->cache->hdr->field_usage_decision_type[idx] !=
+ if (view->cache->hdr->field_usage_decision_type[field] !=
MAIL_CACHE_DECISION_TEMP) {
/* a) forced decision
b) not cached, mail_cache_mark_missing() will handle this
@@ -110,7 +109,7 @@
mail_index_get_header(view->view, &hdr) < 0)
return;
- if (uid < view->cache->field_usage_uid_highwater[idx] ||
+ if (uid < view->cache->field_usage_uid_highwater[field] ||
uid < hdr->day_first_uid[7]) {
/* a) nonordered access within this session. if client doesn't
request messages in growing order, we assume it doesn't
@@ -119,24 +118,22 @@
client with no local cache. if it was just a new client
generating the local cache for the first time, we'll
drop back to TEMP within few months. */
- mail_cache_set_decision_type(view->cache, idx,
+ mail_cache_set_decision_type(view->cache, field,
MAIL_CACHE_DECISION_YES);
} else {
- view->cache->field_usage_uid_highwater[idx] = uid;
+ view->cache->field_usage_uid_highwater[field] = uid;
}
}
-void mail_cache_mark_missing(struct mail_cache_view *view, uint32_t seq,
+void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
enum mail_cache_field field)
{
- unsigned int idx;
uint32_t uid;
if (MAIL_CACHE_IS_UNUSABLE(view->cache))
return;
- idx = mail_cache_field_index(field);
- if (view->cache->hdr->field_usage_decision_type[idx] !=
+ if (view->cache->hdr->field_usage_decision_type[field] !=
MAIL_CACHE_DECISION_NO) {
/* a) forced decision
b) we're already caching it, so it just wasn't in cache */
@@ -144,9 +141,9 @@
}
/* field used the first time */
- mail_cache_set_decision_type(view->cache, idx,
+ mail_cache_set_decision_type(view->cache, field,
MAIL_CACHE_DECISION_TEMP);
if (mail_index_lookup_uid(view->view, seq, &uid) == 0)
- view->cache->field_usage_uid_highwater[idx] = uid;
+ view->cache->field_usage_uid_highwater[field] = uid;
}
Index: mail-cache-lookup.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-lookup.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- mail-cache-lookup.c 4 Jul 2004 21:07:43 -0000 1.11
+++ mail-cache-lookup.c 8 Jul 2004 20:26:15 -0000 1.12
@@ -171,112 +171,163 @@
return 0;
}
-struct mail_cache_record *
-mail_cache_lookup(struct mail_cache_view *view, uint32_t seq)
+int mail_cache_foreach(struct mail_cache_view *view, uint32_t seq,
+ int (*callback)(struct mail_cache_view *view,
+ enum mail_cache_field field,
+ const void *data, size_t data_size,
+ void *context), void *context)
{
- uint32_t offset;
-
- // FIXME: check transactions too
+ const struct mail_cache_record *cache_rec;
+ size_t pos, next_pos, max_size, data_size;
+ uint32_t offset, field;
+ int ret;
if (MAIL_CACHE_IS_UNUSABLE(view->cache))
- return NULL;
+ return 0;
- if (mail_cache_lookup_offset(view, seq, &offset) <= 0)
- return NULL;
+ if ((ret = mail_cache_lookup_offset(view, seq, &offset)) <= 0)
+ return ret;
- return mail_cache_get_record(view->cache, offset);
-}
+ cache_rec = mail_cache_get_record(view->cache, offset);
+ while (cache_rec != NULL) {
+ max_size = cache_rec->size;
+ if (max_size < sizeof(*cache_rec) + sizeof(uint32_t)*2) {
+ mail_cache_set_corrupted(view->cache,
+ "record has invalid size");
+ return -1;
+ }
+ max_size -= sizeof(uint32_t);
-enum mail_cache_field
-mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq)
-{
- struct mail_cache_record *cache_rec;
- enum mail_cache_field fields = 0;
+ for (pos = sizeof(*cache_rec); pos < max_size; ) {
+ field = *((const uint32_t *)
+ CONST_PTR_OFFSET(cache_rec, pos));
+ pos += sizeof(uint32_t);
- cache_rec = mail_cache_lookup(view, seq);
- while (cache_rec != NULL) {
- fields |= cache_rec->fields;
+ data_size = mail_cache_field_sizes[field];
+ if (data_size == (unsigned int)-1) {
+ data_size = *((const uint32_t *)
+ CONST_PTR_OFFSET(cache_rec, pos));
+ pos += sizeof(uint32_t);
+ }
+
+ next_pos = pos + ((data_size + 3) & ~3);
+ if (next_pos > cache_rec->size) {
+ mail_cache_set_corrupted(view->cache,
+ "Record continues outside it's "
+ "allocated size");
+ return -1;
+ }
+
+ ret = callback(view, field,
+ CONST_PTR_OFFSET(cache_rec, pos),
+ data_size, context);
+ if (ret <= 0)
+ return ret;
+
+ pos = next_pos;
+ }
cache_rec = mail_cache_get_record(view->cache,
cache_rec->prev_offset);
}
- return fields;
+ if (view->transaction != NULL) {
+ // FIXME: update
+ }
+ return 1;
}
-static int cache_get_field(struct mail_cache *cache,
- const struct mail_cache_record *cache_rec,
- enum mail_cache_field field, buffer_t *dest_buf)
+static int mail_cache_seq_callback(struct mail_cache_view *view,
+ enum mail_cache_field field,
+ const void *data __attr_unused__,
+ size_t data_size __attr_unused__,
+ void *context __attr_unused__)
{
- unsigned int mask;
- uint32_t data_size;
- size_t offset, prev_offset;
- int i;
+ view->cached_exists[field] = TRUE;
+ return 1;
+}
- offset = sizeof(*cache_rec);
+static int mail_cache_seq(struct mail_cache_view *view, uint32_t seq)
+{
+ int ret;
- for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
- if ((cache_rec->fields & mask) == 0)
- continue;
+ view->cached_exists_seq = seq;
+ memset(view->cached_exists, 0, sizeof(view->cached_exists));
- /* all records are at least 32bit. we have to check this
- before getting data_size. */
- if (offset + sizeof(uint32_t) > cache_rec->size) {
- mail_cache_set_corrupted(cache,
- "Record continues outside it's allocated size");
- return FALSE;
- }
+ ret = mail_cache_foreach(view, seq, mail_cache_seq_callback, NULL);
+ return ret < 0 ? -1 : 0;
+}
- if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
- data_size = mail_cache_field_sizes[i];
- else {
- memcpy(&data_size, CONST_PTR_OFFSET(cache_rec, offset),
- sizeof(data_size));
- offset += sizeof(data_size);
- }
+int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,
+ enum mail_cache_field field)
+{
+ i_assert(field < MAIL_CACHE_FIELD_COUNT);
- prev_offset = offset + ((data_size + 3) & ~3);
- if (prev_offset > cache_rec->size) {
- mail_cache_set_corrupted(cache,
- "Record continues outside it's allocated size");
- return FALSE;
- }
+ if (MAIL_CACHE_IS_UNUSABLE(view->cache))
+ return 0;
- if (field == mask) {
- if (data_size == 0) {
- mail_cache_set_corrupted(cache,
- "Field size is 0");
- return FALSE;
- }
- buffer_append(dest_buf,
- CONST_PTR_OFFSET(cache_rec, offset),
- data_size);
- return TRUE;
- }
- offset = prev_offset;
+ if (view->cached_exists_seq != seq) {
+ if (mail_cache_seq(view, seq) < 0)
+ return -1;
}
+ return view->cached_exists[field];
+}
- i_unreached();
- return FALSE;
+enum mail_cache_decision_type
+mail_cache_field_get_decision(struct mail_cache *cache,
+ enum mail_cache_field field)
+{
+ i_assert(field < MAIL_CACHE_FIELD_COUNT);
+
+ if (MAIL_CACHE_IS_UNUSABLE(cache))
+ return cache->default_field_usage_decision_type[field];
+
+ return cache->hdr->field_usage_decision_type[field];
+}
+
+struct mail_cache_lookup_context {
+ buffer_t *dest_buf;
+ enum mail_cache_field field;
+};
+
+static int
+mail_cache_lookup_callback(struct mail_cache_view *view __attr_unused__,
+ enum mail_cache_field field,
+ const void *data, size_t data_size, void *context)
+{
+ struct mail_cache_lookup_context *ctx = context;
+
+ if (ctx->field != field)
+ return 1;
+
+ buffer_append(ctx->dest_buf, data, data_size);
+ return 0;
}
int mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,
uint32_t seq, enum mail_cache_field field)
{
- struct mail_cache_record *cache_rec;
+ struct mail_cache_lookup_context ctx;
- mail_cache_handle_decisions(view, seq, field);
+ i_assert(field < MAIL_CACHE_FIELD_COUNT);
- cache_rec = mail_cache_lookup(view, seq);
- while (cache_rec != NULL) {
- if ((cache_rec->fields & field) != 0) {
- return cache_get_field(view->cache, cache_rec, field,
- dest_buf);
- }
- cache_rec = mail_cache_get_record(view->cache,
- cache_rec->prev_offset);
+ if (MAIL_CACHE_IS_UNUSABLE(view->cache))
+ return 0;
+
+ mail_cache_decision_lookup(view, seq, field);
+
+ if (view->cached_exists_seq != seq) {
+ if (mail_cache_seq(view, seq) < 0)
+ return -1;
}
- return FALSE;
+ if (!view->cached_exists[field])
+ return 0;
+
+ /* should exist. find it. */
+ ctx.field = field;
+ ctx.dest_buf = dest_buf;
+ return mail_cache_foreach(view, seq, mail_cache_lookup_callback,
+ &ctx) == 0;
}
int mail_cache_lookup_string_field(struct mail_cache_view *view, string_t *dest,
@@ -284,21 +335,24 @@
{
size_t old_size, new_size;
- i_assert((field & MAIL_CACHE_STRING_MASK) != 0);
+ i_assert(field < MAIL_CACHE_FIELD_COUNT);
+
+ if (MAIL_CACHE_IS_UNUSABLE(view->cache))
+ return 0;
old_size = str_len(dest);
if (!mail_cache_lookup_field(view, dest, seq, field))
- return FALSE;
+ return 0;
new_size = str_len(dest);
if (old_size == new_size ||
str_data(dest)[new_size-1] != '\0') {
mail_cache_set_corrupted(view->cache,
"String field %x doesn't end with NUL", field);
- return FALSE;
+ return -1;
}
str_truncate(dest, new_size-1);
- return TRUE;
+ return 1;
}
enum mail_cache_record_flag
Index: mail-cache-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-private.h,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- mail-cache-private.h 4 Jul 2004 20:00:47 -0000 1.7
+++ mail-cache-private.h 8 Jul 2004 20:26:15 -0000 1.8
@@ -35,18 +35,6 @@
#define MAIL_CACHE_IS_UNUSABLE(cache) \
((cache)->hdr == NULL)
-enum mail_cache_decision_type {
- /* Not needed currently */
- MAIL_CACHE_DECISION_NO = 0x00,
- /* Needed only for new mails. Drop when compressing. */
- MAIL_CACHE_DECISION_TEMP = 0x01,
- /* Needed. */
- MAIL_CACHE_DECISION_YES = 0x02,
-
- /* This decision has been forced manually, don't change it. */
- MAIL_CACHE_DECISION_FORCED = 0x80
-};
-
struct mail_cache_header {
/* version is increased only when you can't have backwards
compatibility. */
@@ -69,9 +57,9 @@
};
struct mail_cache_record {
- uint32_t fields; /* enum mail_cache_field */
uint32_t prev_offset;
uint32_t size; /* full record size, including this header */
+ /* array of { uint32_t field; [ uint32_t size; ] { .. } } */
};
struct mail_cache_hole_header {
@@ -101,9 +89,7 @@
uint32_t split_offsets[MAIL_CACHE_HEADERS_COUNT];
const char *const *split_headers[MAIL_CACHE_HEADERS_COUNT];
- enum mail_cache_field default_cache_fields;
- enum mail_cache_field never_cache_fields;
-
+ uint8_t default_field_usage_decision_type[32];
uint32_t field_usage_uid_highwater[32];
unsigned int locked:1;
@@ -115,7 +101,9 @@
struct mail_cache *cache;
struct mail_index_view *view;
- unsigned int broken:1;
+ struct mail_cache_transaction_ctx *transaction;
+ char cached_exists[32];
+ uint32_t cached_exists_seq;
};
extern unsigned int mail_cache_field_sizes[32];
@@ -123,7 +111,6 @@
uint32_t mail_cache_uint32_to_offset(uint32_t offset);
uint32_t mail_cache_offset_to_uint32(uint32_t offset);
-unsigned int mail_cache_field_index(enum mail_cache_field field);
/* Explicitly lock the cache file. Returns -1 if error, 1 if ok, 0 if we
couldn't lock */
@@ -138,8 +125,11 @@
struct mail_cache_record *
mail_cache_get_record(struct mail_cache *cache, uint32_t offset);
-struct mail_cache_record *
-mail_cache_lookup(struct mail_cache_view *view, uint32_t seq);
+int mail_cache_foreach(struct mail_cache_view *view, uint32_t seq,
+ int (*callback)(struct mail_cache_view *view,
+ enum mail_cache_field field,
+ const void *data, size_t data_size,
+ void *context), void *context);
int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx);
void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx);
@@ -154,8 +144,10 @@
/* Mark record in given offset to be deleted. */
int mail_cache_delete(struct mail_cache *cache, uint32_t offset);
-void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq,
- enum mail_cache_field field);
+void mail_cache_decision_lookup(struct mail_cache_view *view, uint32_t seq,
+ enum mail_cache_field field);
+void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
+ enum mail_cache_field field);
void mail_cache_set_syscall_error(struct mail_cache *cache,
const char *function);
Index: mail-cache-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-transaction.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- mail-cache-transaction.c 4 Jul 2004 21:56:12 -0000 1.11
+++ mail-cache-transaction.c 8 Jul 2004 20:26:15 -0000 1.12
@@ -28,7 +28,7 @@
uint32_t last_grow_size;
uint32_t first_seq, last_seq;
- enum mail_cache_field fields;
+ enum mail_cache_field fields[32];
unsigned int changes:1;
};
@@ -55,12 +55,17 @@
ctx->reservations =
buffer_create_dynamic(system_pool, 256, (size_t)-1);
+ i_assert(view->transaction == NULL);
+ view->transaction = ctx;
+
t->cache_trans_ctx = ctx;
return ctx;
}
static void mail_cache_transaction_free(struct mail_cache_transaction_ctx *ctx)
{
+ ctx->view->transaction = NULL;
+
buffer_free(ctx->cache_data);
buffer_free(ctx->cache_data_seq);
buffer_free(ctx->reservations);
@@ -317,7 +322,7 @@
struct mail_cache *cache = ctx->cache;
const struct mail_cache_record *rec, *tmp_rec;
const uint32_t *seq;
- uint32_t write_offset, old_offset, rec_offset;
+ uint32_t write_offset, old_offset, rec_pos;
size_t size, max_size, seq_idx, seq_limit, seq_count;
int commit;
@@ -332,9 +337,10 @@
seq = buffer_get_data(ctx->cache_data_seq, &seq_count);
seq_count /= sizeof(*seq);
+ seq_limit = 0;
- for (seq_idx = 0, rec_offset = 0; rec_offset < ctx->prev_pos;) {
- max_size = ctx->prev_pos - rec_offset;
+ for (seq_idx = 0, rec_pos = 0; rec_pos < ctx->prev_pos;) {
+ max_size = ctx->prev_pos - rec_pos;
write_offset = mail_cache_transaction_get_space(ctx, rec->size,
max_size,
&max_size,
@@ -344,7 +350,7 @@
return ctx->prev_pos == 0 ? 0 : -1;
}
- if (max_size < ctx->prev_pos) {
+ if (rec_pos + max_size < ctx->prev_pos) {
/* see how much we can really write there */
tmp_rec = rec;
for (size = 0; size + tmp_rec->size <= max_size; ) {
@@ -383,7 +389,7 @@
}
write_offset += rec->size;
- rec_offset += rec->size;
+ rec_pos += rec->size;
rec = CONST_PTR_OFFSET(rec, rec->size);
}
}
@@ -391,7 +397,10 @@
/* drop the written data from buffer */
buffer_copy(ctx->cache_data, 0,
ctx->cache_data, ctx->prev_pos, (size_t)-1);
- buffer_set_used_size(ctx->cache_data, size - ctx->prev_pos);
+ buffer_set_used_size(ctx->cache_data,
+ buffer_get_used_size(ctx->cache_data) -
+ ctx->prev_pos);
+ ctx->prev_pos = 0;
buffer_set_used_size(ctx->cache_data_seq, 0);
return 0;
@@ -409,6 +418,7 @@
data = buffer_get_modifyable_data(ctx->cache_data, &size);
rec = PTR_OFFSET(data, ctx->prev_pos);
rec->size = size - ctx->prev_pos;
+ i_assert(rec->size != 0);
buffer_append(ctx->cache_data_seq, &ctx->prev_seq,
sizeof(ctx->prev_seq));
@@ -588,62 +598,23 @@
return offset > 0;
}
-static size_t
-mail_cache_transaction_get_insert_pos(struct mail_cache_transaction_ctx *ctx,
- enum mail_cache_field field)
-{
- const struct mail_cache_record *cache_rec;
- const void *data;
- unsigned int mask;
- uint32_t data_size;
- size_t pos;
- int i;
-
- data = buffer_get_data(ctx->cache_data, NULL);
- cache_rec = CONST_PTR_OFFSET(data, ctx->prev_pos);
-
- pos = ctx->prev_pos + sizeof(*cache_rec);
- for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
- if ((field & mask) != 0)
- return pos;
-
- if ((cache_rec->fields & mask) != 0) {
- if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
- data_size = mail_cache_field_sizes[i];
- else {
- memcpy(&data_size, CONST_PTR_OFFSET(data, pos),
- sizeof(data_size));
- pos += sizeof(data_size);
- }
- pos += (data_size + 3) & ~3;
- }
- }
-
- i_unreached();
- return pos;
-}
-
void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
enum mail_cache_field field,
const void *data, size_t data_size)
{
- struct mail_cache_record *cache_rec;
- unsigned char *buf;
- size_t full_size, pos;
- uint32_t data_size32;
- unsigned int field_idx;
+ uint32_t fixed_size, data_size32;
+ size_t full_size;
+ i_assert(field < MAIL_CACHE_FIELD_COUNT);
i_assert(data_size > 0);
i_assert(data_size < (uint32_t)-1);
- data_size32 = (uint32_t)data_size;
+ mail_cache_decision_add(ctx->view, seq, field);
- if ((field & MAIL_CACHE_FIXED_MASK) != 0) {
- field_idx = mail_cache_field_index(field);
- i_assert(mail_cache_field_sizes[field_idx] == data_size);
- } else if ((field & MAIL_CACHE_STRING_MASK) != 0) {
- i_assert(((char *) data)[data_size-1] == '\0');
- }
+ fixed_size = mail_cache_field_sizes[field];
+ i_assert(fixed_size == (unsigned int)-1 || fixed_size == data_size);
+
+ data_size32 = (uint32_t)data_size;
if (ctx->prev_seq != seq) {
mail_cache_transaction_switch_seq(ctx);
@@ -655,11 +626,12 @@
ctx->first_seq = seq;
if (seq > ctx->last_seq)
ctx->last_seq = seq;
- ctx->fields |= field;
+ ctx->view->cached_exists[field] = TRUE;
+ ctx->fields[field] = TRUE;
}
full_size = (data_size + 3) & ~3;
- if ((field & MAIL_CACHE_FIXED_MASK) == 0)
+ if (fixed_size == (unsigned int)-1)
full_size += sizeof(data_size32);
if (buffer_get_used_size(ctx->cache_data) + full_size >
@@ -669,24 +641,15 @@
return;
}
- /* fields must be ordered. find where to insert it. */
- pos = mail_cache_transaction_get_insert_pos(ctx, field);
- buffer_copy(ctx->cache_data, pos + full_size,
- ctx->cache_data, pos, (size_t)-1);
-
- cache_rec = buffer_get_space_unsafe(ctx->cache_data, ctx->prev_pos,
- sizeof(*cache_rec));
- cache_rec->fields |= field;
-
- /* @UNSAFE */
- buf = buffer_get_space_unsafe(ctx->cache_data, pos, full_size);
- if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
- memcpy(buf, &data_size32, sizeof(data_size32));
- buf += sizeof(data_size32);
+ buffer_append(ctx->cache_data, &field, sizeof(field));
+ if (fixed_size == (unsigned int)-1) {
+ buffer_append(ctx->cache_data, &data_size32,
+ sizeof(data_size32));
}
- memcpy(buf, data, data_size); buf += data_size;
+
+ buffer_append(ctx->cache_data, data, data_size);
if ((data_size & 3) != 0)
- memset(buf, 0, 4 - (data_size & 3));
+ buffer_append(ctx->cache_data, null4, 4 - (data_size & 3));
}
int mail_cache_update_record_flags(struct mail_cache_view *view, uint32_t seq,
Index: mail-cache.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.38
retrieving revision 1.39
diff -u -d -r1.38 -r1.39
--- mail-cache.c 4 Jul 2004 22:09:13 -0000 1.38
+++ mail-cache.c 8 Jul 2004 20:26:15 -0000 1.39
@@ -15,13 +15,13 @@
sizeof(time_t),
sizeof(uoff_t),
- 0, 0, 0, 0, 0, 0, 0, 0,
-
/* variable sized */
(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
+ (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
+ (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1
};
@@ -60,17 +60,6 @@
(((uint32_t)buf[0] & 0x7f) << 23);
}
-unsigned int mail_cache_field_index(enum mail_cache_field field)
-{
- unsigned int i, num;
-
- for (i = 0, num = 1; i < 32; i++, num <<= 1) {
- if (field == num)
- return i;
- }
- i_unreached();
-}
-
void mail_cache_set_syscall_error(struct mail_cache *cache,
const char *function)
{
@@ -284,11 +273,10 @@
}
void mail_cache_set_defaults(struct mail_cache *cache,
- enum mail_cache_field default_cache_fields,
- enum mail_cache_field never_cache_fields)
+ const enum mail_cache_decision_type dec[32])
{
- cache->default_cache_fields = default_cache_fields;
- cache->never_cache_fields = never_cache_fields;
+ memcpy(cache->default_field_usage_decision_type, dec,
+ sizeof(cache->default_field_usage_decision_type));
}
int mail_cache_lock(struct mail_cache *cache)
Index: mail-cache.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.h,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- mail-cache.h 4 Jul 2004 21:07:43 -0000 1.17
+++ mail-cache.h 8 Jul 2004 20:26:15 -0000 1.18
@@ -11,6 +11,18 @@
struct mail_cache_view;
struct mail_cache_transaction_ctx;
+enum mail_cache_decision_type {
+ /* Not needed currently */
+ MAIL_CACHE_DECISION_NO = 0x00,
+ /* Needed only for new mails. Drop when compressing. */
+ MAIL_CACHE_DECISION_TEMP = 0x01,
+ /* Needed. */
+ MAIL_CACHE_DECISION_YES = 0x02,
+
+ /* This decision has been forced manually, don't change it. */
+ MAIL_CACHE_DECISION_FORCED = 0x80
+};
+
enum mail_cache_record_flag {
/* If binary flags are set, it's not checked whether mail is
missing CRs. So this flag may be set as an optimization for
@@ -28,40 +40,23 @@
/* when modifying, remember to update mail_cache_field_sizes[] too */
enum mail_cache_field {
/* fixed size fields */
- MAIL_CACHE_INDEX_FLAGS = 0x00000001,
- MAIL_CACHE_SENT_DATE = 0x00000002,
- MAIL_CACHE_RECEIVED_DATE = 0x00000004,
- MAIL_CACHE_VIRTUAL_FULL_SIZE = 0x00000008,
+ MAIL_CACHE_INDEX_FLAGS = 0,
+ MAIL_CACHE_SENT_DATE,
+ MAIL_CACHE_RECEIVED_DATE,
+ MAIL_CACHE_VIRTUAL_FULL_SIZE,
/* variable sized field */
- MAIL_CACHE_HEADERS1 = 0x40000000,
- MAIL_CACHE_HEADERS2 = 0x20000000,
- MAIL_CACHE_HEADERS3 = 0x10000000,
- MAIL_CACHE_HEADERS4 = 0x08000000,
- MAIL_CACHE_LOCATION = 0x04000000,
- MAIL_CACHE_BODY = 0x02000000,
- MAIL_CACHE_BODYSTRUCTURE = 0x01000000,
- MAIL_CACHE_ENVELOPE = 0x00800000,
- MAIL_CACHE_MESSAGEPART = 0x00400000,
- MAIL_CACHE_UID_STRING = 0x00200000,
+ MAIL_CACHE_HEADERS1,
+ MAIL_CACHE_HEADERS2,
+ MAIL_CACHE_HEADERS3,
+ MAIL_CACHE_HEADERS4,
+ MAIL_CACHE_BODY,
+ MAIL_CACHE_BODYSTRUCTURE,
+ MAIL_CACHE_ENVELOPE,
+ MAIL_CACHE_MESSAGEPART,
+ MAIL_CACHE_UID_STRING,
- MAIL_CACHE_FIXED_MASK = MAIL_CACHE_INDEX_FLAGS |
- MAIL_CACHE_SENT_DATE |
- MAIL_CACHE_RECEIVED_DATE |
- MAIL_CACHE_VIRTUAL_FULL_SIZE,
- MAIL_CACHE_HEADERS_MASK = MAIL_CACHE_HEADERS1 |
- MAIL_CACHE_HEADERS2 |
- MAIL_CACHE_HEADERS3 |
- MAIL_CACHE_HEADERS4,
- MAIL_CACHE_STRING_MASK = MAIL_CACHE_HEADERS_MASK |
- MAIL_CACHE_LOCATION |
- MAIL_CACHE_BODY |
- MAIL_CACHE_BODYSTRUCTURE |
- MAIL_CACHE_ENVELOPE |
- MAIL_CACHE_UID_STRING,
- MAIL_CACHE_BODYSTRUCTURE_MASK = MAIL_CACHE_BODY |
- MAIL_CACHE_BODYSTRUCTURE |
- MAIL_CACHE_MESSAGEPART
+ MAIL_CACHE_FIELD_COUNT
};
struct mail_sent_date {
@@ -75,8 +70,7 @@
void mail_cache_free(struct mail_cache *cache);
void mail_cache_set_defaults(struct mail_cache *cache,
- enum mail_cache_field default_cache_fields,
- enum mail_cache_field never_cache_fields);
+ const enum mail_cache_decision_type dec[32]);
/* Returns TRUE if cache should be compressed. */
int mail_cache_need_compress(struct mail_cache *cache);
@@ -106,9 +100,13 @@
enum mail_cache_field field,
const void *data, size_t data_size);
-/* Return all fields that are currently cached for record. */
-enum mail_cache_field
-mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq);
+/* Retursn TRUE if field exists. */
+int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,
+ enum mail_cache_field field);
+/* Returns current caching decision for given field. */
+enum mail_cache_decision_type
+mail_cache_field_get_decision(struct mail_cache *cache,
+ enum mail_cache_field field);
/* Set data_r and size_r to point to wanted field in cache file.
Returns TRUE if field was found. If field contains multiple fields,
@@ -120,10 +118,6 @@
int mail_cache_lookup_string_field(struct mail_cache_view *view, string_t *dest,
uint32_t seq, enum mail_cache_field field);
-/* Mark given field as missing, ie. it should be cached when possible. */
-void mail_cache_mark_missing(struct mail_cache_view *view, uint32_t uid,
- enum mail_cache_field field);
-
/* Return record flags. */
enum mail_cache_record_flag
mail_cache_get_record_flags(struct mail_cache_view *view, uint32_t seq);
More information about the dovecot-cvs
mailing list