[dovecot-cvs] dovecot/src/lib-index mail-index-private.h, 1.29,
1.30 mail-index-sync-update.c, 1.48,
1.49 mail-index-transaction-private.h, 1.13,
1.14 mail-index-transaction.c, 1.29,
1.30 mail-index-view-sync.c, 1.20, 1.21 mail-index.c, 1.149,
1.150 mail-index.h, 1.129, 1.130 mail-transaction-log-view.c,
1.29, 1.30 mail-transaction-log.c, 1.60,
1.61 mail-transaction-log.h, 1.18, 1.19
cras at dovecot.org
cras at dovecot.org
Sun Oct 3 19:33:02 EEST 2004
Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv20915/lib-index
Modified Files:
mail-index-private.h mail-index-sync-update.c
mail-index-transaction-private.h mail-index-transaction.c
mail-index-view-sync.c mail-index.c mail-index.h
mail-transaction-log-view.c mail-transaction-log.c
mail-transaction-log.h
Log Message:
Index extensions can now specify record alignment, and it's possible to
resize header and records. Fixes mbox crashes with some 64bit systems.
Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- mail-index-private.h 26 Sep 2004 14:15:54 -0000 1.29
+++ mail-index-private.h 3 Oct 2004 16:32:59 -0000 1.30
@@ -21,18 +21,25 @@
((struct mail_index_record *) \
PTR_OFFSET((map)->records, (idx) * (map)->hdr->record_size))
+#define MAIL_INDEX_HEADER_SIZE_ALIGN(size) \
+ (((size) + 7) & ~7)
+
struct mail_index_ext {
const char *name;
uint32_t hdr_offset;
uint32_t hdr_size;
- uint32_t record_offset;
- uint32_t record_size;
+ uint16_t record_offset;
+ uint16_t record_size;
+ uint16_t record_align;
};
struct mail_index_ext_header {
uint32_t hdr_size;
- uint32_t record_offset;
- uint32_t record_size;
+ uint16_t record_offset;
+ uint16_t record_size;
+ uint16_t record_align;
+ uint16_t name_size;
+ /* unsigned char name[] */
};
struct mail_index_map {
@@ -128,10 +135,15 @@
void mail_index_unmap(struct mail_index *index, struct mail_index_map *map);
struct mail_index_map *
mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size);
-uint32_t mail_index_map_register_ext(struct mail_index *index,
- struct mail_index_map *map,
- const char *name, uint32_t hdr_offset,
- uint32_t hdr_size, uint32_t record_size);
+
+uint32_t mail_index_map_lookup_ext(struct mail_index_map *map,
+ const char *name);
+uint32_t
+mail_index_map_register_ext(struct mail_index *index,
+ struct mail_index_map *map, const char *name,
+ uint32_t hdr_offset, uint32_t hdr_size,
+ uint32_t record_offset, uint32_t record_size,
+ uint32_t record_align);
int mail_index_map_get_ext_idx(struct mail_index_map *map,
uint32_t ext_id, uint32_t *idx_r);
Index: mail-index-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.48
retrieving revision 1.49
diff -u -d -r1.48 -r1.49
--- mail-index-sync-update.c 26 Sep 2004 14:15:54 -0000 1.48
+++ mail-index-sync-update.c 3 Oct 2004 16:32:59 -0000 1.49
@@ -11,7 +11,19 @@
#include "mail-transaction-util.h"
#include "mail-cache-private.h"
-#include <time.h>
+#include <stdlib.h>
+
+static void mail_index_sync_replace_map(struct mail_index_view *view,
+ struct mail_index_map *map)
+{
+ mail_index_unmap(view->index, view->map);
+ view->map = map;
+ view->map->refcount++;
+ mail_index_unmap(view->index, view->index->map);
+ view->index->map = map;
+ view->index->hdr = map->hdr;
+ map->write_to_disk = TRUE;
+}
static void
mail_index_header_update_counts(struct mail_index_header *hdr,
@@ -84,6 +96,16 @@
struct mail_index_record *rec;
uint32_t count, seq, seq1, seq2;
+ if (!view->map->write_to_disk) {
+ /* expunges have to be atomic. so we'll have to copy
+ the mapping, do the changes there and then finally
+ replace the whole index file. to avoid extra disk
+ I/O we copy the index into memory rather than to
+ temporary file */
+ map = mail_index_map_to_memory(map,
+ map->hdr->record_size);
+ mail_index_sync_replace_map(view, map);
+ }
i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map));
if (mail_index_lookup_uid_range(view, e->uid1, e->uid2,
@@ -289,44 +311,254 @@
return 1;
}
+static int mail_index_ext_align_cmp(const void *p1, const void *p2)
+{
+ const struct mail_index_ext *const *e1 = p1, *const *e2 = p2;
+
+ return (int)(*e2)->record_align - (int)(*e1)->record_align;
+}
+
+static struct mail_index_map *
+sync_ext_reorder(struct mail_index_map *map, uint32_t ext_id, uint16_t old_size)
+{
+ struct mail_index_map *new_map;
+ struct mail_index_ext *ext, **sorted;
+ struct mail_index_ext_header *ext_hdr;
+ uint16_t *old_offsets, min_align;
+ uint32_t offset, old_records_count, rec_idx;
+ const void *src;
+ void *hdr;
+ size_t i, size;
+
+ t_push();
+ ext = buffer_get_modifyable_data(map->extensions, &size);
+ size /= sizeof(*ext);
+
+ /* @UNSAFE */
+ old_offsets = t_new(uint16_t, size);
+ sorted = t_new(struct mail_index_ext *, size);
+ for (i = 0; i < size; i++) {
+ old_offsets[i] = ext[i].record_offset;
+ ext[i].record_offset = 0;
+ sorted[i] = &ext[i];
+ }
+ qsort(sorted, size, sizeof(struct mail_index_ext *),
+ mail_index_ext_align_cmp);
+
+ /* we simply try to use the extensions with largest alignment
+ requirement first. FIXME: if the extension sizes don't match
+ alignmentation, this may not give the minimal layout. */
+ offset = sizeof(struct mail_index_record);
+ for (;;) {
+ min_align = (uint16_t)-1;
+ for (i = 0; i < size; i++) {
+ if (sorted[i]->record_offset == 0) {
+ if ((offset % sorted[i]->record_align) == 0)
+ break;
+ if (sorted[i]->record_align < min_align)
+ min_align = sorted[i]->record_align;
+ }
+ }
+ if (i == size) {
+ if (min_align == (uint16_t)-1) {
+ /* all done */
+ break;
+ }
+ /* we have to leave space here */
+ i_assert(min_align > 1 && min_align < (uint16_t)-1);
+ offset += min_align - (offset % min_align);
+ } else {
+ sorted[i]->record_offset = offset;
+ offset += sorted[i]->record_size;
+ }
+
+ i_assert(offset < (uint16_t)-1);
+ }
+
+ if ((offset % sizeof(uint32_t)) != 0) {
+ /* keep 32bit alignment */
+ offset += sizeof(uint32_t) - (offset % sizeof(uint32_t));
+ }
+
+ /* create a new mapping without records. a bit kludgy. */
+ old_records_count = map->records_count;
+ map->records_count = 0;
+ new_map = mail_index_map_to_memory(map, offset);
+ map->records_count = old_records_count;
+
+ /* now copy the records to new mapping */
+ src = map->records;
+ offset = 0;
+ for (rec_idx = 0; rec_idx < old_records_count; rec_idx++) {
+ buffer_write(new_map->buffer, offset, src,
+ sizeof(struct mail_index_record));
+ for (i = 0; i < size; i++) {
+ buffer_write(new_map->buffer,
+ offset + ext[i].record_offset,
+ CONST_PTR_OFFSET(src, old_offsets[i]),
+ i == ext_id ? old_size :
+ ext[i].record_size);
+ }
+ src = CONST_PTR_OFFSET(src, map->hdr->record_size);
+ offset += new_map->hdr->record_size;
+ }
+
+ new_map->records = buffer_get_modifyable_data(new_map->buffer, NULL);
+ new_map->records_count = old_records_count;
+
+ /* update record offsets in headers */
+ hdr = buffer_get_modifyable_data(new_map->hdr_copy_buf, NULL);
+ for (i = 0; i < size; i++) {
+ /* kludgy jumping to get to the beginning of header. */
+ offset = ext[i].hdr_offset -
+ MAIL_INDEX_HEADER_SIZE_ALIGN(sizeof(*ext_hdr) +
+ strlen(ext[i].name));
+ ext_hdr = PTR_OFFSET(hdr, offset);
+ i_assert(memcmp((char *)(ext_hdr + 1),
+ ext[i].name, strlen(ext->name)) == 0);
+ ext_hdr->record_offset = ext[i].record_offset;
+ }
+
+ t_pop();
+ return new_map;
+}
+
+static int
+sync_ext_resize(const struct mail_transaction_ext_intro *u, uint32_t ext_id,
+ struct mail_index_sync_map_ctx *ctx)
+{
+ struct mail_index_map *map = ctx->view->map;
+ struct mail_index_ext *ext;
+ struct mail_index_ext_header *ext_hdr;
+ struct mail_index_header *hdr;
+ uint32_t offset, old_size, new_size, old_record_size;
+ int modified = FALSE;
+
+ if (ext_id != u->ext_id) {
+ mail_transaction_log_view_set_corrupted(ctx->view->log_view,
+ "Introduced existing extension with wrong id: %u != %u",
+ u->ext_id, ext_id);
+ return -1;
+ }
+
+ ext = buffer_get_modifyable_data(map->extensions, NULL);
+ ext += ext_id;
+
+ old_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size);
+ new_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size);
+
+ if (new_size < old_size) {
+ /* header shrinked */
+ buffer_delete(map->hdr_copy_buf, ext->hdr_offset + new_size,
+ old_size - new_size);
+ modified = TRUE;
+ } else if (new_size > old_size) {
+ /* header grown */
+ buffer_insert_zero(map->hdr_copy_buf,
+ ext->hdr_offset + old_size,
+ new_size - old_size);
+ modified = TRUE;
+ }
+
+ old_record_size = ext->record_size;
+ ext->hdr_size = u->hdr_size;
+ ext->record_size = u->record_size;
+ ext->record_align = u->record_align;
+
+ if (old_record_size != u->record_size)
+ modified = TRUE;
+
+ if (modified) {
+ hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
+ hdr->header_size = map->hdr_copy_buf->used;
+ map->hdr = hdr;
+
+ /* we need to modify ext header. do some kludgy jumping to
+ get to it. */
+ offset = ext->hdr_offset -
+ MAIL_INDEX_HEADER_SIZE_ALIGN(sizeof(*ext_hdr) +
+ strlen(ext->name));
+ ext_hdr = PTR_OFFSET(hdr, offset);
+ i_assert(memcmp((char *)(ext_hdr + 1),
+ ext->name, strlen(ext->name)) == 0);
+ ext_hdr->hdr_size = ext->hdr_size;
+ ext_hdr->record_offset = ext->record_offset;
+ ext_hdr->record_size = ext->record_size;
+ ext_hdr->record_align = ext->record_align;
+ }
+
+ if (old_record_size != u->record_size) {
+ map = sync_ext_reorder(map, ext_id, old_record_size);
+ mail_index_sync_replace_map(ctx->view, map);
+ modified = TRUE;
+ }
+
+ return 1;
+}
+
static int sync_ext_intro(const struct mail_transaction_ext_intro *u,
void *context)
{
struct mail_index_sync_map_ctx *ctx = context;
+ struct mail_index_map *map = ctx->view->map;
struct mail_index_ext_header ext_hdr;
const struct mail_index_ext *ext;
struct mail_index_header *hdr;
const char *name;
buffer_t *hdr_buf;
- uint32_t ext_id;
+ uint32_t ext_id, hdr_offset;
+ int ret;
t_push();
name = t_strndup(u + 1, u->name_size);
- hdr_buf = ctx->view->map->hdr_copy_buf;
- ext_id = mail_index_map_register_ext(ctx->view->index, ctx->view->map,
- name, hdr_buf->used,
- u->hdr_size, u->record_size);
+ ext_id = mail_index_map_lookup_ext(map, name);
+ if (ext_id != (uint32_t)-1) {
+ /* exists already - are we resizing? */
+ ret = sync_ext_resize(u, ext_id, ctx);
+ t_pop();
+ return ret;
+ }
- ext = ctx->view->index->extensions->data;
- ext += ext_id;
+ hdr_buf = map->hdr_copy_buf;
+ if (MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) != hdr_buf->used) {
+ /* we need to add padding between base header and extensions */
+ buffer_append_zero(hdr_buf,
+ MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) -
+ hdr_buf->used);
+ }
- /* name NUL [padding] ext_hdr [header data] */
- buffer_append(hdr_buf, name, strlen(name)+1);
- if ((hdr_buf->used % 4) != 0)
- buffer_append(hdr_buf, null4, 4 - (hdr_buf->used % 4));
+ /* register record offset initially using record size,
+ sync_ext_reorder() will fix it. */
+ hdr_offset = map->hdr_copy_buf->used + sizeof(ext_hdr) + strlen(name);
+ hdr_offset = MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_offset);
+ ext_id = mail_index_map_register_ext(ctx->view->index, map,
+ name, hdr_offset, u->hdr_size,
+ map->hdr->record_size,
+ u->record_size, u->record_align);
+
+ ext = map->extensions->data;
+ ext += ext_id;
+ /* <ext_hdr> <name> [padding] [header data] */
memset(&ext_hdr, 0, sizeof(ext_hdr));
+ ext_hdr.name_size = strlen(name);
ext_hdr.hdr_size = ext->hdr_size;
ext_hdr.record_offset = ext->record_offset;
ext_hdr.record_size = ext->record_size;
+ ext_hdr.record_align = ext->record_align;
buffer_append(hdr_buf, &ext_hdr, sizeof(ext_hdr));
- buffer_append_zero(hdr_buf, ext->hdr_size);
+ buffer_append(hdr_buf, name, strlen(name));
+ /* header must begin and end in correct alignment */
+ buffer_append_zero(hdr_buf,
+ MAIL_INDEX_HEADER_SIZE_ALIGN(hdr_buf->used) - hdr_buf->used +
+ MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size));
+ i_assert(hdr_buf->used ==
+ hdr_offset + MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size));
hdr = buffer_get_modifyable_data(hdr_buf, NULL);
hdr->header_size = hdr_buf->used;
-
- ctx->view->map->hdr = hdr;
+ map->hdr = hdr;
t_pop();
@@ -336,6 +568,9 @@
u->ext_id, ext_id);
return -1;
}
+
+ map = sync_ext_reorder(map, ext_id, 0);
+ mail_index_sync_replace_map(ctx->view, map);
return 1;
}
@@ -478,18 +713,6 @@
return 0;
}
-static void mail_index_sync_replace_map(struct mail_index_view *view,
- struct mail_index_map *map)
-{
- mail_index_unmap(view->index, view->map);
- view->map = map;
- view->map->refcount++;
- mail_index_unmap(view->index, view->index->map);
- view->index->map = map;
- view->index->hdr = map->hdr;
- map->write_to_disk = TRUE;
-}
-
static void
mail_index_update_day_headers(struct mail_index_header *hdr, uint32_t uid)
{
@@ -577,24 +800,13 @@
first_append_uid = 0;
while ((ret = mail_transaction_log_view_next(view->log_view, &thdr,
&data, &skipped)) > 0) {
- if ((thdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
- !map->write_to_disk) {
- /* expunges have to be atomic. so we'll have to copy
- the mapping, do the changes there and then finally
- replace the whole index file. to avoid extra disk
- I/O we copy the index into memory rather than to
- temporary file */
- map = mail_index_map_to_memory(map,
- map->hdr->record_size);
- mail_index_sync_replace_map(view, map);
- }
-
if ((thdr->type & MAIL_TRANSACTION_APPEND) != 0) {
const struct mail_index_record *rec = data;
if (first_append_uid == 0)
first_append_uid = rec->uid;
+ map = view->map;
count = thdr->size / sizeof(*rec);
if (mail_index_grow(index, map, count) < 0) {
ret = -1;
@@ -614,18 +826,8 @@
ret = -1;
break;
}
- if ((thdr->type & MAIL_TRANSACTION_EXT_INTRO) != 0) {
- const struct mail_index_ext *ext;
- size_t size;
-
- ext = buffer_get_data(map->extensions, &size);
- ext += (size / sizeof(*ext)) - 1;
-
- map = mail_index_map_to_memory(map, ext->record_offset +
- ext->record_size);
- mail_index_sync_replace_map(view, map);
- }
}
+ map = view->map;
if (sync_map_ctx.cache_locked) {
mail_cache_unlock(index->cache);
Index: mail-index-transaction-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction-private.h,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- mail-index-transaction-private.h 26 Sep 2004 14:15:54 -0000 1.13
+++ mail-index-transaction-private.h 3 Oct 2004 16:32:59 -0000 1.14
@@ -20,6 +20,7 @@
unsigned char hdr_mask[sizeof(struct mail_index_header)];
buffer_t *ext_rec_updates; /* buffer[] */
+ buffer_t *ext_resizes; /* struct mail_transaction_ext_intro[] */
buffer_t *ext_intros;
uint32_t ext_intros_max_id;
Index: mail-index-transaction.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- mail-index-transaction.c 26 Sep 2004 14:15:54 -0000 1.29
+++ mail-index-transaction.c 3 Oct 2004 16:32:59 -0000 1.30
@@ -657,12 +657,72 @@
return TRUE;
}
-void mail_index_update_ext(struct mail_index_transaction *t,
- uint32_t seq, uint32_t ext_id, const void *data)
+void mail_index_update_header(struct mail_index_transaction *t,
+ size_t offset, const void *data, size_t size)
+{
+ i_assert(offset < sizeof(t->hdr_change));
+ i_assert(size <= sizeof(t->hdr_change) - offset);
+
+ t->hdr_changed = TRUE;
+ t->log_updates = TRUE;
+
+ memcpy(t->hdr_change + offset, data, size);
+ for (; size > 0; size--)
+ t->hdr_mask[offset++] = 1;
+}
+
+void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
+ uint32_t hdr_size, uint16_t record_size,
+ uint16_t record_align)
+{
+ struct mail_transaction_ext_intro intro;
+ const struct mail_index_ext *ext;
+
+ memset(&intro, 0, sizeof(intro));
+
+ if (!mail_index_map_get_ext_idx(t->view->map, ext_id, &intro.ext_id)) {
+ intro.ext_id = (uint32_t)-1;
+ ext = t->view->index->extensions->data;
+ ext += ext_id;
+ } else {
+ ext = t->view->map->extensions->data;
+ ext += ext_id;
+ }
+
+ /* allow only header size changes if something was already written */
+ i_assert(t->ext_rec_updates == NULL ||
+ (ext->record_size == record_size &&
+ ext->record_align == record_align));
+
+ if (t->ext_resizes == NULL) {
+ t->ext_resizes =
+ buffer_create_dynamic(default_pool, 128, (size_t)-1);
+ }
+
+ intro.hdr_size = hdr_size;
+ intro.record_size = record_size;
+ intro.record_align = record_align;
+ intro.name_size = 1;
+ buffer_write(t->ext_resizes, ext_id * sizeof(intro),
+ &intro, sizeof(intro));
+}
+
+void mail_index_update_header_ext(struct mail_index_transaction *t,
+ uint32_t ext_id, size_t offset,
+ const void *data, size_t size)
+{
+ // FIXME
+}
+
+void mail_index_update_ext(struct mail_index_transaction *t, uint32_t seq,
+ uint32_t ext_id, const void *data)
{
struct mail_index *index = t->view->index;
const struct mail_index_ext *ext;
+ const struct mail_transaction_ext_intro *intro;
buffer_t **buf;
+ uint16_t record_size;
+ size_t size;
i_assert(seq > 0 &&
(seq <= mail_index_view_get_message_count(t->view) ||
@@ -671,8 +731,19 @@
t->log_updates = TRUE;
- ext = index->extensions->data;
- ext += ext_id;
+ if (t->ext_resizes == NULL) {
+ intro = NULL;
+ size = 0;
+ } else {
+ intro = buffer_get_data(t->ext_resizes, &size);
+ }
+ if (ext_id < size / sizeof(*intro) && intro[ext_id].name_size != 0) {
+ /* resized record */
+ record_size = intro[ext_id].record_size;
+ } else {
+ ext = index->extensions->data;
+ record_size = ext[ext_id].record_size;
+ }
if (t->ext_rec_updates == NULL) {
t->ext_rec_updates =
@@ -681,19 +752,5 @@
buf = buffer_get_space_unsafe(t->ext_rec_updates,
ext_id * sizeof(buffer_t *),
sizeof(buffer_t *));
- mail_index_update_seq_buffer(buf, seq, data, ext->record_size, NULL);
-}
-
-void mail_index_update_header(struct mail_index_transaction *t,
- size_t offset, const void *data, size_t size)
-{
- i_assert(offset < sizeof(t->hdr_change));
- i_assert(size <= sizeof(t->hdr_change) - offset);
-
- t->hdr_changed = TRUE;
- t->log_updates = TRUE;
-
- memcpy(t->hdr_change + offset, data, size);
- for (; size > 0; size--)
- t->hdr_mask[offset++] = 1;
+ mail_index_update_seq_buffer(buf, seq, data, record_size, NULL);
}
Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -d -r1.20 -r1.21
--- mail-index-view-sync.c 5 Sep 2004 17:53:45 -0000 1.20
+++ mail-index-view-sync.c 3 Oct 2004 16:32:59 -0000 1.21
@@ -130,7 +130,8 @@
fully synced */
} else {
/* we need a private copy of the map if we don't want to
- sync expunges */
+ sync expunges. we need to sync mapping only if we're not
+ using the latest one. */
if (view->map != view->index->map)
ctx->sync_map_update = TRUE;
Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.149
retrieving revision 1.150
diff -u -d -r1.149 -r1.150
--- mail-index.c 26 Sep 2004 14:15:54 -0000 1.149
+++ mail-index.c 3 Oct 2004 16:32:59 -0000 1.150
@@ -54,7 +54,9 @@
}
uint32_t mail_index_ext_register(struct mail_index *index, const char *name,
- uint32_t hdr_size, uint16_t record_size)
+ uint32_t default_hdr_size,
+ uint16_t default_record_size,
+ uint16_t default_record_align)
{
const struct mail_index_ext *extensions;
struct mail_index_ext ext;
@@ -64,22 +66,17 @@
extensions = buffer_get_data(index->extensions, &ext_count);
ext_count /= sizeof(*extensions);
- /* see if it's there already */
+ /* see if it's already there */
for (i = 0; i < ext_count; i++) {
- if (strcmp(extensions[i].name, name) == 0) {
- i_assert(extensions[i].hdr_size == hdr_size);
- i_assert(extensions[i].record_size == record_size);
+ if (strcmp(extensions[i].name, name) == 0)
return i;
- }
}
- i_assert(hdr_size % 4 == 0);
- i_assert(record_size % 4 == 0);
-
memset(&ext, 0, sizeof(ext));
ext.name = p_strdup(index->extension_pool, name);
- ext.hdr_size = hdr_size;
- ext.record_size = record_size;
+ ext.hdr_size = default_hdr_size;
+ ext.record_size = default_record_size;
+ ext.record_align = default_record_align;
buffer_append(index->extensions, &ext, sizeof(ext));
return ext_count;
@@ -108,27 +105,39 @@
ext_id_map_size, (size_t)-1);
}
-uint32_t mail_index_map_register_ext(struct mail_index *index,
- struct mail_index_map *map,
- const char *name, uint32_t hdr_offset,
- uint32_t hdr_size, uint32_t record_size)
+uint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name)
+{
+ const struct mail_index_ext *extensions;
+ size_t i, size;
+
+ if (map->extensions == NULL)
+ return (uint32_t)-1;
+
+ extensions = buffer_get_data(map->extensions, &size);
+ size /= sizeof(*extensions);
+
+ for (i = 0; i < size; i++) {
+ if (strcmp(extensions[i].name, name) == 0)
+ return i;
+ }
+ return (uint32_t)-1;
+}
+
+uint32_t
+mail_index_map_register_ext(struct mail_index *index,
+ struct mail_index_map *map, const char *name,
+ uint32_t hdr_offset, uint32_t hdr_size,
+ uint32_t record_offset, uint32_t record_size,
+ uint32_t record_align)
{
- const struct mail_index_ext *last_ext;
struct mail_index_ext *ext;
- size_t size;
uint32_t idx, ext_id;
if (map->extensions == NULL) {
mail_index_map_init_extbufs(map, 5);
- last_ext = NULL;
idx = 0;
} else {
- last_ext = buffer_get_data(map->extensions, &size);
- idx = size / sizeof(*last_ext);
- if (idx == 0)
- last_ext = NULL;
- else
- last_ext += idx - 1;
+ idx = map->extensions->used / sizeof(*ext);
}
ext = buffer_append_space_unsafe(map->extensions, sizeof(*ext));
@@ -137,21 +146,31 @@
ext->name = p_strdup(map->extension_pool, name);
ext->hdr_offset = hdr_offset;
ext->hdr_size = hdr_size;
+ ext->record_offset = record_offset;
ext->record_size = record_size;
+ ext->record_align = record_align;
- if (last_ext != NULL) {
- ext->record_offset = last_ext->record_offset +
- last_ext->record_size;
- } else {
- ext->record_offset = sizeof(struct mail_index_record);
- }
-
- ext_id = mail_index_ext_register(index, name, hdr_size, record_size);
+ ext_id = mail_index_ext_register(index, name, hdr_size,
+ record_size, record_align);
buffer_write(map->ext_id_map, ext_id * sizeof(uint32_t),
&idx, sizeof(idx));
return idx;
}
+static int size_check(size_t *size_left, size_t size)
+{
+ if (size > *size_left)
+ return FALSE;
+ *size_left -= size;
+ return TRUE;
+}
+
+static size_t get_align(size_t name_len)
+{
+ size_t size = sizeof(struct mail_index_ext_header) + name_len;
+ return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size;
+}
+
static int mail_index_read_extensions(struct mail_index *index,
struct mail_index_map *map)
{
@@ -159,9 +178,12 @@
unsigned int i, old_count;
const char *name;
uint32_t ext_id, offset, name_offset;
+ size_t size_left;
- offset = map->hdr->base_header_size;
- if (offset == map->hdr->header_size && map->extension_pool == NULL) {
+ /* extension headers always start from 64bit offsets, so if base header
+ doesn't happen to be 64bit aligned we'll skip some bytes */
+ offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr->base_header_size);
+ if (offset >= map->hdr->header_size && map->extension_pool == NULL) {
/* nothing to do, skip allocatations and all */
return 1;
}
@@ -173,39 +195,34 @@
for (i = 0; i < old_count; i++)
buffer_append(map->ext_id_map, &ext_id, sizeof(ext_id));
- name = map->hdr_base;
while (offset < map->hdr->header_size) {
- name_offset = offset;
-
- while (offset < map->hdr->header_size && name[offset] != '\0')
- offset++;
- if (offset == map->hdr->header_size) {
- mail_index_set_error(index, "Corrupted index file %s: "
- "Header extension name doesn't end with NUL",
- index->filepath);
- return -1;
- }
- offset++;
- while (offset < map->hdr->header_size && (offset % 4) != 0)
- offset++;
-
ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
- if (offset + sizeof(*ext_hdr) > map->hdr->header_size ||
- offset + sizeof(*ext_hdr) + ext_hdr->hdr_size >
- map->hdr->header_size) {
+ size_left = map->hdr->header_size - offset;
+ if (!size_check(&size_left, sizeof(*ext_hdr)) ||
+ !size_check(&size_left, ext_hdr->name_size) ||
+ !size_check(&size_left, get_align(ext_hdr->name_size)) ||
+ !size_check(&size_left, ext_hdr->hdr_size)) {
mail_index_set_error(index, "Corrupted index file %s: "
"Header extension goes outside header",
index->filepath);
return -1;
}
+ offset += sizeof(*ext_hdr);
+ name_offset = offset;
+ offset += ext_hdr->name_size + get_align(ext_hdr->name_size);
- mail_index_map_register_ext(index, map, name + name_offset,
- offset + sizeof(*ext_hdr),
- ext_hdr->hdr_size,
- ext_hdr->record_size);
+ t_push();
+ name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
+ ext_hdr->name_size);
+ mail_index_map_register_ext(index, map, name,
+ offset, ext_hdr->hdr_size,
+ ext_hdr->record_offset,
+ ext_hdr->record_size,
+ ext_hdr->record_align);
+ t_pop();
- offset += sizeof(*ext_hdr) + ext_hdr->hdr_size;
+ offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
}
return 1;
}
@@ -663,7 +680,7 @@
/* copy extensions */
if (map->ext_id_map != NULL) {
count = map->ext_id_map->used / sizeof(uint32_t);
- mail_index_map_init_extbufs(mem_map, count);
+ mail_index_map_init_extbufs(mem_map, count + 2);
buffer_append_buf(mem_map->extensions, map->extensions,
0, (size_t)-1);
Index: mail-index.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.129
retrieving revision 1.130
diff -u -d -r1.129 -r1.130
--- mail-index.h 26 Sep 2004 14:15:55 -0000 1.129
+++ mail-index.h 3 Oct 2004 16:32:59 -0000 1.130
@@ -140,11 +140,6 @@
void mail_index_set_permissions(struct mail_index *index,
mode_t mode, gid_t gid);
-/* register index extension. name is a unique identifier for the extension.
- returns identifier for the name. */
-uint32_t mail_index_ext_register(struct mail_index *index, const char *name,
- uint32_t hdr_size, uint16_t record_size);
-
int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags);
void mail_index_close(struct mail_index *index);
@@ -256,10 +251,6 @@
mail_index_lookup()->uid. */
int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
uint32_t *uid_r);
-/* Returns the wanted extension record for given message. If it doesn't exist,
- *data_r is set to NULL. Return values are same as for mail_index_lookup(). */
-int mail_index_lookup_ext(struct mail_index_view *view, uint32_t seq,
- uint32_t ext_id, const void **data_r);
/* Convert UID range to sequence range. If no UIDs are found, sequences are
set to 0. Note that any of the returned sequences may have been expunged
already. */
@@ -288,9 +279,6 @@
/* Update field in header. */
void mail_index_update_header(struct mail_index_transaction *t,
size_t offset, const void *data, size_t size);
-/* Update extension record. */
-void mail_index_update_ext(struct mail_index_transaction *t,
- uint32_t seq, uint32_t ext_id, const void *data);
/* Returns the last error code. */
enum mail_index_error mail_index_get_last_error(struct mail_index *index);
@@ -305,4 +293,36 @@
void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec,
uint8_t *flags, keywords_mask_t keywords);
+/* register index extension. name is a unique identifier for the extension.
+ returns unique identifier for the name. */
+uint32_t mail_index_ext_register(struct mail_index *index, const char *name,
+ uint32_t default_hdr_size,
+ uint16_t default_record_size,
+ uint16_t default_record_align);
+/* Get current extension sizes. Returns 1 if ok, 0 if extension doesn't exist
+ in view. */
+int mail_index_ext_get_size(struct mail_index_view *view, uint32_t ext_id,
+ uint32_t *hdr_size_r, uint16_t *record_size_r,
+ uint16_t *record_align_r);
+/* Resize existing extension data. If size is grown, the new data will be
+ zero-filled. If size is shrinked, the data is simply dropped. */
+void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
+ uint32_t hdr_size, uint16_t record_size,
+ uint16_t record_align);
+
+/* Returns extension header. */
+int mail_index_get_header_ext(struct mail_index_view *view, uint32_t ext_id,
+ const void **data_r, size_t *data_size_r);
+/* Returns the wanted extension record for given message. If it doesn't exist,
+ *data_r is set to NULL. Return values are same as for mail_index_lookup(). */
+int mail_index_lookup_ext(struct mail_index_view *view, uint32_t seq,
+ uint32_t ext_id, const void **data_r);
+/* Update extension header field. */
+void mail_index_update_header_ext(struct mail_index_transaction *t,
+ uint32_t ext_id, size_t offset,
+ const void *data, size_t size);
+/* Update extension record. */
+void mail_index_update_ext(struct mail_index_transaction *t, uint32_t seq,
+ uint32_t ext_id, const void *data);
+
#endif
Index: mail-transaction-log-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- mail-transaction-log-view.c 26 Sep 2004 14:15:55 -0000 1.29
+++ mail-transaction-log-view.c 3 Oct 2004 16:32:59 -0000 1.30
@@ -334,15 +334,26 @@
hdr->type & MAIL_TRANSACTION_TYPE_MASK);
return -1;
} else if (hdr->type == MAIL_TRANSACTION_EXT_INTRO) {
- const struct mail_transaction_ext_intro *intro = data;
+ const struct mail_transaction_ext_intro *intro;
+ uint32_t i;
- if (intro->ext_id > view->max_ext_id)
- view->max_ext_id = intro->ext_id;
- if (intro->name_size >
- hdr_size - sizeof(*hdr) - sizeof(*intro)) {
- mail_transaction_log_file_set_corrupted(file,
- "extension intro: name_size too large");
- return -1;
+ for (i = 0; i < hdr->size; ) {
+ if (i + sizeof(*intro) > hdr->size) {
+ /* should be just extra padding */
+ break;
+ }
+
+ intro = CONST_PTR_OFFSET(data, i);
+ if (intro->ext_id > view->max_ext_id)
+ view->max_ext_id = intro->ext_id;
+ if (intro->name_size >
+ hdr_size - sizeof(*hdr) - sizeof(*intro)) {
+ mail_transaction_log_file_set_corrupted(file,
+ "extension intro: name_size too large");
+ return -1;
+ }
+
+ i += sizeof(*intro) + intro->name_size;
}
} else if (hdr->type == MAIL_TRANSACTION_EXT_REC_UPDATE ||
hdr->type == MAIL_TRANSACTION_EXT_HDR_UPDATE ||
Index: mail-transaction-log.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.60
retrieving revision 1.61
diff -u -d -r1.60 -r1.61
--- mail-transaction-log.c 26 Sep 2004 14:15:55 -0000 1.60
+++ mail-transaction-log.c 3 Oct 2004 16:32:59 -0000 1.61
@@ -986,7 +986,8 @@
t_push();
name = t_strndup((const char *)(intro+1), intro->name_size);
ext_id = mail_index_ext_register(t->view->index, name,
- intro->hdr_size, intro->record_size);
+ intro->hdr_size, intro->record_size,
+ intro->record_align);
pos = ext_id * sizeof(intro->ext_id);
if (pos > t->ext_intros->used) {
/* unused records are -1 */
@@ -1029,9 +1030,20 @@
break;
}
case MAIL_TRANSACTION_EXT_INTRO: {
- const struct mail_transaction_ext_intro *intro = data;
+ const struct mail_transaction_ext_intro *intro;
+ uint32_t i;
- transaction_save_ext_intro(t, intro);
+ for (i = 0; i < hdr->size; ) {
+ if (i + sizeof(*intro) > hdr->size) {
+ /* should be just extra padding */
+ break;
+ }
+
+ intro = CONST_PTR_OFFSET(data, i);
+ transaction_save_ext_intro(t, intro);
+
+ i += sizeof(*intro) + intro->name_size;
+ }
break;
}
}
@@ -1161,50 +1173,107 @@
return buf;
}
-static int
-mail_transaction_log_register_ext(struct mail_transaction_log_file *file,
- struct mail_index_transaction *t,
- uint32_t ext_id, uint32_t *idx_r)
+static uint32_t
+mail_transaction_log_lookup_ext(struct mail_index_transaction *t,
+ uint32_t ext_id)
{
- const struct mail_index_ext *ext;
- struct mail_transaction_ext_intro *intro;
const uint32_t *id_map;
- buffer_t *buf;
+ uint32_t idx;
size_t size;
- int ret;
- /* first check if it's already in nonsynced part of transaction log */
+ /* already in nonsynced part of transaction log? */
if (t->ext_intros != NULL) {
id_map = buffer_get_data(t->ext_intros, &size);
size /= sizeof(*id_map);
- if (ext_id < size && id_map[ext_id] != (uint32_t)-1) {
- *idx_r = id_map[ext_id];
- return 0;
- }
+ if (ext_id < size && id_map[ext_id] != (uint32_t)-1)
+ return id_map[ext_id];
}
- *idx_r = t->ext_intros_max_id++;
+
+ if (mail_index_map_get_ext_idx(t->view->map, ext_id, &idx))
+ return idx;
+
+ return (uint32_t)-1;
+}
+
+static int
+mail_transaction_log_append_ext_intros(struct mail_transaction_log_file *file,
+ struct mail_index_transaction *t)
+{
+ const struct mail_index_ext *ext;
+ struct mail_transaction_ext_intro *intro;
+ buffer_t **updates, *buf;
+ uint32_t idx;
+ size_t i, size;
ext = t->view->index->extensions->data;
- ext += ext_id;
- /* nope, register */
- t_push();
+ if (t->ext_rec_updates != NULL) {
+ updates = buffer_get_modifyable_data(t->ext_rec_updates, &size);
+ size /= sizeof(*updates);
+
+ /* add new extension introductions into resize buffer */
+ for (i = 0; i < size; i++) {
+ if (updates[i] == NULL)
+ continue;
+
+ idx = mail_transaction_log_lookup_ext(t, i);
+ if (idx == (uint32_t)-1) {
+ mail_index_ext_resize(t, i, ext[i].hdr_size,
+ ext[i].record_size,
+ ext[i].record_align);
+ }
+ }
+ }
+
+ if (t->ext_resizes == NULL)
+ return 0;
+
+ /* give IDs to new extensions */
+ intro = buffer_get_modifyable_data(t->ext_resizes, &size);
+ size /= sizeof(*intro);
+ for (i = 0; i < size; i++) {
+ if (intro[i].name_size != 0 && intro[i].ext_id == (uint32_t)-1)
+ intro[i].ext_id = t->ext_intros_max_id++;
+ }
+
+ /* and register them */
buf = buffer_create_dynamic(pool_datastack_create(), 128, (size_t)-1);
- intro = buffer_append_space_unsafe(buf, sizeof(*intro));
- intro->ext_id = *idx_r;
- intro->hdr_size = ext->hdr_size;
- intro->record_size = ext->record_size;
- intro->name_size = strlen(ext->name);
- buffer_append(buf, ext->name, intro->name_size);
+ for (i = 0; i < size; i++) {
+ if (intro[i].name_size != 0) {
+ intro[i].name_size = strlen(ext[i].name);
+ buffer_append(buf, &intro[i], sizeof(*intro));
+ buffer_append(buf, ext[i].name, intro[i].name_size);
+ }
+ }
if ((buf->used % 4) != 0)
- buffer_append(buf, null4, 4 - (buf->used % 4));
+ buffer_append_zero(buf, 4 - (buf->used % 4));
- ret = log_append_buffer(file, buf, NULL, MAIL_TRANSACTION_EXT_INTRO,
- t->view->external);
- t_pop();
- return ret;
+ if (buf->used == 0)
+ return 0;
+
+ return log_append_buffer(file, buf, NULL, MAIL_TRANSACTION_EXT_INTRO,
+ t->view->external);
+}
+
+static uint32_t
+mail_transaction_log_get_ext_idx(struct mail_index_transaction *t,
+ uint32_t ext_id)
+{
+ const struct mail_transaction_ext_intro *intro;
+ uint32_t idx;
+
+ idx = mail_transaction_log_lookup_ext(t, ext_id);
+ if (idx != (uint32_t)-1)
+ return idx;
+
+ i_assert(ext_id < t->ext_resizes->used / sizeof(*intro));
+ intro = t->ext_resizes->data;
+ intro += ext_id;
+
+ i_assert(intro->name_size > 0 && intro->ext_id != (uint32_t)-1);
+ return intro->ext_id;
}
int mail_transaction_log_append(struct mail_index_transaction *t,
@@ -1220,7 +1289,6 @@
uoff_t append_offset;
buffer_t *hdr_buf, **updates;
unsigned int i, lock_id;
- uint32_t idx;
size_t size;
int ret;
@@ -1297,7 +1365,13 @@
}
ret = 0;
- if (t->appends != NULL) {
+
+ /* introduce extensions before appends to avoid having to resize
+ records unneededly */
+ if (mail_transaction_log_append_ext_intros(file, t) < 0)
+ ret = -1;
+
+ if (t->appends != NULL && ret == 0) {
ret = log_append_buffer(file, t->appends, NULL,
MAIL_TRANSACTION_APPEND,
view->external);
@@ -1333,15 +1407,7 @@
if (updates[i] == NULL)
continue;
- if (!mail_index_map_get_ext_idx(index->map, i, &idx)) {
- /* new one */
- ret = mail_transaction_log_register_ext(file, t, i,
- &idx);
- if (ret < 0)
- break;
- }
-
- ext_rec_hdr.ext_id = idx;
+ ext_rec_hdr.ext_id = mail_transaction_log_get_ext_idx(t, i);
ret = log_append_buffer(file, updates[i], hdr_buf,
MAIL_TRANSACTION_EXT_REC_UPDATE,
view->external);
Index: mail-transaction-log.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- mail-transaction-log.h 26 Sep 2004 14:15:55 -0000 1.18
+++ mail-transaction-log.h 3 Oct 2004 16:32:59 -0000 1.19
@@ -72,6 +72,8 @@
uint32_t ext_id; /* must be first */
uint32_t hdr_size;
uint16_t record_size;
+ uint16_t record_align;
+ uint16_t unused_padding;
uint16_t name_size;
/* unsigned char name[]; */
};
More information about the dovecot-cvs
mailing list