dovecot-2.2: dbox: Don't cache pop3.uidl|order unless index head...
dovecot at dovecot.org
dovecot at dovecot.org
Mon May 27 21:03:30 EEST 2013
details: http://hg.dovecot.org/dovecot-2.2/rev/967ef2a7fa6f
changeset: 16403:967ef2a7fa6f
user: Timo Sirainen <tss at iki.fi>
date: Mon May 27 21:03:14 2013 +0300
description:
dbox: Don't cache pop3.uidl|order unless index header indicates there are those.
They exist only when doing a migration, so it's pretty wasteful storing
"doesn't exist" for all other installations.
diffstat:
src/doveadm/doveadm-dump-index.c | 16 +++++++-
src/lib-storage/index/dbox-common/dbox-save.c | 31 ++++++++++++++++
src/lib-storage/index/dbox-common/dbox-save.h | 7 +++
src/lib-storage/index/dbox-common/dbox-storage.c | 13 ++++++
src/lib-storage/index/dbox-common/dbox-storage.h | 9 ++++
src/lib-storage/index/dbox-multi/mdbox-mail.c | 22 ++++++++++-
src/lib-storage/index/dbox-multi/mdbox-save.c | 4 ++
src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 24 +++++++++++-
src/lib-storage/index/dbox-multi/mdbox-storage.c | 15 +++++--
src/lib-storage/index/dbox-multi/mdbox-storage.h | 4 +-
src/lib-storage/index/dbox-single/sdbox-mail.c | 20 +++++++++-
src/lib-storage/index/dbox-single/sdbox-save.c | 4 ++
src/lib-storage/index/dbox-single/sdbox-storage.c | 22 ++++++++---
src/lib-storage/index/dbox-single/sdbox-storage.h | 5 ++-
src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c | 6 ++-
src/lib-storage/index/dbox-single/sdbox-sync.c | 3 +-
16 files changed, 184 insertions(+), 21 deletions(-)
diffs (truncated from 600 to 300 lines):
diff -r bd8ec99cf502 -r 967ef2a7fa6f src/doveadm/doveadm-dump-index.c
--- a/src/doveadm/doveadm-dump-index.c Mon May 27 20:45:08 2013 +0300
+++ b/src/doveadm/doveadm-dump-index.c Mon May 27 21:03:14 2013 +0300
@@ -36,10 +36,14 @@
struct sdbox_index_header {
uint32_t rebuild_count;
guid_128_t mailbox_guid;
+ uint8_t flags;
+ uint8_t unused[3];
};
struct mdbox_index_header {
uint32_t map_uid_validity;
guid_128_t mailbox_guid;
+ uint8_t flags;
+ uint8_t unused[3];
};
struct mdbox_mail_index_record {
uint32_t map_uid;
@@ -122,11 +126,18 @@
const struct mail_index_ext *ext)
{
const void *data;
+ void *buf;
if (strcmp(ext->name, MAIL_INDEX_EXT_KEYWORDS) == 0)
return;
+ /* add some padding, since we don't bother to handle undersized
+ headers correctly */
+ buf = t_malloc0(ext->hdr_size + 128);
data = CONST_PTR_OFFSET(index->map->hdr_base, ext->hdr_offset);
+ memcpy(buf, data, ext->hdr_size);
+ data = buf;
+
if (strcmp(ext->name, "hdr-vsize") == 0) {
const struct index_vsize_header *hdr = data;
@@ -164,6 +175,7 @@
printf(" - map_uid_validity .. = %u\n", hdr->map_uid_validity);
printf(" - mailbox_guid ...... = %s\n",
guid_128_to_string(hdr->mailbox_guid));
+ printf(" - flags ............. = 0x%x\n", hdr->flags);
} else if (strcmp(ext->name, "dbox-hdr") == 0) {
const struct sdbox_index_header *hdr = data;
@@ -171,6 +183,7 @@
printf(" - rebuild_count . = %u\n", hdr->rebuild_count);
printf(" - mailbox_guid .. = %s\n",
guid_128_to_string(hdr->mailbox_guid));
+ printf(" - flags ......... = 0x%x\n", hdr->flags);
} else if (strcmp(ext->name, "modseq") == 0) {
const struct mail_index_modseq_header *hdr = data;
@@ -241,8 +254,9 @@
printf("record_offset = %u\n", ext->record_offset);
printf("record_size . = %u\n", ext->record_size);
printf("record_align = %u\n", ext->record_align);
- if (ext->hdr_size > 0)
+ if (ext->hdr_size > 0) T_BEGIN {
dump_extension_header(index, ext);
+ } T_END;
}
}
diff -r bd8ec99cf502 -r 967ef2a7fa6f src/lib-storage/index/dbox-common/dbox-save.c
--- a/src/lib-storage/index/dbox-common/dbox-save.c Mon May 27 20:45:08 2013 +0300
+++ b/src/lib-storage/index/dbox-common/dbox-save.c Mon May 27 21:03:14 2013 +0300
@@ -163,10 +163,12 @@
i_assert(strchr(mdata->pop3_uidl, '\n') == NULL);
str_printfa(str, "%c%s\n", DBOX_METADATA_POP3_UIDL,
mdata->pop3_uidl);
+ ctx->have_pop3_uidls = TRUE;
}
if (mdata->pop3_order != 0) {
str_printfa(str, "%c%u\n", DBOX_METADATA_POP3_ORDER,
mdata->pop3_order);
+ ctx->have_pop3_orders = TRUE;
}
guid = mdata->guid;
@@ -193,3 +195,32 @@
str_append_c(str, '\n');
o_stream_nsend(output, str_data(str), str_len(str));
}
+
+void dbox_save_update_header_flags(struct dbox_save_context *ctx,
+ struct mail_index_view *sync_view,
+ uint32_t ext_id,
+ unsigned int flags_offset)
+{
+ const void *data;
+ size_t data_size;
+ uint8_t old_flags = 0, flags;
+
+ mail_index_get_header_ext(sync_view, ext_id, &data, &data_size);
+ if (flags_offset < data_size)
+ old_flags = *((const uint8_t *)data + flags_offset);
+ else {
+ /* grow old dbox header */
+ mail_index_ext_resize_hdr(ctx->trans, ext_id, flags_offset+1);
+ }
+
+ flags = old_flags;
+ if (ctx->have_pop3_uidls)
+ flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS;
+ if (ctx->have_pop3_orders)
+ flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS;
+ if (flags != old_flags) {
+ /* flags changed, update them */
+ mail_index_update_header_ext(ctx->trans, ext_id,
+ flags_offset, &flags, 1);
+ }
+}
diff -r bd8ec99cf502 -r 967ef2a7fa6f src/lib-storage/index/dbox-common/dbox-save.h
--- a/src/lib-storage/index/dbox-common/dbox-save.h Mon May 27 20:45:08 2013 +0300
+++ b/src/lib-storage/index/dbox-common/dbox-save.h Mon May 27 21:03:14 2013 +0300
@@ -16,6 +16,8 @@
unsigned int failed:1;
unsigned int finished:1;
+ unsigned int have_pop3_uidls:1;
+ unsigned int have_pop3_orders:1;
};
void dbox_save_begin(struct dbox_save_context *ctx, struct istream *input);
@@ -29,4 +31,9 @@
void dbox_save_add_to_index(struct dbox_save_context *ctx);
+void dbox_save_update_header_flags(struct dbox_save_context *ctx,
+ struct mail_index_view *sync_view,
+ uint32_t ext_id,
+ unsigned int flags_offset);
+
#endif
diff -r bd8ec99cf502 -r 967ef2a7fa6f src/lib-storage/index/dbox-common/dbox-storage.c
--- a/src/lib-storage/index/dbox-common/dbox-storage.c Mon May 27 20:45:08 2013 +0300
+++ b/src/lib-storage/index/dbox-common/dbox-storage.c Mon May 27 21:03:14 2013 +0300
@@ -336,3 +336,16 @@
return -1;
return 0;
}
+
+bool dbox_header_have_flag(struct mailbox *box, uint32_t ext_id,
+ unsigned int flags_offset, uint8_t flag)
+{
+ const void *data;
+ size_t data_size;
+ uint8_t flags = 0;
+
+ mail_index_get_header_ext(box->view, ext_id, &data, &data_size);
+ if (flags_offset < data_size)
+ flags = *((const uint8_t *)data + flags_offset);
+ return (flags & flag) != 0;
+}
diff -r bd8ec99cf502 -r 967ef2a7fa6f src/lib-storage/index/dbox-common/dbox-storage.h
--- a/src/lib-storage/index/dbox-common/dbox-storage.h Mon May 27 20:45:08 2013 +0300
+++ b/src/lib-storage/index/dbox-common/dbox-storage.h Mon May 27 21:03:14 2013 +0300
@@ -23,6 +23,13 @@
/* Flag specifies if the message should be in primary or alternative storage */
#define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND
+enum dbox_index_header_flags {
+ /* messages' metadata contain POP3 UIDLs */
+ DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS = 0x01,
+ /* messages' metadata contain POP3 orders */
+ DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS = 0x02
+};
+
struct dbox_storage_vfuncs {
/* dbox file has zero references now. it should be either freed or
left open in case it's accessed again soon */
@@ -67,5 +74,7 @@
int dbox_mailbox_create(struct mailbox *box,
const struct mailbox_update *update, bool directory);
int dbox_verify_alt_storage(struct mailbox_list *list);
+bool dbox_header_have_flag(struct mailbox *box, uint32_t ext_id,
+ unsigned int flags_offset, uint8_t flag);
#endif
diff -r bd8ec99cf502 -r 967ef2a7fa6f src/lib-storage/index/dbox-multi/mdbox-mail.c
--- a/src/lib-storage/index/dbox-multi/mdbox-mail.c Mon May 27 20:45:08 2013 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c Mon May 27 21:03:14 2013 +0300
@@ -21,6 +21,7 @@
struct mdbox_index_header hdr;
const void *data;
uint32_t uid, cur_map_uid_validity;
+ bool need_resize;
mail_index_lookup_ext(view, seq, mbox->ext_id, &data, NULL);
dbox_rec = data;
@@ -34,7 +35,7 @@
}
if (mbox->map_uid_validity == 0) {
- if (mdbox_read_header(mbox, &hdr) < 0)
+ if (mdbox_read_header(mbox, &hdr, &need_resize) < 0)
return -1;
mbox->map_uid_validity = hdr.map_uid_validity;
}
@@ -190,9 +191,26 @@
*value_r = p_strdup_printf(mail->imail.mail.data_pool, "%u",
refcount);
return 0;
+ case MAIL_FETCH_UIDL_BACKEND:
+ if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id,
+ offsetof(struct mdbox_index_header, flags),
+ DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS)) {
+ *value_r = "";
+ return 0;
+ }
+ break;
+ case MAIL_FETCH_POP3_ORDER:
+ if (!dbox_header_have_flag(&mbox->box, mbox->hdr_ext_id,
+ offsetof(struct mdbox_index_header, flags),
+ DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS)) {
+ *value_r = "";
+ return 0;
+ }
+ break;
default:
- return dbox_mail_get_special(_mail, field, value_r);
+ break;
}
+ return dbox_mail_get_special(_mail, field, value_r);
}
static void
diff -r bd8ec99cf502 -r 967ef2a7fa6f src/lib-storage/index/dbox-multi/mdbox-save.c
--- a/src/lib-storage/index/dbox-multi/mdbox-save.c Mon May 27 20:45:08 2013 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Mon May 27 21:03:14 2013 +0300
@@ -314,6 +314,10 @@
return -1;
}
+ /* update dbox header flags */
+ dbox_save_update_header_flags(&ctx->ctx, ctx->sync_ctx->sync_view,
+ ctx->mbox->hdr_ext_id, offsetof(struct mdbox_index_header, flags));
+
/* assign UIDs for new messages */
hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid,
diff -r bd8ec99cf502 -r 967ef2a7fa6f src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
--- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Mon May 27 20:45:08 2013 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Mon May 27 21:03:14 2013 +0300
@@ -58,6 +58,9 @@
struct mailbox_list *default_list;
struct rebuild_msg_mailbox prev_msg;
+
+ unsigned int have_pop3_uidls:1;
+ unsigned int have_pop3_orders:1;
};
static struct mdbox_storage_rebuild_context *
@@ -126,6 +129,15 @@
return 0;
}
+static void rebuild_scan_metadata(struct mdbox_storage_rebuild_context *ctx,
+ struct dbox_file *file)
+{
+ if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_UIDL) != NULL)
+ ctx->have_pop3_uidls = TRUE;
+ if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_ORDER) != NULL)
+ ctx->have_pop3_orders = TRUE;
+}
+
static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx,
struct dbox_file *file, uint32_t file_id)
{
@@ -177,6 +189,7 @@
ret = 0;
break;
}
+ rebuild_scan_metadata(ctx, file);
rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1);
rec->file_id = file_id;
@@ -476,7 +489,8 @@
memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r)));
}
-static void mdbox_header_update(struct index_rebuild_context *rebuild_ctx,
+static void mdbox_header_update(struct mdbox_storage_rebuild_context *ctx,
+ struct index_rebuild_context *rebuild_ctx,
struct mdbox_mailbox *mbox)
{
struct mdbox_index_header hdr, backup_hdr;
@@ -502,6 +516,11 @@
/* update map's uid-validity */
hdr.map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map);
+ if (ctx->have_pop3_uidls)
+ hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS;
+ if (ctx->have_pop3_orders)
+ hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS;
+
/* and write changes */
mail_index_update_header_ext(rebuild_ctx->trans, mbox->hdr_ext_id, 0,
More information about the dovecot-cvs
mailing list