dovecot-2.2: dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to a...
dovecot at dovecot.org
dovecot at dovecot.org
Mon Sep 29 11:16:32 UTC 2014
details: http://hg.dovecot.org/dovecot-2.2/rev/d2e1b3f6d13b
changeset: 17845:d2e1b3f6d13b
user: Timo Sirainen <tss at iki.fi>
date: Mon Sep 29 14:16:06 2014 +0300
description:
dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily.
diffstat:
src/doveadm/dsync/dsync-brain-mailbox.c | 2 +
src/doveadm/dsync/dsync-brain-private.h | 1 +
src/doveadm/dsync/dsync-brain.c | 2 +
src/doveadm/dsync/dsync-brain.h | 6 ++-
src/doveadm/dsync/dsync-ibc-stream.c | 1 +
src/doveadm/dsync/dsync-mail.c | 26 ++++++++++----
src/doveadm/dsync/dsync-mail.h | 25 ++++++++-----
src/doveadm/dsync/dsync-mailbox-export.c | 21 +++++++----
src/doveadm/dsync/dsync-mailbox-export.h | 3 +-
src/doveadm/dsync/dsync-mailbox-import.c | 56 ++++++++++++++++++++++++-------
10 files changed, 103 insertions(+), 40 deletions(-)
diffs (truncated from 359 to 300 lines):
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-brain-mailbox.c
--- a/src/doveadm/dsync/dsync-brain-mailbox.c Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c Mon Sep 29 14:16:06 2014 +0300
@@ -304,6 +304,8 @@
(brain->local_dsync_box.have_save_guids ||
(brain->backup_send && brain->local_dsync_box.have_guids)))
exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
+ if (brain->no_mail_prefetch)
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
brain->box_exporter = brain->backup_recv ? NULL :
dsync_mailbox_export_init(brain->box, brain->log_scan,
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-brain-private.h
--- a/src/doveadm/dsync/dsync-brain-private.h Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-private.h Mon Sep 29 14:16:06 2014 +0300
@@ -100,6 +100,7 @@
unsigned int sync_visible_namespaces:1;
unsigned int no_mail_sync:1;
unsigned int no_backup_overwrite:1;
+ unsigned int no_mail_prefetch:1;
unsigned int changes_during_sync:1;
unsigned int require_full_resync:1;
unsigned int verbose_proctitle:1;
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-brain.c
--- a/src/doveadm/dsync/dsync-brain.c Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain.c Mon Sep 29 14:16:06 2014 +0300
@@ -137,6 +137,8 @@
brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0;
brain->no_backup_overwrite =
(flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0;
+ brain->no_mail_prefetch =
+ (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0;
}
struct dsync_brain *
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-brain.h
--- a/src/doveadm/dsync/dsync-brain.h Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain.h Mon Sep 29 14:16:06 2014 +0300
@@ -22,7 +22,11 @@
DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE = 0x40,
/* Run storage purge on the remote after syncing.
Useful with e.g. a nightly doveadm backup. */
- DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80
+ DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80,
+ /* Don't prefetch mail bodies until they're actually needed. This works
+ only with pipe ibc. It's useful if most of the mails can be copied
+ directly within filesystem without having to read them. */
+ DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH = 0x100
};
enum dsync_brain_sync_type {
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-ibc-stream.c
--- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-ibc-stream.c Mon Sep 29 14:16:06 2014 +0300
@@ -1681,6 +1681,7 @@
struct dsync_serializer_encoder *encoder;
string_t *str = t_str_new(128);
+ i_assert(!mail->minimal_fields);
i_assert(ibc->value_output == NULL);
str_append_c(str, items[ITEM_MAIL].chr);
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mail.c
--- a/src/doveadm/dsync/dsync-mail.c Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mail.c Mon Sep 29 14:16:06 2014 +0300
@@ -60,10 +60,10 @@
return ret;
}
-int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
- const char **error_field_r)
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+ struct dsync_mail *dmail_r, const char **error_field_r)
{
- const char *guid, *str;
+ const char *guid;
memset(dmail_r, 0, sizeof(*dmail_r));
@@ -76,6 +76,22 @@
dmail_r->input_mail = mail;
dmail_r->input_mail_uid = mail->uid;
+
+ if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) {
+ *error_field_r = "saved-date";
+ return -1;
+ }
+ if (!minimal_fill)
+ return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r);
+ dmail_r->minimal_fields = TRUE;
+ return 0;
+}
+
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+ const char **error_field_r)
+{
+ const char *str;
+
if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) {
*error_field_r = "body";
return -1;
@@ -97,10 +113,6 @@
*error_field_r = "received-date";
return -1;
}
- if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) {
- *error_field_r = "saved-date";
- return -1;
- }
return 0;
}
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mail.h
--- a/src/doveadm/dsync/dsync-mail.h Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mail.h Mon Sep 29 14:16:06 2014 +0300
@@ -10,21 +10,24 @@
/* either GUID="" or uid=0 */
const char *guid;
uint32_t uid;
-
- const char *pop3_uidl;
- unsigned int pop3_order;
- time_t received_date;
time_t saved_date;
- /* Input stream containing the message text, or NULL if all instances
- of the message were already expunged from this mailbox. */
- struct istream *input;
-
/* If non-NULL, we're syncing within the dsync process using ibc-pipe.
This mail can be used to mailbox_copy() the mail. */
struct mail *input_mail;
/* Verify that this equals to input_mail->uid */
uint32_t input_mail_uid;
+
+ /* TRUE if the following fields aren't set, because minimal_fill=TRUE
+ parameter was used. */
+ bool minimal_fields;
+
+ const char *pop3_uidl;
+ unsigned int pop3_order;
+ time_t received_date;
+ /* Input stream containing the message text, or NULL if all instances
+ of the message were already expunged from this mailbox. */
+ struct istream *input;
};
struct dsync_mail_request {
@@ -80,8 +83,10 @@
dsync_mail_get_hash_headers(struct mailbox *box);
int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r);
-int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
- const char **error_field_r);
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+ struct dsync_mail *dmail_r, const char **error_field_r);
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+ const char **error_field_r);
void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
struct dsync_mail_change *dest_r);
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mailbox-export.c
--- a/src/doveadm/dsync/dsync-mailbox-export.c Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-export.c Mon Sep 29 14:16:06 2014 +0300
@@ -57,6 +57,7 @@
unsigned int body_search_initialized:1;
unsigned int auto_export_mails:1;
unsigned int mails_have_guids:1;
+ unsigned int minimal_dmail_fill:1;
unsigned int return_all_mails:1;
};
@@ -474,6 +475,8 @@
(flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0;
exporter->mails_have_guids =
(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
+ exporter->minimal_dmail_fill =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
p_array_init(&exporter->requested_uids, pool, 16);
p_array_init(&exporter->search_uids, pool, 16);
hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
@@ -657,6 +660,7 @@
const struct seq_range *uids;
char *guid;
const char *const_guid;
+ enum mail_fetch_field wanted_fields;
struct dsync_mail_guid_instances *instances;
const struct seq_range *range;
unsigned int i, count;
@@ -714,16 +718,16 @@
array_append_array(&exporter->search_uids, &exporter->requested_uids);
array_clear(&exporter->requested_uids);
+ wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE;
+ if (!exporter->minimal_dmail_fill) {
+ wanted_fields |= MAIL_FETCH_RECEIVED_DATE |
+ MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER |
+ MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY;
+ }
exporter->search_count += seq_range_count(&sarg->value.seqset);
exporter->search_ctx =
mailbox_search_init(exporter->trans, search_args, NULL,
- MAIL_FETCH_GUID |
- MAIL_FETCH_UIDL_BACKEND |
- MAIL_FETCH_POP3_ORDER |
- MAIL_FETCH_RECEIVED_DATE |
- MAIL_FETCH_SAVE_DATE |
- MAIL_FETCH_STREAM_HEADER |
- MAIL_FETCH_STREAM_BODY, NULL);
+ wanted_fields, NULL);
mail_search_args_unref(&search_args);
return array_count(&sarg->value.seqset) > 0 ? 1 : 0;
}
@@ -748,7 +752,8 @@
struct dsync_mail_guid_instances *instances;
const char *error_field;
- if (dsync_mail_fill(mail, &exporter->dsync_mail, &error_field) < 0)
+ if (dsync_mail_fill(mail, exporter->minimal_dmail_fill,
+ &exporter->dsync_mail, &error_field) < 0)
return dsync_mail_error(exporter, mail, error_field);
instances = *exporter->dsync_mail.guid == '\0' ? NULL :
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mailbox-export.h
--- a/src/doveadm/dsync/dsync-mailbox-export.h Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-export.h Mon Sep 29 14:16:06 2014 +0300
@@ -3,7 +3,8 @@
enum dsync_mailbox_exporter_flags {
DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01,
- DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02
+ DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02,
+ DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04
};
struct dsync_mailbox_exporter *
diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mailbox-import.c
--- a/src/doveadm/dsync/dsync-mailbox-import.c Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Sep 29 14:16:06 2014 +0300
@@ -1544,7 +1544,7 @@
if (!mail_set_uid(importer->mail, uid))
return 0;
- if (dsync_mail_fill(importer->mail, dmail_r, &error_field) < 0) {
+ if (dsync_mail_fill(importer->mail, TRUE, dmail_r, &error_field) < 0) {
errstr = mailbox_get_last_error(importer->mail->box, &error);
if (error == MAIL_ERROR_EXPUNGED)
return 0;
@@ -1905,6 +1905,17 @@
return 0;
}
+static void
+dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx,
+ const struct dsync_mail *mail)
+{
+ if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
+ mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
+ if (mail->pop3_order > 0)
+ mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
+ mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
+}
+
static struct mail_save_context *
dsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
const struct dsync_mail *mail,
@@ -1919,11 +1930,9 @@
if (mail->saved_date != 0)
mailbox_save_set_save_date(save_ctx, mail->saved_date);
dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
- if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
- mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
- if (mail->pop3_order > 0)
- mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
- mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
+
+ if (!mail->minimal_fields)
+ dsync_mailbox_save_set_nonminimal(save_ctx, mail);
return save_ctx;
}
@@ -1934,6 +1943,7 @@
struct importer_new_mail **all_newmails_forcopy)
{
struct mail_save_context *save_ctx;
+ struct istream *input;
ssize_t ret;
bool save_failed = FALSE;
@@ -1960,22 +1970,42 @@
return;
}
/* fallback to saving from remote stream */
+ if (mail->minimal_fields) {
+ struct dsync_mail mail2;
+ const char *error_field;
- if (mail->input == NULL) {
+ i_assert(mail->input_mail != NULL);
More information about the dovecot-cvs
mailing list