From dovecot at dovecot.org Thu Jan 3 07:31:07 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Jan 2013 07:31:07 +0200 Subject: dovecot-2.1: virtual storage: get_first_header() always returned... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/abcc9959b757 changeset: 14860:abcc9959b757 user: Timo Sirainen date: Thu Jan 03 07:30:54 2013 +0200 description: virtual storage: get_first_header() always returned "doesn't exist", breaking SORT/THREAD. diffstat: src/plugins/virtual/virtual-mail.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (24 lines): diff -r c9e6ad8a42db -r abcc9959b757 src/plugins/virtual/virtual-mail.c --- a/src/plugins/virtual/virtual-mail.c Fri Dec 07 08:50:35 2012 +0400 +++ b/src/plugins/virtual/virtual-mail.c Thu Jan 03 07:30:54 2013 +0200 @@ -289,15 +289,17 @@ { struct virtual_mail *vmail = (struct virtual_mail *)mail; struct mail_private *p = (struct mail_private *)vmail->backend_mail; + int ret; if (virtual_mail_handle_lost(vmail) < 0) return -1; - if (p->v.get_first_header(vmail->backend_mail, field, - decode_to_utf8, value_r) < 0) { + ret = p->v.get_first_header(vmail->backend_mail, field, + decode_to_utf8, value_r); + if (ret < 0) { virtual_box_copy_error(mail->box, vmail->backend_mail->box); return -1; } - return 0; + return ret; } static int From dovecot at dovecot.org Thu Jan 3 23:04:39 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Jan 2013 23:04:39 +0200 Subject: dovecot-2.2: dsync: Fixed parsing mailbox state from remote dsync. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7d8027a8f74 changeset: 15494:b7d8027a8f74 user: Timo Sirainen date: Thu Jan 03 22:53:18 2013 +0200 description: dsync: Fixed parsing mailbox state from remote dsync. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r ac63858cbe5f -r b7d8027a8f74 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Dec 28 17:02:18 2012 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Thu Jan 03 22:53:18 2013 +0200 @@ -673,12 +673,12 @@ return DSYNC_IBC_RECV_RET_TRYAGAIN; } value = dsync_deserializer_decode_get(decoder, "last_common_uid"); - if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) { + if (str_to_uint32(value, &state_r->last_common_uid) < 0) { dsync_ibc_input_error(ibc, decoder, "Invalid last_common_uid"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } value = dsync_deserializer_decode_get(decoder, "last_common_modseq"); - if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) { + if (str_to_uint64(value, &state_r->last_common_modseq) < 0) { dsync_ibc_input_error(ibc, decoder, "Invalid last_common_modseq"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } From dovecot at dovecot.org Fri Jan 4 01:13:16 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 01:13:16 +0200 Subject: dovecot-2.2: lib-storage: Added support for "private modseqs" fo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/69371578720f changeset: 15495:69371578720f user: Timo Sirainen date: Fri Jan 04 01:12:24 2013 +0200 description: lib-storage: Added support for "private modseqs" for INDEXPVT index. Used by dsync (next commit). diffstat: src/lib-storage/fail-mail.c | 2 + src/lib-storage/index/cydir/cydir-mail.c | 2 + src/lib-storage/index/dbox-multi/mdbox-mail.c | 2 + src/lib-storage/index/dbox-multi/mdbox-storage.c | 6 +- src/lib-storage/index/dbox-single/sdbox-mail.c | 2 + src/lib-storage/index/dbox-single/sdbox-storage.c | 6 +- src/lib-storage/index/imapc/imapc-mail.c | 2 + src/lib-storage/index/index-mail.c | 25 ++++++++++ src/lib-storage/index/index-mail.h | 4 +- src/lib-storage/index/index-status.c | 8 +++ src/lib-storage/index/index-storage.c | 53 ++++++++++++++++++++-- src/lib-storage/index/index-storage.h | 2 +- src/lib-storage/index/index-sync-pvt.c | 49 +-------------------- src/lib-storage/index/maildir/maildir-mail.c | 2 + src/lib-storage/index/mbox/mbox-mail.c | 2 + src/lib-storage/index/pop3c/pop3c-mail.c | 2 + src/lib-storage/index/raw/raw-mail.c | 2 + src/lib-storage/mail-storage-private.h | 5 ++ src/lib-storage/mail-storage.c | 43 ++++++++++++++++++ src/lib-storage/mail-storage.h | 11 ++++- src/lib-storage/mail.c | 14 ++++++ src/plugins/virtual/virtual-mail.c | 2 + 22 files changed, 184 insertions(+), 62 deletions(-) diffs (truncated from 640 to 300 lines): diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/fail-mail.c --- a/src/lib-storage/fail-mail.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/fail-mail.c Fri Jan 04 01:12:24 2013 +0200 @@ -244,6 +244,7 @@ fail_mail_get_keywords, fail_mail_get_keyword_indexes, fail_mail_get_modseq, + fail_mail_get_modseq, fail_mail_get_parts, fail_mail_get_date, fail_mail_get_received_date, @@ -260,6 +261,7 @@ fail_mail_update_flags, fail_mail_update_keywords, fail_mail_update_modseq, + fail_mail_update_modseq, NULL, fail_mail_expunge, fail_mail_set_cache_corrupted, diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/cydir/cydir-mail.c --- a/src/lib-storage/index/cydir/cydir-mail.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/cydir/cydir-mail.c Fri Jan 04 01:12:24 2013 +0200 @@ -140,6 +140,7 @@ index_mail_get_keywords, index_mail_get_keyword_indexes, index_mail_get_modseq, + index_mail_get_pvt_modseq, index_mail_get_parts, index_mail_get_date, cydir_mail_get_received_date, @@ -156,6 +157,7 @@ index_mail_update_flags, index_mail_update_keywords, index_mail_update_modseq, + index_mail_update_pvt_modseq, NULL, index_mail_expunge, index_mail_set_cache_corrupted, diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/dbox-multi/mdbox-mail.c --- a/src/lib-storage/index/dbox-multi/mdbox-mail.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c Fri Jan 04 01:12:24 2013 +0200 @@ -198,6 +198,7 @@ index_mail_get_keywords, index_mail_get_keyword_indexes, index_mail_get_modseq, + index_mail_get_pvt_modseq, index_mail_get_parts, index_mail_get_date, dbox_mail_get_received_date, @@ -214,6 +215,7 @@ mdbox_mail_update_flags, index_mail_update_keywords, index_mail_update_modseq, + index_mail_update_pvt_modseq, NULL, index_mail_expunge, index_mail_set_cache_corrupted, diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/dbox-multi/mdbox-storage.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Fri Jan 04 01:12:24 2013 +0200 @@ -402,9 +402,9 @@ if (mailbox_open(box) < 0) return -1; } - if (update->cache_updates != NULL) - index_storage_mailbox_update_cache(box, update); - return mdbox_write_index_header(box, update, NULL); + if (mdbox_write_index_header(box, update, NULL) < 0) + return -1; + return index_storage_mailbox_update_common(box, update); } struct mail_storage mdbox_storage = { diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/dbox-single/sdbox-mail.c --- a/src/lib-storage/index/dbox-single/sdbox-mail.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-mail.c Fri Jan 04 01:12:24 2013 +0200 @@ -106,6 +106,7 @@ index_mail_get_keywords, index_mail_get_keyword_indexes, index_mail_get_modseq, + index_mail_get_pvt_modseq, index_mail_get_parts, index_mail_get_date, dbox_mail_get_received_date, @@ -122,6 +123,7 @@ index_mail_update_flags, index_mail_update_keywords, index_mail_update_modseq, + index_mail_update_pvt_modseq, NULL, index_mail_expunge, index_mail_set_cache_corrupted, diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/dbox-single/sdbox-storage.c --- a/src/lib-storage/index/dbox-single/sdbox-storage.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Fri Jan 04 01:12:24 2013 +0200 @@ -382,9 +382,9 @@ if (mailbox_open(box) < 0) return -1; } - if (update->cache_updates != NULL) - index_storage_mailbox_update_cache(box, update); - return sdbox_mailbox_create_indexes(box, update, NULL); + if (sdbox_mailbox_create_indexes(box, update, NULL) < 0) + return -1; + return index_storage_mailbox_update_common(box, update); } struct mail_storage sdbox_storage = { diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Fri Jan 04 01:12:24 2013 +0200 @@ -413,6 +413,7 @@ index_mail_get_keywords, index_mail_get_keyword_indexes, index_mail_get_modseq, + index_mail_get_pvt_modseq, index_mail_get_parts, index_mail_get_date, imapc_mail_get_received_date, @@ -429,6 +430,7 @@ index_mail_update_flags, index_mail_update_keywords, index_mail_update_modseq, + index_mail_update_pvt_modseq, NULL, index_mail_expunge, index_mail_set_cache_corrupted, diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/index-mail.c Fri Jan 04 01:12:24 2013 +0200 @@ -195,6 +195,24 @@ return mail->data.modseq; } +uint64_t index_mail_get_pvt_modseq(struct mail *_mail) +{ + struct index_mail *mail = (struct index_mail *)_mail; + + if (mail->data.pvt_modseq != 0) + return mail->data.pvt_modseq; + + if (mailbox_open_index_pvt(_mail->box) <= 0) + return 0; + index_transaction_init_pvt(_mail->transaction); + + mail_index_modseq_enable(_mail->box->index_pvt); + mail->data.pvt_modseq = + mail_index_modseq_lookup(_mail->transaction->view_pvt, + _mail->seq); + return mail->data.pvt_modseq; +} + const char *const *index_mail_get_keywords(struct mail *_mail) { struct index_mail *mail = (struct index_mail *)_mail; @@ -1749,6 +1767,13 @@ min_modseq); } +void index_mail_update_pvt_modseq(struct mail *mail, uint64_t min_pvt_modseq) +{ + index_transaction_init_pvt(mail->transaction); + mail_index_update_modseq(mail->transaction->itrans_pvt, mail->seq, + min_pvt_modseq); +} + void index_mail_expunge(struct mail *mail) { const char *value; diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/index-mail.h Fri Jan 04 01:12:24 2013 +0200 @@ -85,7 +85,7 @@ uint32_t seq; uint32_t cache_flags; - uint64_t modseq; + uint64_t modseq, pvt_modseq; enum index_mail_access_part access_part; /* dont_cache_fields overrides cache_fields */ enum mail_fetch_field cache_fetch_fields, dont_cache_fetch_fields; @@ -193,6 +193,7 @@ enum mail_flags index_mail_get_flags(struct mail *_mail); uint64_t index_mail_get_modseq(struct mail *_mail); +uint64_t index_mail_get_pvt_modseq(struct mail *_mail); const char *const *index_mail_get_keywords(struct mail *_mail); const ARRAY_TYPE(keyword_indexes) * index_mail_get_keyword_indexes(struct mail *_mail); @@ -220,6 +221,7 @@ void index_mail_update_keywords(struct mail *mail, enum modify_type modify_type, struct mail_keywords *keywords); void index_mail_update_modseq(struct mail *mail, uint64_t min_modseq); +void index_mail_update_pvt_modseq(struct mail *mail, uint64_t min_pvt_modseq); void index_mail_expunge(struct mail *mail); void index_mail_precache(struct mail *mail); void index_mail_set_cache_corrupted(struct mail *mail, diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/index-status.c Fri Jan 04 01:12:24 2013 +0200 @@ -144,6 +144,14 @@ status_r->highest_modseq = 1; } } + if ((items & STATUS_HIGHESTPVTMODSEQ) != 0 && box->view_pvt != NULL) { + status_r->highest_pvt_modseq = + mail_index_modseq_get_highest(box->view_pvt); + if (status_r->highest_pvt_modseq == 0) { + /* modseqs not enabled yet, but we can't return 0 */ + status_r->highest_pvt_modseq = 1; + } + } if ((items & STATUS_FIRST_UNSEEN_SEQ) != 0) { if (box->view_pvt == NULL || diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Thu Jan 03 22:53:18 2013 +0200 +++ b/src/lib-storage/index/index-storage.c Fri Jan 04 01:12:24 2013 +0200 @@ -366,8 +366,9 @@ mail_index_alloc_cache_unref(&box->index); } -void index_storage_mailbox_update_cache(struct mailbox *box, - const struct mailbox_update *update) +static void +index_storage_mailbox_update_cache(struct mailbox *box, + const struct mailbox_update *update) { const struct mailbox_cache_field *updates = update->cache_updates; ARRAY(struct mail_cache_field) new_fields; @@ -411,6 +412,49 @@ } } +static int +index_storage_mailbox_update_pvt(struct mailbox *box, + const struct mailbox_update *update) +{ + struct mail_index_transaction *trans; + struct mail_index_view *view; + int ret; + + if ((ret = mailbox_open_index_pvt(box)) <= 0) + return ret; + + mail_index_refresh(box->index_pvt); + view = mail_index_view_open(box->index_pvt); + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + if (update->min_highest_modseq != 0 && + mail_index_modseq_get_highest(view) < update->min_highest_pvt_modseq) { + mail_index_modseq_enable(box->index_pvt); + mail_index_update_highest_modseq(trans, + update->min_highest_pvt_modseq); + } + + if ((ret = mail_index_transaction_commit(&trans)) < 0) + mailbox_set_index_error(box); + mail_index_view_close(&view); + return ret; +} + +int index_storage_mailbox_update_common(struct mailbox *box, + const struct mailbox_update *update) +{ + int ret = 0; + + if (update->cache_updates != NULL) + index_storage_mailbox_update_cache(box, update); + + if (update->min_highest_pvt_modseq != 0) { + if (index_storage_mailbox_update_pvt(box, update) < 0) + ret = -1; + } + return ret; +} + int index_storage_mailbox_update(struct mailbox *box, const struct mailbox_update *update) { @@ -421,8 +465,6 @@ if (mailbox_open(box) < 0) return -1; - if (update->cache_updates != NULL) - index_storage_mailbox_update_cache(box, update); /* make sure we get the latest index info */ mail_index_refresh(box->index); @@ -469,7 +511,8 @@ if ((ret = mail_index_transaction_commit(&trans)) < 0) mailbox_set_index_error(box); mail_index_view_close(&view); - return ret; + return ret < 0 ? -1 : + index_storage_mailbox_update_common(box, update); } int index_storage_mailbox_create(struct mailbox *box, bool directory) diff -r b7d8027a8f74 -r 69371578720f src/lib-storage/index/index-storage.h From dovecot at dovecot.org Fri Jan 4 01:13:16 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 01:13:16 +0200 Subject: dovecot-2.2: dsync: Use private modseqs to support syncing priva... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5d90e2aa1ba8 changeset: 15496:5d90e2aa1ba8 user: Timo Sirainen date: Fri Jan 04 01:13:05 2013 +0200 description: dsync: Use private modseqs to support syncing private flags in shared mailboxes. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 24 +++++- src/doveadm/dsync/dsync-brain-mails.c | 3 + src/doveadm/dsync/dsync-ibc-stream.c | 31 ++++++++- src/doveadm/dsync/dsync-mail.c | 1 + src/doveadm/dsync/dsync-mail.h | 4 + src/doveadm/dsync/dsync-mailbox-export.c | 4 +- src/doveadm/dsync/dsync-mailbox-export.h | 1 - src/doveadm/dsync/dsync-mailbox-import.c | 73 ++++++++++++++++++----- src/doveadm/dsync/dsync-mailbox-import.h | 3 + src/doveadm/dsync/dsync-mailbox-state.c | 5 + src/doveadm/dsync/dsync-mailbox-state.h | 1 + src/doveadm/dsync/dsync-mailbox.h | 2 +- src/doveadm/dsync/dsync-transaction-log-scan.c | 81 +++++++++++++++++-------- src/doveadm/dsync/dsync-transaction-log-scan.h | 3 +- 14 files changed, 176 insertions(+), 60 deletions(-) diffs (truncated from 777 to 300 lines): diff -r 69371578720f -r 5d90e2aa1ba8 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Fri Jan 04 01:12:24 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Fri Jan 04 01:13:05 2013 +0200 @@ -140,17 +140,20 @@ { enum dsync_mailbox_exporter_flags exporter_flags = 0; uint32_t last_common_uid, highest_wanted_uid; - uint64_t last_common_modseq; + uint64_t last_common_modseq, last_common_pvt_modseq; i_assert(brain->log_scan == NULL); last_common_uid = brain->mailbox_state.last_common_uid; last_common_modseq = brain->mailbox_state.last_common_modseq; + last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq; highest_wanted_uid = last_common_uid == 0 ? (uint32_t)-1 : last_common_uid; if (dsync_transaction_log_scan_init(brain->box->view, + brain->box->view_pvt, highest_wanted_uid, last_common_modseq, + last_common_pvt_modseq, &brain->log_scan) < 0) { i_error("Failed to read transaction log for mailbox %s", mailbox_get_vname(brain->box)); @@ -165,7 +168,7 @@ brain->box_exporter = brain->backup_recv ? NULL : dsync_mailbox_export_init(brain->box, brain->log_scan, - last_common_uid, last_common_modseq, + last_common_uid, exporter_flags); return 0; } @@ -176,7 +179,7 @@ enum dsync_mailbox_import_flags import_flags = 0; const struct dsync_mailbox_state *state; uint32_t last_common_uid; - uint64_t last_common_modseq; + uint64_t last_common_modseq, last_common_pvt_modseq; i_assert(brain->box_importer == NULL); i_assert(brain->log_scan != NULL); @@ -191,9 +194,11 @@ if (state != NULL) { last_common_uid = state->last_common_uid; last_common_modseq = state->last_common_modseq; + last_common_pvt_modseq = state->last_common_pvt_modseq; } else { last_common_uid = 0; last_common_modseq = 0; + last_common_pvt_modseq = 0; } if (brain->guid_requests) @@ -208,9 +213,11 @@ brain->box_importer = brain->backup_send ? NULL : dsync_mailbox_import_init(brain->box, brain->log_scan, last_common_uid, last_common_modseq, + last_common_pvt_modseq, remote_dsync_box->uid_next, remote_dsync_box->first_recent_uid, remote_dsync_box->highest_modseq, + remote_dsync_box->highest_pvt_modseq, import_flags); } @@ -228,13 +235,14 @@ } if (brain->box_importer != NULL) { uint32_t last_common_uid; - uint64_t last_common_modseq; + uint64_t last_common_modseq, last_common_pvt_modseq; bool changes_during_sync; i_assert(brain->failed); (void)dsync_mailbox_import_deinit(&brain->box_importer, &last_common_uid, &last_common_modseq, + &last_common_pvt_modseq, &changes_during_sync); } if (brain->log_scan != NULL) @@ -248,7 +256,8 @@ { const enum mailbox_status_items status_items = STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES | - STATUS_FIRST_RECENT_UID | STATUS_HIGHESTMODSEQ; + STATUS_FIRST_RECENT_UID | STATUS_HIGHESTMODSEQ | + STATUS_HIGHESTPVTMODSEQ; const enum mailbox_metadata_items metadata_items = MAILBOX_METADATA_CACHE_FIELDS | MAILBOX_METADATA_GUID; struct mailbox_status status; @@ -283,6 +292,7 @@ dsync_box_r->messages_count = status.messages; dsync_box_r->first_recent_uid = status.first_recent_uid; dsync_box_r->highest_modseq = status.highest_modseq; + dsync_box_r->highest_pvt_modseq = status.highest_pvt_modseq; dsync_box_r->cache_fields = *metadata.cache_fields; return 1; } @@ -300,7 +310,8 @@ return state == NULL || state->last_uidvalidity != dsync_box->uid_validity || state->last_common_uid+1 != dsync_box->uid_next || - state->last_common_modseq != dsync_box->highest_modseq; + state->last_common_modseq != dsync_box->highest_modseq || + state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq; } static int @@ -406,6 +417,7 @@ const struct dsync_mailbox *box2) { return box1->highest_modseq != box2->highest_modseq || + box1->highest_pvt_modseq != box2->highest_pvt_modseq || box1->messages_count != box2->messages_count || box1->uid_next != box2->uid_next || box1->uid_validity != box2->uid_validity || diff -r 69371578720f -r 5d90e2aa1ba8 src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Fri Jan 04 01:12:24 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Fri Jan 04 01:13:05 2013 +0200 @@ -165,10 +165,13 @@ state.last_common_uid = brain->local_dsync_box.uid_next-1; state.last_common_modseq = brain->local_dsync_box.highest_modseq; + state.last_common_pvt_modseq = + brain->local_dsync_box.highest_pvt_modseq; } else { if (dsync_mailbox_import_deinit(&brain->box_importer, &state.last_common_uid, &state.last_common_modseq, + &state.last_common_pvt_modseq, &changes_during_sync) < 0) { i_error("Importing mailbox %s failed", mailbox_get_vname(brain->box)); diff -r 69371578720f -r 5d90e2aa1ba8 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Jan 04 01:12:24 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Fri Jan 04 01:13:05 2013 +0200 @@ -66,7 +66,7 @@ { .name = "mailbox_state", .chr = 'S', .required_keys = "mailbox_guid last_uidvalidity last_common_uid " - "last_common_modseq" + "last_common_modseq last_common_pvt_modseq" }, { .name = "mailbox_tree_node", .chr = 'N', @@ -81,14 +81,14 @@ }, { .name = "mailbox", .chr = 'B', - .required_keys = "mailbox_guid uid_validity uid_next " - "messages_count first_recent_uid highest_modseq", + .required_keys = "mailbox_guid uid_validity uid_next messages_count " + "first_recent_uid highest_modseq highest_pvt_modseq", .optional_keys = "mailbox_lost cache_fields" }, { .name = "mail_change", .chr = 'C', .required_keys = "type uid", - .optional_keys = "guid hdr_hash modseq save_timestamp " + .optional_keys = "guid hdr_hash modseq pvt_modseq save_timestamp " "add_flags remove_flags final_flags " "keywords_reset keyword_changes" }, @@ -642,6 +642,8 @@ dec2str(state->last_common_uid)); dsync_serializer_encode_add(encoder, "last_common_modseq", dec2str(state->last_common_modseq)); + dsync_serializer_encode_add(encoder, "last_common_pvt_modseq", + dec2str(state->last_common_pvt_modseq)); dsync_serializer_encode_finish(&encoder, str); dsync_ibc_stream_send_string(ibc, str); @@ -682,6 +684,11 @@ dsync_ibc_input_error(ibc, decoder, "Invalid last_common_modseq"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } + value = dsync_deserializer_decode_get(decoder, "last_common_pvt_modseq"); + if (str_to_uint64(value, &state_r->last_common_pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_common_pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } return DSYNC_IBC_RECV_RET_OK; } @@ -1002,6 +1009,8 @@ dec2str(dsync_box->first_recent_uid)); dsync_serializer_encode_add(encoder, "highest_modseq", dec2str(dsync_box->highest_modseq)); + dsync_serializer_encode_add(encoder, "highest_pvt_modseq", + dec2str(dsync_box->highest_pvt_modseq)); value = get_cache_fields(ibc, dsync_box); if (value != NULL) @@ -1113,6 +1122,11 @@ dsync_ibc_input_error(ibc, decoder, "Invalid highest_modseq"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } + value = dsync_deserializer_decode_get(decoder, "highest_pvt_modseq"); + if (str_to_uint64(value, &box->highest_pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid highest_pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } p_array_init(&box->cache_fields, pool, 32); if (dsync_deserializer_decode_try(decoder, "cache_fields", &value)) { @@ -1164,6 +1178,10 @@ dsync_serializer_encode_add(encoder, "modseq", dec2str(change->modseq)); } + if (change->pvt_modseq != 0) { + dsync_serializer_encode_add(encoder, "pvt_modseq", + dec2str(change->pvt_modseq)); + } if (change->save_timestamp != 0) { dsync_serializer_encode_add(encoder, "save_timestamp", dec2str(change->save_timestamp)); @@ -1252,6 +1270,11 @@ dsync_ibc_input_error(ibc, decoder, "Invalid modseq"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } + if (dsync_deserializer_decode_try(decoder, "pvt_modseq", &value) && + str_to_uint64(value, &change->pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } if (dsync_deserializer_decode_try(decoder, "save_timestamp", &value) && str_to_time(value, &change->save_timestamp) < 0) { dsync_ibc_input_error(ibc, decoder, "Invalid save_timestamp"); diff -r 69371578720f -r 5d90e2aa1ba8 src/doveadm/dsync/dsync-mail.c --- a/src/doveadm/dsync/dsync-mail.c Fri Jan 04 01:12:24 2013 +0200 +++ b/src/doveadm/dsync/dsync-mail.c Fri Jan 04 01:13:05 2013 +0200 @@ -73,6 +73,7 @@ } dest_r->hdr_hash = p_strdup(pool, src->hdr_hash); dest_r->modseq = src->modseq; + dest_r->pvt_modseq = src->pvt_modseq; dest_r->save_timestamp = src->save_timestamp; dest_r->add_flags = src->add_flags; diff -r 69371578720f -r 5d90e2aa1ba8 src/doveadm/dsync/dsync-mail.h --- a/src/doveadm/dsync/dsync-mail.h Fri Jan 04 01:12:24 2013 +0200 +++ b/src/doveadm/dsync/dsync-mail.h Fri Jan 04 01:13:05 2013 +0200 @@ -49,6 +49,9 @@ /* Message's current modseq (saves, flag changes) */ uint64_t modseq; + /* Message's current private modseq (for private flags in + shared mailboxes, otherwise 0) */ + uint64_t pvt_modseq; /* Message's save timestamp (saves) */ time_t save_timestamp; @@ -57,6 +60,7 @@ /* Flags added/removed since last sync, and final flags containing flags that exist now but haven't changed */ uint8_t add_flags, remove_flags, final_flags; + uint8_t add_pvt_flags, remove_pvt_flags; /* Remove all keywords before applying changes. This is used only with old transaction logs, new ones never reset keywords (just explicitly remove unwanted keywords) */ diff -r 69371578720f -r 5d90e2aa1ba8 src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Fri Jan 04 01:12:24 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Fri Jan 04 01:13:05 2013 +0200 @@ -21,7 +21,6 @@ struct mailbox *box; struct dsync_transaction_log_scan *log_scan; uint32_t last_common_uid; - uint64_t last_common_modseq; struct mailbox_transaction_context *trans; struct mail_search_context *search_ctx; @@ -99,6 +98,7 @@ i_assert((change->add_flags & change->remove_flags) == 0); change->modseq = mail_get_modseq(mail); + change->pvt_modseq = mail_get_pvt_modseq(mail); change->final_flags = mail_get_flags(mail) & MAIL_FLAGS_NONRECENT; keywords = mail_get_keywords(mail); @@ -416,7 +416,6 @@ dsync_mailbox_export_init(struct mailbox *box, struct dsync_transaction_log_scan *log_scan, uint32_t last_common_uid, - uint64_t last_common_modseq, enum dsync_mailbox_exporter_flags flags) { struct dsync_mailbox_exporter *exporter; @@ -429,7 +428,6 @@ exporter->box = box; exporter->log_scan = log_scan; exporter->last_common_uid = last_common_uid; - exporter->last_common_modseq = last_common_modseq; exporter->auto_export_mails = (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0; exporter->mails_have_guids = diff -r 69371578720f -r 5d90e2aa1ba8 src/doveadm/dsync/dsync-mailbox-export.h --- a/src/doveadm/dsync/dsync-mailbox-export.h Fri Jan 04 01:12:24 2013 +0200 From dovecot at dovecot.org Fri Jan 4 01:44:09 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 01:44:09 +0200 Subject: dovecot-2.2: lib-storage: Bypass virtual storage when copying ma... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5bb879b6e3f3 changeset: 15497:5bb879b6e3f3 user: Timo Sirainen date: Fri Jan 04 01:43:59 2013 +0200 description: lib-storage: Bypass virtual storage when copying mail to allow optimizations. As a side effect this also causes all plugins to see the backend mail instead of the virtual mail. This probably isn't bad in most situations, but for those where it's unwanted mail_private.vmail can be used to access the virtual mail. mail_log plugin does this to log the original virtual mailbox name for copy operations. diffstat: src/lib-storage/mail-storage-private.h | 6 ++++++ src/lib-storage/mail-storage.c | 6 +++++- src/plugins/mail-log/mail-log-plugin.c | 9 ++++++++- src/plugins/virtual/virtual-mail.c | 4 ++++ 4 files changed, 23 insertions(+), 2 deletions(-) diffs (82 lines): diff -r 5d90e2aa1ba8 -r 5bb879b6e3f3 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Fri Jan 04 01:13:05 2013 +0200 +++ b/src/lib-storage/mail-storage-private.h Fri Jan 04 01:43:59 2013 +0200 @@ -398,6 +398,12 @@ struct mail_private { struct mail mail; struct mail_vfuncs v, *vlast; + /* normally NULL, but in case this is a "backend mail" for a mail + created by virtual storage, this points back to the original virtual + mail. at least mailbox_copy() bypasses the virtual storage, so this + allows mail_log plugin to log the copy operation using the original + mailbox name. */ + struct mail *vmail; uint32_t seq_pvt; diff -r 5d90e2aa1ba8 -r 5bb879b6e3f3 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Fri Jan 04 01:13:05 2013 +0200 +++ b/src/lib-storage/mail-storage.c Fri Jan 04 01:43:59 2013 +0200 @@ -1942,6 +1942,7 @@ struct mail_save_context *ctx = *_ctx; struct mailbox *box = ctx->transaction->box; struct mail_keywords *keywords = ctx->data.keywords; + struct mail *real_mail; int ret; *_ctx = NULL; @@ -1952,7 +1953,10 @@ return -1; } - ret = ctx->transaction->box->v.copy(ctx, mail); + /* bypass virtual storage, so hard linking can be used whenever + possible */ + real_mail = mail_get_real_mail(mail); + ret = ctx->transaction->box->v.copy(ctx, real_mail); if (keywords != NULL) mailbox_keywords_unref(&keywords); return ret; diff -r 5d90e2aa1ba8 -r 5bb879b6e3f3 src/plugins/mail-log/mail-log-plugin.c --- a/src/plugins/mail-log/mail-log-plugin.c Fri Jan 04 01:13:05 2013 +0200 +++ b/src/plugins/mail-log/mail-log-plugin.c Fri Jan 04 01:43:59 2013 +0200 @@ -360,10 +360,17 @@ { struct mail_log_mail_txn_context *ctx = (struct mail_log_mail_txn_context *)txn; + struct mail_private *src_pmail = (struct mail_private *)src; + struct mailbox *src_box = src->box; const char *desc; + if (src_pmail->vmail != NULL) { + /* copying a mail from virtual storage. src points to the + backend mail, but we want to log the virtual mailbox name. */ + src_box = src_pmail->vmail->box; + } desc = t_strdup_printf("copy from %s", - str_sanitize(mailbox_get_vname(src->box), + str_sanitize(mailbox_get_vname(src_box), MAILBOX_NAME_LOG_LEN)); mail_log_append_mail_message(ctx, dst, MAIL_LOG_EVENT_COPY, desc); diff -r 5d90e2aa1ba8 -r 5bb879b6e3f3 src/plugins/virtual/virtual-mail.c --- a/src/plugins/virtual/virtual-mail.c Fri Jan 04 01:13:05 2013 +0200 +++ b/src/plugins/virtual/virtual-mail.c Fri Jan 04 01:43:59 2013 +0200 @@ -90,6 +90,7 @@ struct virtual_backend_box *bbox) { struct virtual_mail *vmail = (struct virtual_mail *)mail; + struct mail_private *backend_pmail; struct mailbox_transaction_context *backend_trans; struct mailbox_header_lookup_ctx *backend_headers; @@ -102,6 +103,9 @@ backend_headers); if (backend_headers != NULL) mailbox_header_lookup_unref(&backend_headers); + + backend_pmail = (struct mail_private *)vmail->backend_mail; + backend_pmail->vmail = mail; array_append(&vmail->backend_mails, &vmail->backend_mail, 1); return vmail->backend_mail; } From dovecot at dovecot.org Fri Jan 4 03:50:47 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 03:50:47 +0200 Subject: dovecot-2.1: Added buffer_verify_pool() Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/9fda4feb6e2d changeset: 14861:9fda4feb6e2d user: Timo Sirainen date: Fri Jan 04 03:50:18 2013 +0200 description: Added buffer_verify_pool() diffstat: src/lib/buffer.c | 14 ++++++++++++++ src/lib/buffer.h | 6 ++++++ 2 files changed, 20 insertions(+), 0 deletions(-) diffs (35 lines): diff -r abcc9959b757 -r 9fda4feb6e2d src/lib/buffer.c --- a/src/lib/buffer.c Thu Jan 03 07:30:54 2013 +0200 +++ b/src/lib/buffer.c Fri Jan 04 03:50:18 2013 +0200 @@ -314,3 +314,17 @@ return memcmp(buf1->data, buf2->data, buf1->used) == 0; } + +void buffer_verify_pool(buffer_t *_buf) +{ + const struct real_buffer *buf = (const struct real_buffer *)_buf; + void *ret; + + if (buf->pool->datastack_pool) { + /* this doesn't really do anything except verify the + stack frame */ + ret = p_realloc(buf->pool, buf->w_buffer, + buf->alloc, buf->alloc); + i_assert(ret == buf->w_buffer); + } +} diff -r abcc9959b757 -r 9fda4feb6e2d src/lib/buffer.h --- a/src/lib/buffer.h Thu Jan 03 07:30:54 2013 +0200 +++ b/src/lib/buffer.h Fri Jan 04 03:50:18 2013 +0200 @@ -102,4 +102,10 @@ return buf->used; } +/* Crash if buffer was allocated from data stack and stack frame has changed. + This can be used as an assert-like check to verify that it's valid to + increase the buffer size here, instead of crashing only randomly when the + buffer needs to be increased. */ +void buffer_verify_pool(buffer_t *buf); + #endif From dovecot at dovecot.org Fri Jan 4 03:50:47 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 03:50:47 +0200 Subject: dovecot-2.1: DEBUG: Call buffer_verify_pool() on str_c*() Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/a7a45a56211c changeset: 14862:a7a45a56211c user: Timo Sirainen date: Fri Jan 04 03:50:35 2013 +0200 description: DEBUG: Call buffer_verify_pool() on str_c*() diffstat: src/lib/str.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 9fda4feb6e2d -r a7a45a56211c src/lib/str.c --- a/src/lib/str.c Fri Jan 04 03:50:18 2013 +0200 +++ b/src/lib/str.c Fri Jan 04 03:50:35 2013 +0200 @@ -45,6 +45,9 @@ size_t len = str_len(str); size_t alloc = buffer_get_size(str); +#ifdef DEBUG + buffer_verify_pool(str); +#endif if (len == alloc || data[len] != '\0') { buffer_write(str, len, "", 1); /* remove the \0 - we don't want to keep it */ From dovecot at dovecot.org Fri Jan 4 05:29:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 05:29:30 +0200 Subject: dovecot-2.1: acl: Treat empty plugin { acl } setting the same as... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/721aba9128b9 changeset: 14863:721aba9128b9 user: Timo Sirainen date: Fri Jan 04 05:29:25 2013 +0200 description: acl: Treat empty plugin { acl } setting the same as if it didn't exist. This allows using -o plugin/acl= to disable ACLs. diffstat: src/plugins/acl/acl-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r a7a45a56211c -r 721aba9128b9 src/plugins/acl/acl-storage.c --- a/src/plugins/acl/acl-storage.c Fri Jan 04 03:50:35 2013 +0200 +++ b/src/plugins/acl/acl-storage.c Fri Jan 04 05:29:25 2013 +0200 @@ -52,7 +52,7 @@ const char *env; env = mail_user_plugin_getenv(user, "acl"); - if (env != NULL) + if (env != NULL && *env != '\0') acl_mail_user_create(user, env); else { if (user->mail_debug) From dovecot at dovecot.org Fri Jan 4 05:49:37 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 05:49:37 +0200 Subject: dovecot-2.2: configure: Use pkg-config with clucene if available. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/af5265e74b9e changeset: 15498:af5265e74b9e user: Timo Sirainen date: Fri Jan 04 05:49:33 2013 +0200 description: configure: Use pkg-config with clucene if available. diffstat: configure.ac | 5 +++++ src/plugins/fts-lucene/Makefile.am | 5 ++++- 2 files changed, 9 insertions(+), 1 deletions(-) diffs (37 lines): diff -r 5bb879b6e3f3 -r af5265e74b9e configure.ac --- a/configure.ac Fri Jan 04 01:43:59 2013 +0200 +++ b/configure.ac Fri Jan 04 05:49:33 2013 +0200 @@ -2660,6 +2660,11 @@ have_lucene=no if test "$want_lucene" = "yes"; then + PKG_CHECK_MODULES(CLUCENE, libclucene-core,, [ + # no pkg-config file for clucene. fallback to defaults. + # FIXME: we should verify here that this actually works.. + CLUCENE_LIBS="-lclucene-shared -lclucene-core" + ]) if test $want_stemmer != no; then AC_CHECK_LIB(stemmer, sb_stemmer_new, [ have_lucene_stemmer=yes diff -r 5bb879b6e3f3 -r af5265e74b9e src/plugins/fts-lucene/Makefile.am --- a/src/plugins/fts-lucene/Makefile.am Fri Jan 04 01:43:59 2013 +0200 +++ b/src/plugins/fts-lucene/Makefile.am Fri Jan 04 05:49:33 2013 +0200 @@ -8,6 +8,9 @@ -I$(top_srcdir)/src/plugins/fts \ -I$(top_srcdir)/src/doveadm +AM_CXXFLAGS = \ + $(CLUCENE_CFLAGS) + NOPLUGIN_LDFLAGS = lib21_fts_lucene_plugin_la_LDFLAGS = -module -avoid-version lib20_doveadm_fts_lucene_plugin_la_LDFLAGS = -module -avoid-version @@ -28,7 +31,7 @@ endif lib21_fts_lucene_plugin_la_LIBADD = \ - -lclucene-shared -lclucene-core $(TEXTCAT_LIBS) $(STEMMER_LIBS) + $(CLUCENE_LIBS) $(TEXTCAT_LIBS) $(STEMMER_LIBS) lib21_fts_lucene_plugin_la_SOURCES = \ fts-lucene-plugin.c \ From dovecot at dovecot.org Fri Jan 4 21:39:18 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 21:39:18 +0200 Subject: dovecot-2.2: example-config: Added a note about protocol/local/r... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8863e68291bd changeset: 15499:8863e68291bd user: Timo Sirainen date: Fri Jan 04 21:39:07 2013 +0200 description: example-config: Added a note about protocol/local/remote {} sections. diffstat: doc/example-config/dovecot.conf | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r af5265e74b9e -r 8863e68291bd doc/example-config/dovecot.conf --- a/doc/example-config/dovecot.conf Fri Jan 04 05:49:33 2013 +0200 +++ b/doc/example-config/dovecot.conf Fri Jan 04 21:39:07 2013 +0200 @@ -9,6 +9,10 @@ # and tabs are ignored. If you want to use either of these explicitly, put the # value inside quotes, eg.: key = "# char and trailing whitespace " +# Most (but not all) settings can be overridden by different protocols and/or +# source/destination IPs by placing the settings inside sections, for example: +# protocol imap { }, local 127.0.0.1 { }, remote 10.0.0.0/8 { } + # Default values are shown for each setting, it's not required to uncomment # those. These are exceptions to this though: No sections (e.g. namespace {}) # or plugin settings are added by default, they're listed only as examples. From dovecot at dovecot.org Fri Jan 4 23:24:41 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 23:24:41 +0200 Subject: dovecot-2.1: uni_utf8_*(): Treat overlong UTF8 sequences as inva... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/7be4709aab5e changeset: 14864:7be4709aab5e user: Timo Sirainen date: Fri Jan 04 23:24:26 2013 +0200 description: uni_utf8_*(): Treat overlong UTF8 sequences as invalid. diffstat: src/lib/Makefile.am | 1 + src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + src/lib/test-unichar.c | 24 ++++++++++++++++++++++++ src/lib/unichar.c | 28 ++++++++++++++-------------- 5 files changed, 41 insertions(+), 14 deletions(-) diffs (125 lines): diff -r 721aba9128b9 -r 7be4709aab5e src/lib/Makefile.am --- a/src/lib/Makefile.am Fri Jan 04 05:29:25 2013 +0200 +++ b/src/lib/Makefile.am Fri Jan 04 23:24:26 2013 +0200 @@ -273,6 +273,7 @@ test-str-find.c \ test-str-sanitize.c \ test-time-util.c \ + test-unichar.c \ test-utc-mktime.c \ test-var-expand.c diff -r 721aba9128b9 -r 7be4709aab5e src/lib/test-lib.c --- a/src/lib/test-lib.c Fri Jan 04 05:29:25 2013 +0200 +++ b/src/lib/test-lib.c Fri Jan 04 23:24:26 2013 +0200 @@ -30,6 +30,7 @@ test_str_find, test_str_sanitize, test_time_util, + test_unichar, test_utc_mktime, test_var_expand, NULL diff -r 721aba9128b9 -r 7be4709aab5e src/lib/test-lib.h --- a/src/lib/test-lib.h Fri Jan 04 05:29:25 2013 +0200 +++ b/src/lib/test-lib.h Fri Jan 04 23:24:26 2013 +0200 @@ -29,6 +29,7 @@ void test_str_find(void); void test_str_sanitize(void); void test_time_util(void); +void test_unichar(void); void test_utc_mktime(void); void test_var_expand(void); diff -r 721aba9128b9 -r 7be4709aab5e src/lib/test-unichar.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-unichar.c Fri Jan 04 23:24:26 2013 +0200 @@ -0,0 +1,24 @@ +/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "str.h" +#include "unichar.h" + +void test_unichar(void) +{ + static const char *overlong_utf8 = "\xf8\x80\x95\x81\xa1"; + unichar_t chr, chr2; + string_t *str = t_str_new(16); + + test_begin("unichars"); + for (chr = 0; chr <= 0x10ffff; chr++) { + str_truncate(str, 0); + uni_ucs4_to_utf8_c(chr, str); + test_assert(uni_utf8_str_is_valid(str_c(str))); + test_assert(uni_utf8_get_char(str_c(str), &chr2) > 0); + test_assert(chr2 == chr); + } + test_assert(!uni_utf8_str_is_valid(overlong_utf8)); + test_assert(uni_utf8_get_char(overlong_utf8, &chr2) < 0); + test_end(); +} diff -r 721aba9128b9 -r 7be4709aab5e src/lib/unichar.c --- a/src/lib/unichar.c Fri Jan 04 05:29:25 2013 +0200 +++ b/src/lib/unichar.c Fri Jan 04 23:24:26 2013 +0200 @@ -37,8 +37,10 @@ int uni_utf8_get_char_n(const void *_input, size_t max_len, unichar_t *chr_r) { + static unichar_t lowest_valid_chr_table[] = + { 0, 0, 0x80, 0x800, 0x10000, 0x20000, 0x40000 }; const unsigned char *input = _input; - unichar_t chr; + unichar_t chr, lowest_valid_chr; unsigned int i, len; int ret; @@ -75,10 +77,12 @@ return -1; } - if (len <= max_len) + if (len <= max_len) { + lowest_valid_chr = lowest_valid_chr_table[len]; ret = 1; - else { + } else { /* check first if the input is invalid before returning 0 */ + lowest_valid_chr = 0; ret = 0; len = max_len; } @@ -91,6 +95,10 @@ chr <<= 6; chr |= input[i] & 0x3f; } + if (chr < lowest_valid_chr) { + /* overlong encoding */ + return -1; + } *chr_r = chr; return ret; @@ -340,19 +348,11 @@ static inline unsigned int is_valid_utf8_seq(const unsigned char *input, unsigned int size) { - unsigned int i, len; + unichar_t chr; - len = uni_utf8_char_bytes(input[0]); - if (unlikely(len > size || len == 1)) + if (uni_utf8_get_char_n(input, size, &chr) <= 0) return 0; - - /* the rest of the chars should be in 0x80..0xbf range. - anything else is start of a sequence or invalid */ - for (i = 1; i < len; i++) { - if (unlikely(input[i] < 0x80 || input[i] > 0xbf)) - return 0; - } - return len; + return uni_utf8_char_bytes(input[0]); } static int uni_utf8_find_invalid_pos(const unsigned char *input, size_t size, From dovecot at dovecot.org Fri Jan 4 23:59:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 04 Jan 2013 23:59:55 +0200 Subject: dovecot-2.1: quota: Don't crash if shared namespace's owner does... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c91d142130be changeset: 14865:c91d142130be user: Timo Sirainen date: Fri Jan 04 23:59:42 2013 +0200 description: quota: Don't crash if shared namespace's owner doesn't have quota enabled (but you have). diffstat: src/plugins/quota/quota-storage.c | 18 ++++++++++++------ 1 files changed, 12 insertions(+), 6 deletions(-) diffs (50 lines): diff -r 7be4709aab5e -r c91d142130be src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Fri Jan 04 23:24:26 2013 +0200 +++ b/src/plugins/quota/quota-storage.c Fri Jan 04 23:59:42 2013 +0200 @@ -520,6 +520,7 @@ struct quota_mailbox_list *qlist; struct quota *quota = NULL; struct quota_root *root; + struct mail_user *quota_user; bool add; if (QUOTA_USER_CONTEXT(list->ns->user) == NULL) @@ -528,8 +529,14 @@ /* see if we have a quota explicitly defined for this namespace */ quota = quota_get_mail_user_quota(list->ns->user); root = quota_find_root_for_ns(quota, list->ns); - if (root != NULL) + if (root != NULL) { + /* explicit quota root */ root->ns = list->ns; + quota_user = list->ns->user; + } else { + quota_user = list->ns->owner != NULL ? + list->ns->owner : list->ns->user; + } if ((list->ns->flags & NAMESPACE_FLAG_NOQUOTA) != 0) add = FALSE; @@ -538,7 +545,9 @@ explicitly defined for it */ add = root != NULL; } else { - add = TRUE; + /* for shared namespaces add only if the owner has quota + enabled */ + add = QUOTA_USER_CONTEXT(quota_user) != NULL; } if (add) { @@ -550,10 +559,7 @@ v->deinit = quota_mailbox_list_deinit; MODULE_CONTEXT_SET(list, quota_mailbox_list_module, qlist); - /* register to owner's quota roots */ - quota = list->ns->owner != NULL ? - quota_get_mail_user_quota(list->ns->owner) : - quota_get_mail_user_quota(list->ns->user); + quota = quota_get_mail_user_quota(quota_user); quota_add_user_namespace(quota, list->ns); } } From dovecot at dovecot.org Sat Jan 5 00:13:01 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 00:13:01 +0200 Subject: dovecot-2.1: lazy_expunge: Allow expunging from lazy_expunge nam... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/fc521e6378d5 changeset: 14866:fc521e6378d5 user: Timo Sirainen date: Sat Jan 05 00:12:54 2013 +0200 description: lazy_expunge: Allow expunging from lazy_expunge namespace via virtual mailbox. (Instead of copying the message all over again to the lazy_expunge namespace.) diffstat: src/plugins/lazy-expunge/lazy-expunge-plugin.c | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diffs (24 lines): diff -r c91d142130be -r fc521e6378d5 src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Fri Jan 04 23:59:42 2013 +0200 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Sat Jan 05 00:12:54 2013 +0200 @@ -121,9 +121,20 @@ union mail_module_context *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail); struct lazy_expunge_transaction *lt = LAZY_EXPUNGE_CONTEXT(_mail->transaction); + struct lazy_expunge_mailbox_list *llist; + struct mailbox *real_box; struct mail_save_context *save_ctx; const char *error; + /* don't copy the mail if we're expunging from lazy_expunge + namespace (even if it's via a virtual mailbox) */ + real_box = mail_get_real_mail(_mail)->box; + llist = LAZY_EXPUNGE_LIST_CONTEXT(real_box->list); + if (llist != NULL && llist->internal_namespace) { + mmail->super.expunge(_mail); + return; + } + if (lt->dest_box == NULL) { lt->dest_box = mailbox_open_or_create(luser->lazy_ns->list, _mail->box, &error); From dovecot at dovecot.org Sat Jan 5 00:37:37 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 00:37:37 +0200 Subject: dovecot-2.2: auth: Don't keep username duplicated in userdb_repl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/922049229f7f changeset: 15500:922049229f7f user: Timo Sirainen date: Sat Jan 05 00:37:26 2013 +0200 description: auth: Don't keep username duplicated in userdb_reply string. This fixes bugs where userdb_reply is accessed via auth_stream_*() functions that remove/replace existing fields, which may have ended up removing/replacing the username. diffstat: src/auth/auth-master-connection.c | 2 ++ src/auth/auth-request-handler.c | 5 +++-- src/auth/auth-request.c | 22 ---------------------- 3 files changed, 5 insertions(+), 24 deletions(-) diffs (87 lines): diff -r 8863e68291bd -r 922049229f7f src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Fri Jan 04 21:39:07 2013 +0200 +++ b/src/auth/auth-master-connection.c Sat Jan 05 00:37:26 2013 +0200 @@ -276,6 +276,8 @@ break; case USERDB_RESULT_OK: str_printfa(str, "USER\t%u\t", auth_request->id); + str_append_tabescaped(str, auth_request->user); + str_append_c(str, '\t'); str_append(str, auth_stream_reply_export(reply)); break; } diff -r 8863e68291bd -r 922049229f7f src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Fri Jan 04 21:39:07 2013 +0200 +++ b/src/auth/auth-request-handler.c Sat Jan 05 00:37:26 2013 +0200 @@ -661,8 +661,6 @@ auth_stream_reply_add(reply, NULL, dec2str(request->id)); break; case USERDB_RESULT_OK: - auth_stream_reply_add(reply, "USER", NULL); - auth_stream_reply_add(reply, NULL, dec2str(request->id)); if (request->master_user != NULL && auth_stream_reply_find(request->userdb_reply, "master_user") == NULL) { @@ -680,6 +678,9 @@ "anonymous", NULL); } + auth_stream_reply_add(reply, "USER", NULL); + auth_stream_reply_add(reply, NULL, dec2str(request->id)); + auth_stream_reply_add(reply, NULL, request->user); auth_stream_reply_import(reply, auth_stream_reply_export(request->userdb_reply)); diff -r 8863e68291bd -r 922049229f7f src/auth/auth-request.c --- a/src/auth/auth-request.c Fri Jan 04 21:39:07 2013 +0200 +++ b/src/auth/auth-request.c Sat Jan 05 00:37:26 2013 +0200 @@ -438,22 +438,6 @@ result == PASSDB_RESULT_OK); } -static void auth_request_userdb_reply_update_user(struct auth_request *request) -{ - const char *str, *p; - - str = t_strdup(auth_stream_reply_export(request->userdb_reply)); - - /* reset the reply and add the new username */ - auth_stream_reply_reset(request->userdb_reply); - auth_stream_reply_add(request->userdb_reply, NULL, request->user); - - /* add the rest */ - p = strchr(str, '\t'); - if (p != NULL) - auth_stream_reply_import(request->userdb_reply, p + 1); -} - static bool auth_request_master_lookup_finish(struct auth_request *request) { struct auth_passdb *passdb; @@ -468,8 +452,6 @@ request->master_user = request->user; request->user = request->requested_login_user; request->requested_login_user = NULL; - if (request->userdb_reply != NULL) - auth_request_userdb_reply_update_user(request); request->skip_password_check = TRUE; request->passdb_password = NULL; @@ -1261,8 +1243,6 @@ "username changed %s -> %s", request->user, new_value); request->user = p_strdup(request->pool, new_value); - if (request->userdb_reply != NULL) - auth_request_userdb_reply_update_user(request); } return TRUE; } @@ -1367,8 +1347,6 @@ struct userdb_module *module = request->userdb->userdb; request->userdb_reply = auth_stream_reply_init(request->pool); - auth_stream_reply_add(request->userdb_reply, NULL, request->user); - userdb_template_export(module->default_fields_tmpl, request); } From dovecot at dovecot.org Sat Jan 5 00:50:10 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 00:50:10 +0200 Subject: dovecot-2.1: auth: Fixed handling usernames with the same name a... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/3eeb5270963b changeset: 14867:3eeb5270963b user: Timo Sirainen date: Sat Jan 05 00:50:04 2013 +0200 description: auth: Fixed handling usernames with the same name as userdb extra fields. This is a bit kludgy fix, but pretty much guaranteed not to break anything. v2.2 has a cleaner fix, but it's a bit more invasive. diffstat: src/auth/auth-request.c | 4 ++-- src/auth/auth-stream.c | 22 +++++++++++++++++++--- src/auth/auth-stream.h | 1 + 3 files changed, 22 insertions(+), 5 deletions(-) diffs (79 lines): diff -r fc521e6378d5 -r 3eeb5270963b src/auth/auth-request.c --- a/src/auth/auth-request.c Sat Jan 05 00:12:54 2013 +0200 +++ b/src/auth/auth-request.c Sat Jan 05 00:50:04 2013 +0200 @@ -863,7 +863,7 @@ } *result_r = USERDB_RESULT_OK; - *reply_r = auth_stream_reply_init(request->pool); + *reply_r = auth_stream_reply_init_userdb(request->pool); auth_stream_reply_import(*reply_r, value); return TRUE; } @@ -1342,7 +1342,7 @@ { struct userdb_module *module = request->userdb->userdb; - request->userdb_reply = auth_stream_reply_init(request->pool); + request->userdb_reply = auth_stream_reply_init_userdb(request->pool); auth_stream_reply_add(request->userdb_reply, NULL, request->user); userdb_template_export(module->default_fields_tmpl, request); diff -r fc521e6378d5 -r 3eeb5270963b src/auth/auth-stream.c --- a/src/auth/auth-stream.c Sat Jan 05 00:12:54 2013 +0200 +++ b/src/auth/auth-stream.c Sat Jan 05 00:50:04 2013 +0200 @@ -9,6 +9,7 @@ struct auth_stream_reply { string_t *str; + bool userdb; }; struct auth_stream_reply *auth_stream_reply_init(pool_t pool) @@ -20,6 +21,15 @@ return reply; } +struct auth_stream_reply *auth_stream_reply_init_userdb(pool_t pool) +{ + struct auth_stream_reply *reply; + + reply = auth_stream_reply_init(pool); + reply->userdb = TRUE; + return reply; +} + void auth_stream_reply_add(struct auth_stream_reply *reply, const char *key, const char *value) { @@ -44,10 +54,16 @@ auth_stream_reply_find_area(struct auth_stream_reply *reply, const char *key, unsigned int *idx_r, unsigned int *len_r) { - const char *str = str_c(reply->str); - unsigned int i, start, key_len = strlen(key); + const char *p, *str = str_c(reply->str); + unsigned int i = 0, start, key_len = strlen(key); - i = 0; + if (reply->userdb) { + p = strchr(str, '\t'); + if (p == NULL) + return FALSE; + i = p-str+1; + } + while (str[i] != '\0') { start = i; for (; str[i] != '\0'; i++) { diff -r fc521e6378d5 -r 3eeb5270963b src/auth/auth-stream.h --- a/src/auth/auth-stream.h Sat Jan 05 00:12:54 2013 +0200 +++ b/src/auth/auth-stream.h Sat Jan 05 00:50:04 2013 +0200 @@ -4,6 +4,7 @@ struct auth_request; struct auth_stream_reply *auth_stream_reply_init(pool_t pool); +struct auth_stream_reply *auth_stream_reply_init_userdb(pool_t pool); void auth_stream_reply_add(struct auth_stream_reply *reply, const char *key, const char *value); void auth_stream_reply_reset(struct auth_stream_reply *reply); From dovecot at dovecot.org Sat Jan 5 01:14:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:33 +0200 Subject: dovecot-2.2: lib-lda: Avoid doing a costly mailbox sync when del... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9cdeab12f3e1 changeset: 15502:9cdeab12f3e1 user: Timo Sirainen date: Tue Oct 30 15:05:45 2012 +0200 description: lib-lda: Avoid doing a costly mailbox sync when delivering mail to multiple recipients. diffstat: src/lib-lda/mail-deliver.c | 3 ++- src/lib-storage/index/index-sync.c | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletions(-) diffs (34 lines): diff -r 9d63f882194d -r 9cdeab12f3e1 src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Mon Oct 29 19:04:36 2012 +0200 +++ b/src/lib-lda/mail-deliver.c Tue Oct 30 15:05:45 2012 +0200 @@ -331,7 +331,8 @@ ctx->saved_mail = TRUE; mail_deliver_log(ctx, "saved mail to %s", mailbox_name); - if (ctx->save_dest_mail && mailbox_sync(box, 0) == 0) { + if (ctx->save_dest_mail && + mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) == 0) { range = array_idx(&changes.saved_uids, 0); i_assert(range[0].seq1 == range[0].seq2); diff -r 9d63f882194d -r 9cdeab12f3e1 src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Mon Oct 29 19:04:36 2012 +0200 +++ b/src/lib-storage/index/index-sync.c Tue Oct 30 15:05:45 2012 +0200 @@ -31,6 +31,17 @@ ioloop_time < ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL) return FALSE; + if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 && + (box->flags & MAILBOX_FLAG_SAVEONLY) != 0) { + /* lib-lda is syncing the mailbox after saving a mail. + it only wants to find the new mail for potentially copying + to other mailboxes. that's mainly an optimization, and since + the mail was most likely already added to index we don't + need to do a full sync to find it. the main benefit here is + to avoid a very costly sync with a large Maildir/new/ */ + return FALSE; + } + if (ibox->notify_to != NULL) timeout_reset(ibox->notify_to); ibox->sync_last_check = ioloop_time; From dovecot at dovecot.org Sat Jan 5 01:14:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:33 +0200 Subject: dovecot-2.2: login proxy: For connect() failures log also the us... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9d63f882194d changeset: 15501:9d63f882194d user: Timo Sirainen date: Mon Oct 29 19:04:36 2012 +0200 description: login proxy: For connect() failures log also the used local IP:port if available. diffstat: src/login-common/login-proxy.c | 41 +++++++++++++++++++++++++++++------------ 1 files changed, 29 insertions(+), 12 deletions(-) diffs (74 lines): diff -r 67e9cb0b06ec -r 9d63f882194d src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Mon Oct 29 16:36:59 2012 +0200 +++ b/src/login-common/login-proxy.c Mon Oct 29 19:04:36 2012 +0200 @@ -5,6 +5,7 @@ #include "istream.h" #include "ostream.h" #include "llist.h" +#include "str.h" #include "str-sanitize.h" #include "time-util.h" #include "master-service.h" @@ -194,16 +195,33 @@ proxy->state_rec = NULL; } +static void +proxy_log_connect_error(struct login_proxy *proxy) +{ + string_t *str = t_str_new(128); + struct ip_addr local_ip; + unsigned int local_port; + + str_printfa(str, "proxy(%s): connect(%s, %u) failed: %m (after %u secs", + proxy->client->virtual_user, + proxy->host, proxy->port, + (unsigned int)(ioloop_time - proxy->created.tv_sec)); + + if (proxy->server_fd != -1 && + net_getsockname(proxy->server_fd, &local_ip, &local_port) == 0) { + str_printfa(str, ", local=%s:%u", + net_ip2addr(&local_ip), local_port); + } + + str_append_c(str, ')'); + i_error("%s", str_c(str)); +} + static void proxy_wait_connect(struct login_proxy *proxy) { - int err; - - err = net_geterror(proxy->server_fd); - if (err != 0) { - i_error("proxy(%s): connect(%s, %u) failed: %s (after %u secs)", - proxy->client->virtual_user, - proxy->host, proxy->port, strerror(err), - (unsigned int)(ioloop_time - proxy->created.tv_sec)); + errno = net_geterror(proxy->server_fd); + if (errno != 0) { + proxy_log_connect_error(proxy); proxy_fail_connect(proxy); login_proxy_free(&proxy); return; @@ -229,8 +247,8 @@ static void proxy_connect_timeout(struct login_proxy *proxy) { - i_error("proxy(%s): connect(%s, %u) timed out", - proxy->client->virtual_user, proxy->host, proxy->port); + errno = ETIMEDOUT; + proxy_log_connect_error(proxy); proxy_fail_connect(proxy); login_proxy_free(&proxy); } @@ -252,8 +270,7 @@ proxy->server_fd = net_connect_ip(&proxy->ip, proxy->port, NULL); if (proxy->server_fd == -1) { - i_error("proxy(%s): connect(%s, %u) failed: %m", - proxy->client->virtual_user, proxy->host, proxy->port); + proxy_log_connect_error(proxy); login_proxy_free(&proxy); return -1; } From dovecot at dovecot.org Sat Jan 5 01:14:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:33 +0200 Subject: dovecot-2.2: Makefile: Add -lssl to installed dovecot-config's L... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7d931927e4ac changeset: 15503:7d931927e4ac user: Timo Sirainen date: Wed Oct 31 13:24:49 2012 +0200 description: Makefile: Add -lssl to installed dovecot-config's LIBDOVECOT_STORAGE if needed. diffstat: Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 9cdeab12f3e1 -r 7d931927e4ac Makefile.am --- a/Makefile.am Tue Oct 30 15:05:45 2012 +0200 +++ b/Makefile.am Wed Oct 31 13:24:49 2012 +0200 @@ -66,7 +66,7 @@ -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ - -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1=-ldovecot-storage|" \ + -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ -e "s|^\(LIBDOVECOT_INCLUDE\)=.*$$|\1=-I$(pkgincludedir)|" \ > $(DESTDIR)$(pkglibdir)/dovecot-config From dovecot at dovecot.org Sat Jan 5 01:14:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:33 +0200 Subject: dovecot-2.2: lib-index: Recent cache file changes broke writing. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2de37734d0b9 changeset: 15504:2de37734d0b9 user: Timo Sirainen date: Sat Nov 03 18:43:00 2012 +0200 description: lib-index: Recent cache file changes broke writing. diffstat: src/lib-index/mail-cache.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7d931927e4ac -r 2de37734d0b9 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Wed Oct 31 13:24:49 2012 +0200 +++ b/src/lib-index/mail-cache.c Sat Nov 03 18:43:00 2012 +0200 @@ -684,7 +684,7 @@ file_cache_invalidate(cache->file_cache, 0, sizeof(struct mail_cache_header)); } - if (mail_cache_map(cache, 0, 0, &data) == 0) + if (mail_cache_map(cache, 0, 0, &data) > 0) cache->hdr_copy = *cache->hdr; else { (void)mail_cache_unlock(cache); From dovecot at dovecot.org Sat Jan 5 01:14:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:33 +0200 Subject: dovecot-2.2: lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY may have c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8252bcd7c489 changeset: 15505:8252bcd7c489 user: Timo Sirainen date: Sun Nov 04 17:26:24 2012 +0200 description: lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY may have caused stale data to be read from cache. diffstat: src/lib-index/mail-cache.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 2de37734d0b9 -r 8252bcd7c489 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Sat Nov 03 18:43:00 2012 +0200 +++ b/src/lib-index/mail-cache.c Sun Nov 04 17:26:24 2012 +0200 @@ -751,6 +751,8 @@ if (cache->file_cache != NULL) file_cache_write(cache->file_cache, data, size, offset); + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); return 0; } From dovecot at dovecot.org Sat Jan 5 01:14:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:33 +0200 Subject: dovecot-2.2: lib-ssl-iostream: [io]streams were added an unneces... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ba1287f46c98 changeset: 15506:ba1287f46c98 user: Timo Sirainen date: Mon Nov 05 14:28:24 2012 +0200 description: lib-ssl-iostream: [io]streams were added an unnecessary reference, causing memory leaks. diffstat: src/lib-ssl-iostream/iostream-openssl.c | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diffs (13 lines): diff -r 8252bcd7c489 -r ba1287f46c98 src/lib-ssl-iostream/iostream-openssl.c --- a/src/lib-ssl-iostream/iostream-openssl.c Sun Nov 04 17:26:24 2012 +0200 +++ b/src/lib-ssl-iostream/iostream-openssl.c Mon Nov 05 14:28:24 2012 +0200 @@ -207,9 +207,6 @@ SSL_set_bio(ssl_io->ssl, bio_int, bio_int); SSL_set_ex_data(ssl_io->ssl, dovecot_ssl_extdata_index, ssl_io); - i_stream_ref(ssl_io->plain_input); - o_stream_ref(ssl_io->plain_output); - T_BEGIN { ret = ssl_iostream_set(ssl_io, set); } T_END; From dovecot at dovecot.org Sat Jan 5 01:14:33 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:33 +0200 Subject: dovecot-2.2: imap: Avoid sending multiple unnecessary VANISHED l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bcd2d140ea10 changeset: 15507:bcd2d140ea10 user: Timo Sirainen date: Tue Nov 06 22:13:13 2012 +0200 description: imap: Avoid sending multiple unnecessary VANISHED lines when sync had lots of changes. diffstat: src/imap/imap-sync.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r ba1287f46c98 -r bcd2d140ea10 src/imap/imap-sync.c --- a/src/imap/imap-sync.c Mon Nov 05 14:28:24 2012 +0200 +++ b/src/imap/imap-sync.c Tue Nov 06 22:13:13 2012 +0200 @@ -472,8 +472,10 @@ ctx->seq = 0; } - if (array_is_created(&ctx->expunges)) - imap_sync_vanished(ctx); + if (ret > 0) { + if (array_is_created(&ctx->expunges)) + imap_sync_vanished(ctx); + } return ret; } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: mdbox: Don't leave extra records to dovecot.map.ind... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6c1b4b9f527c changeset: 15508:6c1b4b9f527c user: Timo Sirainen date: Wed Nov 07 17:05:47 2012 +0200 description: mdbox: Don't leave extra records to dovecot.map.index if mailbox index locking fails. Fixes errors such as: Corrupted dbox file /mdbox/storage/m.1 (around offset=2652): EOF reading msg header (got 0/30 bytes) diffstat: src/lib-storage/index/dbox-multi/mdbox-save.c | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) diffs (37 lines): diff -r bcd2d140ea10 -r 6c1b4b9f527c src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Tue Nov 06 22:13:13 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Nov 07 17:05:47 2012 +0200 @@ -296,16 +296,6 @@ mdbox_transaction_save_rollback(_ctx); return -1; } - - /* assign map UIDs for newly saved messages. they're written to - transaction log immediately within this function, but the map - is left locked. */ - if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, - &last_map_uid) < 0) { - mdbox_transaction_save_rollback(_ctx); - return -1; - } - /* lock the mailbox after map to avoid deadlocks. if we've noticed any corruption, deal with it later, otherwise we won't have up-to-date atomic->sync_view */ @@ -318,6 +308,16 @@ return -1; } + /* assign map UIDs for newly saved messages after we've successfully + acquired all the locks. the transaction is now very unlikely to + fail. the UIDs are written to the transaction log immediately within + this function, but the map is left locked. */ + if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, + &last_map_uid) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + /* assign UIDs for new messages */ hdr = mail_index_get_header(ctx->sync_ctx->sync_view); mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: var_expand*(): Don't use a data stack frame when ex... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3a33e686fc38 changeset: 15509:3a33e686fc38 user: Timo Sirainen date: Wed Nov 07 17:37:16 2012 +0200 description: var_expand*(): Don't use a data stack frame when expanding long %{variables} This avoids potential crashes if the destination string is also allocated from data stack and requires growing. diffstat: src/lib/var-expand.c | 50 ++++++++++++++++++++++++-------------------------- 1 files changed, 24 insertions(+), 26 deletions(-) diffs (71 lines): diff -r 6c1b4b9f527c -r 3a33e686fc38 src/lib/var-expand.c --- a/src/lib/var-expand.c Wed Nov 07 17:05:47 2012 +0200 +++ b/src/lib/var-expand.c Wed Nov 07 17:37:16 2012 +0200 @@ -160,7 +160,7 @@ const void *key_start, unsigned int key_len, void *context) { const struct var_expand_table *t; - const char *value = NULL; + const char *key, *value = NULL; if (table != NULL) { for (t = table; !TABLE_LAST(t); t++) { @@ -171,35 +171,33 @@ } } } + key = t_strndup(key_start, key_len); /* built-in variables: */ - T_BEGIN { - const char *key = t_strndup(key_start, key_len); + switch (key_len) { + case 3: + if (strcmp(key, "pid") == 0) + value = my_pid; + else if (strcmp(key, "uid") == 0) + value = dec2str(geteuid()); + else if (strcmp(key, "gid") == 0) + value = dec2str(getegid()); + break; + case 8: + if (strcmp(key, "hostname") == 0) + value = my_hostname; + break; + } - switch (key_len) { - case 3: - if (strcmp(key, "pid") == 0) - value = my_pid; - else if (strcmp(key, "uid") == 0) - value = dec2str(geteuid()); - else if (strcmp(key, "gid") == 0) - value = dec2str(getegid()); - break; - case 8: - if (strcmp(key, "hostname") == 0) - value = my_hostname; - break; - } - if (value == NULL) { - const char *data = strchr(key, ':'); + if (value == NULL) { + const char *data = strchr(key, ':'); - if (data != NULL) - key = t_strdup_until(key, data++); - else - data = ""; - value = var_expand_func(func_table, key, data, context); - } - } T_END; + if (data != NULL) + key = t_strdup_until(key, data++); + else + data = ""; + value = var_expand_func(func_table, key, data, context); + } return value; } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: Makefile: Add -lssl to installed dovecot-config's L... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22caa1317eb1 changeset: 15510:22caa1317eb1 user: Timo Sirainen date: Wed Nov 07 17:44:32 2012 +0200 description: Makefile: Add -lssl to installed dovecot-config's LIBDOVECOT_LOGIN if needed. diffstat: Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3a33e686fc38 -r 22caa1317eb1 Makefile.am --- a/Makefile.am Wed Nov 07 17:37:16 2012 +0200 +++ b/Makefile.am Wed Nov 07 17:44:32 2012 +0200 @@ -63,7 +63,7 @@ grep -v '^LIBDOVECOT_.*_INCLUDE' dovecot-config | \ grep -v '^LIBDOVECOT.*_DEPS' | sed \ -e "s|^\(LIBDOVECOT\)=.*$$|\1='-L$(pkglibdir) -ldovecot'|" \ - -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login|" \ + -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login $(SSL_LIBS)|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: auth: Give a better error message for "client doesn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c811aab61355 changeset: 15511:c811aab61355 user: Timo Sirainen date: Wed Nov 07 18:14:20 2012 +0200 description: auth: Give a better error message for "client doesn't have lookup permissions". diffstat: src/auth/auth-master-connection.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 22caa1317eb1 -r c811aab61355 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Nov 07 17:44:32 2012 +0200 +++ b/src/auth/auth-master-connection.c Wed Nov 07 18:14:20 2012 +0200 @@ -239,7 +239,8 @@ auth_request_log_error(auth_request, "userdb", "client doesn't have lookup permissions for this user: %s " - "(change userdb socket permissions)", reason); + "(to bypass this check, set: service auth { unix_listener %s { mode=0777 } })", + reason, conn->path); return -1; } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: auth: Added a way to set default values for nonexis... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c722bd39098b changeset: 15512:c722bd39098b user: Timo Sirainen date: Fri Nov 23 08:32:13 2012 +0200 description: auth: Added a way to set default values for nonexistent LDAP attributes. %{ldap:attrName:default_value} expands to default_value if attrName doesn't exist. diffstat: src/auth/db-ldap.c | 28 +++++++++++++++++++++------- 1 files changed, 21 insertions(+), 7 deletions(-) diffs (61 lines): diff -r c811aab61355 -r c722bd39098b src/auth/db-ldap.c --- a/src/auth/db-ldap.c Wed Nov 07 18:14:20 2012 +0200 +++ b/src/auth/db-ldap.c Fri Nov 23 08:32:13 2012 +0200 @@ -990,7 +990,7 @@ char *ldap_attr; if (*data != '\0') { - ldap_attr = p_strdup(ctx->pool, data); + ldap_attr = p_strdup(ctx->pool, t_strcut(data, ':')); array_append(&ctx->attr_names, &ldap_attr, 1); } return NULL; @@ -1200,28 +1200,42 @@ return ctx; } +static const char *db_ldap_field_get_default(const char *data) +{ + const char *p; + + p = strchr(data, ':'); + if (p == NULL) + return ""; + else { + /* default value given */ + return p+1; + } +} + static const char *db_ldap_field_expand(const char *data, void *context) { struct db_ldap_result_iterate_context *ctx = context; struct db_ldap_value *ldap_value; + const char *field_name = t_strcut(data, ':'); - ldap_value = hash_table_lookup(ctx->ldap_attrs, data); + ldap_value = hash_table_lookup(ctx->ldap_attrs, field_name); if (ldap_value == NULL) { - /* ldap attribute wasn't requested */ + /* requested ldap attribute wasn't returned at all */ if (ctx->debug) - str_printfa(ctx->debug, "; %s missing", data); - return ""; + str_printfa(ctx->debug, "; %s missing", field_name); + return db_ldap_field_get_default(data); } ldap_value->used = TRUE; if (ldap_value->values[0] == NULL) { /* no value for ldap attribute */ - return ""; + return db_ldap_field_get_default(data); } if (ldap_value->values[1] != NULL) { auth_request_log_warning(ctx->auth_request, "ldap", "Multiple values found for '%s', using value '%s'", - data, ldap_value->values[0]); + field_name, ldap_value->values[0]); } return ldap_value->values[0]; } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: login: Try to avoid busy-looping on SSL_accept() wh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e95479f439aa changeset: 15513:e95479f439aa user: Timo Sirainen date: Fri Nov 23 08:52:06 2012 +0200 description: login: Try to avoid busy-looping on SSL_accept() when client doesn't behave nicely. diffstat: src/login-common/ssl-proxy-openssl.c | 17 +++++++++++------ 1 files changed, 11 insertions(+), 6 deletions(-) diffs (63 lines): diff -r c722bd39098b -r e95479f439aa src/login-common/ssl-proxy-openssl.c --- a/src/login-common/ssl-proxy-openssl.c Fri Nov 23 08:32:13 2012 +0200 +++ b/src/login-common/ssl-proxy-openssl.c Fri Nov 23 08:52:06 2012 +0200 @@ -394,8 +394,9 @@ return ssl_err2str(err, data, flags); } -static void ssl_handle_error(struct ssl_proxy *proxy, int ret, - const char *func_name) +static void +ssl_handle_error(struct ssl_proxy *proxy, int ret, bool remove_wrong_direction, + const char *func_name) { const char *errstr = NULL; int err; @@ -408,9 +409,13 @@ switch (err) { case SSL_ERROR_WANT_READ: ssl_set_io(proxy, SSL_ADD_INPUT); + if (remove_wrong_direction) + ssl_set_io(proxy, SSL_REMOVE_OUTPUT); break; case SSL_ERROR_WANT_WRITE: ssl_set_io(proxy, SSL_ADD_OUTPUT); + if (remove_wrong_direction) + ssl_set_io(proxy, SSL_REMOVE_INPUT); break; case SSL_ERROR_SYSCALL: /* eat up the error queue */ @@ -458,13 +463,13 @@ if (proxy->client_proxy) { ret = SSL_connect(proxy->ssl); if (ret != 1) { - ssl_handle_error(proxy, ret, "SSL_connect()"); + ssl_handle_error(proxy, ret, TRUE, "SSL_connect()"); return; } } else { ret = SSL_accept(proxy->ssl); if (ret != 1) { - ssl_handle_error(proxy, ret, "SSL_accept()"); + ssl_handle_error(proxy, ret, TRUE, "SSL_accept()"); return; } } @@ -491,7 +496,7 @@ sizeof(proxy->plainout_buf) - proxy->plainout_size); if (ret <= 0) { - ssl_handle_error(proxy, ret, "SSL_read()"); + ssl_handle_error(proxy, ret, FALSE, "SSL_read()"); break; } else { i_free_and_null(proxy->last_error); @@ -507,7 +512,7 @@ ret = SSL_write(proxy->ssl, proxy->sslout_buf, proxy->sslout_size); if (ret <= 0) - ssl_handle_error(proxy, ret, "SSL_write()"); + ssl_handle_error(proxy, ret, FALSE, "SSL_write()"); else { i_free_and_null(proxy->last_error); proxy->sslout_size -= ret; From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: man: Added -d parameter to doveadm expunge. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1ac96aad5027 changeset: 15514:1ac96aad5027 user: Timo Sirainen date: Tue Nov 27 02:20:37 2012 +0200 description: man: Added -d parameter to doveadm expunge. Patch by Pascal Volk. diffstat: doc/man/doveadm-expunge.1.in | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (22 lines): diff -r e95479f439aa -r 1ac96aad5027 doc/man/doveadm-expunge.1.in --- a/doc/man/doveadm-expunge.1.in Fri Nov 23 08:52:06 2012 +0200 +++ b/doc/man/doveadm-expunge.1.in Tue Nov 27 02:20:37 2012 +0200 @@ -1,5 +1,5 @@ -.\" Copyright (c) 2010 Dovecot authors, see the included COPYING file -.TH DOVEADM\-EXPUNGE 1 "2010-11-25" "Dovecot v2.1" "Dovecot" +.\" Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file +.TH DOVEADM\-EXPUNGE 1 "2012-11-27" "Dovecot v2.1" "Dovecot" .SH NAME doveadm\-expunge \- Expunge messages matching given search query .\"------------------------------------------------------------------------ @@ -47,6 +47,10 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- +.TP +.B \-d +Delete the mailbox if it is empty after expunging. +.\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- @INCLUDE:option-u-user@ From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: quota: mailbox_get_status(STATUS_CHECK_OVER_QUOTA) ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bebe54e1d640 changeset: 15515:bebe54e1d640 user: Timo Sirainen date: Tue Nov 27 02:41:53 2012 +0200 description: quota: mailbox_get_status(STATUS_CHECK_OVER_QUOTA) returned wrong value on success. This fixes lmtp_rcpt_check_quota=yes diffstat: src/plugins/quota/quota-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1ac96aad5027 -r bebe54e1d640 src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Tue Nov 27 02:20:37 2012 +0200 +++ b/src/plugins/quota/quota-storage.c Tue Nov 27 02:41:53 2012 +0200 @@ -87,7 +87,7 @@ if ((items & ~STATUS_CHECK_OVER_QUOTA) == 0) { /* don't bother calling parent, it may unnecessarily try to open the mailbox */ - return ret; + return ret < 0 ? -1 : 0; } } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: liblib: Added uni_utf8_short_*() for handling UTF8 ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/172295f5a78b changeset: 15516:172295f5a78b user: Timo Sirainen date: Tue Nov 27 03:48:15 2012 +0200 description: liblib: Added uni_utf8_short_*() for handling UTF8 data where [56]-byte sequences are invalid. diffstat: src/lib/unichar.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/unichar.h | 7 ++++++ 2 files changed, 71 insertions(+), 0 deletions(-) diffs (86 lines): diff -r bebe54e1d640 -r 172295f5a78b src/lib/unichar.c --- a/src/lib/unichar.c Tue Nov 27 02:41:53 2012 +0200 +++ b/src/lib/unichar.c Tue Nov 27 03:48:15 2012 +0200 @@ -420,3 +420,67 @@ return uni_utf8_find_invalid_pos(data, size, &i) == 0; } + +static int +uni_utf8_short_find_invalid_pos(const unsigned char *input, size_t size, + size_t *pos_r) +{ + size_t i, len; + + /* find the first invalid utf8 sequence */ + for (i = 0; i < size;) { + if (input[i] < 0x80) + i++; + else { + len = is_valid_utf8_seq(input + i, size-i); + if (unlikely(len == 0 || len > 4)) { + *pos_r = i; + return -1; + } + i += len; + } + } + return 0; +} + +bool uni_utf8_short_get_valid_data(const unsigned char *input, size_t size, + buffer_t *buf) +{ + size_t i, len; + + if (uni_utf8_short_find_invalid_pos(input, size, &i) == 0) + return TRUE; + + /* broken utf-8 input - skip the broken characters */ + while (i < size) { + if (input[i] < 0x80) { + buffer_append_c(buf, input[i++]); + continue; + } + + len = is_valid_utf8_seq(input + i, size-i); + if (len == 0 || len > 4) { + i += I_MAX(len, 1); + output_add_replacement_char(buf); + continue; + } + buffer_append(buf, input + i, len); + i += len; + } + return FALSE; +} + +bool uni_utf8_short_str_is_valid(const char *str) +{ + size_t i; + + return uni_utf8_short_find_invalid_pos((const unsigned char *)str, + strlen(str), &i) == 0; +} + +bool uni_utf8_short_data_is_valid(const unsigned char *data, size_t size) +{ + size_t i; + + return uni_utf8_find_invalid_pos(data, size, &i) == 0; +} diff -r bebe54e1d640 -r 172295f5a78b src/lib/unichar.h --- a/src/lib/unichar.h Tue Nov 27 02:41:53 2012 +0200 +++ b/src/lib/unichar.h Tue Nov 27 03:48:15 2012 +0200 @@ -88,4 +88,11 @@ /* Returns TRUE if data contains only valid UTF-8 input. */ bool uni_utf8_data_is_valid(const unsigned char *data, size_t size); +/* Same as the non-short variants, but assume 5-byte and 6-byte UTF8 + sequences are illegal. */ +bool uni_utf8_short_get_valid_data(const unsigned char *input, size_t size, + buffer_t *buf); +bool uni_utf8_short_str_is_valid(const char *str); +bool uni_utf8_short_data_is_valid(const unsigned char *data, size_t size); + #endif From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: fts: Added FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 to req... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/01550514f189 changeset: 15517:01550514f189 user: Timo Sirainen date: Tue Nov 27 03:49:25 2012 +0200 description: fts: Added FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 to require sending only short UTF8 data to backend. diffstat: src/plugins/fts/fts-api-private.h | 4 +++- src/plugins/fts/fts-build-mail.c | 7 +++++-- src/plugins/fts/fts-parser.c | 24 ++++++++++++++++++------ src/plugins/fts/fts-parser.h | 3 ++- 4 files changed, 28 insertions(+), 10 deletions(-) diffs (116 lines): diff -r 172295f5a78b -r 01550514f189 src/plugins/fts/fts-api-private.h --- a/src/plugins/fts/fts-api-private.h Tue Nov 27 03:48:15 2012 +0200 +++ b/src/plugins/fts/fts-api-private.h Tue Nov 27 03:49:25 2012 +0200 @@ -59,7 +59,9 @@ /* Send only fully indexable words rather than randomly sized blocks */ FTS_BACKEND_FLAG_BUILD_FULL_WORDS = 0x04, /* Fuzzy search works */ - FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08 + FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08, + /* Don't allow 5-byte or 6-byte UTF8 sequences */ + FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 = 0x10 }; struct fts_backend { diff -r 172295f5a78b -r 01550514f189 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Tue Nov 27 03:48:15 2012 +0200 +++ b/src/plugins/fts/fts-build-mail.c Tue Nov 27 03:49:25 2012 +0200 @@ -144,6 +144,7 @@ struct mail_storage *storage; const char *content_type; struct fts_backend_build_key key; + bool require_short_utf8; i_assert(ctx->body_parser == NULL); @@ -158,9 +159,11 @@ return FALSE; } - + require_short_utf8 = (ctx->update_ctx->backend->flags & + FTS_BACKEND_FLAG_BUILD_SHORT_UTF8) != 0; + storage = mailbox_get_storage(ctx->mail->box); - if (fts_parser_init(mail_storage_get_user(storage), + if (fts_parser_init(mail_storage_get_user(storage), require_short_utf8, content_type, ctx->content_disposition, &ctx->body_parser)) { /* extract text using the the returned parser */ diff -r 172295f5a78b -r 01550514f189 src/plugins/fts/fts-parser.c --- a/src/plugins/fts/fts-parser.c Tue Nov 27 03:48:15 2012 +0200 +++ b/src/plugins/fts/fts-parser.c Tue Nov 27 03:49:25 2012 +0200 @@ -11,7 +11,7 @@ &fts_parser_script }; -bool fts_parser_init(struct mail_user *user, +bool fts_parser_init(struct mail_user *user, bool require_short_utf8, const char *content_type, const char *content_disposition, struct fts_parser **parser_r) { @@ -20,8 +20,10 @@ for (i = 0; i < N_ELEMENTS(parsers); i++) { *parser_r = parsers[i]->try_init(user, content_type, content_disposition); - if (*parser_r != NULL) + if (*parser_r != NULL) { + (*parser_r)->require_short_utf8 = require_short_utf8; return TRUE; + } } return FALSE; } @@ -56,11 +58,15 @@ void fts_parser_more(struct fts_parser *parser, struct message_block *block) { + bool valid_utf8; + if (parser->v.more != NULL) parser->v.more(parser, block); - if (!uni_utf8_data_is_valid(block->data, block->size) || - data_has_nuls(block->data, block->size)) { + valid_utf8 = parser->require_short_utf8 ? + uni_utf8_short_data_is_valid(block->data, block->size) : + uni_utf8_data_is_valid(block->data, block->size); + if (!valid_utf8 || data_has_nuls(block->data, block->size)) { /* output isn't valid UTF-8. make it. */ if (parser->utf8_output == NULL) { parser->utf8_output = @@ -68,8 +74,14 @@ } else { buffer_set_used_size(parser->utf8_output, 0); } - (void)uni_utf8_get_valid_data(block->data, block->size, - parser->utf8_output); + if (parser->require_short_utf8) { + (void)uni_utf8_short_get_valid_data(block->data, + block->size, + parser->utf8_output); + } else { + (void)uni_utf8_get_valid_data(block->data, block->size, + parser->utf8_output); + } replace_nul_bytes(parser->utf8_output); block->data = parser->utf8_output->data; block->size = parser->utf8_output->used; diff -r 172295f5a78b -r 01550514f189 src/plugins/fts/fts-parser.h --- a/src/plugins/fts/fts-parser.h Tue Nov 27 03:48:15 2012 +0200 +++ b/src/plugins/fts/fts-parser.h Tue Nov 27 03:49:25 2012 +0200 @@ -15,12 +15,13 @@ struct fts_parser { struct fts_parser_vfuncs v; buffer_t *utf8_output; + bool require_short_utf8; }; extern struct fts_parser_vfuncs fts_parser_html; extern struct fts_parser_vfuncs fts_parser_script; -bool fts_parser_init(struct mail_user *user, +bool fts_parser_init(struct mail_user *user, bool require_short_utf8, const char *content_type, const char *content_disposition, struct fts_parser **parser_r); struct fts_parser *fts_parser_text_init(void); From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: fts-solr: Marked as FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/339e654f371e changeset: 15518:339e654f371e user: Timo Sirainen date: Tue Nov 27 03:49:36 2012 +0200 description: fts-solr: Marked as FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 2 +- src/plugins/fts-solr/fts-backend-solr.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diffs (25 lines): diff -r 01550514f189 -r 339e654f371e src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Tue Nov 27 03:49:25 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Tue Nov 27 03:49:36 2012 +0200 @@ -810,7 +810,7 @@ struct fts_backend fts_backend_solr_old = { .name = "solr_old", - .flags = 0, + .flags = FTS_BACKEND_FLAG_BUILD_SHORT_UTF8, { fts_backend_solr_alloc, diff -r 01550514f189 -r 339e654f371e src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 03:49:25 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 03:49:36 2012 +0200 @@ -879,7 +879,8 @@ struct fts_backend fts_backend_solr = { .name = "solr", - .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH, + .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH | + FTS_BACKEND_FLAG_BUILD_SHORT_UTF8, { fts_backend_solr_alloc, From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: doveadm: Include username in internal errors and "n... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6f19c535110e changeset: 15519:6f19c535110e user: Timo Sirainen date: Tue Nov 27 06:19:10 2012 +0200 description: doveadm: Include username in internal errors and "no such user" errors when running via server. diffstat: src/doveadm/doveadm-mail-server.c | 27 ++++++++++++++++++++++----- 1 files changed, 22 insertions(+), 5 deletions(-) diffs (73 lines): diff -r 339e654f371e -r 6f19c535110e src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Tue Nov 27 03:49:36 2012 +0200 +++ b/src/doveadm/doveadm-mail-server.c Tue Nov 27 06:19:10 2012 +0200 @@ -22,6 +22,11 @@ #define DOVEADM_MAIL_SERVER_FAILED() \ (internal_failure || master_service_is_killed(master_service)) +struct doveadm_mail_server_cmd { + struct server_connection *conn; + char *username; +}; + static struct hash_table *servers; static pool_t server_pool; static struct doveadm_mail_cmd_context *cmd_ctx; @@ -80,16 +85,22 @@ static void doveadm_cmd_callback(enum server_cmd_reply reply, void *context) { - struct server_connection *conn = context; - struct doveadm_server *server; + struct doveadm_mail_server_cmd *servercmd = context; + struct doveadm_server *server = + server_connection_get_server(servercmd->conn); + const char *username = t_strdup(servercmd->username); + + i_free(servercmd->username); + i_free(servercmd); switch (reply) { case SERVER_CMD_REPLY_INTERNAL_FAILURE: + i_error("%s: Internal failure for %s", server->name, username); internal_failure = TRUE; master_service_stop(master_service); return; case SERVER_CMD_REPLY_UNKNOWN_USER: - i_error("No such user"); + i_error("%s: No such user: %s", server->name, username); if (cmd_ctx->exit_code == 0) cmd_ctx->exit_code = EX_NOUSER; break; @@ -100,8 +111,8 @@ break; } - server = server_connection_get_server(conn); if (array_count(&server->queue) > 0) { + struct server_connection *conn; char *const *usernamep = array_idx(&server->queue, 0); char *username = *usernamep; @@ -119,6 +130,7 @@ static void doveadm_mail_server_handle(struct server_connection *conn, const char *username) { + struct doveadm_mail_server_cmd *servercmd; string_t *cmd; unsigned int i; @@ -138,7 +150,12 @@ str_tabescape_write(cmd, cmd_ctx->full_args[i]); } str_append_c(cmd, '\n'); - server_connection_cmd(conn, str_c(cmd), doveadm_cmd_callback, conn); + + servercmd = i_new(struct doveadm_mail_server_cmd, 1); + servercmd->conn = conn; + servercmd->username = i_strdup(username); + server_connection_cmd(conn, str_c(cmd), + doveadm_cmd_callback, servercmd); } static void doveadm_server_flush_one(struct doveadm_server *server) From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: doveadm-server: Avoid eating CPU when client discon... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0dc3f56e6468 changeset: 15521:0dc3f56e6468 user: Timo Sirainen date: Tue Nov 27 06:20:44 2012 +0200 description: doveadm-server: Avoid eating CPU when client disconnects before authentication is done. diffstat: src/doveadm/client-connection.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (16 lines): diff -r 275a57b8dc70 -r 0dc3f56e6468 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Tue Nov 27 06:19:57 2012 +0200 +++ b/src/doveadm/client-connection.c Tue Nov 27 06:20:44 2012 +0200 @@ -239,8 +239,11 @@ const unsigned char *data; size_t size; - if ((line = i_stream_read_next_line(conn->input)) == NULL) + if ((line = i_stream_read_next_line(conn->input)) == NULL) { + if (conn->input->eof) + return -1; return 0; + } if (*conn->set->doveadm_password == '\0') { i_error("doveadm_password not set, " From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: doveadm: Don't disconnect from server too early whi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/275a57b8dc70 changeset: 15520:275a57b8dc70 user: Timo Sirainen date: Tue Nov 27 06:19:57 2012 +0200 description: doveadm: Don't disconnect from server too early while there are still commands. diffstat: src/doveadm/server-connection.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 6f19c535110e -r 275a57b8dc70 src/doveadm/server-connection.c --- a/src/doveadm/server-connection.c Tue Nov 27 06:19:10 2012 +0200 +++ b/src/doveadm/server-connection.c Tue Nov 27 06:19:57 2012 +0200 @@ -269,8 +269,10 @@ server_connection_callback(conn, reply); } else i_error("doveadm server sent broken input"); - /* we're finished, close the connection */ - server_connection_destroy(&conn); + if (conn->callback == NULL) { + /* we're finished, close the connection */ + server_connection_destroy(&conn); + } break; } } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: doveadm-server: Reset headers in print output betwe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fdc509644d05 changeset: 15522:fdc509644d05 user: Timo Sirainen date: Tue Nov 27 06:21:18 2012 +0200 description: doveadm-server: Reset headers in print output between running commands. diffstat: src/doveadm/client-connection.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 0dc3f56e6468 -r fdc509644d05 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Tue Nov 27 06:20:44 2012 +0200 +++ b/src/doveadm/client-connection.c Tue Nov 27 06:21:18 2012 +0200 @@ -139,6 +139,10 @@ o_stream_send(conn->output, "\n+\n", 3); } pool_unref(&ctx->pool); + + /* clear all headers */ + doveadm_print_deinit(); + doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); } static bool client_is_allowed_command(const struct doveadm_settings *set, From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: Reversed recent "short utf8" changes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f584aae3b566 changeset: 15523:f584aae3b566 user: Timo Sirainen date: Tue Nov 27 07:50:06 2012 +0200 description: Reversed recent "short utf8" changes. Solr code needs to parse the UTF8 input explicitly anyway to encode the XML characters. And all the character checks were already done in it. diffstat: src/lib/unichar.c | 64 ----------------------------- src/lib/unichar.h | 7 --- src/plugins/fts-solr/fts-backend-solr-old.c | 2 +- src/plugins/fts-solr/fts-backend-solr.c | 3 +- src/plugins/fts/fts-api-private.h | 4 +- src/plugins/fts/fts-build-mail.c | 7 +-- src/plugins/fts/fts-parser.c | 24 ++-------- src/plugins/fts/fts-parser.h | 3 +- 8 files changed, 12 insertions(+), 102 deletions(-) diffs (227 lines): diff -r fdc509644d05 -r f584aae3b566 src/lib/unichar.c --- a/src/lib/unichar.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/lib/unichar.c Tue Nov 27 07:50:06 2012 +0200 @@ -420,67 +420,3 @@ return uni_utf8_find_invalid_pos(data, size, &i) == 0; } - -static int -uni_utf8_short_find_invalid_pos(const unsigned char *input, size_t size, - size_t *pos_r) -{ - size_t i, len; - - /* find the first invalid utf8 sequence */ - for (i = 0; i < size;) { - if (input[i] < 0x80) - i++; - else { - len = is_valid_utf8_seq(input + i, size-i); - if (unlikely(len == 0 || len > 4)) { - *pos_r = i; - return -1; - } - i += len; - } - } - return 0; -} - -bool uni_utf8_short_get_valid_data(const unsigned char *input, size_t size, - buffer_t *buf) -{ - size_t i, len; - - if (uni_utf8_short_find_invalid_pos(input, size, &i) == 0) - return TRUE; - - /* broken utf-8 input - skip the broken characters */ - while (i < size) { - if (input[i] < 0x80) { - buffer_append_c(buf, input[i++]); - continue; - } - - len = is_valid_utf8_seq(input + i, size-i); - if (len == 0 || len > 4) { - i += I_MAX(len, 1); - output_add_replacement_char(buf); - continue; - } - buffer_append(buf, input + i, len); - i += len; - } - return FALSE; -} - -bool uni_utf8_short_str_is_valid(const char *str) -{ - size_t i; - - return uni_utf8_short_find_invalid_pos((const unsigned char *)str, - strlen(str), &i) == 0; -} - -bool uni_utf8_short_data_is_valid(const unsigned char *data, size_t size) -{ - size_t i; - - return uni_utf8_find_invalid_pos(data, size, &i) == 0; -} diff -r fdc509644d05 -r f584aae3b566 src/lib/unichar.h --- a/src/lib/unichar.h Tue Nov 27 06:21:18 2012 +0200 +++ b/src/lib/unichar.h Tue Nov 27 07:50:06 2012 +0200 @@ -88,11 +88,4 @@ /* Returns TRUE if data contains only valid UTF-8 input. */ bool uni_utf8_data_is_valid(const unsigned char *data, size_t size); -/* Same as the non-short variants, but assume 5-byte and 6-byte UTF8 - sequences are illegal. */ -bool uni_utf8_short_get_valid_data(const unsigned char *input, size_t size, - buffer_t *buf); -bool uni_utf8_short_str_is_valid(const char *str); -bool uni_utf8_short_data_is_valid(const unsigned char *data, size_t size); - #endif diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Tue Nov 27 07:50:06 2012 +0200 @@ -810,7 +810,7 @@ struct fts_backend fts_backend_solr_old = { .name = "solr_old", - .flags = FTS_BACKEND_FLAG_BUILD_SHORT_UTF8, + .flags = 0, { fts_backend_solr_alloc, diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 07:50:06 2012 +0200 @@ -879,8 +879,7 @@ struct fts_backend fts_backend_solr = { .name = "solr", - .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH | - FTS_BACKEND_FLAG_BUILD_SHORT_UTF8, + .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH, { fts_backend_solr_alloc, diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts/fts-api-private.h --- a/src/plugins/fts/fts-api-private.h Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts/fts-api-private.h Tue Nov 27 07:50:06 2012 +0200 @@ -59,9 +59,7 @@ /* Send only fully indexable words rather than randomly sized blocks */ FTS_BACKEND_FLAG_BUILD_FULL_WORDS = 0x04, /* Fuzzy search works */ - FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08, - /* Don't allow 5-byte or 6-byte UTF8 sequences */ - FTS_BACKEND_FLAG_BUILD_SHORT_UTF8 = 0x10 + FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08 }; struct fts_backend { diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts/fts-build-mail.c Tue Nov 27 07:50:06 2012 +0200 @@ -144,7 +144,6 @@ struct mail_storage *storage; const char *content_type; struct fts_backend_build_key key; - bool require_short_utf8; i_assert(ctx->body_parser == NULL); @@ -159,11 +158,9 @@ return FALSE; } - require_short_utf8 = (ctx->update_ctx->backend->flags & - FTS_BACKEND_FLAG_BUILD_SHORT_UTF8) != 0; - + storage = mailbox_get_storage(ctx->mail->box); - if (fts_parser_init(mail_storage_get_user(storage), require_short_utf8, + if (fts_parser_init(mail_storage_get_user(storage), content_type, ctx->content_disposition, &ctx->body_parser)) { /* extract text using the the returned parser */ diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts/fts-parser.c --- a/src/plugins/fts/fts-parser.c Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts/fts-parser.c Tue Nov 27 07:50:06 2012 +0200 @@ -11,7 +11,7 @@ &fts_parser_script }; -bool fts_parser_init(struct mail_user *user, bool require_short_utf8, +bool fts_parser_init(struct mail_user *user, const char *content_type, const char *content_disposition, struct fts_parser **parser_r) { @@ -20,10 +20,8 @@ for (i = 0; i < N_ELEMENTS(parsers); i++) { *parser_r = parsers[i]->try_init(user, content_type, content_disposition); - if (*parser_r != NULL) { - (*parser_r)->require_short_utf8 = require_short_utf8; + if (*parser_r != NULL) return TRUE; - } } return FALSE; } @@ -58,15 +56,11 @@ void fts_parser_more(struct fts_parser *parser, struct message_block *block) { - bool valid_utf8; - if (parser->v.more != NULL) parser->v.more(parser, block); - valid_utf8 = parser->require_short_utf8 ? - uni_utf8_short_data_is_valid(block->data, block->size) : - uni_utf8_data_is_valid(block->data, block->size); - if (!valid_utf8 || data_has_nuls(block->data, block->size)) { + if (!uni_utf8_data_is_valid(block->data, block->size) || + data_has_nuls(block->data, block->size)) { /* output isn't valid UTF-8. make it. */ if (parser->utf8_output == NULL) { parser->utf8_output = @@ -74,14 +68,8 @@ } else { buffer_set_used_size(parser->utf8_output, 0); } - if (parser->require_short_utf8) { - (void)uni_utf8_short_get_valid_data(block->data, - block->size, - parser->utf8_output); - } else { - (void)uni_utf8_get_valid_data(block->data, block->size, - parser->utf8_output); - } + (void)uni_utf8_get_valid_data(block->data, block->size, + parser->utf8_output); replace_nul_bytes(parser->utf8_output); block->data = parser->utf8_output->data; block->size = parser->utf8_output->used; diff -r fdc509644d05 -r f584aae3b566 src/plugins/fts/fts-parser.h --- a/src/plugins/fts/fts-parser.h Tue Nov 27 06:21:18 2012 +0200 +++ b/src/plugins/fts/fts-parser.h Tue Nov 27 07:50:06 2012 +0200 @@ -15,13 +15,12 @@ struct fts_parser { struct fts_parser_vfuncs v; buffer_t *utf8_output; - bool require_short_utf8; }; extern struct fts_parser_vfuncs fts_parser_html; extern struct fts_parser_vfuncs fts_parser_script; -bool fts_parser_init(struct mail_user *user, bool require_short_utf8, +bool fts_parser_init(struct mail_user *user, const char *content_type, const char *content_disposition, struct fts_parser **parser_r); struct fts_parser *fts_parser_text_init(void); From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: fts-solr: Removed deprecated waitFlush parameter fr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0f3972fa6dba changeset: 15524:0f3972fa6dba user: Timo Sirainen date: Tue Nov 27 08:08:50 2012 +0200 description: fts-solr: Removed deprecated waitFlush parameter from commit. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r f584aae3b566 -r 0f3972fa6dba src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 07:50:06 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 08:08:50 2012 +0200 @@ -352,8 +352,7 @@ visible to the following search */ if (ctx->expunges) fts_backend_solr_expunge_flush(ctx); - str = t_strdup_printf("", + str = t_strdup_printf("", ctx->documents_added ? "true" : "false"); if (solr_connection_post(solr_conn, str) < 0) ret = -1; From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: auth: userdb passwd lookups should have been done v... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8e5d9d88e250 changeset: 15525:8e5d9d88e250 user: Timo Sirainen date: Tue Nov 27 08:43:15 2012 +0200 description: auth: userdb passwd lookups should have been done via worker processes by default. diffstat: src/auth/userdb-passwd.c | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diffs (18 lines): diff -r 0f3972fa6dba -r 8e5d9d88e250 src/auth/userdb-passwd.c --- a/src/auth/userdb-passwd.c Tue Nov 27 08:08:50 2012 +0200 +++ b/src/auth/userdb-passwd.c Tue Nov 27 08:43:15 2012 +0200 @@ -216,11 +216,10 @@ module = p_new(pool, struct passwd_userdb_module, 1); module->module.cache_key = USER_CACHE_KEY; module->tmpl = userdb_template_build(pool, "passwd", args); + module->module.blocking = TRUE; - if (userdb_template_remove(module->tmpl, "blocking", &value)) { - module->module.blocking = value == NULL || - strcasecmp(value, "yes") == 0; - } + if (userdb_template_remove(module->tmpl, "blocking", &value)) + module->module.blocking = strcasecmp(value, "yes") == 0; /* FIXME: backwards compatibility */ if (!userdb_template_is_empty(module->tmpl)) i_warning("userdb passwd: Move templates args to override_fields setting"); From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: login: Reverted previous fix attempt for SSL_accept... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/174d7e974326 changeset: 15526:174d7e974326 user: Timo Sirainen date: Tue Nov 27 09:13:57 2012 +0200 description: login: Reverted previous fix attempt for SSL_accept() busy-looping. Apparently didn't work. Probably this patch doesn't help anything then, so just remove the extra complexity. diffstat: src/login-common/ssl-proxy-openssl.c | 17 ++++++----------- 1 files changed, 6 insertions(+), 11 deletions(-) diffs (63 lines): diff -r 8e5d9d88e250 -r 174d7e974326 src/login-common/ssl-proxy-openssl.c --- a/src/login-common/ssl-proxy-openssl.c Tue Nov 27 08:43:15 2012 +0200 +++ b/src/login-common/ssl-proxy-openssl.c Tue Nov 27 09:13:57 2012 +0200 @@ -394,9 +394,8 @@ return ssl_err2str(err, data, flags); } -static void -ssl_handle_error(struct ssl_proxy *proxy, int ret, bool remove_wrong_direction, - const char *func_name) +static void ssl_handle_error(struct ssl_proxy *proxy, int ret, + const char *func_name) { const char *errstr = NULL; int err; @@ -409,13 +408,9 @@ switch (err) { case SSL_ERROR_WANT_READ: ssl_set_io(proxy, SSL_ADD_INPUT); - if (remove_wrong_direction) - ssl_set_io(proxy, SSL_REMOVE_OUTPUT); break; case SSL_ERROR_WANT_WRITE: ssl_set_io(proxy, SSL_ADD_OUTPUT); - if (remove_wrong_direction) - ssl_set_io(proxy, SSL_REMOVE_INPUT); break; case SSL_ERROR_SYSCALL: /* eat up the error queue */ @@ -463,13 +458,13 @@ if (proxy->client_proxy) { ret = SSL_connect(proxy->ssl); if (ret != 1) { - ssl_handle_error(proxy, ret, TRUE, "SSL_connect()"); + ssl_handle_error(proxy, ret, "SSL_connect()"); return; } } else { ret = SSL_accept(proxy->ssl); if (ret != 1) { - ssl_handle_error(proxy, ret, TRUE, "SSL_accept()"); + ssl_handle_error(proxy, ret, "SSL_accept()"); return; } } @@ -496,7 +491,7 @@ sizeof(proxy->plainout_buf) - proxy->plainout_size); if (ret <= 0) { - ssl_handle_error(proxy, ret, FALSE, "SSL_read()"); + ssl_handle_error(proxy, ret, "SSL_read()"); break; } else { i_free_and_null(proxy->last_error); @@ -512,7 +507,7 @@ ret = SSL_write(proxy->ssl, proxy->sslout_buf, proxy->sslout_size); if (ret <= 0) - ssl_handle_error(proxy, ret, FALSE, "SSL_write()"); + ssl_handle_error(proxy, ret, "SSL_write()"); else { i_free_and_null(proxy->last_error); proxy->sslout_size -= ret; From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: lib-index: Added v2.2+ forwards compatibility to do... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/55e84bb452e6 changeset: 15527:55e84bb452e6 user: Timo Sirainen date: Tue Nov 27 09:53:33 2012 +0200 description: lib-index: Added v2.2+ forwards compatibility to dovecot.index.cache diffstat: src/lib-index/mail-cache-private.h | 3 ++- src/lib-index/mail-cache-transaction.c | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diffs (36 lines): diff -r 174d7e974326 -r 55e84bb452e6 src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Tue Nov 27 09:13:57 2012 +0200 +++ b/src/lib-index/mail-cache-private.h Tue Nov 27 09:53:33 2012 +0200 @@ -47,7 +47,8 @@ compatibility. */ uint8_t version; uint8_t compat_sizeof_uoff_t; - uint8_t unused[2]; + uint8_t minor_version; + uint8_t unused; uint32_t indexid; uint32_t file_seq; diff -r 174d7e974326 -r 55e84bb452e6 src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Tue Nov 27 09:13:57 2012 +0200 +++ b/src/lib-index/mail-cache-transaction.c Tue Nov 27 09:53:33 2012 +0200 @@ -315,6 +315,9 @@ i_assert(cache->locked); + if (hdr->minor_version > 0) + return FALSE; /* this is record_count field in v2.2+ */ + offset = hdr->hole_offset; prev_offset = 0; while (offset != 0) { if (pread_full(cache->fd, &hole, sizeof(hole), offset) <= 0) { @@ -481,7 +484,8 @@ /* we can just set used_file_size back */ cache->hdr_modified = TRUE; cache->hdr_copy.used_file_size = offset; - } else if (size >= MAIL_CACHE_MIN_HOLE_SIZE) { + } else if (size >= MAIL_CACHE_MIN_HOLE_SIZE && + cache->hdr_copy.minor_version == 0) { /* set it up as a hole */ hole.next_offset = cache->hdr_copy.hole_offset; hole.size = size; From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: mailbox-alias plugin: Error handling fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fd2d24171afd changeset: 15528:fd2d24171afd user: Timo Sirainen date: Tue Nov 27 10:02:59 2012 +0200 description: mailbox-alias plugin: Error handling fix. diffstat: src/plugins/mailbox-alias/mailbox-alias-plugin.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 55e84bb452e6 -r fd2d24171afd src/plugins/mailbox-alias/mailbox-alias-plugin.c --- a/src/plugins/mailbox-alias/mailbox-alias-plugin.c Tue Nov 27 09:53:33 2012 +0200 +++ b/src/plugins/mailbox-alias/mailbox-alias-plugin.c Tue Nov 27 10:02:59 2012 +0200 @@ -114,7 +114,7 @@ if (mailbox_symlink_exists(list, alias->new_vname, &existence) < 0) ret = -1; - if (existence == MAILBOX_SYMLINK_EXISTENCE_SYMLINK) + else if (existence == MAILBOX_SYMLINK_EXISTENCE_SYMLINK) return 1; } } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: mdbox: When rebuilding storage, fsck the map index ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8770940057b9 changeset: 15529:8770940057b9 user: Timo Sirainen date: Tue Nov 27 10:39:26 2012 +0200 description: mdbox: When rebuilding storage, fsck the map index first to make sure it's valid. This fixes assert-crash when the map index contained records with UIDs in wrong order. diffstat: src/lib-storage/index/dbox-multi/mdbox-map.c | 12 +++++++----- src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 6 ++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diffs (49 lines): diff -r fd2d24171afd -r 8770940057b9 src/lib-storage/index/dbox-multi/mdbox-map.c --- a/src/lib-storage/index/dbox-multi/mdbox-map.c Tue Nov 27 10:02:59 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Tue Nov 27 10:39:26 2012 +0200 @@ -243,7 +243,8 @@ int mdbox_map_refresh(struct mdbox_map *map) { struct mail_index_view_sync_ctx *ctx; - bool delayed_expunges; + bool delayed_expunges, fscked; + int ret = 0; /* some open files may have read partially written mails. now that map syncing makes the new mails visible, we need to make sure the @@ -262,14 +263,15 @@ ctx = mail_index_view_sync_begin(map->view, MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); - if (mail_index_reset_fscked(map->view->index)) - mdbox_storage_set_corrupted(map->storage); + fscked = mail_index_reset_fscked(map->view->index); if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) { mail_storage_set_internal_error(MAP_STORAGE(map)); mail_index_reset_error(map->index); - return -1; + ret = -1; } - return 0; + if (fscked) + mdbox_storage_set_corrupted(map->storage); + return ret; } static void diff -r fd2d24171afd -r 8770940057b9 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Tue Nov 27 10:02:59 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Tue Nov 27 10:39:26 2012 +0200 @@ -836,6 +836,12 @@ if (mdbox_map_atomic_lock(ctx->atomic) < 0) return -1; + /* fsck the map just in case its UIDs are broken */ + if (mail_index_fsck(ctx->storage->map->index) < 0) { + mail_storage_set_internal_error(&ctx->storage->storage.storage); + return -1; + } + /* get old map header */ mail_index_get_header_ext(ctx->atomic->sync_view, ctx->storage->map->map_ext_id, From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: fts_parser: Minor error logging improvement Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a9e7f9333f57 changeset: 15530:a9e7f9333f57 user: Timo Sirainen date: Tue Nov 27 10:48:11 2012 +0200 description: fts_parser: Minor error logging improvement diffstat: src/plugins/fts/fts-parser-script.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (20 lines): diff -r 8770940057b9 -r a9e7f9333f57 src/plugins/fts/fts-parser-script.c --- a/src/plugins/fts/fts-parser-script.c Tue Nov 27 10:39:26 2012 +0200 +++ b/src/plugins/fts/fts-parser-script.c Tue Nov 27 10:48:11 2012 +0200 @@ -99,10 +99,13 @@ content->content_type = args[0]; content->extensions = (const void *)(args+1); } + if (!eof_seen) { + if (input->v_offset == 0) + i_error("parser script didn't send any data"); + else + i_error("parser script didn't send empty EOF line"); + } i_stream_destroy(&input); - - if (!eof_seen) - i_error("parser script didn't send empty EOF line"); return 0; } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: lib-storage: Added struct mail_user.nonexistent fla... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fe1be6725304 changeset: 15531:fe1be6725304 user: Timo Sirainen date: Tue Nov 27 10:49:18 2012 +0200 description: lib-storage: Added struct mail_user.nonexistent flag, which is filled by userdb lookup. diffstat: src/lib-storage/mail-user.c | 19 +++++++++++-------- src/lib-storage/mail-user.h | 3 +++ 2 files changed, 14 insertions(+), 8 deletions(-) diffs (50 lines): diff -r a9e7f9333f57 -r fe1be6725304 src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Tue Nov 27 10:48:11 2012 +0200 +++ b/src/lib-storage/mail-user.c Tue Nov 27 10:49:18 2012 +0200 @@ -289,9 +289,6 @@ if (user->remote_ip != NULL) info.remote_ip = *user->remote_ip; - if (mail_user_auth_master_conn == NULL) - return 0; - userdb_pool = pool_alloconly_create("userdb lookup", 2048); ret = auth_master_user_lookup(mail_user_auth_master_conn, user->username, &info, userdb_pool, @@ -313,12 +310,18 @@ return user->_home != NULL ? 1 : 0; } - ret = mail_user_userdb_lookup_home(user); - if (ret < 0) + if (mail_user_auth_master_conn == NULL) { + /* no userdb connection. we can only use mail_home setting. */ + user->_home = user->set->mail_home; + } else if ((ret = mail_user_userdb_lookup_home(user)) < 0) { + /* userdb lookup failed */ return -1; - - if (ret > 0 && user->_home == NULL && *user->set->mail_home != '\0') { - /* no home in userdb, fallback to mail_home setting */ + } else if (ret == 0) { + /* user doesn't exist */ + user->nonexistent = TRUE; + } else if (user->_home == NULL && *user->set->mail_home != '\0') { + /* no home returned by userdb lookup, fallback to mail_home + setting. */ user->_home = user->set->mail_home; } user->home_looked_up = TRUE; diff -r a9e7f9333f57 -r fe1be6725304 src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Tue Nov 27 10:48:11 2012 +0200 +++ b/src/lib-storage/mail-user.h Tue Nov 27 10:49:18 2012 +0200 @@ -42,6 +42,9 @@ /* Module-specific contexts. See mail_storage_module_id. */ ARRAY_DEFINE(module_contexts, union mail_user_module_context *); + /* User doesn't exist (as reported by userdb lookup when looking + up home) */ + unsigned int nonexistent:1; /* Either home is set or there is no home for the user. */ unsigned int home_looked_up:1; /* User is an administrator. Allow operations not normally allowed From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: lib-storage: Handle better when attempting to acces... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7bca3be44b8f changeset: 15532:7bca3be44b8f user: Timo Sirainen date: Tue Nov 27 10:49:41 2012 +0200 description: lib-storage: Handle better when attempting to access shared mailboxes for nonexistent users. diffstat: src/lib-storage/index/shared/shared-storage.c | 18 ++++++++++-------- 1 files changed, 10 insertions(+), 8 deletions(-) diffs (29 lines): diff -r fe1be6725304 -r 7bca3be44b8f src/lib-storage/index/shared/shared-storage.c --- a/src/lib-storage/index/shared/shared-storage.c Tue Nov 27 10:49:18 2012 +0200 +++ b/src/lib-storage/index/shared/shared-storage.c Tue Nov 27 10:49:41 2012 +0200 @@ -257,15 +257,17 @@ user->unexpanded_set); owner->autocreated = TRUE; if (mail_user_init(owner, &error) < 0) { - mailbox_list_set_critical(list, - "Couldn't create namespace '%s' for user %s: %s", - ns->prefix, userdomain, error); - mail_user_unref(&owner); - return -1; - } - if (!var_has_key(storage->location, 'h', "home")) + if (!owner->nonexistent) { + mailbox_list_set_critical(list, + "Couldn't create namespace '%s' for user %s: %s", + ns->prefix, userdomain, error); + mail_user_unref(&owner); + return -1; + } + ret = 0; + } else if (!var_has_key(storage->location, 'h', "home")) { ret = 1; - else { + } else { /* we'll need to look up the user's home directory */ if ((ret = mail_user_get_home(owner, &tab[3].value)) < 0) { mailbox_list_set_critical(list, "Namespace '%s': " From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: fts-solr: XML-encode non-body fields as well Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6a97faf3e500 changeset: 15533:6a97faf3e500 user: Timo Sirainen date: Tue Nov 27 23:06:38 2012 +0200 description: fts-solr: XML-encode non-body fields as well diffstat: src/plugins/fts-solr/fts-backend-solr.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7bca3be44b8f -r 6a97faf3e500 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 10:49:41 2012 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Nov 27 23:06:38 2012 +0200 @@ -305,7 +305,7 @@ } array_foreach_modifiable(&ctx->fields, field) { str_printfa(ctx->cmd, "", field->key); - str_append_str(ctx->cmd, field->value); + xml_encode_data(ctx->cmd, str_data(field->value), str_len(field->value)); str_append(ctx->cmd, ""); str_truncate(field->value, 0); } From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: lib-auth: Added auth_master_get_socket_path() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/755a2f08cde5 changeset: 15534:755a2f08cde5 user: Timo Sirainen date: Wed Nov 28 03:32:01 2012 +0200 description: lib-auth: Added auth_master_get_socket_path() diffstat: src/lib-auth/auth-master.c | 5 +++++ src/lib-auth/auth-master.h | 3 +++ 2 files changed, 8 insertions(+), 0 deletions(-) diffs (28 lines): diff -r 6a97faf3e500 -r 755a2f08cde5 src/lib-auth/auth-master.c --- a/src/lib-auth/auth-master.c Tue Nov 27 23:06:38 2012 +0200 +++ b/src/lib-auth/auth-master.c Wed Nov 28 03:32:01 2012 +0200 @@ -107,6 +107,11 @@ i_free(conn); } +const char *auth_master_get_socket_path(struct auth_master_connection *conn) +{ + return conn->auth_socket_path; +} + static void auth_request_lookup_abort(struct auth_master_connection *conn) { io_loop_stop(conn->ioloop); diff -r 6a97faf3e500 -r 755a2f08cde5 src/lib-auth/auth-master.h --- a/src/lib-auth/auth-master.h Tue Nov 27 23:06:38 2012 +0200 +++ b/src/lib-auth/auth-master.h Wed Nov 28 03:32:01 2012 +0200 @@ -27,6 +27,9 @@ auth_master_init(const char *auth_socket_path, enum auth_master_flags flags); void auth_master_deinit(struct auth_master_connection **conn); +/* Returns the auth_socket_path */ +const char *auth_master_get_socket_path(struct auth_master_connection *conn); + /* Do a USER lookup. Returns -1 = error, 0 = user not found, 1 = ok. When returning -1 and fields[0] isn't NULL, it contains an error message that should be shown to user. */ From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: lib-index: Fixed memory leak with MAIL_INDEX_OPEN_F... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e62938129955 changeset: 15536:e62938129955 user: Timo Sirainen date: Wed Nov 28 03:55:25 2012 +0200 description: lib-index: Fixed memory leak with MAIL_INDEX_OPEN_FLAG_SAVEONLY diffstat: src/lib-index/mail-cache.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 8bc9d40d77c4 -r e62938129955 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Wed Nov 28 03:32:31 2012 +0200 +++ b/src/lib-index/mail-cache.c Wed Nov 28 03:55:25 2012 +0200 @@ -552,6 +552,8 @@ mail_cache_file_close(cache); + if (cache->read_buf != NULL) + buffer_free(&cache->read_buf); hash_table_destroy(&cache->field_name_hash); pool_unref(&cache->field_pool); i_free(cache->field_file_map); From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: doveadm: Log better errors for failing passdb proxy... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8bc9d40d77c4 changeset: 15535:8bc9d40d77c4 user: Timo Sirainen date: Wed Nov 28 03:32:31 2012 +0200 description: doveadm: Log better errors for failing passdb proxy lookups. diffstat: src/doveadm/doveadm-mail-server.c | 15 +++++++++++---- 1 files changed, 11 insertions(+), 4 deletions(-) diffs (45 lines): diff -r 755a2f08cde5 -r 8bc9d40d77c4 src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Wed Nov 28 03:32:01 2012 +0200 +++ b/src/doveadm/doveadm-mail-server.c Wed Nov 28 03:32:31 2012 +0200 @@ -177,7 +177,7 @@ struct auth_master_connection *auth_conn; struct auth_user_info info; pool_t pool; - const char *proxy_host, *const *fields; + const char *auth_socket_path, *proxy_host, *const *fields; unsigned int i; bool proxying; int ret; @@ -195,14 +195,15 @@ pool = pool_alloconly_create("auth lookup", 1024); auth_conn = mail_storage_service_get_auth_conn(ctx->storage_service); + auth_socket_path = auth_master_get_socket_path(auth_conn); ret = auth_master_pass_lookup(auth_conn, input->username, &info, pool, &fields); if (ret < 0) { *error_r = fields[0] != NULL ? t_strdup(fields[0]) : "passdb lookup failed"; - *error_r = t_strdup_printf("%s (to see if user is proxied, " + *error_r = t_strdup_printf("%s: %s (to see if user is proxied, " "because doveadm_proxy_port is set)", - *error_r); + auth_socket_path, *error_r); } else if (ret == 0) { /* user not found from passdb. it could be in userdb though, so just continue with the default host */ @@ -218,7 +219,13 @@ if (!proxying) ret = 0; else if (proxy_host == NULL) { - *error_r = "Proxy is missing destination host"; + *error_r = t_strdup_printf("%s: Proxy is missing destination host", + auth_socket_path); + if (strstr(auth_socket_path, "/auth-userdb") != NULL) { + *error_r = t_strdup_printf( + "%s (maybe set auth_socket_path=director-userdb)", + *error_r); + } ret = -1; } else { *host_r = t_strdup_printf("%s:%u", proxy_host, From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: lib-index: Fixed assert-crash caused by recent cach... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/348e9bad84f0 changeset: 15537:348e9bad84f0 user: Timo Sirainen date: Wed Nov 28 03:56:06 2012 +0200 description: lib-index: Fixed assert-crash caused by recent cache file changes. diffstat: src/lib-index/mail-cache.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (46 lines): diff -r e62938129955 -r 348e9bad84f0 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Wed Nov 28 03:55:25 2012 +0200 +++ b/src/lib-index/mail-cache.c Wed Nov 28 03:56:06 2012 +0200 @@ -281,6 +281,7 @@ !MAIL_CACHE_IS_UNUSABLE(cache) && cache->hdr->file_seq != 0 ? cache->hdr->file_seq : 0; + cache->hdr = NULL; return -1; } } @@ -296,6 +297,7 @@ } else { i_assert(cache->hdr != NULL); } + i_assert(cache->hdr->file_seq != 0); if (offset + size > cache->mmap_length) return 0; @@ -365,8 +367,6 @@ return mail_cache_map_with_read(cache, offset, size, data_r); if (cache->file_cache != NULL) { - cache->hdr = NULL; - ret = file_cache_read(cache->file_cache, offset, size); if (ret < 0) { /* In case of ESTALE we'll simply fail without error @@ -378,6 +378,7 @@ offsets. */ if (errno != ESTALE) mail_cache_set_syscall_error(cache, "read()"); + cache->hdr = NULL; return -1; } @@ -385,7 +386,8 @@ &cache->mmap_length); *data_r = offset > cache->mmap_length ? NULL : CONST_PTR_OFFSET(data, offset); - return mail_cache_map_finish(cache, offset, size, data, TRUE); + return mail_cache_map_finish(cache, offset, size, + offset == 0 ? data : NULL, TRUE); } if (offset < cache->mmap_length && From dovecot at dovecot.org Sat Jan 5 01:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:34 +0200 Subject: dovecot-2.2: stats: Fixed crash when handling multiple users wit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/31ed1f5cc8f1 changeset: 15538:31ed1f5cc8f1 user: Timo Sirainen date: Wed Nov 28 04:21:03 2012 +0200 description: stats: Fixed crash when handling multiple users within same process. diffstat: src/plugins/stats/stats-plugin.c | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diffs (20 lines): diff -r 348e9bad84f0 -r 31ed1f5cc8f1 src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Wed Nov 28 03:56:06 2012 +0200 +++ b/src/plugins/stats/stats-plugin.c Wed Nov 28 04:21:03 2012 +0200 @@ -587,9 +587,13 @@ stats_global_user = user; } else if (stats_user_count == 1) { /* second user connection. we'll need to start doing - per-io callback tracking now. */ - stats_add_session(stats_global_user); - stats_global_user = NULL; + per-io callback tracking now. (we might have been doing it + also previously but just temporarily quickly dropped to + having 1 user, in which case stats_global_user=NULL) */ + if (stats_global_user != NULL) { + stats_add_session(stats_global_user); + stats_global_user = NULL; + } } stats_user_count++; From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: stats: Log username and service for invalid UPDATE-... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c81f1ca3cda6 changeset: 15539:c81f1ca3cda6 user: Timo Sirainen date: Wed Nov 28 04:29:58 2012 +0200 description: stats: Log username and service for invalid UPDATE-SESSIONs diffstat: src/stats/mail-session.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (23 lines): diff -r 31ed1f5cc8f1 -r c81f1ca3cda6 src/stats/mail-session.c --- a/src/stats/mail-session.c Wed Nov 28 04:21:03 2012 +0200 +++ b/src/stats/mail-session.c Wed Nov 28 04:29:58 2012 +0200 @@ -255,13 +255,16 @@ return -1; if (mail_stats_parse(args+1, &stats, error_r) < 0) { - *error_r = t_strconcat("UPDATE-SESSION: ", *error_r, NULL); + *error_r = t_strdup_printf("UPDATE-SESSION %s %s: %s", + session->user->name, + session->service, *error_r); return -1; } if (!mail_stats_diff(&session->stats, &stats, &diff_stats, &error)) { - *error_r = t_strconcat("UPDATE-SESSION: stats shrank: ", - error, NULL); + *error_r = t_strdup_printf("UPDATE-SESSION %s %s: stats shrank: %s", + session->user->name, + session->service, error); return -1; } mail_session_refresh(session, &diff_stats); From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: doveadm: Don't truncate printed output that contain... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6fde42ec5c81 changeset: 15540:6fde42ec5c81 user: Timo Sirainen date: Thu Nov 29 08:35:51 2012 +0200 description: doveadm: Don't truncate printed output that contain NUL bytes. doveadm fetch text output especially could have been truncated. diffstat: src/doveadm/doveadm-print-flow.c | 2 +- src/doveadm/doveadm-print-pager.c | 2 +- src/doveadm/doveadm-print-tab.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diffs (36 lines): diff -r c81f1ca3cda6 -r 6fde42ec5c81 src/doveadm/doveadm-print-flow.c --- a/src/doveadm/doveadm-print-flow.c Wed Nov 28 04:29:58 2012 +0200 +++ b/src/doveadm/doveadm-print-flow.c Thu Nov 29 08:35:51 2012 +0200 @@ -63,7 +63,7 @@ if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) printf("%s=", hdr->title); } - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); if (size == 0) { flow_next_hdr(); ctx->streaming = FALSE; diff -r c81f1ca3cda6 -r 6fde42ec5c81 src/doveadm/doveadm-print-pager.c --- a/src/doveadm/doveadm-print-pager.c Wed Nov 28 04:29:58 2012 +0200 +++ b/src/doveadm/doveadm-print-pager.c Thu Nov 29 08:35:51 2012 +0200 @@ -56,7 +56,7 @@ ctx->streaming = TRUE; printf("%s:\n", hdr->title); } - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); if (size == 0) { pager_next_hdr(); ctx->streaming = FALSE; diff -r c81f1ca3cda6 -r 6fde42ec5c81 src/doveadm/doveadm-print-tab.c --- a/src/doveadm/doveadm-print-tab.c Wed Nov 28 04:29:58 2012 +0200 +++ b/src/doveadm/doveadm-print-tab.c Thu Nov 29 08:35:51 2012 +0200 @@ -51,7 +51,7 @@ } if (ctx.header_idx > 0) printf("\t"); - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); } static void doveadm_print_tab_flush(void) From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: Released v2.1.11. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/741d800a192f changeset: 15541:741d800a192f user: Timo Sirainen date: Thu Nov 29 08:41:21 2012 +0200 description: Released v2.1.11. diffstat: NEWS | 19 +++++++++++++++++++ configure.in | 2 +- 2 files changed, 20 insertions(+), 1 deletions(-) diffs (36 lines): diff -r 6fde42ec5c81 -r 741d800a192f NEWS --- a/NEWS Thu Nov 29 08:35:51 2012 +0200 +++ b/NEWS Thu Nov 29 08:41:21 2012 +0200 @@ -1,3 +1,22 @@ +v2.1.11 2012-11-29 Timo Sirainen + + * lmtp/lda: dovecot.index.cache file is no longer fully mapped to + memory, allowing mail deliveries to work even if the file is huge. + * auth: userdb passwd lookups are now done by auth worker processes + instead of auth master process (as it was documented, but + accidentally didn't work that way). + + + lmtp: lmtp_rcpt_check_quota=yes setting checks quota on RCPT TO. + - lmtp: After successful proxying RCPT TO, the next one to a + nonexistent user gave tempfail error instead of "user not found". + - lmtp proxy: Fixed hanging if remote server was down. + - imap: Fixed crash when SEARCH contained multiple KEYWORD parameters. + - doveadm: Various fixes to handling doveadm-server connections. + - -i parameter for Dovecot tools didn't work correctly. + - director was somewhat broken in v2.1.10. This version also includes + various reliability enhancements. + - auth: passdb imap was broken in v2.1.10. + v2.1.10 2012-09-18 Timo Sirainen + imap: Implemented THREAD=ORDEREDSUBJECT extension. diff -r 6fde42ec5c81 -r 741d800a192f configure.in --- a/configure.in Thu Nov 29 08:35:51 2012 +0200 +++ b/configure.in Thu Nov 29 08:41:21 2012 +0200 @@ -1,5 +1,5 @@ AC_PREREQ([2.59]) -AC_INIT([Dovecot],[2.1.10],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.1.11],[dovecot at dovecot.org]) AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: Added tag 2.1.11 for changeset 741d800a192f Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/15ec2d078a17 changeset: 15542:15ec2d078a17 user: Timo Sirainen date: Thu Nov 29 08:41:21 2012 +0200 description: Added tag 2.1.11 for changeset 741d800a192f diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 741d800a192f -r 15ec2d078a17 .hgtags --- a/.hgtags Thu Nov 29 08:41:21 2012 +0200 +++ b/.hgtags Thu Nov 29 08:41:21 2012 +0200 @@ -87,3 +87,4 @@ 7e5f36fd989d27a2fb48250adbab8fa54ddb6083 2.1.8 bc86680293d256d5a8009690caeb73ab2e34e359 2.1.9 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 +741d800a192fa23572bb14196df2a8917cf20614 2.1.11 From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: Added signature for changeset 741d800a192f Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b101277a3d26 changeset: 15543:b101277a3d26 user: Timo Sirainen date: Thu Nov 29 08:41:28 2012 +0200 description: Added signature for changeset 741d800a192f diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 15ec2d078a17 -r b101277a3d26 .hgsigs --- a/.hgsigs Thu Nov 29 08:41:21 2012 +0200 +++ b/.hgsigs Thu Nov 29 08:41:28 2012 +0200 @@ -50,3 +50,4 @@ 7e5f36fd989d27a2fb48250adbab8fa54ddb6083 0 iEYEABECAAYFAk/yVakACgkQyUhSUUBVismekwCfSEVQjd6fwdChjd53LSt03b4UWKoAoIxd/IjLatTISlHm44iiQwzRKByo bc86680293d256d5a8009690caeb73ab2e34e359 0 iEYEABECAAYFAlAZaTUACgkQyUhSUUBVisnTAACfU1pB34RrXEyLnpnL4Ee/oeNBYcoAnRWxTqx870Efjwf+eBPzafO0C/NU 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT +741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: expire plugin: Don't crash when doveadm is run with... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/75dc4cb4bfe0 changeset: 15545:75dc4cb4bfe0 user: Timo Sirainen date: Fri Nov 30 03:14:10 2012 +0200 description: expire plugin: Don't crash when doveadm is run with proxying diffstat: src/doveadm/client-connection.c | 1 + src/doveadm/doveadm-mail.h | 2 ++ src/plugins/expire/doveadm-expire.c | 5 ++++- 3 files changed, 7 insertions(+), 1 deletions(-) diffs (38 lines): diff -r f77e87298a66 -r 75dc4cb4bfe0 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Thu Nov 29 12:43:54 2012 +0200 +++ b/src/doveadm/client-connection.c Fri Nov 30 03:14:10 2012 +0200 @@ -57,6 +57,7 @@ ctx = doveadm_mail_cmd_init(cmd, set); ctx->full_args = (const void *)(argv + 1); + ctx->proxying = TRUE; ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | diff -r f77e87298a66 -r 75dc4cb4bfe0 src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Thu Nov 29 12:43:54 2012 +0200 +++ b/src/doveadm/doveadm-mail.h Fri Nov 30 03:14:10 2012 +0200 @@ -62,6 +62,8 @@ /* if non-zero, exit with this code */ int exit_code; + /* This command is being called by a remote doveadm client. */ + unsigned int proxying:1; /* We're handling only a single user */ unsigned int iterate_single_user:1; /* We're going through all users (not set for wildcard usernames) */ diff -r f77e87298a66 -r 75dc4cb4bfe0 src/plugins/expire/doveadm-expire.c --- a/src/plugins/expire/doveadm-expire.c Thu Nov 29 12:43:54 2012 +0200 +++ b/src/plugins/expire/doveadm-expire.c Fri Nov 30 03:14:10 2012 +0200 @@ -374,7 +374,10 @@ if (expire_dict == NULL) return; - if (ctx->iterate_single_user) { + /* doveadm proxying uses expire database only locally. the remote + doveadm handles each user one at a time (even though + iterate_single_user=FALSE) */ + if (ctx->iterate_single_user || ctx->proxying) { if (doveadm_debug) { i_debug("expire: Iterating only a single user, " "ignoring expire database"); From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: Makefile: dovecot-config was missing quotes in LIBD... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f77e87298a66 changeset: 15544:f77e87298a66 user: Timo Sirainen date: Thu Nov 29 12:43:54 2012 +0200 description: Makefile: dovecot-config was missing quotes in LIBDOVECOT_LOGIN diffstat: Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b101277a3d26 -r f77e87298a66 Makefile.am --- a/Makefile.am Thu Nov 29 08:41:28 2012 +0200 +++ b/Makefile.am Thu Nov 29 12:43:54 2012 +0200 @@ -63,7 +63,7 @@ grep -v '^LIBDOVECOT_.*_INCLUDE' dovecot-config | \ grep -v '^LIBDOVECOT.*_DEPS' | sed \ -e "s|^\(LIBDOVECOT\)=.*$$|\1='-L$(pkglibdir) -ldovecot'|" \ - -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login $(SSL_LIBS)|" \ + -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1='-ldovecot-login $(SSL_LIBS)'|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: Released v2.1.12. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/75bfda4a7c6c changeset: 15546:75bfda4a7c6c user: Timo Sirainen date: Fri Nov 30 08:56:44 2012 +0200 description: Released v2.1.12. diffstat: NEWS | 4 ++++ configure.in | 2 +- 2 files changed, 5 insertions(+), 1 deletions(-) diffs (21 lines): diff -r 75dc4cb4bfe0 -r 75bfda4a7c6c NEWS --- a/NEWS Fri Nov 30 03:14:10 2012 +0200 +++ b/NEWS Fri Nov 30 08:56:44 2012 +0200 @@ -1,3 +1,7 @@ +v2.1.12 2012-11-30 Timo Sirainen + + - dovecot-config in v2.1.11 caused build problems with Pigeonhole + v2.1.11 2012-11-29 Timo Sirainen * lmtp/lda: dovecot.index.cache file is no longer fully mapped to diff -r 75dc4cb4bfe0 -r 75bfda4a7c6c configure.in --- a/configure.in Fri Nov 30 03:14:10 2012 +0200 +++ b/configure.in Fri Nov 30 08:56:44 2012 +0200 @@ -1,5 +1,5 @@ AC_PREREQ([2.59]) -AC_INIT([Dovecot],[2.1.11],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.1.12],[dovecot at dovecot.org]) AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: Added tag 2.1.12 for changeset 75bfda4a7c6c Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/23ed55ce16d0 changeset: 15547:23ed55ce16d0 user: Timo Sirainen date: Fri Nov 30 08:56:44 2012 +0200 description: Added tag 2.1.12 for changeset 75bfda4a7c6c diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 75bfda4a7c6c -r 23ed55ce16d0 .hgtags --- a/.hgtags Fri Nov 30 08:56:44 2012 +0200 +++ b/.hgtags Fri Nov 30 08:56:44 2012 +0200 @@ -88,3 +88,4 @@ bc86680293d256d5a8009690caeb73ab2e34e359 2.1.9 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 741d800a192fa23572bb14196df2a8917cf20614 2.1.11 +75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: Added signature for changeset 75bfda4a7c6c Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b60872b87d26 changeset: 15548:b60872b87d26 user: Timo Sirainen date: Fri Nov 30 08:56:55 2012 +0200 description: Added signature for changeset 75bfda4a7c6c diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 23ed55ce16d0 -r b60872b87d26 .hgsigs --- a/.hgsigs Fri Nov 30 08:56:44 2012 +0200 +++ b/.hgsigs Fri Nov 30 08:56:55 2012 +0200 @@ -51,3 +51,4 @@ bc86680293d256d5a8009690caeb73ab2e34e359 0 iEYEABECAAYFAlAZaTUACgkQyUhSUUBVisnTAACfU1pB34RrXEyLnpnL4Ee/oeNBYcoAnRWxTqx870Efjwf+eBPzafO0C/NU 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT 741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy +75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99 From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: fts-solr: Fixed memory leak Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cfa6fc9d2856 changeset: 15549:cfa6fc9d2856 user: Timo Sirainen date: Sat Dec 08 09:35:33 2012 +0200 description: fts-solr: Fixed memory leak diffstat: src/plugins/fts-solr/solr-connection.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b60872b87d26 -r cfa6fc9d2856 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Fri Nov 30 08:56:55 2012 +0200 +++ b/src/plugins/fts-solr/solr-connection.c Sat Dec 08 09:35:33 2012 +0200 @@ -192,6 +192,7 @@ curl_slist_free_all(conn->headers_post); curl_multi_cleanup(conn->curlm); curl_easy_cleanup(conn->curl); + XML_ParserFree(conn->xml_parser); i_free(conn->last_sent_url); i_free(conn->url); i_free(conn); From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: imap: If mailbox doesn't have read ACL, hide APPEND... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f74f1b14975d changeset: 15550:f74f1b14975d user: Timo Sirainen date: Sat Dec 08 10:43:21 2012 +0200 description: imap: If mailbox doesn't have read ACL, hide APPENDUID/COPYUID resp-codes. diffstat: src/imap/cmd-append.c | 2 +- src/imap/cmd-copy.c | 6 ++++-- src/lib-storage/mail-storage.h | 3 +++ src/plugins/acl/acl-mailbox.c | 17 ++++++++++++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diffs (88 lines): diff -r cfa6fc9d2856 -r f74f1b14975d src/imap/cmd-append.c --- a/src/imap/cmd-append.c Sat Dec 08 09:35:33 2012 +0200 +++ b/src/imap/cmd-append.c Sat Dec 08 10:43:21 2012 +0200 @@ -290,7 +290,7 @@ msg = t_str_new(256); save_count = seq_range_count(&changes.saved_uids); - if (save_count == 0) { + if (save_count == 0 || changes.no_read_perm) { /* not supported by backend (virtual) */ str_append(msg, "OK Append completed."); } else { diff -r cfa6fc9d2856 -r f74f1b14975d src/imap/cmd-copy.c --- a/src/imap/cmd-copy.c Sat Dec 08 09:35:33 2012 +0200 +++ b/src/imap/cmd-copy.c Sat Dec 08 10:43:21 2012 +0200 @@ -125,8 +125,10 @@ else if (copy_count == 0) { str_append(msg, "OK No messages copied."); pool_unref(&changes.pool); - } else if (seq_range_count(&changes.saved_uids) == 0) { - /* not supported by backend (virtual) */ + } else if (seq_range_count(&changes.saved_uids) == 0 || + changes.no_read_perm) { + /* not supported by backend (virtual) or no read permissions + for mailbox */ str_append(msg, "OK Copy completed."); pool_unref(&changes.pool); } else { diff -r cfa6fc9d2856 -r f74f1b14975d src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Sat Dec 08 09:35:33 2012 +0200 +++ b/src/lib-storage/mail-storage.h Sat Dec 08 10:43:21 2012 +0200 @@ -268,6 +268,9 @@ /* TRUE if anything actually changed with this commit */ bool changed; + /* User doesn't have read ACL for the mailbox, so don't show the + uid_validity / saved_uids. */ + bool no_read_perm; }; struct mailbox_sync_rec { diff -r cfa6fc9d2856 -r f74f1b14975d src/plugins/acl/acl-mailbox.c --- a/src/plugins/acl/acl-mailbox.c Sat Dec 08 09:35:33 2012 +0200 +++ b/src/plugins/acl/acl-mailbox.c Sat Dec 08 10:43:21 2012 +0200 @@ -21,6 +21,7 @@ struct acl_object *aclobj; bool skip_acl_checks; bool acl_enabled; + bool no_read_right; }; struct acl_transaction_context { @@ -408,13 +409,19 @@ { struct acl_mailbox *abox = ACL_CONTEXT(ctx->box); void *at = ACL_CONTEXT(ctx); + int ret; if (at != NULL) { abox->module_ctx.super.transaction_rollback(ctx); return -1; } - return abox->module_ctx.super.transaction_commit(ctx, changes_r); + ret = abox->module_ctx.super.transaction_commit(ctx, changes_r); + if (abox->no_read_right) { + /* don't allow IMAP client to see what UIDs the messages got */ + changes_r->no_read_perm = TRUE; + } + return ret; } static int acl_mailbox_exists(struct mailbox *box, bool auto_boxes, @@ -473,6 +480,14 @@ } return -1; } + if (open_right != ACL_STORAGE_RIGHT_READ) { + ret = acl_object_have_right(abox->aclobj, + idx_arr[ACL_STORAGE_RIGHT_READ]); + if (ret < 0) + return -1; + if (ret == 0) + abox->no_read_right = TRUE; + } return 0; } From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: doveadm: table formatter supports now writing by st... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c1e47738e7c3 changeset: 15551:c1e47738e7c3 user: Timo Sirainen date: Sat Dec 08 10:48:23 2012 +0200 description: doveadm: table formatter supports now writing by streaming. This fixes it to work with doveadm server, which sometimes may use streaming. diffstat: src/doveadm/doveadm-print-table.c | 17 ++++++++++++++--- 1 files changed, 14 insertions(+), 3 deletions(-) diffs (56 lines): diff -r f74f1b14975d -r c1e47738e7c3 src/doveadm/doveadm-print-table.c --- a/src/doveadm/doveadm-print-table.c Sat Dec 08 10:43:21 2012 +0200 +++ b/src/doveadm/doveadm-print-table.c Sat Dec 08 10:48:23 2012 +0200 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "str.h" #include "doveadm-print-private.h" #include @@ -24,6 +25,7 @@ pool_t pool; ARRAY_DEFINE(headers, struct doveadm_print_table_header); ARRAY_TYPE(const_string) buffered_values; + string_t *stream; unsigned int hdr_idx; unsigned int columns; @@ -179,10 +181,17 @@ } static void -doveadm_print_table_print_stream(const unsigned char *value ATTR_UNUSED, - size_t size ATTR_UNUSED) +doveadm_print_table_print_stream(const unsigned char *value, size_t size) { - i_fatal("table formatter doesn't support multi-line values"); + if (memchr(value, '\n', size) != NULL) + i_fatal("table formatter doesn't support multi-line values"); + + if (size != 0) + str_append_n(ctx->stream, value, size); + else { + doveadm_print_table_print(str_c(ctx->stream)); + str_truncate(ctx->stream, 0); + } } static void doveadm_print_table_flush(void) @@ -199,6 +208,7 @@ pool = pool_alloconly_create("doveadm print table", 2048); ctx = p_new(pool, struct doveadm_print_table_context, 1); ctx->pool = pool; + ctx->stream = str_new(default_pool, 128); p_array_init(&ctx->headers, pool, 16); i_array_init(&ctx->buffered_values, 64); ctx->columns = DEFAULT_COLUMNS; @@ -211,6 +221,7 @@ static void doveadm_print_table_deinit(void) { + str_free(&ctx->stream); array_free(&ctx->buffered_values); pool_unref(&ctx->pool); ctx = NULL; From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: lib-master: Ignore mountpoints under /Volumes by de... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/657cb4c35a0e changeset: 15552:657cb4c35a0e user: Timo Sirainen date: Sat Dec 08 11:12:10 2012 +0200 description: lib-master: Ignore mountpoints under /Volumes by default. diffstat: src/lib-master/mountpoint-list.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r c1e47738e7c3 -r 657cb4c35a0e src/lib-master/mountpoint-list.c --- a/src/lib-master/mountpoint-list.c Sat Dec 08 10:48:23 2012 +0200 +++ b/src/lib-master/mountpoint-list.c Sat Dec 08 11:12:10 2012 +0200 @@ -57,6 +57,7 @@ "/proc", "/var/run", "/run", + "/Volumes", /* OSX */ NULL }; From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: lib-master: Ignore /Volumes and /private/tmp mountp... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/265adb00eacd changeset: 15553:265adb00eacd user: Timo Sirainen date: Sat Dec 08 11:13:51 2012 +0200 description: lib-master: Ignore /Volumes and /private/tmp mountpoints with OSX (only). diffstat: src/lib-master/mountpoint-list.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 657cb4c35a0e -r 265adb00eacd src/lib-master/mountpoint-list.c --- a/src/lib-master/mountpoint-list.c Sat Dec 08 11:12:10 2012 +0200 +++ b/src/lib-master/mountpoint-list.c Sat Dec 08 11:13:51 2012 +0200 @@ -57,7 +57,10 @@ "/proc", "/var/run", "/run", - "/Volumes", /* OSX */ +#ifdef __APPLE__ + "/Volumes", + "/private/tmp", +#endif NULL }; From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: virtual plugin: Don't fail if mailbox patterns don'... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f13f2725882e changeset: 15554:f13f2725882e user: Timo Sirainen date: Tue Dec 11 20:04:10 2012 +0200 description: virtual plugin: Don't fail if mailbox patterns don't match anything. diffstat: src/plugins/virtual/virtual-config.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (28 lines): diff -r 265adb00eacd -r f13f2725882e src/plugins/virtual/virtual-config.c --- a/src/plugins/virtual/virtual-config.c Sat Dec 08 11:13:51 2012 +0200 +++ b/src/plugins/virtual/virtual-config.c Tue Dec 11 20:04:10 2012 +0200 @@ -27,6 +27,7 @@ char sep; bool have_wildcards; + bool have_mailbox_defines; }; static struct mail_search_args * @@ -169,6 +170,7 @@ bbox->name++; ctx->mbox->save_bbox = bbox; } + ctx->have_mailbox_defines = TRUE; array_append(&ctx->mbox->backend_boxes, &bbox, 1); return 0; } @@ -419,7 +421,7 @@ if (ret == 0 && ctx.have_wildcards) ret = virtual_config_expand_wildcards(&ctx); - if (ret == 0 && array_count(&mbox->backend_boxes) == 0) { + if (ret == 0 && !ctx.have_mailbox_defines) { mail_storage_set_critical(storage, "%s: No mailboxes defined", path); ret = -1; From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: Added stat_first_parent() helper function. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/40f9e2e339a6 changeset: 15555:40f9e2e339a6 user: Timo Sirainen date: Thu Dec 13 12:13:42 2012 +0200 description: Added stat_first_parent() helper function. diffstat: src/lib/mkdir-parents.c | 20 ++++++++++++++++++++ src/lib/mkdir-parents.h | 8 ++++++++ 2 files changed, 28 insertions(+), 0 deletions(-) diffs (50 lines): diff -r f13f2725882e -r 40f9e2e339a6 src/lib/mkdir-parents.c --- a/src/lib/mkdir-parents.c Tue Dec 11 20:04:10 2012 +0200 +++ b/src/lib/mkdir-parents.c Thu Dec 13 12:13:42 2012 +0200 @@ -125,3 +125,23 @@ { return mkdir_parents_chown(path, mode, (uid_t)-1, (gid_t)-1); } + +int stat_first_parent(const char *path, const char **root_dir_r, + struct stat *st_r) +{ + const char *p; + + while (stat(path, st_r) < 0) { + if (errno != ENOENT || strcmp(path, "/") == 0) { + *root_dir_r = path; + return -1; + } + p = strrchr(path, '/'); + if (p == NULL) + path = "/"; + else + path = t_strdup_until(path, p); + } + *root_dir_r = path; + return 0; +} diff -r f13f2725882e -r 40f9e2e339a6 src/lib/mkdir-parents.h --- a/src/lib/mkdir-parents.h Tue Dec 11 20:04:10 2012 +0200 +++ b/src/lib/mkdir-parents.h Thu Dec 13 12:13:42 2012 +0200 @@ -1,6 +1,8 @@ #ifndef MKDIR_PARENTS_H #define MKDIR_PARENTS_H +#include + /* Create path and all the directories under it if needed. Permissions for existing directories isn't changed. Returns 0 if ok. If directory already exists, returns -1 with errno=EEXIST. */ @@ -20,4 +22,10 @@ int mkdir_chgrp(const char *path, mode_t mode, gid_t gid, const char *gid_origin); +/* stat() the path or its first parent that exists. Returns 0 if ok, -1 if + failed. root_dir is set to the last stat()ed directory (on success and + on failure). */ +int stat_first_parent(const char *path, const char **root_dir_r, + struct stat *st_r); + #endif From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: lib-storage: Use stat_first_parent() instead of doi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c0ad6afa4128 changeset: 15556:c0ad6afa4128 user: Timo Sirainen date: Thu Dec 13 12:14:03 2012 +0200 description: lib-storage: Use stat_first_parent() instead of doing it ourself. diffstat: src/lib-storage/mailbox-list.c | 26 +++----------------------- 1 files changed, 3 insertions(+), 23 deletions(-) diffs (44 lines): diff -r 40f9e2e339a6 -r c0ad6afa4128 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Thu Dec 13 12:13:42 2012 +0200 +++ b/src/lib-storage/mailbox-list.c Thu Dec 13 12:14:03 2012 +0200 @@ -765,27 +765,6 @@ } } -static int -mailbox_list_stat_parent(const char *path, const char **root_dir_r, - struct stat *st_r, const char **error_r) -{ - const char *p; - - while (stat(path, st_r) < 0) { - if (errno != ENOENT || strcmp(path, "/") == 0) { - *error_r = t_strdup_printf("stat(%s) failed: %m", path); - return -1; - } - p = strrchr(path, '/'); - if (p == NULL) - path = "/"; - else - path = t_strdup_until(path, p); - } - *root_dir_r = path; - return 0; -} - static const char * get_expanded_path(const char *unexpanded_start, const char *unexpanded_stop, const char *expanded_full) @@ -873,9 +852,10 @@ /* up to this directory get the permissions from the first parent directory that exists, if it has setgid bit enabled. */ - if (mailbox_list_stat_parent(expanded, &root_dir, &st, - error_r) < 0) + if (stat_first_parent(expanded, &root_dir, &st) < 0) { + *error_r = t_strdup_printf("stat(%s) failed: %m", root_dir); return -1; + } if ((st.st_mode & S_ISGID) != 0 && root_dir != expanded) { if (mkdir_parents_chgrp(expanded, st.st_mode, (gid_t)-1, root_dir) < 0 && From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: dict-file: Automatically mkdir missing parent direc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3c6a13c0d525 changeset: 15557:3c6a13c0d525 user: Timo Sirainen date: Thu Dec 13 12:14:57 2012 +0200 description: dict-file: Automatically mkdir missing parent directories if they don't exist. diffstat: src/lib-dict/dict-file.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 files changed, 39 insertions(+), 0 deletions(-) diffs (70 lines): diff -r c0ad6afa4128 -r 3c6a13c0d525 src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Thu Dec 13 12:14:03 2012 +0200 +++ b/src/lib-dict/dict-file.c Thu Dec 13 12:14:57 2012 +0200 @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "hash.h" +#include "mkdir-parents.h" #include "file-lock.h" #include "file-dotlock.h" #include "nfs-workarounds.h" @@ -424,6 +425,33 @@ return fd_copy_stat_permissions(&src_st, dest_fd, dest_path); } +static int file_dict_mkdir(struct file_dict *dict) +{ + const char *path, *p, *root; + struct stat st; + mode_t mode = 0700; + + p = strrchr(dict->path, '/'); + if (p == NULL) + return 0; + path = t_strdup_until(dict->path, p); + + if (stat_first_parent(path, &root, &st) < 0) { + i_error("stat(%s) failed: %m", root); + return -1; + } + if ((st.st_mode & S_ISGID) != 0) { + /* preserve parent's permissions when it has setgid bit */ + mode = st.st_mode; + } + + if (mkdir_parents(path, mode) < 0) { + i_error("mkdir_parents(%s) failed: %m", path); + return -1; + } + return 0; +} + static int file_dict_lock(struct file_dict *dict, struct file_lock **lock_r) { @@ -435,6 +463,11 @@ if (dict->fd == -1) { /* quota file doesn't exist yet, we need to create it */ dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + if (dict->fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict) < 0) + return -1; + dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + } if (dict->fd == -1) { i_error("creat(%s) failed: %m", dict->path); return -1; @@ -485,6 +518,12 @@ case FILE_LOCK_METHOD_DOTLOCK: fd = file_dotlock_open(&file_dict_dotlock_settings, dict->path, 0, &dotlock); + if (fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict) < 0) + return -1; + fd = file_dotlock_open(&file_dict_dotlock_settings, + dict->path, 0, &dotlock); + } if (fd == -1) { i_error("file dict commit: file_dotlock_open(%s) failed: %m", dict->path); From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: auth: When auto-loading auth mechanisms from plugin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4ec13ce76f6e changeset: 15558:4ec13ce76f6e user: Timo Sirainen date: Tue Dec 18 18:57:41 2012 +0200 description: auth: When auto-loading auth mechanisms from plugins, generate the name better. diffstat: src/auth/mech.c | 17 ++++++++++++++++- 1 files changed, 16 insertions(+), 1 deletions(-) diffs (41 lines): diff -r 3c6a13c0d525 -r 4ec13ce76f6e src/auth/mech.c --- a/src/auth/mech.c Thu Dec 13 12:14:57 2012 +0200 +++ b/src/auth/mech.c Tue Dec 18 18:57:41 2012 +0200 @@ -7,6 +7,7 @@ #include "passdb.h" #include +#include static struct mech_module_list *mech_modules; @@ -112,6 +113,20 @@ reg->modules = list; } +static const char *mech_get_plugin_name(const char *name) +{ + string_t *str = t_str_new(32); + + str_append(str, "mech_"); + for (; *name != '\0'; name++) { + if (*name == '-') + str_append_c(str, '_'); + else + str_append_c(str, i_tolower(*name)); + } + return str_c(str); +} + struct mechanisms_register * mech_register_init(const struct auth_settings *set) { @@ -139,7 +154,7 @@ mech = mech_module_find(name); if (mech == NULL) { /* maybe it's a plugin. try to load it. */ - auth_module_load(t_strconcat("mech_", name, NULL)); + auth_module_load(mech_get_plugin_name(name)); mech = mech_module_find(name); } if (mech == NULL) From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: indexer-worker: Don't assert-crash with some mailbo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/04704d1eb6c3 changeset: 15559:04704d1eb6c3 user: Timo Sirainen date: Tue Dec 18 20:48:36 2012 +0200 description: indexer-worker: Don't assert-crash with some mailbox names in some configurations. Virtual mailbox name shouldn't be accessed directly using mailbox_list_*() functions, since they expect a storage name. In some configurations this could have caused assert-crashes with mailbox names that contained invalid characters. diffstat: src/indexer/master-connection.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 4ec13ce76f6e -r 04704d1eb6c3 src/indexer/master-connection.c --- a/src/indexer/master-connection.c Tue Dec 18 18:57:41 2012 +0200 +++ b/src/indexer/master-connection.c Tue Dec 18 20:48:36 2012 +0200 @@ -129,10 +129,10 @@ return -1; } - path = mailbox_list_get_path(ns->list, mailbox, + path = mailbox_list_get_path(ns->list, NULL, MAILBOX_LIST_PATH_TYPE_INDEX); if (*path == '\0') { - i_info("Indexes disabled for Mailbox %s, skipping", mailbox); + i_info("Indexes disabled for mailbox %s, skipping", mailbox); return 0; } From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: lib-index: If sure mail_index_alloc_cache_destroy_u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b53ab1c11843 changeset: 15560:b53ab1c11843 user: Timo Sirainen date: Tue Dec 18 21:15:19 2012 +0200 description: lib-index: If sure mail_index_alloc_cache_destroy_unrefed() closes indexes it alone keeps open. This makes sure that all index files are closed within process when mailbox is deleted. diffstat: src/lib-index/mail-index-alloc-cache.c | 15 +++++++++++++-- 1 files changed, 13 insertions(+), 2 deletions(-) diffs (47 lines): diff -r 04704d1eb6c3 -r b53ab1c11843 src/lib-index/mail-index-alloc-cache.c --- a/src/lib-index/mail-index-alloc-cache.c Tue Dec 18 20:48:36 2012 +0200 +++ b/src/lib-index/mail-index-alloc-cache.c Tue Dec 18 21:15:19 2012 +0200 @@ -22,6 +22,7 @@ struct mail_index *index; char *mailbox_path; int refcount; + bool referenced; dev_t index_dir_dev; ino_t index_dir_ino; @@ -58,7 +59,7 @@ static void mail_index_alloc_cache_list_free(struct mail_index_alloc_cache_list *list) { - if (list->index->open_count > 0) + if (list->referenced) mail_index_close(list->index); mail_index_free(&list->index); i_free(list->mailbox_path); @@ -166,6 +167,15 @@ } else { if (rec->refcount == 0) seen_ref0 = TRUE; + if (all && rec->index->open_count == 1 && + rec->referenced) { + /* we're the only one keeping this index open. + we might be here, because the caller is + deleting this mailbox and wants its indexes + to be closed. so close it. */ + rec->referenced = FALSE; + mail_index_close(rec->index); + } list = &(*list)->next; } } @@ -228,8 +238,9 @@ list->index_dir_dev = st.st_dev; } } - if (list != NULL) { + if (list != NULL && !list->referenced) { /* keep it referenced for ourself */ + list->referenced = TRUE; index->open_count++; } } From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: example-config: Added missing dovecot-dict-auth.con... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/310b3a77a9d1 changeset: 15561:310b3a77a9d1 user: Timo Sirainen date: Tue Dec 18 21:20:38 2012 +0200 description: example-config: Added missing dovecot-dict-auth.conf.ext diffstat: doc/example-config/Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b53ab1c11843 -r 310b3a77a9d1 doc/example-config/Makefile.am --- a/doc/example-config/Makefile.am Tue Dec 18 21:15:19 2012 +0200 +++ b/doc/example-config/Makefile.am Tue Dec 18 21:20:38 2012 +0200 @@ -13,6 +13,7 @@ example_DATA = \ dovecot.conf \ dovecot-db.conf.ext \ + dovecot-dict-auth.conf.ext \ dovecot-dict-sql.conf.ext \ dovecot-ldap.conf.ext \ dovecot-sql.conf.ext From dovecot at dovecot.org Sat Jan 5 01:14:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:35 +0200 Subject: dovecot-2.2: config: Notify process creation success to master o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bc11033ef035 changeset: 15562:bc11033ef035 user: Timo Sirainen date: Tue Dec 18 21:37:57 2012 +0200 description: config: Notify process creation success to master only after parsing config file diffstat: src/config/main.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 310b3a77a9d1 -r bc11033ef035 src/config/main.c --- a/src/config/main.c Tue Dec 18 21:20:38 2012 +0200 +++ b/src/config/main.c Tue Dec 18 21:37:57 2012 +0200 @@ -26,13 +26,17 @@ restrict_access_by_env(NULL, FALSE); restrict_access_allow_coredumps(TRUE); - master_service_init_finish(master_service); config_parse_load_modules(); path = master_service_get_config_path(master_service); if (config_parse_file(path, TRUE, "", &error) <= 0) i_fatal("%s", error); + /* notify about our success only after successfully parsing the + config file, so if the parsing fails, master won't immediately + just recreate this process (and fail again and so on). */ + master_service_init_finish(master_service); + master_service_run(master_service, client_connected); config_connections_destroy_all(); From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: raw storage: Make sure "from envelope" isn't return... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/579984fdb6e5 changeset: 15563:579984fdb6e5 user: Timo Sirainen date: Tue Dec 18 21:45:08 2012 +0200 description: raw storage: Make sure "from envelope" isn't returned as NULL. diffstat: src/lib-storage/index/raw/raw-mail.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r bc11033ef035 -r 579984fdb6e5 src/lib-storage/index/raw/raw-mail.c --- a/src/lib-storage/index/raw/raw-mail.c Tue Dec 18 21:37:57 2012 +0200 +++ b/src/lib-storage/index/raw/raw-mail.c Tue Dec 18 21:45:08 2012 +0200 @@ -101,7 +101,8 @@ switch (field) { case MAIL_FETCH_FROM_ENVELOPE: - *value_r = mbox->envelope_sender; + *value_r = mbox->envelope_sender != NULL ? + mbox->envelope_sender : ""; return 0; case MAIL_FETCH_UIDL_FILE_NAME: *value_r = mbox->have_filename ? From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: lib-index: Make sure a corrupted mail_cache_header_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2f848393f78e changeset: 15564:2f848393f78e user: Timo Sirainen date: Tue Dec 18 22:05:55 2012 +0200 description: lib-index: Make sure a corrupted mail_cache_header_fields.size doesn't cause crashes. diffstat: src/lib-index/mail-cache-fields.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (31 lines): diff -r 579984fdb6e5 -r 2f848393f78e src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Tue Dec 18 21:45:08 2012 +0200 +++ b/src/lib-index/mail-cache-fields.c Tue Dec 18 22:05:55 2012 +0200 @@ -206,7 +206,7 @@ const struct mail_cache_header_fields *field_hdr; struct mail_cache_header_fields tmp_field_hdr; const void *data; - uint32_t offset = 0, next_offset; + uint32_t offset = 0, next_offset, field_hdr_size; unsigned int next_count = 0; bool invalidate = FALSE; int ret; @@ -276,14 +276,16 @@ cache->need_compress_file_seq = cache->hdr->file_seq; if (field_hdr_r != NULL) { + /* detect corrupted size later */ + field_hdr_size = I_MAX(field_hdr->size, sizeof(*field_hdr)); if (cache->file_cache != NULL && invalidate) { /* if this isn't the first header in file and we hadn't read this before, we can't trust that the cached data is valid */ file_cache_invalidate(cache->file_cache, offset, - field_hdr->size); + field_hdr_size); } - ret = mail_cache_map(cache, offset, field_hdr->size, &data); + ret = mail_cache_map(cache, offset, field_hdr_size, &data); if (ret < 0) return -1; if (ret == 0) { From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: auth: Added CLEAR as yet another alias to PLAIN/CLE... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bde8963a3774 changeset: 15566:bde8963a3774 user: Timo Sirainen date: Tue Dec 18 22:13:48 2012 +0200 description: auth: Added CLEAR as yet another alias to PLAIN/CLEARTEXT password scheme. This apparently is used by Sun LDAP server. diffstat: src/auth/password-scheme.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r bded819417d9 -r bde8963a3774 src/auth/password-scheme.c --- a/src/auth/password-scheme.c Tue Dec 18 22:07:36 2012 +0200 +++ b/src/auth/password-scheme.c Tue Dec 18 22:13:48 2012 +0200 @@ -817,6 +817,7 @@ { "SSHA256", PW_ENCODING_BASE64, 0, ssha256_verify, ssha256_generate }, { "SSHA512", PW_ENCODING_BASE64, 0, ssha512_verify, ssha512_generate }, { "PLAIN", PW_ENCODING_NONE, 0, NULL, plain_generate }, + { "CLEAR", PW_ENCODING_NONE, 0, NULL, plain_generate }, { "CLEARTEXT", PW_ENCODING_NONE, 0, NULL, plain_generate }, { "PLAIN-TRUNC", PW_ENCODING_NONE, 0, plain_trunc_verify, plain_generate }, { "CRAM-MD5", PW_ENCODING_HEX, CRAM_MD5_CONTEXTLEN, From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY was buggy ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bded819417d9 changeset: 15565:bded819417d9 user: Timo Sirainen date: Tue Dec 18 22:07:36 2012 +0200 description: lib-index: MAIL_INDEX_OPEN_FLAG_SAVEONLY was buggy when reading data near end of dovecot.index.cache. We assumed that we read as much as we requested, even if the file was smaller. diffstat: src/lib-index/mail-cache.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 2f848393f78e -r bded819417d9 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Dec 18 22:05:55 2012 +0200 +++ b/src/lib-index/mail-cache.c Tue Dec 18 22:07:36 2012 +0200 @@ -346,11 +346,12 @@ buffer_set_used_size(cache->read_buf, ret); cache->read_offset = offset; - cache->mmap_length = offset + size; + cache->mmap_length = offset + cache->read_buf->used; *data_r = data; hdr_data = offset == 0 ? *data_r : NULL; - return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE); + return mail_cache_map_finish(cache, offset, + cache->read_buf->used, hdr_data, TRUE); } int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: lib-index: If map_with_read gets disabled, reset th... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/32ce915e046a changeset: 15567:32ce915e046a user: Timo Sirainen date: Wed Dec 19 15:00:45 2012 +0200 description: lib-index: If map_with_read gets disabled, reset the cache buffers properly. diffstat: src/lib-index/mail-cache-compress.c | 8 +++++++- src/lib-index/mail-cache.c | 1 + 2 files changed, 8 insertions(+), 1 deletions(-) diffs (29 lines): diff -r bde8963a3774 -r 32ce915e046a src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Tue Dec 18 22:13:48 2012 +0200 +++ b/src/lib-index/mail-cache-compress.c Wed Dec 19 15:00:45 2012 +0200 @@ -462,7 +462,13 @@ return 0; /* compression isn't very efficient with small read()s */ - cache->map_with_read = FALSE; + if (cache->map_with_read) { + cache->map_with_read = FALSE; + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + } if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) { /* we're using dotlocking, cache file creation itself creates diff -r bde8963a3774 -r 32ce915e046a src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Dec 18 22:13:48 2012 +0200 +++ b/src/lib-index/mail-cache.c Wed Dec 19 15:00:45 2012 +0200 @@ -394,6 +394,7 @@ if (offset < cache->mmap_length && size <= cache->mmap_length - offset) { /* already mapped */ + i_assert(cache->mmap_base != NULL); *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); return 1; } From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: lib-index: Fixed sequence lookup of newly created m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7f4835cff04a changeset: 15568:7f4835cff04a user: lion at ubuntu.ubuntu-domain date: Fri Dec 07 08:50:35 2012 +0400 description: lib-index: Fixed sequence lookup of newly created mails in transaction view. diffstat: src/lib-index/mail-index-transaction-view.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 32ce915e046a -r 7f4835cff04a src/lib-index/mail-index-transaction-view.c --- a/src/lib-index/mail-index-transaction-view.c Wed Dec 19 15:00:45 2012 +0200 +++ b/src/lib-index/mail-index-transaction-view.c Fri Dec 07 08:50:35 2012 +0400 @@ -203,7 +203,7 @@ if (first_uid <= rec->uid) break; } - if (seq > tview->t->last_new_seq) { + if (seq > tview->t->last_new_seq || rec->uid > last_uid) { /* no messages in range */ return; } From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: virtual storage: get_first_header() always returned... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/abcc9959b757 changeset: 15570:abcc9959b757 user: Timo Sirainen date: Thu Jan 03 07:30:54 2013 +0200 description: virtual storage: get_first_header() always returned "doesn't exist", breaking SORT/THREAD. diffstat: src/plugins/virtual/virtual-mail.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (24 lines): diff -r c9e6ad8a42db -r abcc9959b757 src/plugins/virtual/virtual-mail.c --- a/src/plugins/virtual/virtual-mail.c Fri Dec 07 08:50:35 2012 +0400 +++ b/src/plugins/virtual/virtual-mail.c Thu Jan 03 07:30:54 2013 +0200 @@ -289,15 +289,17 @@ { struct virtual_mail *vmail = (struct virtual_mail *)mail; struct mail_private *p = (struct mail_private *)vmail->backend_mail; + int ret; if (virtual_mail_handle_lost(vmail) < 0) return -1; - if (p->v.get_first_header(vmail->backend_mail, field, - decode_to_utf8, value_r) < 0) { + ret = p->v.get_first_header(vmail->backend_mail, field, + decode_to_utf8, value_r); + if (ret < 0) { virtual_box_copy_error(mail->box, vmail->backend_mail->box); return -1; } - return 0; + return ret; } static int From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: lib-index: Optimized single sequence lookup of newl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c9e6ad8a42db changeset: 15569:c9e6ad8a42db user: lion at ubuntu.ubuntu-domain date: Fri Dec 07 08:50:35 2012 +0400 description: lib-index: Optimized single sequence lookup of newly created mails in transaction view. diffstat: src/lib-index/mail-index-transaction-view.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 7f4835cff04a -r c9e6ad8a42db src/lib-index/mail-index-transaction-view.c --- a/src/lib-index/mail-index-transaction-view.c Fri Dec 07 08:50:35 2012 +0400 +++ b/src/lib-index/mail-index-transaction-view.c Fri Dec 07 08:50:35 2012 +0400 @@ -208,6 +208,12 @@ return; } *first_seq_r = seq; + + if (rec->uid == last_uid) { + /* one seq in range */ + *last_seq_r = seq; + return; + } } seq = tview->t->last_new_seq; From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: Added buffer_verify_pool() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9fda4feb6e2d changeset: 15571:9fda4feb6e2d user: Timo Sirainen date: Fri Jan 04 03:50:18 2013 +0200 description: Added buffer_verify_pool() diffstat: src/lib/buffer.c | 14 ++++++++++++++ src/lib/buffer.h | 6 ++++++ 2 files changed, 20 insertions(+), 0 deletions(-) diffs (35 lines): diff -r abcc9959b757 -r 9fda4feb6e2d src/lib/buffer.c --- a/src/lib/buffer.c Thu Jan 03 07:30:54 2013 +0200 +++ b/src/lib/buffer.c Fri Jan 04 03:50:18 2013 +0200 @@ -314,3 +314,17 @@ return memcmp(buf1->data, buf2->data, buf1->used) == 0; } + +void buffer_verify_pool(buffer_t *_buf) +{ + const struct real_buffer *buf = (const struct real_buffer *)_buf; + void *ret; + + if (buf->pool->datastack_pool) { + /* this doesn't really do anything except verify the + stack frame */ + ret = p_realloc(buf->pool, buf->w_buffer, + buf->alloc, buf->alloc); + i_assert(ret == buf->w_buffer); + } +} diff -r abcc9959b757 -r 9fda4feb6e2d src/lib/buffer.h --- a/src/lib/buffer.h Thu Jan 03 07:30:54 2013 +0200 +++ b/src/lib/buffer.h Fri Jan 04 03:50:18 2013 +0200 @@ -102,4 +102,10 @@ return buf->used; } +/* Crash if buffer was allocated from data stack and stack frame has changed. + This can be used as an assert-like check to verify that it's valid to + increase the buffer size here, instead of crashing only randomly when the + buffer needs to be increased. */ +void buffer_verify_pool(buffer_t *buf); + #endif From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: DEBUG: Call buffer_verify_pool() on str_c*() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a7a45a56211c changeset: 15572:a7a45a56211c user: Timo Sirainen date: Fri Jan 04 03:50:35 2013 +0200 description: DEBUG: Call buffer_verify_pool() on str_c*() diffstat: src/lib/str.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 9fda4feb6e2d -r a7a45a56211c src/lib/str.c --- a/src/lib/str.c Fri Jan 04 03:50:18 2013 +0200 +++ b/src/lib/str.c Fri Jan 04 03:50:35 2013 +0200 @@ -45,6 +45,9 @@ size_t len = str_len(str); size_t alloc = buffer_get_size(str); +#ifdef DEBUG + buffer_verify_pool(str); +#endif if (len == alloc || data[len] != '\0') { buffer_write(str, len, "", 1); /* remove the \0 - we don't want to keep it */ From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: acl: Treat empty plugin { acl } setting the same as... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/721aba9128b9 changeset: 15573:721aba9128b9 user: Timo Sirainen date: Fri Jan 04 05:29:25 2013 +0200 description: acl: Treat empty plugin { acl } setting the same as if it didn't exist. This allows using -o plugin/acl= to disable ACLs. diffstat: src/plugins/acl/acl-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r a7a45a56211c -r 721aba9128b9 src/plugins/acl/acl-storage.c --- a/src/plugins/acl/acl-storage.c Fri Jan 04 03:50:35 2013 +0200 +++ b/src/plugins/acl/acl-storage.c Fri Jan 04 05:29:25 2013 +0200 @@ -52,7 +52,7 @@ const char *env; env = mail_user_plugin_getenv(user, "acl"); - if (env != NULL) + if (env != NULL && *env != '\0') acl_mail_user_create(user, env); else { if (user->mail_debug) From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: uni_utf8_*(): Treat overlong UTF8 sequences as inva... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7be4709aab5e changeset: 15574:7be4709aab5e user: Timo Sirainen date: Fri Jan 04 23:24:26 2013 +0200 description: uni_utf8_*(): Treat overlong UTF8 sequences as invalid. diffstat: src/lib/Makefile.am | 1 + src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + src/lib/test-unichar.c | 24 ++++++++++++++++++++++++ src/lib/unichar.c | 28 ++++++++++++++-------------- 5 files changed, 41 insertions(+), 14 deletions(-) diffs (125 lines): diff -r 721aba9128b9 -r 7be4709aab5e src/lib/Makefile.am --- a/src/lib/Makefile.am Fri Jan 04 05:29:25 2013 +0200 +++ b/src/lib/Makefile.am Fri Jan 04 23:24:26 2013 +0200 @@ -273,6 +273,7 @@ test-str-find.c \ test-str-sanitize.c \ test-time-util.c \ + test-unichar.c \ test-utc-mktime.c \ test-var-expand.c diff -r 721aba9128b9 -r 7be4709aab5e src/lib/test-lib.c --- a/src/lib/test-lib.c Fri Jan 04 05:29:25 2013 +0200 +++ b/src/lib/test-lib.c Fri Jan 04 23:24:26 2013 +0200 @@ -30,6 +30,7 @@ test_str_find, test_str_sanitize, test_time_util, + test_unichar, test_utc_mktime, test_var_expand, NULL diff -r 721aba9128b9 -r 7be4709aab5e src/lib/test-lib.h --- a/src/lib/test-lib.h Fri Jan 04 05:29:25 2013 +0200 +++ b/src/lib/test-lib.h Fri Jan 04 23:24:26 2013 +0200 @@ -29,6 +29,7 @@ void test_str_find(void); void test_str_sanitize(void); void test_time_util(void); +void test_unichar(void); void test_utc_mktime(void); void test_var_expand(void); diff -r 721aba9128b9 -r 7be4709aab5e src/lib/test-unichar.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-unichar.c Fri Jan 04 23:24:26 2013 +0200 @@ -0,0 +1,24 @@ +/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "str.h" +#include "unichar.h" + +void test_unichar(void) +{ + static const char *overlong_utf8 = "\xf8\x80\x95\x81\xa1"; + unichar_t chr, chr2; + string_t *str = t_str_new(16); + + test_begin("unichars"); + for (chr = 0; chr <= 0x10ffff; chr++) { + str_truncate(str, 0); + uni_ucs4_to_utf8_c(chr, str); + test_assert(uni_utf8_str_is_valid(str_c(str))); + test_assert(uni_utf8_get_char(str_c(str), &chr2) > 0); + test_assert(chr2 == chr); + } + test_assert(!uni_utf8_str_is_valid(overlong_utf8)); + test_assert(uni_utf8_get_char(overlong_utf8, &chr2) < 0); + test_end(); +} diff -r 721aba9128b9 -r 7be4709aab5e src/lib/unichar.c --- a/src/lib/unichar.c Fri Jan 04 05:29:25 2013 +0200 +++ b/src/lib/unichar.c Fri Jan 04 23:24:26 2013 +0200 @@ -37,8 +37,10 @@ int uni_utf8_get_char_n(const void *_input, size_t max_len, unichar_t *chr_r) { + static unichar_t lowest_valid_chr_table[] = + { 0, 0, 0x80, 0x800, 0x10000, 0x20000, 0x40000 }; const unsigned char *input = _input; - unichar_t chr; + unichar_t chr, lowest_valid_chr; unsigned int i, len; int ret; @@ -75,10 +77,12 @@ return -1; } - if (len <= max_len) + if (len <= max_len) { + lowest_valid_chr = lowest_valid_chr_table[len]; ret = 1; - else { + } else { /* check first if the input is invalid before returning 0 */ + lowest_valid_chr = 0; ret = 0; len = max_len; } @@ -91,6 +95,10 @@ chr <<= 6; chr |= input[i] & 0x3f; } + if (chr < lowest_valid_chr) { + /* overlong encoding */ + return -1; + } *chr_r = chr; return ret; @@ -340,19 +348,11 @@ static inline unsigned int is_valid_utf8_seq(const unsigned char *input, unsigned int size) { - unsigned int i, len; + unichar_t chr; - len = uni_utf8_char_bytes(input[0]); - if (unlikely(len > size || len == 1)) + if (uni_utf8_get_char_n(input, size, &chr) <= 0) return 0; - - /* the rest of the chars should be in 0x80..0xbf range. - anything else is start of a sequence or invalid */ - for (i = 1; i < len; i++) { - if (unlikely(input[i] < 0x80 || input[i] > 0xbf)) - return 0; - } - return len; + return uni_utf8_char_bytes(input[0]); } static int uni_utf8_find_invalid_pos(const unsigned char *input, size_t size, From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: quota: Don't crash if shared namespace's owner does... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c91d142130be changeset: 15575:c91d142130be user: Timo Sirainen date: Fri Jan 04 23:59:42 2013 +0200 description: quota: Don't crash if shared namespace's owner doesn't have quota enabled (but you have). diffstat: src/plugins/quota/quota-storage.c | 18 ++++++++++++------ 1 files changed, 12 insertions(+), 6 deletions(-) diffs (50 lines): diff -r 7be4709aab5e -r c91d142130be src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Fri Jan 04 23:24:26 2013 +0200 +++ b/src/plugins/quota/quota-storage.c Fri Jan 04 23:59:42 2013 +0200 @@ -520,6 +520,7 @@ struct quota_mailbox_list *qlist; struct quota *quota = NULL; struct quota_root *root; + struct mail_user *quota_user; bool add; if (QUOTA_USER_CONTEXT(list->ns->user) == NULL) @@ -528,8 +529,14 @@ /* see if we have a quota explicitly defined for this namespace */ quota = quota_get_mail_user_quota(list->ns->user); root = quota_find_root_for_ns(quota, list->ns); - if (root != NULL) + if (root != NULL) { + /* explicit quota root */ root->ns = list->ns; + quota_user = list->ns->user; + } else { + quota_user = list->ns->owner != NULL ? + list->ns->owner : list->ns->user; + } if ((list->ns->flags & NAMESPACE_FLAG_NOQUOTA) != 0) add = FALSE; @@ -538,7 +545,9 @@ explicitly defined for it */ add = root != NULL; } else { - add = TRUE; + /* for shared namespaces add only if the owner has quota + enabled */ + add = QUOTA_USER_CONTEXT(quota_user) != NULL; } if (add) { @@ -550,10 +559,7 @@ v->deinit = quota_mailbox_list_deinit; MODULE_CONTEXT_SET(list, quota_mailbox_list_module, qlist); - /* register to owner's quota roots */ - quota = list->ns->owner != NULL ? - quota_get_mail_user_quota(list->ns->owner) : - quota_get_mail_user_quota(list->ns->user); + quota = quota_get_mail_user_quota(quota_user); quota_add_user_namespace(quota, list->ns); } } From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: lazy_expunge: Allow expunging from lazy_expunge nam... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fc521e6378d5 changeset: 15576:fc521e6378d5 user: Timo Sirainen date: Sat Jan 05 00:12:54 2013 +0200 description: lazy_expunge: Allow expunging from lazy_expunge namespace via virtual mailbox. (Instead of copying the message all over again to the lazy_expunge namespace.) diffstat: src/plugins/lazy-expunge/lazy-expunge-plugin.c | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diffs (24 lines): diff -r c91d142130be -r fc521e6378d5 src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Fri Jan 04 23:59:42 2013 +0200 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Sat Jan 05 00:12:54 2013 +0200 @@ -121,9 +121,20 @@ union mail_module_context *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail); struct lazy_expunge_transaction *lt = LAZY_EXPUNGE_CONTEXT(_mail->transaction); + struct lazy_expunge_mailbox_list *llist; + struct mailbox *real_box; struct mail_save_context *save_ctx; const char *error; + /* don't copy the mail if we're expunging from lazy_expunge + namespace (even if it's via a virtual mailbox) */ + real_box = mail_get_real_mail(_mail)->box; + llist = LAZY_EXPUNGE_LIST_CONTEXT(real_box->list); + if (llist != NULL && llist->internal_namespace) { + mmail->super.expunge(_mail); + return; + } + if (lt->dest_box == NULL) { lt->dest_box = mailbox_open_or_create(luser->lazy_ns->list, _mail->box, &error); From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: auth: Fixed handling usernames with the same name a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3eeb5270963b changeset: 15577:3eeb5270963b user: Timo Sirainen date: Sat Jan 05 00:50:04 2013 +0200 description: auth: Fixed handling usernames with the same name as userdb extra fields. This is a bit kludgy fix, but pretty much guaranteed not to break anything. v2.2 has a cleaner fix, but it's a bit more invasive. diffstat: src/auth/auth-request.c | 4 ++-- src/auth/auth-stream.c | 22 +++++++++++++++++++--- src/auth/auth-stream.h | 1 + 3 files changed, 22 insertions(+), 5 deletions(-) diffs (79 lines): diff -r fc521e6378d5 -r 3eeb5270963b src/auth/auth-request.c --- a/src/auth/auth-request.c Sat Jan 05 00:12:54 2013 +0200 +++ b/src/auth/auth-request.c Sat Jan 05 00:50:04 2013 +0200 @@ -863,7 +863,7 @@ } *result_r = USERDB_RESULT_OK; - *reply_r = auth_stream_reply_init(request->pool); + *reply_r = auth_stream_reply_init_userdb(request->pool); auth_stream_reply_import(*reply_r, value); return TRUE; } @@ -1342,7 +1342,7 @@ { struct userdb_module *module = request->userdb->userdb; - request->userdb_reply = auth_stream_reply_init(request->pool); + request->userdb_reply = auth_stream_reply_init_userdb(request->pool); auth_stream_reply_add(request->userdb_reply, NULL, request->user); userdb_template_export(module->default_fields_tmpl, request); diff -r fc521e6378d5 -r 3eeb5270963b src/auth/auth-stream.c --- a/src/auth/auth-stream.c Sat Jan 05 00:12:54 2013 +0200 +++ b/src/auth/auth-stream.c Sat Jan 05 00:50:04 2013 +0200 @@ -9,6 +9,7 @@ struct auth_stream_reply { string_t *str; + bool userdb; }; struct auth_stream_reply *auth_stream_reply_init(pool_t pool) @@ -20,6 +21,15 @@ return reply; } +struct auth_stream_reply *auth_stream_reply_init_userdb(pool_t pool) +{ + struct auth_stream_reply *reply; + + reply = auth_stream_reply_init(pool); + reply->userdb = TRUE; + return reply; +} + void auth_stream_reply_add(struct auth_stream_reply *reply, const char *key, const char *value) { @@ -44,10 +54,16 @@ auth_stream_reply_find_area(struct auth_stream_reply *reply, const char *key, unsigned int *idx_r, unsigned int *len_r) { - const char *str = str_c(reply->str); - unsigned int i, start, key_len = strlen(key); + const char *p, *str = str_c(reply->str); + unsigned int i = 0, start, key_len = strlen(key); - i = 0; + if (reply->userdb) { + p = strchr(str, '\t'); + if (p == NULL) + return FALSE; + i = p-str+1; + } + while (str[i] != '\0') { start = i; for (; str[i] != '\0'; i++) { diff -r fc521e6378d5 -r 3eeb5270963b src/auth/auth-stream.h --- a/src/auth/auth-stream.h Sat Jan 05 00:12:54 2013 +0200 +++ b/src/auth/auth-stream.h Sat Jan 05 00:50:04 2013 +0200 @@ -4,6 +4,7 @@ struct auth_request; struct auth_stream_reply *auth_stream_reply_init(pool_t pool); +struct auth_stream_reply *auth_stream_reply_init_userdb(pool_t pool); void auth_stream_reply_add(struct auth_stream_reply *reply, const char *key, const char *value); void auth_stream_reply_reset(struct auth_stream_reply *reply); From dovecot at dovecot.org Sat Jan 5 01:14:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Jan 2013 01:14:36 +0200 Subject: dovecot-2.2: Merged changes from v2.1 tree. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/13e74bd5ac8c changeset: 15578:13e74bd5ac8c user: Timo Sirainen date: Sat Jan 05 01:14:11 2013 +0200 description: Merged changes from v2.1 tree. diffstat: .hgsigs | 2 + .hgtags | 2 + Makefile.am | 4 +- NEWS | 23 +++++++ doc/example-config/Makefile.am | 1 + doc/man/doveadm-expunge.1.in | 8 +- src/auth/auth-master-connection.c | 3 +- src/auth/db-ldap.c | 28 ++++++-- src/auth/password-scheme.c | 1 + src/auth/userdb-passwd.c | 7 +- src/config/main.c | 6 +- src/doveadm/client-connection.c | 10 ++- src/doveadm/doveadm-mail-server.c | 42 ++++++++++-- src/doveadm/doveadm-mail.h | 2 + src/doveadm/doveadm-print-flow.c | 2 +- src/doveadm/doveadm-print-pager.c | 2 +- src/doveadm/doveadm-print-tab.c | 2 +- src/doveadm/doveadm-print-table.c | 17 ++++- src/doveadm/server-connection.c | 6 +- src/imap/cmd-append.c | 2 +- src/imap/cmd-copy.c | 6 +- src/indexer/master-connection.c | 2 +- src/lib-auth/auth-master.c | 5 + src/lib-auth/auth-master.h | 3 + src/lib-dict/dict-file.c | 39 ++++++++++++ src/lib-index/mail-cache-compress.c | 8 ++- src/lib-index/mail-cache-fields.c | 8 +- src/lib-index/mail-cache.c | 18 ++++- src/lib-index/mail-index-alloc-cache.c | 15 ++++- src/lib-index/mail-index-transaction-view.c | 8 ++- src/lib-lda/mail-deliver.c | 3 +- src/lib-master/mountpoint-list.c | 4 + src/lib-storage/index/dbox-multi/mdbox-map.c | 12 ++- src/lib-storage/index/dbox-multi/mdbox-save.c | 20 +++--- src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 6 + src/lib-storage/index/index-sync.c | 11 +++ src/lib-storage/index/raw/raw-mail.c | 3 +- src/lib-storage/mail-storage.h | 3 + src/lib-storage/mailbox-list.c | 25 +------- src/lib/Makefile.am | 1 + src/lib/buffer.c | 14 ++++ src/lib/buffer.h | 6 + src/lib/mkdir-parents.c | 20 ++++++ src/lib/mkdir-parents.h | 8 ++ src/lib/str.c | 3 + src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + src/lib/test-unichar.c | 24 +++++++ src/lib/unichar.c | 28 ++++---- src/lib/var-expand.c | 50 +++++++-------- src/login-common/login-proxy.c | 41 +++++++++--- src/plugins/acl/acl-mailbox.c | 17 +++++- src/plugins/acl/acl-storage.c | 2 +- src/plugins/expire/doveadm-expire.c | 5 +- src/plugins/fts-solr/fts-backend-solr.c | 5 +- src/plugins/fts-solr/solr-connection.c | 1 + src/plugins/fts/fts-parser-script.c | 9 +- src/plugins/lazy-expunge/lazy-expunge-plugin.c | 11 +++ src/plugins/mailbox-alias/mailbox-alias-plugin.c | 2 +- src/plugins/quota/quota-storage.c | 20 ++++-- src/plugins/stats/stats-plugin.c | 10 ++- src/plugins/virtual/virtual-config.c | 4 +- src/plugins/virtual/virtual-mail.c | 8 +- src/stats/mail-session.c | 9 +- 64 files changed, 500 insertions(+), 169 deletions(-) diffs (truncated from 1725 to 300 lines): diff -r 922049229f7f -r 13e74bd5ac8c .hgsigs --- a/.hgsigs Sat Jan 05 00:37:26 2013 +0200 +++ b/.hgsigs Sat Jan 05 01:14:11 2013 +0200 @@ -51,3 +51,5 @@ bc86680293d256d5a8009690caeb73ab2e34e359 0 iEYEABECAAYFAlAZaTUACgkQyUhSUUBVisnTAACfU1pB34RrXEyLnpnL4Ee/oeNBYcoAnRWxTqx870Efjwf+eBPzafO0C/NU 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT f5941f3ac7622361634b6cba464da79cc883d1bb 0 iEYEABECAAYFAlCO62QACgkQyUhSUUBViskUtQCffWRQpSqaf+iCOipsTWE1D93TwVEAnAhxx1aezuqDVAsjWoYZkrIufO28 +741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy +75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99 diff -r 922049229f7f -r 13e74bd5ac8c .hgtags --- a/.hgtags Sat Jan 05 00:37:26 2013 +0200 +++ b/.hgtags Sat Jan 05 01:14:11 2013 +0200 @@ -88,3 +88,5 @@ bc86680293d256d5a8009690caeb73ab2e34e359 2.1.9 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 f5941f3ac7622361634b6cba464da79cc883d1bb 2.2.alpha1 +741d800a192fa23572bb14196df2a8917cf20614 2.1.11 +75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 diff -r 922049229f7f -r 13e74bd5ac8c Makefile.am --- a/Makefile.am Sat Jan 05 00:37:26 2013 +0200 +++ b/Makefile.am Sat Jan 05 01:14:11 2013 +0200 @@ -64,12 +64,12 @@ grep -v '^LIBDOVECOT_.*_INCLUDE' dovecot-config | \ grep -v '^LIBDOVECOT.*_DEPS' | sed \ -e "s|^\(LIBDOVECOT\)=.*$$|\1='-L$(pkglibdir) -ldovecot'|" \ - -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login|" \ + -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1='-ldovecot-login $(SSL_LIBS)'|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ -e "s|^\(LIBDOVECOT_SSL\)=.*$$|\1=-ldovecot-ssl|" \ -e "s|^\(LIBDOVECOT_COMPRESS\)=.*$$|\1=-ldovecot-compression|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ - -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1=-ldovecot-storage|" \ + -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ -e "s|^\(LIBDOVECOT_INCLUDE\)=.*$$|\1=-I$(pkgincludedir)|" \ > $(DESTDIR)$(pkglibdir)/dovecot-config diff -r 922049229f7f -r 13e74bd5ac8c NEWS --- a/NEWS Sat Jan 05 00:37:26 2013 +0200 +++ b/NEWS Sat Jan 05 01:14:11 2013 +0200 @@ -30,6 +30,29 @@ + LMTP proxy: Implemented XCLIENT extension for passing remote IP address through proxy. +v2.1.12 2012-11-30 Timo Sirainen + + - dovecot-config in v2.1.11 caused build problems with Pigeonhole + +v2.1.11 2012-11-29 Timo Sirainen + + * lmtp/lda: dovecot.index.cache file is no longer fully mapped to + memory, allowing mail deliveries to work even if the file is huge. + * auth: userdb passwd lookups are now done by auth worker processes + instead of auth master process (as it was documented, but + accidentally didn't work that way). + + + lmtp: lmtp_rcpt_check_quota=yes setting checks quota on RCPT TO. + - lmtp: After successful proxying RCPT TO, the next one to a + nonexistent user gave tempfail error instead of "user not found". + - lmtp proxy: Fixed hanging if remote server was down. + - imap: Fixed crash when SEARCH contained multiple KEYWORD parameters. + - doveadm: Various fixes to handling doveadm-server connections. + - -i parameter for Dovecot tools didn't work correctly. + - director was somewhat broken in v2.1.10. This version also includes + various reliability enhancements. + - auth: passdb imap was broken in v2.1.10. + v2.1.10 2012-09-18 Timo Sirainen + imap: Implemented THREAD=ORDEREDSUBJECT extension. diff -r 922049229f7f -r 13e74bd5ac8c doc/example-config/Makefile.am --- a/doc/example-config/Makefile.am Sat Jan 05 00:37:26 2013 +0200 +++ b/doc/example-config/Makefile.am Sat Jan 05 01:14:11 2013 +0200 @@ -13,6 +13,7 @@ example_DATA = \ dovecot.conf \ dovecot-db.conf.ext \ + dovecot-dict-auth.conf.ext \ dovecot-dict-sql.conf.ext \ dovecot-ldap.conf.ext \ dovecot-sql.conf.ext diff -r 922049229f7f -r 13e74bd5ac8c doc/man/doveadm-expunge.1.in --- a/doc/man/doveadm-expunge.1.in Sat Jan 05 00:37:26 2013 +0200 +++ b/doc/man/doveadm-expunge.1.in Sat Jan 05 01:14:11 2013 +0200 @@ -1,5 +1,5 @@ -.\" Copyright (c) 2010 Dovecot authors, see the included COPYING file -.TH DOVEADM\-EXPUNGE 1 "2010-11-25" "Dovecot v2.1" "Dovecot" +.\" Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file +.TH DOVEADM\-EXPUNGE 1 "2012-11-27" "Dovecot v2.1" "Dovecot" .SH NAME doveadm\-expunge \- Expunge messages matching given search query .\"------------------------------------------------------------------------ @@ -47,6 +47,10 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- +.TP +.B \-d +Delete the mailbox if it is empty after expunging. +.\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- @INCLUDE:option-u-user@ diff -r 922049229f7f -r 13e74bd5ac8c src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/auth/auth-master-connection.c Sat Jan 05 01:14:11 2013 +0200 @@ -240,7 +240,8 @@ auth_request_log_error(auth_request, "userdb", "client doesn't have lookup permissions for this user: %s " - "(change userdb socket permissions)", reason); + "(to bypass this check, set: service auth { unix_listener %s { mode=0777 } })", + reason, conn->path); return -1; } diff -r 922049229f7f -r 13e74bd5ac8c src/auth/db-ldap.c --- a/src/auth/db-ldap.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/auth/db-ldap.c Sat Jan 05 01:14:11 2013 +0200 @@ -990,7 +990,7 @@ char *ldap_attr; if (*data != '\0') { - ldap_attr = p_strdup(ctx->pool, data); + ldap_attr = p_strdup(ctx->pool, t_strcut(data, ':')); array_append(&ctx->attr_names, &ldap_attr, 1); } return NULL; @@ -1198,28 +1198,42 @@ return ctx; } +static const char *db_ldap_field_get_default(const char *data) +{ + const char *p; + + p = strchr(data, ':'); + if (p == NULL) + return ""; + else { + /* default value given */ + return p+1; + } +} + static const char *db_ldap_field_expand(const char *data, void *context) { struct db_ldap_result_iterate_context *ctx = context; struct db_ldap_value *ldap_value; + const char *field_name = t_strcut(data, ':'); - ldap_value = hash_table_lookup(ctx->ldap_attrs, data); + ldap_value = hash_table_lookup(ctx->ldap_attrs, field_name); if (ldap_value == NULL) { - /* ldap attribute wasn't requested */ + /* requested ldap attribute wasn't returned at all */ if (ctx->debug) - str_printfa(ctx->debug, "; %s missing", data); - return ""; + str_printfa(ctx->debug, "; %s missing", field_name); + return db_ldap_field_get_default(data); } ldap_value->used = TRUE; if (ldap_value->values[0] == NULL) { /* no value for ldap attribute */ - return ""; + return db_ldap_field_get_default(data); } if (ldap_value->values[1] != NULL) { auth_request_log_warning(ctx->auth_request, "ldap", "Multiple values found for '%s', using value '%s'", - data, ldap_value->values[0]); + field_name, ldap_value->values[0]); } return ldap_value->values[0]; } diff -r 922049229f7f -r 13e74bd5ac8c src/auth/password-scheme.c --- a/src/auth/password-scheme.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/auth/password-scheme.c Sat Jan 05 01:14:11 2013 +0200 @@ -818,6 +818,7 @@ { "SSHA256", PW_ENCODING_BASE64, 0, ssha256_verify, ssha256_generate }, { "SSHA512", PW_ENCODING_BASE64, 0, ssha512_verify, ssha512_generate }, { "PLAIN", PW_ENCODING_NONE, 0, NULL, plain_generate }, + { "CLEAR", PW_ENCODING_NONE, 0, NULL, plain_generate }, { "CLEARTEXT", PW_ENCODING_NONE, 0, NULL, plain_generate }, { "PLAIN-TRUNC", PW_ENCODING_NONE, 0, plain_trunc_verify, plain_generate }, { "CRAM-MD5", PW_ENCODING_HEX, CRAM_MD5_CONTEXTLEN, diff -r 922049229f7f -r 13e74bd5ac8c src/auth/userdb-passwd.c --- a/src/auth/userdb-passwd.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/auth/userdb-passwd.c Sat Jan 05 01:14:11 2013 +0200 @@ -217,11 +217,10 @@ module = p_new(pool, struct passwd_userdb_module, 1); module->module.cache_key = USER_CACHE_KEY; module->tmpl = userdb_template_build(pool, "passwd", args); + module->module.blocking = TRUE; - if (userdb_template_remove(module->tmpl, "blocking", &value)) { - module->module.blocking = value == NULL || - strcasecmp(value, "yes") == 0; - } + if (userdb_template_remove(module->tmpl, "blocking", &value)) + module->module.blocking = strcasecmp(value, "yes") == 0; /* FIXME: backwards compatibility */ if (!userdb_template_is_empty(module->tmpl)) i_warning("userdb passwd: Move templates args to override_fields setting"); diff -r 922049229f7f -r 13e74bd5ac8c src/config/main.c --- a/src/config/main.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/config/main.c Sat Jan 05 01:14:11 2013 +0200 @@ -26,13 +26,17 @@ restrict_access_by_env(NULL, FALSE); restrict_access_allow_coredumps(TRUE); - master_service_init_finish(master_service); config_parse_load_modules(); path = master_service_get_config_path(master_service); if (config_parse_file(path, TRUE, NULL, &error) <= 0) i_fatal("%s", error); + /* notify about our success only after successfully parsing the + config file, so if the parsing fails, master won't immediately + just recreate this process (and fail again and so on). */ + master_service_init_finish(master_service); + master_service_run(master_service, client_connected); config_connections_destroy_all(); diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/client-connection.c Sat Jan 05 01:14:11 2013 +0200 @@ -57,6 +57,7 @@ ctx = doveadm_mail_cmd_init(cmd, set); ctx->full_args = (const void *)(argv + 1); + ctx->proxying = TRUE; ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | @@ -139,6 +140,10 @@ o_stream_nsend(conn->output, "\n+\n", 3); } pool_unref(&ctx->pool); + + /* clear all headers */ + doveadm_print_deinit(); + doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); } static bool client_is_allowed_command(const struct doveadm_settings *set, @@ -239,8 +244,11 @@ const unsigned char *data; size_t size; - if ((line = i_stream_read_next_line(conn->input)) == NULL) + if ((line = i_stream_read_next_line(conn->input)) == NULL) { + if (conn->input->eof) + return -1; return 0; + } if (*conn->set->doveadm_password == '\0') { i_error("doveadm_password not set, " diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/doveadm-mail-server.c Sat Jan 05 01:14:11 2013 +0200 @@ -22,6 +22,11 @@ #define DOVEADM_MAIL_SERVER_FAILED() \ (internal_failure || master_service_is_killed(master_service)) +struct doveadm_mail_server_cmd { + struct server_connection *conn; + char *username; +}; + static HASH_TABLE(char *, struct doveadm_server *) servers; static pool_t server_pool; static struct doveadm_mail_cmd_context *cmd_ctx; @@ -78,16 +83,22 @@ static void doveadm_cmd_callback(enum server_cmd_reply reply, void *context) { - struct server_connection *conn = context; - struct doveadm_server *server; + struct doveadm_mail_server_cmd *servercmd = context; + struct doveadm_server *server = + server_connection_get_server(servercmd->conn); + const char *username = t_strdup(servercmd->username); + + i_free(servercmd->username); + i_free(servercmd); switch (reply) { case SERVER_CMD_REPLY_INTERNAL_FAILURE: + i_error("%s: Internal failure for %s", server->name, username); internal_failure = TRUE; master_service_stop(master_service); return; case SERVER_CMD_REPLY_UNKNOWN_USER: - i_error("No such user"); + i_error("%s: No such user: %s", server->name, username); From dovecot at dovecot.org Mon Jan 7 11:00:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Jan 2013 11:00:27 +0200 Subject: dovecot-2.1: Released v2.1.13. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/cf9d62fd0b14 changeset: 14868:cf9d62fd0b14 user: Timo Sirainen date: Mon Jan 07 10:55:17 2013 +0200 description: Released v2.1.13. diffstat: NEWS | 7 +++++++ configure.in | 2 +- 2 files changed, 8 insertions(+), 1 deletions(-) diffs (24 lines): diff -r 3eeb5270963b -r cf9d62fd0b14 NEWS --- a/NEWS Sat Jan 05 00:50:04 2013 +0200 +++ b/NEWS Mon Jan 07 10:55:17 2013 +0200 @@ -1,3 +1,10 @@ +v2.1.13 2013-01-06 Timo Sirainen + + - Some fixes to cache file changes in v2.1.11. + - fts-solr: Overlong UTF8 sequences in mails were rejected by Solr and + caused the mails to not be indexed. + - virtual storage: Sorting mailbox by from/to/cc/bcc didn't work. + v2.1.12 2012-11-30 Timo Sirainen - dovecot-config in v2.1.11 caused build problems with Pigeonhole diff -r 3eeb5270963b -r cf9d62fd0b14 configure.in --- a/configure.in Sat Jan 05 00:50:04 2013 +0200 +++ b/configure.in Mon Jan 07 10:55:17 2013 +0200 @@ -1,5 +1,5 @@ AC_PREREQ([2.59]) -AC_INIT([Dovecot],[2.1.12],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.1.13],[dovecot at dovecot.org]) AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) From dovecot at dovecot.org Mon Jan 7 11:00:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Jan 2013 11:00:27 +0200 Subject: dovecot-2.1: Added tag 2.1.13 for changeset cf9d62fd0b14 Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/a6ce877f1cd7 changeset: 14869:a6ce877f1cd7 user: Timo Sirainen date: Mon Jan 07 10:55:17 2013 +0200 description: Added tag 2.1.13 for changeset cf9d62fd0b14 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r cf9d62fd0b14 -r a6ce877f1cd7 .hgtags --- a/.hgtags Mon Jan 07 10:55:17 2013 +0200 +++ b/.hgtags Mon Jan 07 10:55:17 2013 +0200 @@ -89,3 +89,4 @@ 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 741d800a192fa23572bb14196df2a8917cf20614 2.1.11 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 +cf9d62fd0b143efa8e49fac998eb78a648cdd8a9 2.1.13 From dovecot at dovecot.org Mon Jan 7 11:00:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Jan 2013 11:00:27 +0200 Subject: dovecot-2.1: Added signature for changeset cf9d62fd0b14 Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/36eb80cd02ea changeset: 14870:36eb80cd02ea user: Timo Sirainen date: Mon Jan 07 10:55:27 2013 +0200 description: Added signature for changeset cf9d62fd0b14 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r a6ce877f1cd7 -r 36eb80cd02ea .hgsigs --- a/.hgsigs Mon Jan 07 10:55:17 2013 +0200 +++ b/.hgsigs Mon Jan 07 10:55:27 2013 +0200 @@ -52,3 +52,4 @@ 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT 741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99 +cf9d62fd0b143efa8e49fac998eb78a648cdd8a9 0 iEYEABECAAYFAlDqjXUACgkQyUhSUUBViskUEwCfYTWHeDmPr8HfxSBQN17SD5IwDygAnROhb3IVTm9niDun4gxPxbHLo/Pe From dovecot at dovecot.org Mon Jan 7 12:24:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Jan 2013 12:24:34 +0200 Subject: dovecot-2.2: lib-http: Various fixes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d1eaa348482f changeset: 15579:d1eaa348482f user: Timo Sirainen date: Mon Jan 07 12:24:29 2013 +0200 description: lib-http: Various fixes Patch by Stephan Bosch. diffstat: src/lib-http/http-client-connection.c | 7 +- src/lib-http/http-client-private.h | 6 +- src/lib-http/http-client-request.c | 115 ++++++++++++++++++++++++++------- src/lib-http/http-transfer-chunked.c | 16 ++++ src/lib-http/test-http-client.c | 21 ++++++ 5 files changed, 135 insertions(+), 30 deletions(-) diffs (truncated from 332 to 300 lines): diff -r 13e74bd5ac8c -r d1eaa348482f src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Jan 05 01:14:11 2013 +0200 +++ b/src/lib-http/http-client-connection.c Mon Jan 07 12:24:29 2013 +0200 @@ -232,6 +232,7 @@ conn->payload_continue = FALSE; if (conn->peer->no_payload_sync) req->payload_sync = FALSE; + array_append(&conn->request_wait_list, &req, 1); http_client_request_ref(req); @@ -254,7 +255,7 @@ period before sending the payload body. */ if (req->payload_sync) { - i_assert(req->input_size > 0); + i_assert(req->payload_size > 0); i_assert(conn->to_response == NULL); conn->to_response = timeout_add(HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS, http_client_connection_continue_timeout, conn); @@ -432,6 +433,10 @@ if (conn->to_response != NULL) timeout_remove(&conn->to_response); + /* Got some response; cancel response timeout */ + if (conn->to_response != NULL) + timeout_remove(&conn->to_response); + /* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21 Section 7.2: diff -r 13e74bd5ac8c -r d1eaa348482f src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sat Jan 05 01:14:11 2013 +0200 +++ b/src/lib-http/http-client-private.h Mon Jan 07 12:24:29 2013 +0200 @@ -56,8 +56,9 @@ struct http_client_connection *conn; string_t *headers; - struct istream *input; - uoff_t input_size, input_offset; + struct istream *payload_input; + uoff_t payload_size, payload_offset; + struct ostream *payload_output; unsigned int attempts; unsigned int redirects; @@ -68,6 +69,7 @@ enum http_request_state state; unsigned int payload_sync:1; + unsigned int payload_chunked:1; unsigned int ssl:1; unsigned int urgent:1; }; diff -r 13e74bd5ac8c -r d1eaa348482f src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Jan 05 01:14:11 2013 +0200 +++ b/src/lib-http/http-client-request.c Mon Jan 07 12:24:29 2013 +0200 @@ -9,6 +9,7 @@ #include "ostream.h" #include "http-url.h" #include "http-response-parser.h" +#include "http-transfer.h" #include "http-client-private.h" @@ -86,8 +87,10 @@ http_client_request_debug(req, "Destroy (requests left=%d)", client->pending_requests); - if (req->input != NULL) - i_stream_unref(&req->input); + if (req->payload_input != NULL) + i_stream_unref(&req->payload_input); + if (req->payload_output != NULL) + o_stream_unref(&req->payload_output); str_free(&req->headers); pool_unref(&req->pool); *_req = NULL; @@ -131,18 +134,19 @@ struct istream *input, bool sync) { i_assert(req->state == HTTP_REQUEST_STATE_NEW); - i_assert(req->input == NULL); + i_assert(req->payload_input == NULL); i_stream_ref(input); - req->input = input; - if (i_stream_get_size(input, TRUE, &req->input_size) <= 0) - i_unreached(); //FIXME - req->input_offset = input->v_offset; + req->payload_input = input; + if (i_stream_get_size(input, TRUE, &req->payload_size) <= 0) { + req->payload_size = 0; + req->payload_chunked = TRUE; + } + req->payload_offset = input->v_offset; /* prepare request payload sync using 100 Continue response from server */ - if (req->input_size > 0 && sync) { + if ((req->payload_chunked || req->payload_size > 0) && sync) req->payload_sync = TRUE; - } } void http_client_request_submit(struct http_client_request *req) @@ -161,21 +165,24 @@ int http_client_request_send_more(struct http_client_request *req) { struct http_client_connection *conn = req->conn; - struct ostream *output = conn->conn.output; + struct ostream *output = req->payload_output; int ret = 0; - i_assert(req->input != NULL); + i_assert(req->payload_input != NULL); o_stream_set_max_buffer_size(output, 0); - if (o_stream_send_istream(output, req->input) < 0) + if (o_stream_send_istream(output, req->payload_input) < 0) ret = -1; o_stream_set_max_buffer_size(output, (size_t)-1); - if (!i_stream_have_bytes_left(req->input)) { - if (req->input->v_offset != req->input_size) { + if (!i_stream_have_bytes_left(req->payload_input)) { + if (!req->payload_chunked && + req->payload_input->v_offset - req->payload_offset != req->payload_size) { i_error("stream input size changed"); //FIXME return -1; } + o_stream_unref(&output); + req->payload_output = NULL; req->state = HTTP_REQUEST_STATE_WAITING; conn->output_locked = FALSE; http_client_request_debug(req, "Sent all payload"); @@ -212,9 +219,15 @@ if (req->payload_sync) { str_append(rtext, "Expect: 100-continue\r\n"); } - if (req->input_size != 0) { + if (req->payload_chunked) { + str_append(rtext, "Transfer-Encoding: chunked\r\n"); + req->payload_output = + http_transfer_chunked_ostream_create(output); + } else if (req->payload_size != 0) { str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n", - req->input_size); + req->payload_size); + req->payload_output = output; + o_stream_ref(output); } iov[0].iov_base = str_data(rtext); @@ -231,7 +244,7 @@ http_client_request_debug(req, "Sent"); - if (ret >= 0 && req->input_size != 0) { + if (ret >= 0 && req->payload_output != NULL) { if (!req->payload_sync) { if (http_client_request_send_more(req) < 0) ret = -1; @@ -342,14 +355,28 @@ } /* rewind payload stream */ - if (req->input != NULL && req->input_size > 0 && status != 303) { - if (req->input->v_offset != req->input_offset && !req->input->seekable) { + if (req->payload_input != NULL && req->payload_size > 0 && status != 303) { + if (req->payload_input->v_offset != req->payload_offset && + !req->payload_input->seekable) { http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Redirect failed: Cannot resend payload; stream is not seekable"); return; } else { - i_stream_seek(req->input, req->input_offset); + i_stream_seek(req->payload_input, req->payload_offset); + } + } + + /* rewind payload stream */ + if (req->payload_input != NULL && req->payload_size > 0 && status != 303) { + if (req->payload_input->v_offset != req->payload_offset && + !req->payload_input->seekable) { + http_client_request_error(req, + HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Redirect failed: Cannot resend payload; stream is not seekable"); + return; + } else { + i_stream_seek(req->payload_input, req->payload_offset); } } @@ -380,10 +407,30 @@ req->method = p_strdup(req->pool, "GET"); /* drop payload */ - if (req->input != NULL) - i_stream_unref(&req->input); - req->input_size = 0; - req->input_offset = 0; + if (req->payload_input != NULL) + i_stream_unref(&req->payload_input); + req->payload_size = 0; + req->payload_offset = 0; + } + + /* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21 + Section-7.4.4 + + -> A 303 `See Other' redirect status response is handled a bit differently. + Basically, the response content is located elsewhere, but the original + (POST) request is handled already. + */ + if (status == 303 && strcasecmp(req->method, "HEAD") != 0 && + strcasecmp(req->method, "GET") != 0) { + // FIXME: should we provide the means to skip this step? The original + // request was already handled at this point. + req->method = p_strdup(req->pool, "GET"); + + /* drop payload */ + if (req->payload_input != NULL) + i_stream_unref(&req->payload_input); + req->payload_size = 0; + req->payload_offset = 0; } /* resubmit */ @@ -397,14 +444,28 @@ http_client_request_debug(req, "Resubmitting request"); /* rewind payload stream */ - if (req->input != NULL && req->input_size > 0) { - if (req->input->v_offset != req->input_offset && !req->input->seekable) { + if (req->payload_input != NULL && req->payload_size > 0) { + if (req->payload_input->v_offset != req->payload_offset && + !req->payload_input->seekable) { http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Resubmission failed: Cannot resend payload; stream is not seekable"); return; } else { - i_stream_seek(req->input, req->input_offset); + i_stream_seek(req->payload_input, req->payload_offset); + } + } + + /* rewind payload stream */ + if (req->payload_input != NULL && req->payload_size > 0) { + if (req->payload_input->v_offset != req->payload_offset && + !req->payload_input->seekable) { + http_client_request_error(req, + HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Resubmission failed: Cannot resend payload; stream is not seekable"); + return; + } else { + i_stream_seek(req->payload_input, req->payload_offset); } } diff -r 13e74bd5ac8c -r d1eaa348482f src/lib-http/http-transfer-chunked.c --- a/src/lib-http/http-transfer-chunked.c Sat Jan 05 01:14:11 2013 +0200 +++ b/src/lib-http/http-transfer-chunked.c Mon Jan 07 12:24:29 2013 +0200 @@ -475,6 +475,21 @@ return -1; } +static void +http_transfer_chunked_istream_destroy(struct iostream_private *stream) +{ + struct http_transfer_chunked_istream *tcstream = + (struct http_transfer_chunked_istream *)stream; + + if (tcstream->header_parser != NULL) + http_header_parser_deinit(&tcstream->header_parser); + + // FIXME: copied from istream.c; there's got to be a better way. + i_free(tcstream->istream.w_buffer); + if (tcstream->istream.parent != NULL) + i_stream_unref(&tcstream->istream.parent); +} + struct istream * http_transfer_chunked_istream_create(struct istream *input) { @@ -485,6 +500,7 @@ tcstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + tcstream->istream.iostream.destroy = http_transfer_chunked_istream_destroy; tcstream->istream.read = http_transfer_chunked_istream_read; tcstream->istream.istream.readable_fd = FALSE; diff -r 13e74bd5ac8c -r d1eaa348482f src/lib-http/test-http-client.c --- a/src/lib-http/test-http-client.c Sat Jan 05 01:14:11 2013 +0200 +++ b/src/lib-http/test-http-client.c Mon Jan 07 12:24:29 2013 +0200 @@ -208,6 +208,17 @@ test_req = i_new(struct http_test_request, 1); From dovecot at dovecot.org Mon Jan 7 12:33:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Jan 2013 12:33:58 +0200 Subject: dovecot-2.2: Added tag 2.2.beta1 for changeset 86bccdf46d17 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ebec57410960 changeset: 15581:ebec57410960 user: Timo Sirainen date: Mon Jan 07 12:24:58 2013 +0200 description: Added tag 2.2.beta1 for changeset 86bccdf46d17 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 86bccdf46d17 -r ebec57410960 .hgtags --- a/.hgtags Mon Jan 07 12:24:58 2013 +0200 +++ b/.hgtags Mon Jan 07 12:24:58 2013 +0200 @@ -90,3 +90,4 @@ f5941f3ac7622361634b6cba464da79cc883d1bb 2.2.alpha1 741d800a192fa23572bb14196df2a8917cf20614 2.1.11 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 +86bccdf46d172524ca19a1a8a16a50ac30a6743c 2.2.beta1 From dovecot at dovecot.org Mon Jan 7 12:33:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Jan 2013 12:33:58 +0200 Subject: dovecot-2.2: Released v2.2.beta1. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/86bccdf46d17 changeset: 15580:86bccdf46d17 user: Timo Sirainen date: Mon Jan 07 12:24:58 2013 +0200 description: Released v2.2.beta1. diffstat: configure.ac | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d1eaa348482f -r 86bccdf46d17 configure.ac --- a/configure.ac Mon Jan 07 12:24:29 2013 +0200 +++ b/configure.ac Mon Jan 07 12:24:58 2013 +0200 @@ -2,7 +2,7 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.alpha1],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.2.beta1],[dovecot at dovecot.org]) AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv0($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Mon Jan 7 12:33:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 07 Jan 2013 12:33:58 +0200 Subject: dovecot-2.2: Added signature for changeset 86bccdf46d17 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/44b7fe18e5b6 changeset: 15582:44b7fe18e5b6 user: Timo Sirainen date: Mon Jan 07 12:25:01 2013 +0200 description: Added signature for changeset 86bccdf46d17 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r ebec57410960 -r 44b7fe18e5b6 .hgsigs --- a/.hgsigs Mon Jan 07 12:24:58 2013 +0200 +++ b/.hgsigs Mon Jan 07 12:25:01 2013 +0200 @@ -53,3 +53,4 @@ f5941f3ac7622361634b6cba464da79cc883d1bb 0 iEYEABECAAYFAlCO62QACgkQyUhSUUBViskUtQCffWRQpSqaf+iCOipsTWE1D93TwVEAnAhxx1aezuqDVAsjWoYZkrIufO28 741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99 +86bccdf46d172524ca19a1a8a16a50ac30a6743c 0 iEYEABECAAYFAlDqonoACgkQyUhSUUBVisnqqACfaqdR6GxUAJznotKT9WHIUVhVgcIAoJIEa0SBzlGIWThmLvtQByF9vXcc From dovecot at dovecot.org Tue Jan 8 03:03:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 03:03:36 +0200 Subject: dovecot-2.2: Makefile: Removed test-http-client from "make check". Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/425dc9afd023 changeset: 15583:425dc9afd023 user: Timo Sirainen date: Tue Jan 08 03:03:21 2013 +0200 description: Makefile: Removed test-http-client from "make check". Not really a good idea to connect to remote servers (that might fail) during a normal make check. diffstat: src/lib-http/Makefile.am | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 44b7fe18e5b6 -r 425dc9afd023 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Mon Jan 07 12:25:01 2013 +0200 +++ b/src/lib-http/Makefile.am Tue Jan 08 03:03:21 2013 +0200 @@ -37,10 +37,12 @@ test-http-url \ test-http-header-parser \ test-http-transfer \ - test-http-response-parser \ + test-http-response-parser + +test_nocheck_programs = \ test-http-client -noinst_PROGRAMS = $(test_programs) +noinst_PROGRAMS = $(test_programs) $(test_nocheck_programs) test_libs = \ ../lib-test/libtest.la \ From dovecot at dovecot.org Tue Jan 8 03:19:24 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 03:19:24 +0200 Subject: dovecot-2.1: net_connect_unix(): If path is too long, return ENA... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/e4cc3f2b66e3 changeset: 14871:e4cc3f2b66e3 user: Timo Sirainen date: Tue Jan 08 03:19:14 2013 +0200 description: net_connect_unix(): If path is too long, return ENAMETOOLONG if it's available. diffstat: src/lib/network.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 36eb80cd02ea -r e4cc3f2b66e3 src/lib/network.c --- a/src/lib/network.c Mon Jan 07 10:55:27 2013 +0200 +++ b/src/lib/network.c Tue Jan 08 03:19:14 2013 +0200 @@ -278,7 +278,11 @@ sa.un.sun_family = AF_UNIX; if (i_strocpy(sa.un.sun_path, path, sizeof(sa.un.sun_path)) < 0) { /* too long path */ +#ifdef ENAMETOOLONG + errno = ENAMETOOLONG; +#else errno = EINVAL; +#endif return -1; } From dovecot at dovecot.org Tue Jan 8 03:28:53 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 03:28:53 +0200 Subject: dovecot-2.2: run-test.sh didn't exit with failure code when the ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22e39b6bef32 changeset: 15584:22e39b6bef32 user: Timo Sirainen date: Tue Jan 08 03:25:55 2013 +0200 description: run-test.sh didn't exit with failure code when the test failed diffstat: run-test.sh | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (18 lines): diff -r 425dc9afd023 -r 22e39b6bef32 run-test.sh --- a/run-test.sh Tue Jan 08 03:03:21 2013 +0200 +++ b/run-test.sh Tue Jan 08 03:25:55 2013 +0200 @@ -5,11 +5,14 @@ if valgrind --version | grep '^valgrind-3.[012]'; then # RHEL 5.4 still has Valgrind v3.2 valgrind -q --log-file-exactly=test.out.$$ $* + ret=$? else # v3.3+ valgrind -q --log-file=test.out.$$ $* + ret=$? fi if [ -s test.out.$$ ]; then cat test.out.$$ exit 1 fi +exit $ret From dovecot at dovecot.org Tue Jan 8 03:32:16 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 03:32:16 +0200 Subject: dovecot-2.2: test-imap-bodystructure: Fixed the test to pass wit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6a19df272459 changeset: 15585:6a19df272459 user: Timo Sirainen date: Tue Jan 08 03:31:36 2013 +0200 description: test-imap-bodystructure: Fixed the test to pass with new imap-quote logic. diffstat: src/lib-imap/test-imap-bodystructure.c | 6 ++---- 1 files changed, 2 insertions(+), 4 deletions(-) diffs (18 lines): diff -r 22e39b6bef32 -r 6a19df272459 src/lib-imap/test-imap-bodystructure.c --- a/src/lib-imap/test-imap-bodystructure.c Tue Jan 08 03:25:55 2013 +0200 +++ b/src/lib-imap/test-imap-bodystructure.c Tue Jan 08 03:31:36 2013 +0200 @@ -54,12 +54,10 @@ "Root MIME epilogue\n"; static const char testmsg_bodystructure[] = -"(\"text\" \"x-myown\" (\"charset\" \"us-ascii\" \"foo\" {13}\r\n" -"quoted\"string) \"\" \"hellodescription\" \"7bit\" 7 1 \"Q2hlY2sgSW50ZWdyaXR5IQ==\" (\"inline\" (\"foo\" \"bar\")) (\"en\" \"fi\" \"se\") \"http://example.com/test.txt\")(\"message\" \"rfc822\" NIL NIL NIL \"7bit\" 331 (\"Sun, 12 Aug 2012 12:34:56 +0300\" \"submsg\" ((NIL NIL \"sub\" \"domain.org\")) ((NIL NIL \"sub\" \"domain.org\")) ((NIL NIL \"sub\" \"domain.org\")) NIL NIL NIL NIL NIL) ((\"text\" \"html\" (\"charset\" \"us-ascii\") NIL NIL \"8bit\" 20 1 NIL NIL NIL NIL)(\"text\" \"plain\" (\"charset\" \"us-ascii\") NIL NIL \"7bit\" 21 1 NIL NIL NIL NIL) \"alternative\" (\"boundary\" \"sub1\") NIL NIL NIL) 19 NIL NIL NIL NIL) \"mixed\" (\"boundary\" \"foo bar\") NIL NIL NIL"; +"(\"text\" \"x-myown\" (\"charset\" \"us-ascii\" \"foo\" \"quoted\\\"string\") \"\" \"hellodescription\" \"7bit\" 7 1 \"Q2hlY2sgSW50ZWdyaXR5IQ==\" (\"inline\" (\"foo\" \"bar\")) (\"en\" \"fi\" \"se\") \"http://example.com/test.txt\")(\"message\" \"rfc822\" NIL NIL NIL \"7bit\" 331 (\"Sun, 12 Aug 2012 12:34:56 +0300\" \"submsg\" ((NIL NIL \"sub\" \"domain.org\")) ((NIL NIL \"sub\" \"domain.org\")) ((NIL NIL \"sub\" \"domain.org\")) NIL NIL NIL NIL NIL) ((\"text\" \"html\" (\"charset\" \"us-ascii\") NIL NIL \"8bit\" 20 1 NIL NIL NIL NIL)(\"text\" \"plain\" (\"charset\" \"us-ascii\") NIL NIL \"7bit\" 21 1 NIL NIL NIL NIL) \"alternative\" (\"boundary\" \"sub1\") NIL NIL NIL) 19 NIL NIL NIL NIL) \"mixed\" (\"boundary\" \"foo bar\") NIL NIL NIL"; static const char testmsg_body[] = -"(\"text\" \"x-myown\" (\"charset\" \"us-ascii\" \"foo\" {13}\r\n" -"quoted\"string) \"\" \"hellodescription\" \"7bit\" 7 1)(\"message\" \"rfc822\" NIL NIL NIL \"7bit\" 331 (\"Sun, 12 Aug 2012 12:34:56 +0300\" \"submsg\" ((NIL NIL \"sub\" \"domain.org\")) ((NIL NIL \"sub\" \"domain.org\")) ((NIL NIL \"sub\" \"domain.org\")) NIL NIL NIL NIL NIL) ((\"text\" \"html\" (\"charset\" \"us-ascii\") NIL NIL \"8bit\" 20 1)(\"text\" \"plain\" (\"charset\" \"us-ascii\") NIL NIL \"7bit\" 21 1) \"alternative\") 19) \"mixed\""; +"(\"text\" \"x-myown\" (\"charset\" \"us-ascii\" \"foo\" \"quoted\\\"string\") \"\" \"hellodescription\" \"7bit\" 7 1)(\"message\" \"rfc822\" NIL NIL NIL \"7bit\" 331 (\"Sun, 12 Aug 2012 12:34:56 +0300\" \"submsg\" ((NIL NIL \"sub\" \"domain.org\")) ((NIL NIL \"sub\" \"domain.org\")) ((NIL NIL \"sub\" \"domain.org\")) NIL NIL NIL NIL NIL) ((\"text\" \"html\" (\"charset\" \"us-ascii\") NIL NIL \"8bit\" 20 1)(\"text\" \"plain\" (\"charset\" \"us-ascii\") NIL NIL \"7bit\" 21 1) \"alternative\") 19) \"mixed\""; static struct message_part *msg_parse(pool_t pool, bool parse_bodystructure) { From dovecot at dovecot.org Tue Jan 8 03:39:47 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 03:39:47 +0200 Subject: dovecot-2.2: lib-index: Fixed failing transaction log test. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/051affd69699 changeset: 15586:051affd69699 user: Timo Sirainen date: Tue Jan 08 03:39:34 2013 +0200 description: lib-index: Fixed failing transaction log test. diffstat: src/lib-index/test-mail-transaction-log-append.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (25 lines): diff -r 6a19df272459 -r 051affd69699 src/lib-index/test-mail-transaction-log-append.c --- a/src/lib-index/test-mail-transaction-log-append.c Tue Jan 08 03:31:36 2013 +0200 +++ b/src/lib-index/test-mail-transaction-log-append.c Tue Jan 08 03:39:34 2013 +0200 @@ -44,6 +44,7 @@ struct mail_transaction_log_append_ctx *ctx; const struct mail_transaction_header *hdr; const unsigned int *bufp; + const struct mail_transaction_boundary *bound; test_assert(mail_transaction_log_append_begin(log->index, MAIL_TRANSACTION_EXTERNAL, &ctx) == 0); mail_transaction_log_append_add(ctx, MAIL_TRANSACTION_APPEND, @@ -58,6 +59,13 @@ test_assert(file->sync_offset == file->buffer_offset + file->buffer->used); hdr = file->buffer->data; + test_assert(hdr->type == (MAIL_TRANSACTION_BOUNDARY | + MAIL_TRANSACTION_EXTERNAL)); + test_assert(mail_index_offset_to_uint32(hdr->size) == sizeof(*hdr) + sizeof(*bound)); + bound = (const void *)(hdr + 1); + test_assert(bound->size == file->buffer->used); + hdr = (const void *)(bound + 1); + test_assert(hdr->type == (MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL)); test_assert(mail_index_offset_to_uint32(hdr->size) == sizeof(*hdr) + sizeof(buf[0])); From dovecot at dovecot.org Tue Jan 8 03:44:20 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 03:44:20 +0200 Subject: dovecot-2.1: DEBUG: buffer_verify_pool() crashed with static siz... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/427dd81baa8d changeset: 14873:427dd81baa8d user: Timo Sirainen date: Tue Jan 08 03:44:14 2013 +0200 description: DEBUG: buffer_verify_pool() crashed with static sized buffers. diffstat: src/lib/buffer.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3e3e0b482a36 -r 427dd81baa8d src/lib/buffer.c --- a/src/lib/buffer.c Tue Jan 08 03:25:55 2013 +0200 +++ b/src/lib/buffer.c Tue Jan 08 03:44:14 2013 +0200 @@ -320,7 +320,7 @@ const struct real_buffer *buf = (const struct real_buffer *)_buf; void *ret; - if (buf->pool->datastack_pool) { + if (buf->pool != NULL && buf->pool->datastack_pool) { /* this doesn't really do anything except verify the stack frame */ ret = p_realloc(buf->pool, buf->w_buffer, From dovecot at dovecot.org Tue Jan 8 03:44:20 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 03:44:20 +0200 Subject: dovecot-2.1: run-test.sh didn't exit with failure code when the ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/3e3e0b482a36 changeset: 14872:3e3e0b482a36 user: Timo Sirainen date: Tue Jan 08 03:25:55 2013 +0200 description: run-test.sh didn't exit with failure code when the test failed diffstat: run-test.sh | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (18 lines): diff -r e4cc3f2b66e3 -r 3e3e0b482a36 run-test.sh --- a/run-test.sh Tue Jan 08 03:19:14 2013 +0200 +++ b/run-test.sh Tue Jan 08 03:25:55 2013 +0200 @@ -5,11 +5,14 @@ if valgrind --version | grep '^valgrind-3.[012]'; then # RHEL 5.4 still has Valgrind v3.2 valgrind -q --log-file-exactly=test.out.$$ $* + ret=$? else # v3.3+ valgrind -q --log-file=test.out.$$ $* + ret=$? fi if [ -s test.out.$$ ]; then cat test.out.$$ exit 1 fi +exit $ret From dovecot at dovecot.org Tue Jan 8 04:00:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 04:00:14 +0200 Subject: dovecot-2.2: dsync: Assert-crashfix for handling keywords. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf361d83bea6 changeset: 15587:cf361d83bea6 user: Timo Sirainen date: Tue Jan 08 04:00:04 2013 +0200 description: dsync: Assert-crashfix for handling keywords. diffstat: src/doveadm/dsync/dsync-mailbox-export.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 051affd69699 -r cf361d83bea6 src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Tue Jan 08 03:39:34 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Tue Jan 08 04:00:04 2013 +0200 @@ -113,7 +113,7 @@ if (!final_keyword_check(change, keywords[i])) { const char *keyword_change = p_strdup_printf(exporter->pool, "%c%s", - KEYWORD_CHANGE_FINAL, + KEYWORD_CHANGE_ADD, keywords[i]); array_append(&change->keyword_changes, &keyword_change, 1); From dovecot at dovecot.org Tue Jan 8 04:33:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 04:33:21 +0200 Subject: dovecot-2.2: dsync: Avoid infinite looping. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/475a494b2fab changeset: 15588:475a494b2fab user: Timo Sirainen date: Tue Jan 08 04:33:01 2013 +0200 description: dsync: Avoid infinite looping. diffstat: src/doveadm/dsync/dsync-brain-mails.c | 26 ++++++++++++++------------ 1 files changed, 14 insertions(+), 12 deletions(-) diffs (52 lines): diff -r cf361d83bea6 -r 475a494b2fab src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Tue Jan 08 04:00:04 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Tue Jan 08 04:33:01 2013 +0200 @@ -114,7 +114,7 @@ return TRUE; } -static void dsync_brain_send_mail_request(struct dsync_brain *brain) +static bool dsync_brain_send_mail_request(struct dsync_brain *brain) { const struct dsync_mail_request *request; @@ -122,17 +122,19 @@ while ((request = dsync_mailbox_import_next_request(brain->box_importer)) != NULL) { if (dsync_ibc_send_mail_request(brain->ibc, request) == 0) - return; + return TRUE; } - if (brain->box_recv_state > DSYNC_BOX_STATE_CHANGES) { - dsync_ibc_send_end_of_list(brain->ibc); - if (brain->box_exporter != NULL) - brain->box_send_state = DSYNC_BOX_STATE_MAILS; - else { - i_assert(brain->box_recv_state != DSYNC_BOX_STATE_DONE); - brain->box_send_state = DSYNC_BOX_STATE_DONE; - } + if (brain->box_recv_state <= DSYNC_BOX_STATE_CHANGES) + return FALSE; + + dsync_ibc_send_end_of_list(brain->ibc); + if (brain->box_exporter != NULL) + brain->box_send_state = DSYNC_BOX_STATE_MAILS; + else { + i_assert(brain->box_recv_state != DSYNC_BOX_STATE_DONE); + brain->box_send_state = DSYNC_BOX_STATE_DONE; } + return TRUE; } static void dsync_brain_sync_half_finished(struct dsync_brain *brain) @@ -293,8 +295,8 @@ changed = TRUE; break; case DSYNC_BOX_STATE_MAIL_REQUESTS: - dsync_brain_send_mail_request(brain); - changed = TRUE; + if (dsync_brain_send_mail_request(brain)) + changed = TRUE; break; case DSYNC_BOX_STATE_MAILS: if (!dsync_ibc_is_send_queue_full(brain->ibc)) { From dovecot at dovecot.org Tue Jan 8 04:58:18 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 04:58:18 +0200 Subject: dovecot-2.2: dsync: Avoid updating modseqs unnecessarily. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df5b25b11592 changeset: 15589:df5b25b11592 user: Timo Sirainen date: Tue Jan 08 04:58:08 2013 +0200 description: dsync: Avoid updating modseqs unnecessarily. lib-index perhaps should do this automatically as well. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-) diffs (29 lines): diff -r 475a494b2fab -r df5b25b11592 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Tue Jan 08 04:33:01 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Tue Jan 08 04:58:08 2013 +0200 @@ -696,8 +696,10 @@ mail_update_flags(mail, MODIFY_REPLACE, change->add_flags | change->final_flags); - mail_update_modseq(mail, change->modseq); - mail_update_pvt_modseq(mail, change->pvt_modseq); + if (mail_get_modseq(mail) < change->modseq) + mail_update_modseq(mail, change->modseq); + if (mail_get_pvt_modseq(mail) < change->pvt_modseq) + mail_update_pvt_modseq(mail, change->pvt_modseq); } static void @@ -770,8 +772,10 @@ /* merge keywords */ merge_keywords(mail, &local_keyword_changes, &change->keyword_changes, prefer_remote); - mail_update_modseq(mail, change->modseq); - mail_update_pvt_modseq(mail, change->pvt_modseq); + if (mail_get_modseq(mail) < change->modseq) + mail_update_modseq(mail, change->modseq); + if (mail_get_pvt_modseq(mail) < change->pvt_modseq) + mail_update_pvt_modseq(mail, change->pvt_modseq); } static void From dovecot at dovecot.org Tue Jan 8 05:19:42 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 05:19:42 +0200 Subject: dovecot-2.2: dsync: Fixed syncing expunge for last mail in mailbox. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ea3c1bd6d434 changeset: 15590:ea3c1bd6d434 user: Timo Sirainen date: Tue Jan 08 05:19:37 2013 +0200 description: dsync: Fixed syncing expunge for last mail in mailbox. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 73 ++++++++++++++++++------------- 1 files changed, 43 insertions(+), 30 deletions(-) diffs (98 lines): diff -r df5b25b11592 -r ea3c1bd6d434 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Tue Jan 08 04:58:08 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Tue Jan 08 05:19:37 2013 +0200 @@ -916,19 +916,56 @@ return strcmp(change->hdr_hash, hdr_hash) == 0 ? 1 : 0; } +static bool +dsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const struct dsync_mail_change *local_change; + guid_128_t guid_128, change_guid_128; + + if (*change->guid == '\0') { + /* remote doesn't support GUIDs, can't verify expunge */ + return FALSE; + } + + /* local message is expunged. see if we can find its GUID from + transaction log and check if the GUIDs match. The GUID in + log is a 128bit GUID, so we may need to convert the remote's + GUID string to 128bit GUID first. */ + local_change = hash_table_lookup(importer->local_changes, + POINTER_CAST(change->uid)); + if (local_change == NULL || local_change->guid == NULL) + return FALSE; + if (guid_128_from_string(local_change->guid, guid_128) < 0) + i_unreached(); + + mail_generate_guid_128_hash(change->guid, change_guid_128); + if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) { + /* mismatch - found the first non-common UID */ + dsync_mailbox_common_uid_found(importer); + } else { + importer->last_common_uid = change->uid; + } + return TRUE; +} + static void dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer, const struct dsync_mail_change *change) { - const struct dsync_mail_change *local_change; - guid_128_t guid_128, change_guid_128; int ret; /* try to find the matching local mail */ if (!importer_next_mail(importer, change->uid)) { - /* no more local mails. use the last message with a matching - GUID as the last common UID. */ - dsync_mailbox_common_uid_found(importer); + /* no more local mails. we can still try to match + expunged mails though. */ + if (change->guid == NULL || + !dsync_mailbox_find_common_expunged_uid(importer, change)) { + /* couldn't match it for an expunged mail. use the last + message with a matching GUID as the last common + UID. */ + dsync_mailbox_common_uid_found(importer); + } return; } @@ -951,31 +988,7 @@ } return; } - - if (*change->guid == '\0') { - /* remote doesn't support GUIDs, can't verify expunge */ - return; - } - - /* local message is expunged. see if we can find its GUID from - transaction log and check if the GUIDs match. The GUID in - log is a 128bit GUID, so we may need to convert the remote's - GUID string to 128bit GUID first. */ - local_change = hash_table_lookup(importer->local_changes, - POINTER_CAST(change->uid)); - if (local_change == NULL || local_change->guid == NULL) - return; - if (guid_128_from_string(local_change->guid, guid_128) < 0) - i_unreached(); - - mail_generate_guid_128_hash(change->guid, change_guid_128); - if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) { - /* mismatch - found the first non-common UID */ - dsync_mailbox_common_uid_found(importer); - } else { - importer->last_common_uid = change->uid; - } - return; + dsync_mailbox_find_common_expunged_uid(importer, change); } void dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, From dovecot at dovecot.org Tue Jan 8 05:23:10 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 05:23:10 +0200 Subject: dovecot-2.2: dsync: Crashfix for handling expunges from end of m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a7be0c9a00df changeset: 15591:a7be0c9a00df user: Timo Sirainen date: Tue Jan 08 05:23:06 2013 +0200 description: dsync: Crashfix for handling expunges from end of mailbox. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r ea3c1bd6d434 -r a7be0c9a00df src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Tue Jan 08 05:19:37 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Tue Jan 08 05:23:06 2013 +0200 @@ -822,7 +822,8 @@ i_assert(change->uid <= importer->last_common_uid); if (dsync_import_set_mail(importer, change)) mail_expunge(importer->mail); - } else if (change->uid < importer->cur_mail->uid) { + } else if (importer->cur_mail == NULL || + change->uid < importer->cur_mail->uid) { /* already expunged locally, we can ignore this. uid=last_common_uid if we managed to verify from transaction log that the GUIDs match */ From dovecot at dovecot.org Tue Jan 8 05:37:53 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 05:37:53 +0200 Subject: dovecot-2.1: DEBUG: buffer_verify_pool() crashed with empty buffer. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/e098f2f90fb2 changeset: 14874:e098f2f90fb2 user: Timo Sirainen date: Tue Jan 08 05:37:43 2013 +0200 description: DEBUG: buffer_verify_pool() crashed with empty buffer. diffstat: src/lib/buffer.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 427dd81baa8d -r e098f2f90fb2 src/lib/buffer.c --- a/src/lib/buffer.c Tue Jan 08 03:44:14 2013 +0200 +++ b/src/lib/buffer.c Tue Jan 08 05:37:43 2013 +0200 @@ -320,7 +320,7 @@ const struct real_buffer *buf = (const struct real_buffer *)_buf; void *ret; - if (buf->pool != NULL && buf->pool->datastack_pool) { + if (buf->pool != NULL && buf->pool->datastack_pool && buf->alloc > 0) { /* this doesn't really do anything except verify the stack frame */ ret = p_realloc(buf->pool, buf->w_buffer, From dovecot at dovecot.org Tue Jan 8 06:42:09 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 06:42:09 +0200 Subject: dovecot-2.2: dsync: Minor fix for talking with remote dsync. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0949ddc3a892 changeset: 15592:0949ddc3a892 user: Timo Sirainen date: Tue Jan 08 06:41:56 2013 +0200 description: dsync: Minor fix for talking with remote dsync. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 5 +---- 1 files changed, 1 insertions(+), 4 deletions(-) diffs (22 lines): diff -r a7be0c9a00df -r 0949ddc3a892 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Tue Jan 08 05:23:06 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Tue Jan 08 06:41:56 2013 +0200 @@ -331,8 +331,7 @@ } /* try reading some */ - switch (i_stream_read(ibc->input)) { - case -1: + if (i_stream_read(ibc->input) == -1) { if (ibc->input->stream_errno != 0) { errno = ibc->input->stream_errno; i_error("read(%s) failed: %m", ibc->name); @@ -342,8 +341,6 @@ } dsync_ibc_stream_stop(ibc); return -1; - case 0: - return 0; } *line_r = i_stream_next_line(ibc->input); if (*line_r == NULL) { From dovecot at dovecot.org Tue Jan 8 07:43:40 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 07:43:40 +0200 Subject: dovecot-2.2: dsync: Fixes to handling output stream buffering. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5bb80a6491eb changeset: 15594:5bb80a6491eb user: Timo Sirainen date: Tue Jan 08 07:43:29 2013 +0200 description: dsync: Fixes to handling output stream buffering. diffstat: src/doveadm/dsync/dsync-brain-mails.c | 39 ++++++++++++++++------------------ src/doveadm/dsync/dsync-brain.c | 2 - src/doveadm/dsync/dsync-ibc-pipe.c | 1 - src/doveadm/dsync/dsync-ibc-private.h | 1 - src/doveadm/dsync/dsync-ibc-stream.c | 21 +++--------------- src/doveadm/dsync/dsync-ibc.c | 6 ----- src/doveadm/dsync/dsync-ibc.h | 1 - 7 files changed, 22 insertions(+), 49 deletions(-) diffs (174 lines): diff -r e0d4d8f9d097 -r 5bb80a6491eb src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Tue Jan 08 07:42:38 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Tue Jan 08 07:43:29 2013 +0200 @@ -283,31 +283,28 @@ break; } - if (brain->failed) - return TRUE; - - switch (brain->box_send_state) { - case DSYNC_BOX_STATE_MAILBOX: - /* wait for mailbox to be received first */ - break; - case DSYNC_BOX_STATE_CHANGES: - dsync_brain_send_mail_change(brain); - changed = TRUE; - break; - case DSYNC_BOX_STATE_MAIL_REQUESTS: - if (dsync_brain_send_mail_request(brain)) + if (!dsync_ibc_is_send_queue_full(brain->ibc) && !brain->failed) { + switch (brain->box_send_state) { + case DSYNC_BOX_STATE_MAILBOX: + /* wait for mailbox to be received first */ + break; + case DSYNC_BOX_STATE_CHANGES: + dsync_brain_send_mail_change(brain); changed = TRUE; - break; - case DSYNC_BOX_STATE_MAILS: - if (!dsync_ibc_is_send_queue_full(brain->ibc)) { + break; + case DSYNC_BOX_STATE_MAIL_REQUESTS: + if (dsync_brain_send_mail_request(brain)) + changed = TRUE; + break; + case DSYNC_BOX_STATE_MAILS: if (dsync_brain_send_mail(brain)) changed = TRUE; + break; + case DSYNC_BOX_STATE_RECV_LAST_COMMON: + i_unreached(); + case DSYNC_BOX_STATE_DONE: + break; } - break; - case DSYNC_BOX_STATE_RECV_LAST_COMMON: - i_unreached(); - case DSYNC_BOX_STATE_DONE: - break; } if (brain->debug) { i_debug("brain %c: out box '%s' recv_state=%s send_state=%s changed=%d", diff -r e0d4d8f9d097 -r 5bb80a6491eb src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Tue Jan 08 07:42:38 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Tue Jan 08 07:43:29 2013 +0200 @@ -332,8 +332,6 @@ T_BEGIN { ret = dsync_brain_run_real(brain, changed_r); } T_END; - if (!brain->failed) - dsync_ibc_flush(brain->ibc); return ret; } diff -r e0d4d8f9d097 -r 5bb80a6491eb src/doveadm/dsync/dsync-ibc-pipe.c --- a/src/doveadm/dsync/dsync-ibc-pipe.c Tue Jan 08 07:42:38 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-pipe.c Tue Jan 08 07:43:29 2013 +0200 @@ -453,7 +453,6 @@ dsync_ibc_pipe_recv_mail_request, dsync_ibc_pipe_send_mail, dsync_ibc_pipe_recv_mail, - NULL, dsync_ibc_pipe_is_send_queue_full, dsync_ibc_pipe_has_pending_data }; diff -r e0d4d8f9d097 -r 5bb80a6491eb src/doveadm/dsync/dsync-ibc-private.h --- a/src/doveadm/dsync/dsync-ibc-private.h Tue Jan 08 07:42:38 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-private.h Tue Jan 08 07:43:29 2013 +0200 @@ -61,7 +61,6 @@ (*recv_mail)(struct dsync_ibc *ibc, struct dsync_mail **mail_r); - void (*flush)(struct dsync_ibc *ibc); bool (*is_send_queue_full)(struct dsync_ibc *ibc); bool (*has_pending_data)(struct dsync_ibc *ibc); }; diff -r e0d4d8f9d097 -r 5bb80a6491eb src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Tue Jan 08 07:42:38 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Tue Jan 08 07:43:29 2013 +0200 @@ -170,7 +170,9 @@ if (dsync_ibc_stream_read_mail_stream(ibc) == 0) return; } + o_stream_cork(ibc->output); ibc->ibc.io_callback(ibc->ibc.io_context); + o_stream_uncork(ibc->output); } static int dsync_ibc_stream_send_mail_stream(struct dsync_ibc_stream *ibc) @@ -295,8 +297,7 @@ } } T_END; o_stream_nsend_str(ibc->output, ".\n"); - - dsync_ibc_flush(&ibc->ibc); + o_stream_uncork(ibc->output); } static void dsync_ibc_stream_deinit(struct dsync_ibc *_ibc) @@ -1390,11 +1391,7 @@ ibc->mail_output_last = '\0'; ibc->mail_output = mail->input; i_stream_ref(ibc->mail_output); - if (dsync_ibc_stream_send_mail_stream(ibc) == 0) { - /* flush callback isn't being called while output - stream is corked */ - o_stream_uncork(ibc->output); - } + (void)dsync_ibc_stream_send_mail_stream(ibc); } } @@ -1492,15 +1489,6 @@ return DSYNC_IBC_RECV_RET_OK; } -static void dsync_ibc_stream_flush(struct dsync_ibc *_ibc) -{ - struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; - - o_stream_uncork(ibc->output); - if (ibc->mail_output == NULL) - o_stream_cork(ibc->output); -} - static bool dsync_ibc_stream_is_send_queue_full(struct dsync_ibc *_ibc) { struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; @@ -1543,7 +1531,6 @@ dsync_ibc_stream_recv_mail_request, dsync_ibc_stream_send_mail, dsync_ibc_stream_recv_mail, - dsync_ibc_stream_flush, dsync_ibc_stream_is_send_queue_full, dsync_ibc_stream_has_pending_data }; diff -r e0d4d8f9d097 -r 5bb80a6491eb src/doveadm/dsync/dsync-ibc.c --- a/src/doveadm/dsync/dsync-ibc.c Tue Jan 08 07:42:38 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc.c Tue Jan 08 07:43:29 2013 +0200 @@ -178,12 +178,6 @@ return ibc->v.recv_mail(ibc, mail_r); } -void dsync_ibc_flush(struct dsync_ibc *ibc) -{ - if (ibc->v.flush != NULL) - ibc->v.flush(ibc); -} - bool dsync_ibc_has_failed(struct dsync_ibc *ibc) { return ibc->failed; diff -r e0d4d8f9d097 -r 5bb80a6491eb src/doveadm/dsync/dsync-ibc.h --- a/src/doveadm/dsync/dsync-ibc.h Tue Jan 08 07:42:38 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc.h Tue Jan 08 07:43:29 2013 +0200 @@ -110,7 +110,6 @@ enum dsync_ibc_recv_ret dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r); -void dsync_ibc_flush(struct dsync_ibc *ibc); bool dsync_ibc_has_failed(struct dsync_ibc *ibc); bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc); bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc); From dovecot at dovecot.org Tue Jan 8 07:43:40 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 07:43:40 +0200 Subject: dovecot-2.2: replicator: Send only one sync command to doveadm-s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e0d4d8f9d097 changeset: 15593:e0d4d8f9d097 user: Timo Sirainen date: Tue Jan 08 07:42:38 2013 +0200 description: replicator: Send only one sync command to doveadm-server per connection. This fixes setuid()/setgid() problems when multiple UIDs are used. diffstat: src/replication/replicator/doveadm-connection.c | 21 ++++++++++++++++----- 1 files changed, 16 insertions(+), 5 deletions(-) diffs (80 lines): diff -r 0949ddc3a892 -r e0d4d8f9d097 src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Tue Jan 08 06:41:56 2013 +0200 +++ b/src/replication/replicator/doveadm-connection.c Tue Jan 08 07:42:38 2013 +0200 @@ -29,6 +29,7 @@ time_t last_connect_failure; unsigned int handshaked:1; unsigned int end_of_print:1; + unsigned int cmd_sent:1; }; struct doveadm_connection *doveadm_connection_init(const char *path) @@ -55,7 +56,7 @@ callback(reply, context); } -static void doveadm_disconnect(struct doveadm_connection *conn) +static void doveadm_close(struct doveadm_connection *conn) { if (conn->fd == -1) return; @@ -66,7 +67,13 @@ if (close(conn->fd) < 0) i_error("close(doveadm) failed: %m"); conn->fd = -1; + conn->end_of_print = FALSE; + conn->cmd_sent = FALSE; +} +static void doveadm_disconnect(struct doveadm_connection *conn) +{ + doveadm_close(conn); if (conn->callback != NULL) doveadm_callback(conn, DOVEADM_REPLY_FAIL); } @@ -102,6 +109,9 @@ conn->end_of_print = TRUE; return 0; } + line = t_strdup(line); + doveadm_close(conn); + if (line[0] == '+') doveadm_callback(conn, DOVEADM_REPLY_OK); else if (line[0] == '-') { @@ -113,9 +123,9 @@ i_error("%s: Invalid input: %s", conn->path, line); return -1; } - conn->end_of_print = FALSE; /* FIXME: disconnect after each request for now. - doveadm server's getopt() handling seems to break otherwise */ + doveadm server's getopt() handling seems to break otherwise. + also with multiple UIDs doveadm-server fails because setid() fails */ return -1; } @@ -158,7 +168,7 @@ static void doveadm_fail_timeout(struct doveadm_connection *conn) { - doveadm_callback(conn, DOVEADM_REPLY_FAIL); + doveadm_disconnect(conn); } void doveadm_connection_sync(struct doveadm_connection *conn, @@ -170,6 +180,7 @@ i_assert(callback != NULL); i_assert(!doveadm_connection_is_busy(conn)); + conn->cmd_sent = TRUE; conn->callback = callback; conn->context = context; @@ -192,5 +203,5 @@ bool doveadm_connection_is_busy(struct doveadm_connection *conn) { - return conn->callback != NULL; + return conn->cmd_sent; } From dovecot at dovecot.org Tue Jan 8 10:25:31 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 10:25:31 +0200 Subject: dovecot-2.2: dsync: Print remote's stderr line-buffered. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6ff774dc0b6e changeset: 15595:6ff774dc0b6e user: Timo Sirainen date: Tue Jan 08 10:25:21 2013 +0200 description: dsync: Print remote's stderr line-buffered. Especially when debugging is enabled there can be a lot of output, which makes this useful. diffstat: src/doveadm/dsync/doveadm-dsync.c | 25 ++++++++++++++----------- 1 files changed, 14 insertions(+), 11 deletions(-) diffs (65 lines): diff -r 5bb80a6491eb -r 6ff774dc0b6e src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Tue Jan 08 07:43:29 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Tue Jan 08 10:25:21 2013 +0200 @@ -40,6 +40,7 @@ int fd_in, fd_out, fd_err; struct io *io_err; + struct istream *err_stream; unsigned int lock_timeout; @@ -55,18 +56,13 @@ static void remote_error_input(struct dsync_cmd_context *ctx) { - char buf[1024]; - ssize_t ret; + const char *line; - ret = read(ctx->fd_err, buf, sizeof(buf)-1); - if (ret == -1) { + while ((line = i_stream_read_next_line(ctx->err_stream)) != NULL) + i_error("remote: %s", line); + + if (ctx->err_stream->eof) io_remove(&ctx->io_err); - return; - } - if (ret > 0) { - buf[ret-1] = '\0'; - i_error("remote: %s", buf); - } } static void @@ -107,7 +103,8 @@ ctx->fd_in = fd_out[0]; ctx->fd_out = fd_in[1]; ctx->fd_err = fd_err[0]; - ctx->io_err = io_add(ctx->fd_err, IO_READ, remote_error_input, ctx); + ctx->err_stream = i_stream_create_fd(ctx->fd_err, IO_BLOCK_SIZE, FALSE); + i_stream_set_return_partial_line(ctx->err_stream, TRUE); } static void @@ -374,6 +371,8 @@ mail_user_set_get_temp_prefix(temp_prefix, user->set); ibc = cmd_dsync_icb_stream_init(ctx, ctx->remote_name, str_c(temp_prefix)); + ctx->io_err = io_add(ctx->fd_err, IO_READ, + remote_error_input, ctx); } brain_flags = DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS | @@ -411,6 +410,10 @@ dsync_ibc_deinit(&ibc); if (ibc2 != NULL) dsync_ibc_deinit(&ibc2); + if (ctx->err_stream != NULL) { + remote_error_input(ctx); /* print any pending errors */ + i_stream_destroy(&ctx->err_stream); + } if (ctx->io_err != NULL) io_remove(&ctx->io_err); if (ctx->fd_in != -1) { From dovecot at dovecot.org Tue Jan 8 10:57:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 10:57:38 +0200 Subject: dovecot-2.2: dsync: Fixes to printing remote's stderr output. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f0c11f532e3b changeset: 15596:f0c11f532e3b user: Timo Sirainen date: Tue Jan 08 10:57:34 2013 +0200 description: dsync: Fixes to printing remote's stderr output. diffstat: src/doveadm/dsync/doveadm-dsync.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (15 lines): diff -r 6ff774dc0b6e -r f0c11f532e3b src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Tue Jan 08 10:25:21 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Tue Jan 08 10:57:34 2013 +0200 @@ -59,9 +59,9 @@ const char *line; while ((line = i_stream_read_next_line(ctx->err_stream)) != NULL) - i_error("remote: %s", line); + fprintf(stderr, "%s\n", line); - if (ctx->err_stream->eof) + if (ctx->err_stream->eof && ctx->io_err != NULL) io_remove(&ctx->io_err); } From dovecot at dovecot.org Tue Jan 8 16:24:31 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Jan 2013 16:24:31 +0200 Subject: dovecot-2.2: dsync: Make remote's stderr fd nonblocking to avoid... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2ae4a40b3b24 changeset: 15597:2ae4a40b3b24 user: Timo Sirainen date: Tue Jan 08 16:24:20 2013 +0200 description: dsync: Make remote's stderr fd nonblocking to avoid hanging. diffstat: src/doveadm/dsync/doveadm-dsync.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r f0c11f532e3b -r 2ae4a40b3b24 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Tue Jan 08 10:57:34 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Tue Jan 08 16:24:20 2013 +0200 @@ -103,6 +103,8 @@ ctx->fd_in = fd_out[0]; ctx->fd_out = fd_in[1]; ctx->fd_err = fd_err[0]; + + fd_set_nonblock(ctx->fd_err, TRUE); ctx->err_stream = i_stream_create_fd(ctx->fd_err, IO_BLOCK_SIZE, FALSE); i_stream_set_return_partial_line(ctx->err_stream, TRUE); } From pigeonhole at rename-it.nl Wed Jan 9 01:38:22 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 09 Jan 2013 00:38:22 +0100 Subject: dovecot-2.2-pigeonhole: Adjusted to changes in Dovecot. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/50c067006c67 changeset: 1700:50c067006c67 user: Stephan Bosch date: Wed Jan 09 00:38:07 2013 +0100 description: Adjusted to changes in Dovecot. diffstat: src/lib-sieve/edit-mail.c | 16 ++++++++++++++++ 1 files changed, 16 insertions(+), 0 deletions(-) diffs (47 lines): diff -r 613258f9f541 -r 50c067006c67 src/lib-sieve/edit-mail.c --- a/src/lib-sieve/edit-mail.c Mon Oct 29 22:42:34 2012 +0100 +++ b/src/lib-sieve/edit-mail.c Wed Jan 09 00:38:07 2013 +0100 @@ -1078,6 +1078,13 @@ return edmail->wrapped->v.get_modseq(&edmail->wrapped->mail); } +static uint64_t edit_mail_get_pvt_modseq(struct mail *mail) +{ + struct edit_mail *edmail = (struct edit_mail *)mail; + + return edmail->wrapped->v.get_pvt_modseq(&edmail->wrapped->mail); +} + static int edit_mail_get_parts (struct mail *mail, struct message_part **parts_r) { @@ -1397,6 +1404,13 @@ edmail->wrapped->v.update_modseq(&edmail->wrapped->mail, min_modseq); } +static void edit_mail_update_pvt_modseq(struct mail *mail, uint64_t min_pvt_modseq) +{ + struct edit_mail *edmail = (struct edit_mail *)mail; + + edmail->wrapped->v.update_pvt_modseq(&edmail->wrapped->mail, min_pvt_modseq); +} + static void edit_mail_update_pop3_uidl(struct mail *mail, const char *uidl) { struct edit_mail *edmail = (struct edit_mail *)mail; @@ -1431,6 +1445,7 @@ edit_mail_get_keywords, edit_mail_get_keyword_indexes, edit_mail_get_modseq, + edit_mail_get_pvt_modseq, edit_mail_get_parts, edit_mail_get_date, edit_mail_get_received_date, @@ -1447,6 +1462,7 @@ edit_mail_update_flags, edit_mail_update_keywords, edit_mail_update_modseq, + edit_mail_update_pvt_modseq, edit_mail_update_pop3_uidl, edit_mail_expunge, edit_mail_set_cache_corrupted, From pigeonhole at rename-it.nl Wed Jan 9 01:55:58 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 09 Jan 2013 00:55:58 +0100 Subject: dovecot-2.2-pigeonhole: Fixed SMTP crash. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c937a4060ee2 changeset: 1701:c937a4060ee2 user: Stephan Bosch date: Wed Jan 09 00:55:49 2013 +0100 description: Fixed SMTP crash. By Timo Sirainen. diffstat: src/lib-sieve/sieve-smtp.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r 50c067006c67 -r c937a4060ee2 src/lib-sieve/sieve-smtp.c --- a/src/lib-sieve/sieve-smtp.c Wed Jan 09 00:38:07 2013 +0100 +++ b/src/lib-sieve/sieve-smtp.c Wed Jan 09 00:55:49 2013 +0100 @@ -19,8 +19,7 @@ if ( senv->smtp_open == NULL || senv->smtp_close == NULL ) return NULL; - return senv->smtp_open - (senv->script_context, destination, return_path, output_r); + return senv->smtp_open(senv, destination, return_path, output_r); } bool sieve_smtp_close From dovecot at dovecot.org Wed Jan 9 05:35:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:35:35 +0200 Subject: dovecot-2.1: dbox: Give a name to the file's output stream (for ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/bdd1499989c6 changeset: 14875:bdd1499989c6 user: Timo Sirainen date: Wed Jan 09 05:30:06 2013 +0200 description: dbox: Give a name to the file's output stream (for improved error logging). diffstat: src/lib-storage/index/dbox-common/dbox-file.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r e098f2f90fb2 -r bdd1499989c6 src/lib-storage/index/dbox-common/dbox-file.c --- a/src/lib-storage/index/dbox-common/dbox-file.c Tue Jan 08 05:37:43 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Wed Jan 09 05:30:06 2013 +0200 @@ -469,6 +469,7 @@ ctx->file = file; if (file->fd != -1) { ctx->output = o_stream_create_fd_file(file->fd, 0, FALSE); + o_stream_set_name(ctx->output, file->cur_path); o_stream_cork(ctx->output); } return ctx; From dovecot at dovecot.org Wed Jan 9 05:35:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:35:35 +0200 Subject: dovecot-2.1: lib-storage: dest_mail wasn't reset if previous sav... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b87a7c4d775f changeset: 14876:b87a7c4d775f user: Timo Sirainen date: Wed Jan 09 05:33:32 2013 +0200 description: lib-storage: dest_mail wasn't reset if previous save was aborted. This could have happened only with dsync. diffstat: src/lib-storage/index/index-mail.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (19 lines): diff -r bdd1499989c6 -r b87a7c4d775f src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Jan 09 05:30:06 2013 +0200 +++ b/src/lib-storage/index/index-mail.c Wed Jan 09 05:33:32 2013 +0200 @@ -1387,8 +1387,13 @@ { struct index_mail *mail = (struct index_mail *)_mail; - if (mail->data.seq == seq) - return; + if (mail->data.seq == seq) { + if (!saving) + return; + /* we started saving a mail, aborted it, and now we're saving + another mail with the same sequence. make sure the mail + gets reset. */ + } mail->mail.v.close(&mail->mail.mail); From dovecot at dovecot.org Wed Jan 9 05:35:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:35:35 +0200 Subject: dovecot-2.1: mdbox: Allow reflink-copying a mail when wanted GUI... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/df06c834d7e8 changeset: 14877:df06c834d7e8 user: Timo Sirainen date: Wed Jan 09 05:35:20 2013 +0200 description: mdbox: Allow reflink-copying a mail when wanted GUID matches the current GUID. diffstat: src/lib-storage/index/dbox-multi/mdbox-save.c | 28 ++++++++++++++++++-------- 1 files changed, 19 insertions(+), 9 deletions(-) diffs (57 lines): diff -r b87a7c4d775f -r df06c834d7e8 src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Jan 09 05:33:32 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Jan 09 05:35:20 2013 +0200 @@ -412,14 +412,14 @@ struct dbox_save_mail *save_mail; struct mdbox_mailbox *src_mbox; struct mdbox_mail_index_record rec; - const void *data; + const void *guid_data; + guid_128_t wanted_guid; bool expunged; ctx->ctx.finished = TRUE; if (mail->box->storage != _ctx->transaction->box->storage || - _ctx->transaction->box->disable_reflink_copy_to || - _ctx->guid != NULL) + _ctx->transaction->box->disable_reflink_copy_to) return mail_storage_copy(_ctx, mail); src_mbox = (struct mdbox_mailbox *)mail->box; @@ -429,6 +429,20 @@ &rec.map_uid) < 0) return -1; + mail_index_lookup_ext(mail->transaction->view, mail->seq, + src_mbox->guid_ext_id, &guid_data, &expunged); + if (guid_data == NULL || guid_128_is_empty(guid_data)) { + /* missing GUID, something's broken. don't copy using + refcounting. */ + return mail_storage_copy(_ctx, mail); + } else if (_ctx->guid != NULL && + (guid_128_from_string(_ctx->guid, wanted_guid) < 0 || + memcmp(guid_data, wanted_guid, sizeof(wanted_guid)) != 0)) { + /* GUID change requested. we can't do it with refcount + copying */ + return mail_storage_copy(_ctx, mail); + } + /* remember the map_uid so we can later increase its refcount */ if (!array_is_created(&ctx->copy_map_uids)) i_array_init(&ctx->copy_map_uids, 32); @@ -439,12 +453,8 @@ mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, ctx->mbox->ext_id, &rec, NULL); - mail_index_lookup_ext(mail->transaction->view, mail->seq, - src_mbox->guid_ext_id, &data, &expunged); - if (data != NULL) { - mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, - ctx->mbox->guid_ext_id, data, NULL); - } + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, guid_data, NULL); index_copy_cache_fields(_ctx, mail, ctx->ctx.seq); save_mail = array_append_space(&ctx->mails); From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: Released v2.1.13. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf9d62fd0b14 changeset: 15598:cf9d62fd0b14 user: Timo Sirainen date: Mon Jan 07 10:55:17 2013 +0200 description: Released v2.1.13. diffstat: NEWS | 7 +++++++ configure.in | 2 +- 2 files changed, 8 insertions(+), 1 deletions(-) diffs (24 lines): diff -r 3eeb5270963b -r cf9d62fd0b14 NEWS --- a/NEWS Sat Jan 05 00:50:04 2013 +0200 +++ b/NEWS Mon Jan 07 10:55:17 2013 +0200 @@ -1,3 +1,10 @@ +v2.1.13 2013-01-06 Timo Sirainen + + - Some fixes to cache file changes in v2.1.11. + - fts-solr: Overlong UTF8 sequences in mails were rejected by Solr and + caused the mails to not be indexed. + - virtual storage: Sorting mailbox by from/to/cc/bcc didn't work. + v2.1.12 2012-11-30 Timo Sirainen - dovecot-config in v2.1.11 caused build problems with Pigeonhole diff -r 3eeb5270963b -r cf9d62fd0b14 configure.in --- a/configure.in Sat Jan 05 00:50:04 2013 +0200 +++ b/configure.in Mon Jan 07 10:55:17 2013 +0200 @@ -1,5 +1,5 @@ AC_PREREQ([2.59]) -AC_INIT([Dovecot],[2.1.12],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.1.13],[dovecot at dovecot.org]) AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: Added tag 2.1.13 for changeset cf9d62fd0b14 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a6ce877f1cd7 changeset: 15599:a6ce877f1cd7 user: Timo Sirainen date: Mon Jan 07 10:55:17 2013 +0200 description: Added tag 2.1.13 for changeset cf9d62fd0b14 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r cf9d62fd0b14 -r a6ce877f1cd7 .hgtags --- a/.hgtags Mon Jan 07 10:55:17 2013 +0200 +++ b/.hgtags Mon Jan 07 10:55:17 2013 +0200 @@ -89,3 +89,4 @@ 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 741d800a192fa23572bb14196df2a8917cf20614 2.1.11 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 +cf9d62fd0b143efa8e49fac998eb78a648cdd8a9 2.1.13 From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: net_connect_unix(): If path is too long, return ENA... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4cc3f2b66e3 changeset: 15601:e4cc3f2b66e3 user: Timo Sirainen date: Tue Jan 08 03:19:14 2013 +0200 description: net_connect_unix(): If path is too long, return ENAMETOOLONG if it's available. diffstat: src/lib/network.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 36eb80cd02ea -r e4cc3f2b66e3 src/lib/network.c --- a/src/lib/network.c Mon Jan 07 10:55:27 2013 +0200 +++ b/src/lib/network.c Tue Jan 08 03:19:14 2013 +0200 @@ -278,7 +278,11 @@ sa.un.sun_family = AF_UNIX; if (i_strocpy(sa.un.sun_path, path, sizeof(sa.un.sun_path)) < 0) { /* too long path */ +#ifdef ENAMETOOLONG + errno = ENAMETOOLONG; +#else errno = EINVAL; +#endif return -1; } From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: Added signature for changeset cf9d62fd0b14 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/36eb80cd02ea changeset: 15600:36eb80cd02ea user: Timo Sirainen date: Mon Jan 07 10:55:27 2013 +0200 description: Added signature for changeset cf9d62fd0b14 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r a6ce877f1cd7 -r 36eb80cd02ea .hgsigs --- a/.hgsigs Mon Jan 07 10:55:17 2013 +0200 +++ b/.hgsigs Mon Jan 07 10:55:27 2013 +0200 @@ -52,3 +52,4 @@ 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT 741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99 +cf9d62fd0b143efa8e49fac998eb78a648cdd8a9 0 iEYEABECAAYFAlDqjXUACgkQyUhSUUBViskUEwCfYTWHeDmPr8HfxSBQN17SD5IwDygAnROhb3IVTm9niDun4gxPxbHLo/Pe From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: run-test.sh didn't exit with failure code when the ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3e3e0b482a36 changeset: 15602:3e3e0b482a36 user: Timo Sirainen date: Tue Jan 08 03:25:55 2013 +0200 description: run-test.sh didn't exit with failure code when the test failed diffstat: run-test.sh | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (18 lines): diff -r e4cc3f2b66e3 -r 3e3e0b482a36 run-test.sh --- a/run-test.sh Tue Jan 08 03:19:14 2013 +0200 +++ b/run-test.sh Tue Jan 08 03:25:55 2013 +0200 @@ -5,11 +5,14 @@ if valgrind --version | grep '^valgrind-3.[012]'; then # RHEL 5.4 still has Valgrind v3.2 valgrind -q --log-file-exactly=test.out.$$ $* + ret=$? else # v3.3+ valgrind -q --log-file=test.out.$$ $* + ret=$? fi if [ -s test.out.$$ ]; then cat test.out.$$ exit 1 fi +exit $ret From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: DEBUG: buffer_verify_pool() crashed with static siz... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/427dd81baa8d changeset: 15603:427dd81baa8d user: Timo Sirainen date: Tue Jan 08 03:44:14 2013 +0200 description: DEBUG: buffer_verify_pool() crashed with static sized buffers. diffstat: src/lib/buffer.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3e3e0b482a36 -r 427dd81baa8d src/lib/buffer.c --- a/src/lib/buffer.c Tue Jan 08 03:25:55 2013 +0200 +++ b/src/lib/buffer.c Tue Jan 08 03:44:14 2013 +0200 @@ -320,7 +320,7 @@ const struct real_buffer *buf = (const struct real_buffer *)_buf; void *ret; - if (buf->pool->datastack_pool) { + if (buf->pool != NULL && buf->pool->datastack_pool) { /* this doesn't really do anything except verify the stack frame */ ret = p_realloc(buf->pool, buf->w_buffer, From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: dbox: Give a name to the file's output stream (for ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bdd1499989c6 changeset: 15605:bdd1499989c6 user: Timo Sirainen date: Wed Jan 09 05:30:06 2013 +0200 description: dbox: Give a name to the file's output stream (for improved error logging). diffstat: src/lib-storage/index/dbox-common/dbox-file.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r e098f2f90fb2 -r bdd1499989c6 src/lib-storage/index/dbox-common/dbox-file.c --- a/src/lib-storage/index/dbox-common/dbox-file.c Tue Jan 08 05:37:43 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Wed Jan 09 05:30:06 2013 +0200 @@ -469,6 +469,7 @@ ctx->file = file; if (file->fd != -1) { ctx->output = o_stream_create_fd_file(file->fd, 0, FALSE); + o_stream_set_name(ctx->output, file->cur_path); o_stream_cork(ctx->output); } return ctx; From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: DEBUG: buffer_verify_pool() crashed with empty buffer. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e098f2f90fb2 changeset: 15604:e098f2f90fb2 user: Timo Sirainen date: Tue Jan 08 05:37:43 2013 +0200 description: DEBUG: buffer_verify_pool() crashed with empty buffer. diffstat: src/lib/buffer.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 427dd81baa8d -r e098f2f90fb2 src/lib/buffer.c --- a/src/lib/buffer.c Tue Jan 08 03:44:14 2013 +0200 +++ b/src/lib/buffer.c Tue Jan 08 05:37:43 2013 +0200 @@ -320,7 +320,7 @@ const struct real_buffer *buf = (const struct real_buffer *)_buf; void *ret; - if (buf->pool != NULL && buf->pool->datastack_pool) { + if (buf->pool != NULL && buf->pool->datastack_pool && buf->alloc > 0) { /* this doesn't really do anything except verify the stack frame */ ret = p_realloc(buf->pool, buf->w_buffer, From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: lib-storage: dest_mail wasn't reset if previous sav... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b87a7c4d775f changeset: 15606:b87a7c4d775f user: Timo Sirainen date: Wed Jan 09 05:33:32 2013 +0200 description: lib-storage: dest_mail wasn't reset if previous save was aborted. This could have happened only with dsync. diffstat: src/lib-storage/index/index-mail.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (19 lines): diff -r bdd1499989c6 -r b87a7c4d775f src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Jan 09 05:30:06 2013 +0200 +++ b/src/lib-storage/index/index-mail.c Wed Jan 09 05:33:32 2013 +0200 @@ -1387,8 +1387,13 @@ { struct index_mail *mail = (struct index_mail *)_mail; - if (mail->data.seq == seq) - return; + if (mail->data.seq == seq) { + if (!saving) + return; + /* we started saving a mail, aborted it, and now we're saving + another mail with the same sequence. make sure the mail + gets reset. */ + } mail->mail.v.close(&mail->mail.mail); From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: mdbox: Allow reflink-copying a mail when wanted GUI... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df06c834d7e8 changeset: 15607:df06c834d7e8 user: Timo Sirainen date: Wed Jan 09 05:35:20 2013 +0200 description: mdbox: Allow reflink-copying a mail when wanted GUID matches the current GUID. diffstat: src/lib-storage/index/dbox-multi/mdbox-save.c | 28 ++++++++++++++++++-------- 1 files changed, 19 insertions(+), 9 deletions(-) diffs (57 lines): diff -r b87a7c4d775f -r df06c834d7e8 src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Jan 09 05:33:32 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Jan 09 05:35:20 2013 +0200 @@ -412,14 +412,14 @@ struct dbox_save_mail *save_mail; struct mdbox_mailbox *src_mbox; struct mdbox_mail_index_record rec; - const void *data; + const void *guid_data; + guid_128_t wanted_guid; bool expunged; ctx->ctx.finished = TRUE; if (mail->box->storage != _ctx->transaction->box->storage || - _ctx->transaction->box->disable_reflink_copy_to || - _ctx->guid != NULL) + _ctx->transaction->box->disable_reflink_copy_to) return mail_storage_copy(_ctx, mail); src_mbox = (struct mdbox_mailbox *)mail->box; @@ -429,6 +429,20 @@ &rec.map_uid) < 0) return -1; + mail_index_lookup_ext(mail->transaction->view, mail->seq, + src_mbox->guid_ext_id, &guid_data, &expunged); + if (guid_data == NULL || guid_128_is_empty(guid_data)) { + /* missing GUID, something's broken. don't copy using + refcounting. */ + return mail_storage_copy(_ctx, mail); + } else if (_ctx->guid != NULL && + (guid_128_from_string(_ctx->guid, wanted_guid) < 0 || + memcmp(guid_data, wanted_guid, sizeof(wanted_guid)) != 0)) { + /* GUID change requested. we can't do it with refcount + copying */ + return mail_storage_copy(_ctx, mail); + } + /* remember the map_uid so we can later increase its refcount */ if (!array_is_created(&ctx->copy_map_uids)) i_array_init(&ctx->copy_map_uids, 32); @@ -439,12 +453,8 @@ mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, ctx->mbox->ext_id, &rec, NULL); - mail_index_lookup_ext(mail->transaction->view, mail->seq, - src_mbox->guid_ext_id, &data, &expunged); - if (data != NULL) { - mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, - ctx->mbox->guid_ext_id, data, NULL); - } + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, guid_data, NULL); index_copy_cache_fields(_ctx, mail, ctx->ctx.seq); save_mail = array_append_space(&ctx->mails); From dovecot at dovecot.org Wed Jan 9 05:39:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:39:06 +0200 Subject: dovecot-2.2: Merged changes from v2.1 tree. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f06419a40a06 changeset: 15608:f06419a40a06 user: Timo Sirainen date: Wed Jan 09 05:38:13 2013 +0200 description: Merged changes from v2.1 tree. diffstat: .hgsigs | 1 + .hgtags | 1 + NEWS | 7 ++++++ src/lib-storage/index/dbox-common/dbox-file.c | 1 + src/lib-storage/index/dbox-multi/mdbox-save.c | 28 ++++++++++++++++++-------- src/lib-storage/index/index-mail.c | 9 ++++++- src/lib/buffer.c | 2 +- src/lib/net.c | 4 +++ 8 files changed, 41 insertions(+), 12 deletions(-) diffs (148 lines): diff -r 2ae4a40b3b24 -r f06419a40a06 .hgsigs --- a/.hgsigs Tue Jan 08 16:24:20 2013 +0200 +++ b/.hgsigs Wed Jan 09 05:38:13 2013 +0200 @@ -54,3 +54,4 @@ 741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99 86bccdf46d172524ca19a1a8a16a50ac30a6743c 0 iEYEABECAAYFAlDqonoACgkQyUhSUUBVisnqqACfaqdR6GxUAJznotKT9WHIUVhVgcIAoJIEa0SBzlGIWThmLvtQByF9vXcc +cf9d62fd0b143efa8e49fac998eb78a648cdd8a9 0 iEYEABECAAYFAlDqjXUACgkQyUhSUUBViskUEwCfYTWHeDmPr8HfxSBQN17SD5IwDygAnROhb3IVTm9niDun4gxPxbHLo/Pe diff -r 2ae4a40b3b24 -r f06419a40a06 .hgtags --- a/.hgtags Tue Jan 08 16:24:20 2013 +0200 +++ b/.hgtags Wed Jan 09 05:38:13 2013 +0200 @@ -90,4 +90,5 @@ f5941f3ac7622361634b6cba464da79cc883d1bb 2.2.alpha1 741d800a192fa23572bb14196df2a8917cf20614 2.1.11 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 +cf9d62fd0b143efa8e49fac998eb78a648cdd8a9 2.1.13 86bccdf46d172524ca19a1a8a16a50ac30a6743c 2.2.beta1 diff -r 2ae4a40b3b24 -r f06419a40a06 NEWS --- a/NEWS Tue Jan 08 16:24:20 2013 +0200 +++ b/NEWS Wed Jan 09 05:38:13 2013 +0200 @@ -30,6 +30,13 @@ + LMTP proxy: Implemented XCLIENT extension for passing remote IP address through proxy. +v2.1.13 2013-01-06 Timo Sirainen + + - Some fixes to cache file changes in v2.1.11. + - fts-solr: Overlong UTF8 sequences in mails were rejected by Solr and + caused the mails to not be indexed. + - virtual storage: Sorting mailbox by from/to/cc/bcc didn't work. + v2.1.12 2012-11-30 Timo Sirainen - dovecot-config in v2.1.11 caused build problems with Pigeonhole diff -r 2ae4a40b3b24 -r f06419a40a06 src/lib-storage/index/dbox-common/dbox-file.c --- a/src/lib-storage/index/dbox-common/dbox-file.c Tue Jan 08 16:24:20 2013 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Wed Jan 09 05:38:13 2013 +0200 @@ -469,6 +469,7 @@ ctx->file = file; if (file->fd != -1) { ctx->output = o_stream_create_fd_file(file->fd, 0, FALSE); + o_stream_set_name(ctx->output, file->cur_path); o_stream_cork(ctx->output); } return ctx; diff -r 2ae4a40b3b24 -r f06419a40a06 src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Tue Jan 08 16:24:20 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Jan 09 05:38:13 2013 +0200 @@ -411,14 +411,14 @@ struct dbox_save_mail *save_mail; struct mdbox_mailbox *src_mbox; struct mdbox_mail_index_record rec; - const void *data; + const void *guid_data; + guid_128_t wanted_guid; bool expunged; ctx->ctx.finished = TRUE; if (mail->box->storage != _ctx->transaction->box->storage || - _ctx->transaction->box->disable_reflink_copy_to || - _ctx->data.guid != NULL) + _ctx->transaction->box->disable_reflink_copy_to) return mail_storage_copy(_ctx, mail); src_mbox = (struct mdbox_mailbox *)mail->box; @@ -428,6 +428,20 @@ &rec.map_uid) < 0) return -1; + mail_index_lookup_ext(mail->transaction->view, mail->seq, + src_mbox->guid_ext_id, &guid_data, &expunged); + if (guid_data == NULL || guid_128_is_empty(guid_data)) { + /* missing GUID, something's broken. don't copy using + refcounting. */ + return mail_storage_copy(_ctx, mail); + } else if (_ctx->data.guid != NULL && + (guid_128_from_string(_ctx->data.guid, wanted_guid) < 0 || + memcmp(guid_data, wanted_guid, sizeof(wanted_guid)) != 0)) { + /* GUID change requested. we can't do it with refcount + copying */ + return mail_storage_copy(_ctx, mail); + } + /* remember the map_uid so we can later increase its refcount */ if (!array_is_created(&ctx->copy_map_uids)) i_array_init(&ctx->copy_map_uids, 32); @@ -438,12 +452,8 @@ mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, ctx->mbox->ext_id, &rec, NULL); - mail_index_lookup_ext(mail->transaction->view, mail->seq, - src_mbox->guid_ext_id, &data, &expunged); - if (data != NULL) { - mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, - ctx->mbox->guid_ext_id, data, NULL); - } + mail_index_update_ext(ctx->ctx.trans, ctx->ctx.seq, + ctx->mbox->guid_ext_id, guid_data, NULL); index_copy_cache_fields(_ctx, mail, ctx->ctx.seq); save_mail = array_append_space(&ctx->mails); diff -r 2ae4a40b3b24 -r f06419a40a06 src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Tue Jan 08 16:24:20 2013 +0200 +++ b/src/lib-storage/index/index-mail.c Wed Jan 09 05:38:13 2013 +0200 @@ -1459,8 +1459,13 @@ { struct index_mail *mail = (struct index_mail *)_mail; - if (mail->data.seq == seq) - return; + if (mail->data.seq == seq) { + if (!saving) + return; + /* we started saving a mail, aborted it, and now we're saving + another mail with the same sequence. make sure the mail + gets reset. */ + } mail->mail.v.close(&mail->mail.mail); diff -r 2ae4a40b3b24 -r f06419a40a06 src/lib/buffer.c --- a/src/lib/buffer.c Tue Jan 08 16:24:20 2013 +0200 +++ b/src/lib/buffer.c Wed Jan 09 05:38:13 2013 +0200 @@ -325,7 +325,7 @@ const struct real_buffer *buf = (const struct real_buffer *)_buf; void *ret; - if (buf->pool->datastack_pool) { + if (buf->pool != NULL && buf->pool->datastack_pool && buf->alloc > 0) { /* this doesn't really do anything except verify the stack frame */ ret = p_realloc(buf->pool, buf->w_buffer, diff -r 2ae4a40b3b24 -r f06419a40a06 src/lib/net.c --- a/src/lib/net.c Tue Jan 08 16:24:20 2013 +0200 +++ b/src/lib/net.c Wed Jan 09 05:38:13 2013 +0200 @@ -279,7 +279,11 @@ sa.un.sun_family = AF_UNIX; if (i_strocpy(sa.un.sun_path, path, sizeof(sa.un.sun_path)) < 0) { /* too long path */ +#ifdef ENAMETOOLONG + errno = ENAMETOOLONG; +#else errno = EINVAL; +#endif return -1; } From dovecot at dovecot.org Wed Jan 9 05:43:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:43:21 +0200 Subject: dovecot-2.2: lib-index: Comment update Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7bbc5c50c309 changeset: 15609:7bbc5c50c309 user: Timo Sirainen date: Wed Jan 09 05:29:13 2013 +0200 description: lib-index: Comment update diffstat: src/lib-index/mail-index.h | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) diffs (19 lines): diff -r f06419a40a06 -r 7bbc5c50c309 src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Wed Jan 09 05:38:13 2013 +0200 +++ b/src/lib-index/mail-index.h Wed Jan 09 05:29:13 2013 +0200 @@ -441,10 +441,11 @@ /* Append a new record to index. */ void mail_index_append(struct mail_index_transaction *t, uint32_t uid, uint32_t *seq_r); -/* Assign UIDs for mails with uid=0 or uid details: http://hg.dovecot.org/dovecot-2.2/rev/3d3625d18231 changeset: 15610:3d3625d18231 user: Timo Sirainen date: Wed Jan 09 05:28:58 2013 +0200 description: o_stream_send_istream(): Fixed copying within same file, when istream's size is unknown. This fixes copying a mail from mdbox's m.* file to the end of the same file. diffstat: src/lib/ostream-file.c | 12 +++++++++--- 1 files changed, 9 insertions(+), 3 deletions(-) diffs (29 lines): diff -r 7bbc5c50c309 -r 3d3625d18231 src/lib/ostream-file.c --- a/src/lib/ostream-file.c Wed Jan 09 05:29:13 2013 +0200 +++ b/src/lib/ostream-file.c Wed Jan 09 05:28:58 2013 +0200 @@ -814,16 +814,22 @@ { struct file_ostream *foutstream = (struct file_ostream *)outstream; uoff_t in_size; - off_t in_abs_offset, ret; + off_t in_abs_offset, ret = 0; if (same_stream) { /* copying data within same fd. we'll have to be careful with seeks and overlapping writes. */ - if ((ret = i_stream_get_size(instream, TRUE, &in_size)) <= 0) { - outstream->ostream.stream_errno = ret == 0 ? ESPIPE : + if ((ret = i_stream_get_size(instream, TRUE, &in_size)) < 0) { + outstream->ostream.stream_errno = instream->stream_errno; return -1; } + /* if we couldn't find out the size, it means that instream + isn't a regular file_istream. we can be reasonably sure that + we can copy it safely the regular way. (there's really no + other possibility, other than failing completely.) */ + } + if (ret > 0) { i_assert(instream->v_offset <= in_size); in_abs_offset = instream->real_stream->abs_start_offset + From dovecot at dovecot.org Wed Jan 9 05:43:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:43:21 +0200 Subject: dovecot-2.2: dsync: Don't wrongly log errors about missing messa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cc77431b09b4 changeset: 15611:cc77431b09b4 user: Timo Sirainen date: Wed Jan 09 05:42:03 2013 +0200 description: dsync: Don't wrongly log errors about missing message bodies. Also added some debug logging related to it. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 2 + src/doveadm/dsync/dsync-mailbox-import.c | 46 ++++++++++++++++++++----------- src/doveadm/dsync/dsync-mailbox-import.h | 3 +- 3 files changed, 34 insertions(+), 17 deletions(-) diffs (125 lines): diff -r 3d3625d18231 -r cc77431b09b4 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Wed Jan 09 05:28:58 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Wed Jan 09 05:42:03 2013 +0200 @@ -209,6 +209,8 @@ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS; if (brain->backup_recv) import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES; + if (brain->debug) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_DEBUG; brain->box_importer = brain->backup_send ? NULL : dsync_mailbox_import_init(brain->box, brain->log_scan, diff -r 3d3625d18231 -r cc77431b09b4 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 05:28:58 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 05:42:03 2013 +0200 @@ -73,6 +73,7 @@ uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq; unsigned int failed:1; + unsigned int debug:1; unsigned int last_common_uid_found:1; unsigned int cur_uid_has_change:1; unsigned int cur_mail_saved:1; @@ -165,6 +166,7 @@ (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0; importer->revert_local_changes = (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0; + importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0; mailbox_get_open_status(importer->box, STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ, @@ -1489,20 +1491,26 @@ } static unsigned int -dsync_mailbox_import_count_missing_guid_imports(HASH_TABLE_TYPE(guid_new_mail) imports) +dsync_mailbox_import_count_missing_guid_imports(struct dsync_mailbox_importer *importer) { struct hash_iterate_context *iter; const char *key; struct importer_new_mail *mail; unsigned int msgs_left = 0; - iter = hash_table_iterate_init(imports); - while (hash_table_iterate(iter, imports, &key, &mail)) { + iter = hash_table_iterate_init(importer->import_guids); + while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) { for (; mail != NULL; mail = mail->next) { - if (!mail->uid_in_local) { - msgs_left++; - break; + if (mail->uid_in_local || mail->skip) + continue; + + if (importer->debug) { + i_debug("Mailbox %s: Missing mail GUID=%s (UID=%u)", + mailbox_get_vname(importer->box), + mail->guid, mail->uid); } + msgs_left++; + break; } } hash_table_iterate_deinit(&iter); @@ -1510,20 +1518,26 @@ } static unsigned int -dsync_mailbox_import_count_missing_uid_imports(HASH_TABLE_TYPE(uid_new_mail) imports) +dsync_mailbox_import_count_missing_uid_imports(struct dsync_mailbox_importer *importer) { struct hash_iterate_context *iter; void *key; struct importer_new_mail *mail; unsigned int msgs_left = 0; - iter = hash_table_iterate_init(imports); - while (hash_table_iterate(iter, imports, &key, &mail)) { + iter = hash_table_iterate_init(importer->import_uids); + while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) { for (; mail != NULL; mail = mail->next) { - if (!mail->uid_in_local) { - msgs_left++; - break; + if (mail->uid_in_local || mail->skip) + continue; + + if (importer->debug) { + i_debug("Mailbox %s: Missing mail UID=%u", + mailbox_get_vname(importer->box), + mail->uid); } + msgs_left++; + break; } } hash_table_iterate_deinit(&iter); @@ -1546,10 +1560,10 @@ if (!importer->new_uids_assigned) dsync_mailbox_import_assign_new_uids(importer); - msgs_left = - dsync_mailbox_import_count_missing_guid_imports(importer->import_guids) + - dsync_mailbox_import_count_missing_uid_imports(importer->import_uids); - if (!importer->failed && msgs_left > 0) { + msgs_left = importer->failed ? 0 : + dsync_mailbox_import_count_missing_guid_imports(importer) + + dsync_mailbox_import_count_missing_uid_imports(importer); + if (msgs_left > 0) { i_error("%s: Remote didn't send %u expected message bodies", mailbox_get_vname(importer->box), msgs_left); } diff -r 3d3625d18231 -r cc77431b09b4 src/doveadm/dsync/dsync-mailbox-import.h --- a/src/doveadm/dsync/dsync-mailbox-import.h Wed Jan 09 05:28:58 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.h Wed Jan 09 05:42:03 2013 +0200 @@ -5,7 +5,8 @@ DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN = 0x01, DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS = 0x02, DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x04, - DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES = 0x08 + DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES = 0x08, + DSYNC_MAILBOX_IMPORT_FLAG_DEBUG = 0x10 }; struct mailbox; From dovecot at dovecot.org Wed Jan 9 05:43:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 05:43:21 +0200 Subject: dovecot-2.2: dsync: Fixed handling UID renumbering when new mail... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a0493fa018fa changeset: 15612:a0493fa018fa user: Timo Sirainen date: Wed Jan 09 05:43:09 2013 +0200 description: dsync: Fixed handling UID renumbering when new mail was saved during dsync. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 65 ++++++++++++++++++++++--------- 1 files changed, 46 insertions(+), 19 deletions(-) diffs (92 lines): diff -r cc77431b09b4 -r a0493fa018fa src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 05:42:03 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 05:43:09 2013 +0200 @@ -1393,11 +1393,28 @@ { struct seq_range_iter iter; const uint32_t *wanted_uids; - uint32_t saved_uid, highest_unwanted_uid = 0; + uint32_t saved_uid, highest_wanted_uid = 0; uint32_t seq1, seq2, lowest_saved_uid = (uint32_t)-1; + uint32_t lowest_unwanted_uid = (uint32_t)-1; unsigned int i, n, wanted_count; int ret = 0; + /* wanted_uids contains the UIDs we tried to save mails with. + if nothing changed during dsync, we should have the expected UIDs + (changes->saved_uids) and all is well. + + if any new messages got inserted during dsync, we'll need to fix up + the UIDs and let the next dsync fix up the other side. for example: + + remote uids = 5,7,9 = wanted_uids + remote uidnext = 12 + locally added new uid=5 -> + saved_uids = 10,7,9 + + we'll now need to reassign UIDs 5 and 10. or more generally, we + need to reassign UIDs [original local uidnext .. lowest saved_uid-1] + and [lowest unwanted uid .. remote uidnext-1] */ + /* find the highest wanted UID that doesn't match what we got */ wanted_uids = array_get(&importer->wanted_uids, &wanted_count); seq_range_array_iter_init(&iter, &changes->saved_uids); i = n = 0; @@ -1405,30 +1422,40 @@ i_assert(i < wanted_count); if (lowest_saved_uid > saved_uid) lowest_saved_uid = saved_uid; - if (saved_uid != wanted_uids[i]) { - if (highest_unwanted_uid < wanted_uids[i]) - highest_unwanted_uid = wanted_uids[i]; + if (saved_uid == wanted_uids[i]) { + if (highest_wanted_uid < saved_uid) + highest_wanted_uid = saved_uid; + } else { + if (lowest_unwanted_uid > saved_uid) + lowest_unwanted_uid = saved_uid; } i++; } + i_assert(lowest_unwanted_uid == (uint32_t)-1 || + lowest_unwanted_uid == highest_wanted_uid+1 || + highest_wanted_uid == 0); - if (highest_unwanted_uid == 0 && i > 0 && - importer->local_uid_next <= lowest_saved_uid-1) { - /* we didn't see any unwanted UIDs, but we'll still need to - verify that messages didn't just get saved locally to a gap - that we left in local_uid_next..(lowest_saved_uid-1) */ - highest_unwanted_uid = lowest_saved_uid-1; + if (importer->local_uid_next != lowest_saved_uid && + lowest_saved_uid != (uint32_t)-1) { + /* [original local uidnext .. lowest saved_uid-1] */ + mailbox_get_seq_range(importer->box, importer->local_uid_next, + lowest_saved_uid-1, &seq1, &seq2); + if (seq1 > 0) { + ret = reassign_uids_in_seq_range(importer->box, + seq1, seq2); + *changes_during_sync_r = TRUE; + } } - if (highest_unwanted_uid == 0) - seq1 = seq2 = 0; - else { - mailbox_get_seq_range(importer->box, importer->local_uid_next, - highest_unwanted_uid, &seq1, &seq2); - } - if (seq1 > 0) { - ret = reassign_uids_in_seq_range(importer->box, seq1, seq2); - *changes_during_sync_r = TRUE; + if (lowest_unwanted_uid < importer->remote_uid_next) { + /* [highest wanted_uid+1 .. remote uidnext-1] */ + mailbox_get_seq_range(importer->box, lowest_unwanted_uid, + importer->remote_uid_next-1, &seq1, &seq2); + if (seq1 > 0) { + ret = reassign_uids_in_seq_range(importer->box, + seq1, seq2); + *changes_during_sync_r = TRUE; + } } return ret; } From dovecot at dovecot.org Wed Jan 9 06:57:46 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 06:57:46 +0200 Subject: dovecot-2.1: mdbox rebuild: Use mail size instead of record size... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b0bd7b2ff1c5 changeset: 14878:b0bd7b2ff1c5 user: Timo Sirainen date: Wed Jan 09 06:57:34 2013 +0200 description: mdbox rebuild: Use mail size instead of record size when guessing if the mails are the same. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 26 ++++++++++----- 1 files changed, 17 insertions(+), 9 deletions(-) diffs (72 lines): diff -r df06c834d7e8 -r b0bd7b2ff1c5 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 05:35:20 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 06:57:34 2013 +0200 @@ -24,7 +24,8 @@ guid_128_t guid_128; uint32_t file_id; uint32_t offset; - uint32_t size; + uint32_t rec_size; + uoff_t mail_size; uint32_t map_uid; uint16_t refcount; @@ -106,9 +107,9 @@ if ((*m1)->offset > (*m2)->offset) return 1; - if ((*m1)->size < (*m2)->size) + if ((*m1)->rec_size < (*m2)->rec_size) return -1; - if ((*m1)->size > (*m2)->size) + if ((*m1)->rec_size > (*m2)->rec_size) return 1; return 0; } @@ -175,7 +176,8 @@ rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1); rec->file_id = file_id; rec->offset = offset; - rec->size = file->input->v_offset - offset; + rec->rec_size = file->input->v_offset - offset; + rec->mail_size = dbox_file_get_plaintext_size(file); mail_generate_guid_128_hash(guid, rec->guid_128); i_assert(!guid_128_is_empty(rec->guid_128)); array_append(&ctx->msgs, &rec, 1); @@ -183,9 +185,15 @@ old_rec = hash_table_lookup(ctx->guid_hash, rec->guid_128); if (old_rec == NULL) hash_table_insert(ctx->guid_hash, rec->guid_128, rec); - else if (rec->size == old_rec->size) { - /* duplicate. save this as a refcount=0 to map, - so it will eventually be deleted. */ + else if (rec->mail_size == old_rec->mail_size) { + /* two mails' GUID and size are the same, which quite + likely means that their contents are the same as + well. we'll compare the mail sizes instead of the + record sizes, because the records' metadata may + differ. + + save this duplicate mail with refcount=0 to the map, + so it will eventually be purged. */ rec->seen_zero_ref_in_map = TRUE; } else { /* duplicate GUID, but not a duplicate message. */ @@ -290,7 +298,7 @@ rec.file_id = msgs[i]->file_id; rec.offset = msgs[i]->offset; - rec.size = msgs[i]->size; + rec.size = msgs[i]->rec_size; msgs[i]->map_uid = next_uid++; mail_index_append(ctx->atomic->sync_trans, @@ -326,7 +334,7 @@ the (file_id, offset, size) triplet */ search_msg.file_id = rec.rec.file_id; search_msg.offset = rec.rec.offset; - search_msg.size = rec.rec.size; + search_msg.rec_size = rec.rec.size; pos = bsearch(&search_msgp, msgs, count, sizeof(*msgs), mdbox_rebuild_msg_offset_cmp); if (pos == NULL || (*pos)->map_uid != 0) { From dovecot at dovecot.org Wed Jan 9 06:59:57 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 06:59:57 +0200 Subject: dovecot-2.1: mdbox rebuild: Don't always duplicate messages that... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/6ff1678c289f changeset: 14879:6ff1678c289f user: Timo Sirainen date: Wed Jan 09 06:59:51 2013 +0200 description: mdbox rebuild: Don't always duplicate messages that have conflicting GUIDs. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 36 ++++++++++++--- 1 files changed, 29 insertions(+), 7 deletions(-) diffs (83 lines): diff -r b0bd7b2ff1c5 -r 6ff1678c289f src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 06:57:34 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 06:59:51 2013 +0200 @@ -21,6 +21,8 @@ #include struct mdbox_rebuild_msg { + struct mdbox_rebuild_msg *guid_hash_next; + guid_128_t guid_128; uint32_t file_id; uint32_t offset; @@ -198,10 +200,13 @@ } else { /* duplicate GUID, but not a duplicate message. */ i_error("mdbox %s: Duplicate GUID %s in " - "m.%u:%u and m.%u:%u", + "m.%u:%u (size=%"PRIuUOFF_T") and m.%u:%u " + "(size=%"PRIuUOFF_T")", ctx->storage->storage_dir, guid, - old_rec->file_id, old_rec->offset, - rec->file_id, rec->offset); + old_rec->file_id, old_rec->offset, old_rec->mail_size, + rec->file_id, rec->offset, rec->mail_size); + rec->guid_hash_next = old_rec->guid_hash_next; + old_rec->guid_hash_next = rec; } } if (ret < 0) @@ -369,6 +374,20 @@ return pos == NULL ? NULL : *pos; } +static bool +guid_hash_have_map_uid(struct mdbox_rebuild_msg **recp, uint32_t map_uid) +{ + struct mdbox_rebuild_msg *rec; + + for (rec = *recp; rec != NULL; rec = rec->guid_hash_next) { + if (rec->map_uid == map_uid) { + *recp = rec; + return TRUE; + } + } + return FALSE; +} + static void rebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx, struct dbox_sync_rebuild_context *rebuild_ctx, @@ -411,17 +430,20 @@ still try to look it up using map_uid. */ rec = map_uid == 0 ? NULL : rebuild_lookup_map_uid(ctx, map_uid); - } else if (map_uid != rec->map_uid) { + } else if (!guid_hash_have_map_uid(&rec, map_uid)) { /* message's GUID and map_uid point to different physical messages. assume that GUID is correct and map_uid is wrong. */ + map_uid = rec->map_uid; } else { - /* everything was ok */ + /* everything was ok. use this specific record's + map_uid to avoid duplicating mails in case the same + GUID exists multiple times */ } if (rec != NULL) T_BEGIN { /* keep this message. add it to mailbox index. */ - i_assert(rec->map_uid != 0); + i_assert(map_uid != 0); rec->refcount++; mail_index_lookup_uid(view, old_seq, &uid); @@ -429,7 +451,7 @@ dbox_sync_rebuild_index_metadata(rebuild_ctx, new_seq, uid); - new_dbox_rec.map_uid = rec->map_uid; + new_dbox_rec.map_uid = map_uid; mail_index_update_ext(trans, new_seq, mbox->ext_id, &new_dbox_rec, NULL); mail_index_update_ext(trans, new_seq, mbox->guid_ext_id, From dovecot at dovecot.org Wed Jan 9 07:01:46 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 07:01:46 +0200 Subject: dovecot-2.1: mdbox rebuild: Update to previous commit Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/1ab8e0e699f7 changeset: 14880:1ab8e0e699f7 user: Timo Sirainen date: Wed Jan 09 07:01:41 2013 +0200 description: mdbox rebuild: Update to previous commit diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 6ff1678c289f -r 1ab8e0e699f7 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 06:59:51 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 07:01:41 2013 +0200 @@ -430,6 +430,7 @@ still try to look it up using map_uid. */ rec = map_uid == 0 ? NULL : rebuild_lookup_map_uid(ctx, map_uid); + map_uid = rec->map_uid; } else if (!guid_hash_have_map_uid(&rec, map_uid)) { /* message's GUID and map_uid point to different physical messages. assume that GUID is correct and From dovecot at dovecot.org Wed Jan 9 07:02:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 07:02:36 +0200 Subject: dovecot-2.2: mdbox rebuild: Use mail size instead of record size... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0bd7b2ff1c5 changeset: 15613:b0bd7b2ff1c5 user: Timo Sirainen date: Wed Jan 09 06:57:34 2013 +0200 description: mdbox rebuild: Use mail size instead of record size when guessing if the mails are the same. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 26 ++++++++++----- 1 files changed, 17 insertions(+), 9 deletions(-) diffs (72 lines): diff -r df06c834d7e8 -r b0bd7b2ff1c5 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 05:35:20 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 06:57:34 2013 +0200 @@ -24,7 +24,8 @@ guid_128_t guid_128; uint32_t file_id; uint32_t offset; - uint32_t size; + uint32_t rec_size; + uoff_t mail_size; uint32_t map_uid; uint16_t refcount; @@ -106,9 +107,9 @@ if ((*m1)->offset > (*m2)->offset) return 1; - if ((*m1)->size < (*m2)->size) + if ((*m1)->rec_size < (*m2)->rec_size) return -1; - if ((*m1)->size > (*m2)->size) + if ((*m1)->rec_size > (*m2)->rec_size) return 1; return 0; } @@ -175,7 +176,8 @@ rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1); rec->file_id = file_id; rec->offset = offset; - rec->size = file->input->v_offset - offset; + rec->rec_size = file->input->v_offset - offset; + rec->mail_size = dbox_file_get_plaintext_size(file); mail_generate_guid_128_hash(guid, rec->guid_128); i_assert(!guid_128_is_empty(rec->guid_128)); array_append(&ctx->msgs, &rec, 1); @@ -183,9 +185,15 @@ old_rec = hash_table_lookup(ctx->guid_hash, rec->guid_128); if (old_rec == NULL) hash_table_insert(ctx->guid_hash, rec->guid_128, rec); - else if (rec->size == old_rec->size) { - /* duplicate. save this as a refcount=0 to map, - so it will eventually be deleted. */ + else if (rec->mail_size == old_rec->mail_size) { + /* two mails' GUID and size are the same, which quite + likely means that their contents are the same as + well. we'll compare the mail sizes instead of the + record sizes, because the records' metadata may + differ. + + save this duplicate mail with refcount=0 to the map, + so it will eventually be purged. */ rec->seen_zero_ref_in_map = TRUE; } else { /* duplicate GUID, but not a duplicate message. */ @@ -290,7 +298,7 @@ rec.file_id = msgs[i]->file_id; rec.offset = msgs[i]->offset; - rec.size = msgs[i]->size; + rec.size = msgs[i]->rec_size; msgs[i]->map_uid = next_uid++; mail_index_append(ctx->atomic->sync_trans, @@ -326,7 +334,7 @@ the (file_id, offset, size) triplet */ search_msg.file_id = rec.rec.file_id; search_msg.offset = rec.rec.offset; - search_msg.size = rec.rec.size; + search_msg.rec_size = rec.rec.size; pos = bsearch(&search_msgp, msgs, count, sizeof(*msgs), mdbox_rebuild_msg_offset_cmp); if (pos == NULL || (*pos)->map_uid != 0) { From dovecot at dovecot.org Wed Jan 9 07:02:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 07:02:36 +0200 Subject: dovecot-2.2: mdbox rebuild: Don't always duplicate messages that... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6ff1678c289f changeset: 15614:6ff1678c289f user: Timo Sirainen date: Wed Jan 09 06:59:51 2013 +0200 description: mdbox rebuild: Don't always duplicate messages that have conflicting GUIDs. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 36 ++++++++++++--- 1 files changed, 29 insertions(+), 7 deletions(-) diffs (83 lines): diff -r b0bd7b2ff1c5 -r 6ff1678c289f src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 06:57:34 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 06:59:51 2013 +0200 @@ -21,6 +21,8 @@ #include struct mdbox_rebuild_msg { + struct mdbox_rebuild_msg *guid_hash_next; + guid_128_t guid_128; uint32_t file_id; uint32_t offset; @@ -198,10 +200,13 @@ } else { /* duplicate GUID, but not a duplicate message. */ i_error("mdbox %s: Duplicate GUID %s in " - "m.%u:%u and m.%u:%u", + "m.%u:%u (size=%"PRIuUOFF_T") and m.%u:%u " + "(size=%"PRIuUOFF_T")", ctx->storage->storage_dir, guid, - old_rec->file_id, old_rec->offset, - rec->file_id, rec->offset); + old_rec->file_id, old_rec->offset, old_rec->mail_size, + rec->file_id, rec->offset, rec->mail_size); + rec->guid_hash_next = old_rec->guid_hash_next; + old_rec->guid_hash_next = rec; } } if (ret < 0) @@ -369,6 +374,20 @@ return pos == NULL ? NULL : *pos; } +static bool +guid_hash_have_map_uid(struct mdbox_rebuild_msg **recp, uint32_t map_uid) +{ + struct mdbox_rebuild_msg *rec; + + for (rec = *recp; rec != NULL; rec = rec->guid_hash_next) { + if (rec->map_uid == map_uid) { + *recp = rec; + return TRUE; + } + } + return FALSE; +} + static void rebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx, struct dbox_sync_rebuild_context *rebuild_ctx, @@ -411,17 +430,20 @@ still try to look it up using map_uid. */ rec = map_uid == 0 ? NULL : rebuild_lookup_map_uid(ctx, map_uid); - } else if (map_uid != rec->map_uid) { + } else if (!guid_hash_have_map_uid(&rec, map_uid)) { /* message's GUID and map_uid point to different physical messages. assume that GUID is correct and map_uid is wrong. */ + map_uid = rec->map_uid; } else { - /* everything was ok */ + /* everything was ok. use this specific record's + map_uid to avoid duplicating mails in case the same + GUID exists multiple times */ } if (rec != NULL) T_BEGIN { /* keep this message. add it to mailbox index. */ - i_assert(rec->map_uid != 0); + i_assert(map_uid != 0); rec->refcount++; mail_index_lookup_uid(view, old_seq, &uid); @@ -429,7 +451,7 @@ dbox_sync_rebuild_index_metadata(rebuild_ctx, new_seq, uid); - new_dbox_rec.map_uid = rec->map_uid; + new_dbox_rec.map_uid = map_uid; mail_index_update_ext(trans, new_seq, mbox->ext_id, &new_dbox_rec, NULL); mail_index_update_ext(trans, new_seq, mbox->guid_ext_id, From dovecot at dovecot.org Wed Jan 9 07:02:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 07:02:36 +0200 Subject: dovecot-2.2: mdbox rebuild: Update to previous commit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1ab8e0e699f7 changeset: 15615:1ab8e0e699f7 user: Timo Sirainen date: Wed Jan 09 07:01:41 2013 +0200 description: mdbox rebuild: Update to previous commit diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 6ff1678c289f -r 1ab8e0e699f7 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 06:59:51 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 07:01:41 2013 +0200 @@ -430,6 +430,7 @@ still try to look it up using map_uid. */ rec = map_uid == 0 ? NULL : rebuild_lookup_map_uid(ctx, map_uid); + map_uid = rec->map_uid; } else if (!guid_hash_have_map_uid(&rec, map_uid)) { /* message's GUID and map_uid point to different physical messages. assume that GUID is correct and From dovecot at dovecot.org Wed Jan 9 07:02:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 07:02:36 +0200 Subject: dovecot-2.2: Merged changes from v2.1 tree. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ec9b98df8c72 changeset: 15616:ec9b98df8c72 user: Timo Sirainen date: Wed Jan 09 07:02:28 2013 +0200 description: Merged changes from v2.1 tree. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 63 +++++++++++---- 1 files changed, 47 insertions(+), 16 deletions(-) diffs (146 lines): diff -r a0493fa018fa -r ec9b98df8c72 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 05:43:09 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Wed Jan 09 07:02:28 2013 +0200 @@ -21,10 +21,13 @@ #include struct mdbox_rebuild_msg { + struct mdbox_rebuild_msg *guid_hash_next; + guid_128_t guid_128; uint32_t file_id; uint32_t offset; - uint32_t size; + uint32_t rec_size; + uoff_t mail_size; uint32_t map_uid; uint16_t refcount; @@ -106,9 +109,9 @@ if ((*m1)->offset > (*m2)->offset) return 1; - if ((*m1)->size < (*m2)->size) + if ((*m1)->rec_size < (*m2)->rec_size) return -1; - if ((*m1)->size > (*m2)->size) + if ((*m1)->rec_size > (*m2)->rec_size) return 1; return 0; } @@ -176,7 +179,8 @@ rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1); rec->file_id = file_id; rec->offset = offset; - rec->size = file->input->v_offset - offset; + rec->rec_size = file->input->v_offset - offset; + rec->mail_size = dbox_file_get_plaintext_size(file); mail_generate_guid_128_hash(guid, rec->guid_128); i_assert(!guid_128_is_empty(rec->guid_128)); array_append(&ctx->msgs, &rec, 1); @@ -185,17 +189,26 @@ old_rec = hash_table_lookup(ctx->guid_hash, guid_p); if (old_rec == NULL) hash_table_insert(ctx->guid_hash, guid_p, rec); - else if (rec->size == old_rec->size) { - /* duplicate. save this as a refcount=0 to map, - so it will eventually be deleted. */ + else if (rec->mail_size == old_rec->mail_size) { + /* two mails' GUID and size are the same, which quite + likely means that their contents are the same as + well. we'll compare the mail sizes instead of the + record sizes, because the records' metadata may + differ. + + save this duplicate mail with refcount=0 to the map, + so it will eventually be purged. */ rec->seen_zero_ref_in_map = TRUE; } else { /* duplicate GUID, but not a duplicate message. */ i_error("mdbox %s: Duplicate GUID %s in " - "m.%u:%u and m.%u:%u", + "m.%u:%u (size=%"PRIuUOFF_T") and m.%u:%u " + "(size=%"PRIuUOFF_T")", ctx->storage->storage_dir, guid, - old_rec->file_id, old_rec->offset, - rec->file_id, rec->offset); + old_rec->file_id, old_rec->offset, old_rec->mail_size, + rec->file_id, rec->offset, rec->mail_size); + rec->guid_hash_next = old_rec->guid_hash_next; + old_rec->guid_hash_next = rec; } } if (ret < 0) @@ -292,7 +305,7 @@ rec.file_id = msgs[i]->file_id; rec.offset = msgs[i]->offset; - rec.size = msgs[i]->size; + rec.size = msgs[i]->rec_size; msgs[i]->map_uid = next_uid++; mail_index_append(ctx->atomic->sync_trans, @@ -326,7 +339,7 @@ the (file_id, offset, size) triplet */ search_msg.file_id = rec.rec.file_id; search_msg.offset = rec.rec.offset; - search_msg.size = rec.rec.size; + search_msg.rec_size = rec.rec.size; pos = array_bsearch(&ctx->msgs, &search_msgp, mdbox_rebuild_msg_offset_cmp); if (pos == NULL || (*pos)->map_uid != 0) { @@ -361,6 +374,20 @@ return pos == NULL ? NULL : *pos; } +static bool +guid_hash_have_map_uid(struct mdbox_rebuild_msg **recp, uint32_t map_uid) +{ + struct mdbox_rebuild_msg *rec; + + for (rec = *recp; rec != NULL; rec = rec->guid_hash_next) { + if (rec->map_uid == map_uid) { + *recp = rec; + return TRUE; + } + } + return FALSE; +} + static void rebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx, struct index_rebuild_context *rebuild_ctx, @@ -405,17 +432,21 @@ still try to look it up using map_uid. */ rec = map_uid == 0 ? NULL : rebuild_lookup_map_uid(ctx, map_uid); - } else if (map_uid != rec->map_uid) { + map_uid = rec->map_uid; + } else if (!guid_hash_have_map_uid(&rec, map_uid)) { /* message's GUID and map_uid point to different physical messages. assume that GUID is correct and map_uid is wrong. */ + map_uid = rec->map_uid; } else { - /* everything was ok */ + /* everything was ok. use this specific record's + map_uid to avoid duplicating mails in case the same + GUID exists multiple times */ } if (rec != NULL) T_BEGIN { /* keep this message. add it to mailbox index. */ - i_assert(rec->map_uid != 0); + i_assert(map_uid != 0); rec->refcount++; mail_index_lookup_uid(view, old_seq, &uid); @@ -423,7 +454,7 @@ index_rebuild_index_metadata(rebuild_ctx, new_seq, uid); - new_dbox_rec.map_uid = rec->map_uid; + new_dbox_rec.map_uid = map_uid; mail_index_update_ext(trans, new_seq, mbox->ext_id, &new_dbox_rec, NULL); mail_index_update_ext(trans, new_seq, mbox->guid_ext_id, From dovecot at dovecot.org Wed Jan 9 07:54:04 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 07:54:04 +0200 Subject: dovecot-2.2: dsync: Error handling fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2c4394d0360b changeset: 15617:2c4394d0360b user: Timo Sirainen date: Wed Jan 09 07:53:53 2013 +0200 description: dsync: Error handling fix. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 12 +++++++----- 1 files changed, 7 insertions(+), 5 deletions(-) diffs (36 lines): diff -r ec9b98df8c72 -r 2c4394d0360b src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 07:02:28 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 07:53:53 2013 +0200 @@ -1386,7 +1386,7 @@ return ret; } -static bool +static int reassign_unwanted_uids(struct dsync_mailbox_importer *importer, const struct mail_transaction_commit_changes *changes, bool *changes_during_sync_r) @@ -1441,8 +1441,9 @@ mailbox_get_seq_range(importer->box, importer->local_uid_next, lowest_saved_uid-1, &seq1, &seq2); if (seq1 > 0) { - ret = reassign_uids_in_seq_range(importer->box, - seq1, seq2); + if (reassign_uids_in_seq_range(importer->box, + seq1, seq2) < 0) + ret = -1; *changes_during_sync_r = TRUE; } } @@ -1452,8 +1453,9 @@ mailbox_get_seq_range(importer->box, lowest_unwanted_uid, importer->remote_uid_next-1, &seq1, &seq2); if (seq1 > 0) { - ret = reassign_uids_in_seq_range(importer->box, - seq1, seq2); + if (reassign_uids_in_seq_range(importer->box, + seq1, seq2) < 0) + ret = -1; *changes_during_sync_r = TRUE; } } From dovecot at dovecot.org Wed Jan 9 09:11:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 09:11:14 +0200 Subject: dovecot-2.2: dsync: Mailbox syncing fix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b338c4a08f42 changeset: 15618:b338c4a08f42 user: Timo Sirainen date: Wed Jan 09 09:11:09 2013 +0200 description: dsync: Mailbox syncing fix diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 16 +++++++++++++--- 1 files changed, 13 insertions(+), 3 deletions(-) diffs (43 lines): diff -r 2c4394d0360b -r b338c4a08f42 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 07:53:53 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 09:11:09 2013 +0200 @@ -1076,6 +1076,7 @@ { struct importer_new_mail *newmail, *const *newmailp; uint32_t common_uid_next, new_uid; + bool linked_uid; common_uid_next = I_MAX(importer->local_uid_next, importer->remote_uid_next); @@ -1094,19 +1095,28 @@ if (newmail->uid_is_usable) { /* keep the UID */ new_uid = newmail->uid; + linked_uid = FALSE; } else if (newmail->link != NULL && - newmail->link->uid_is_usable) + newmail->link->uid_is_usable) { new_uid = newmail->link->uid; - else + linked_uid = TRUE; + } else { new_uid = common_uid_next++; + linked_uid = FALSE; + } if (newmail->uid_in_local && newmail->uid != new_uid) { /* local UID changed, reassign it by copying */ dsync_msg_update_uid(importer, newmail->uid, new_uid); + } else if (linked_uid && newmail->link->uid_in_local) { + /* the linked message already exists. we'll just need + to forget about this message. */ + i_assert(!newmail->uid_in_local); + newmail->skip = TRUE; } newmail->uid = new_uid; - if (newmail->link != NULL) { + if (newmail->link != NULL && !newmail->skip) { /* skip the linked mail */ newmail->link->skip = TRUE; } From dovecot at dovecot.org Wed Jan 9 11:03:56 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 09 Jan 2013 11:03:56 +0200 Subject: dovecot-2.2: dsync: Another fix to handling expunges from the en... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b27245a6adde changeset: 15619:b27245a6adde user: Timo Sirainen date: Wed Jan 09 11:03:43 2013 +0200 description: dsync: Another fix to handling expunges from the end of mailbox. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r b338c4a08f42 -r b27245a6adde src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 09:11:09 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Jan 09 11:03:43 2013 +0200 @@ -962,6 +962,11 @@ if (!importer_next_mail(importer, change->uid)) { /* no more local mails. we can still try to match expunged mails though. */ + if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* mail doesn't exist remotely either, don't bother + looking it up locally. */ + return; + } if (change->guid == NULL || !dsync_mailbox_find_common_expunged_uid(importer, change)) { /* couldn't match it for an expunged mail. use the last From dovecot at dovecot.org Thu Jan 10 06:52:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 06:52:48 +0200 Subject: dovecot-2.2: doveadm: Added assert. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/90a603209bad changeset: 15620:90a603209bad user: Timo Sirainen date: Thu Jan 10 06:51:19 2013 +0200 description: doveadm: Added assert. diffstat: src/doveadm/doveadm-print.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b27245a6adde -r 90a603209bad src/doveadm/doveadm-print.c --- a/src/doveadm/doveadm-print.c Wed Jan 09 11:03:43 2013 +0200 +++ b/src/doveadm/doveadm-print.c Thu Jan 10 06:51:19 2013 +0200 @@ -59,6 +59,7 @@ unsigned int count; headers = array_get(&ctx->headers, &count); + i_assert(count > 0); for (;;) { if (ctx->header_idx == count) ctx->header_idx = 0; From dovecot at dovecot.org Thu Jan 10 06:52:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 06:52:48 +0200 Subject: dovecot-2.2: dsync: Fixed printing output state when running via... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/29e5702b23ab changeset: 15621:29e5702b23ab user: Timo Sirainen date: Thu Jan 10 06:52:37 2013 +0200 description: dsync: Fixed printing output state when running via doveadm-server. diffstat: src/doveadm/dsync/doveadm-dsync.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (40 lines): diff -r 90a603209bad -r 29e5702b23ab src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 06:51:19 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 06:52:37 2013 +0200 @@ -18,6 +18,7 @@ #include "mailbox-list.h" #include "doveadm-settings.h" #include "doveadm-mail.h" +#include "doveadm-print.h" #include "dsync-brain.h" #include "dsync-ibc.h" #include "doveadm-dsync.h" @@ -404,7 +405,7 @@ if (ctx->state_input != NULL) { string_t *str = t_str_new(128); dsync_brain_get_state(brain, str); - printf("%s\n", str_c(str)); + doveadm_print(str_c(str)); } if (dsync_brain_deinit(&brain) < 0) @@ -566,6 +567,9 @@ ctx->ctx.v.prerun = cmd_dsync_prerun; ctx->ctx.v.run = cmd_dsync_run; ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + doveadm_print_header("state", "state", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); return &ctx->ctx; } @@ -649,6 +653,9 @@ ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; ctx->fd_in = STDIN_FILENO; ctx->fd_out = STDOUT_FILENO; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + doveadm_print_header("state", "state", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); return &ctx->ctx; } From dovecot at dovecot.org Thu Jan 10 07:32:00 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 07:32:00 +0200 Subject: dovecot-2.2: mdbox: Crashfix for storage rebuild Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/42a88f557662 changeset: 15622:42a88f557662 user: Timo Sirainen date: Thu Jan 10 07:31:51 2013 +0200 description: mdbox: Crashfix for storage rebuild diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 29e5702b23ab -r 42a88f557662 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Thu Jan 10 06:52:37 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Thu Jan 10 07:31:51 2013 +0200 @@ -432,7 +432,7 @@ still try to look it up using map_uid. */ rec = map_uid == 0 ? NULL : rebuild_lookup_map_uid(ctx, map_uid); - map_uid = rec->map_uid; + map_uid = rec == NULL ? 0 : rec->map_uid; } else if (!guid_hash_have_map_uid(&rec, map_uid)) { /* message's GUID and map_uid point to different physical messages. assume that GUID is correct and From dovecot at dovecot.org Thu Jan 10 07:39:12 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 07:39:12 +0200 Subject: dovecot-2.2: dsync: Fixed parsing state string. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1d7d160015cb changeset: 15623:1d7d160015cb user: Timo Sirainen date: Thu Jan 10 07:39:02 2013 +0200 description: dsync: Fixed parsing state string. diffstat: src/doveadm/dsync/dsync-mailbox-state.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 42a88f557662 -r 1d7d160015cb src/doveadm/dsync/dsync-mailbox-state.c --- a/src/doveadm/dsync/dsync-mailbox-state.c Thu Jan 10 07:31:51 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-state.c Thu Jan 10 07:39:02 2013 +0200 @@ -7,7 +7,7 @@ #include "hash.h" #include "dsync-mailbox-state.h" -#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8) +#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8) static void put_uint32(buffer_t *output, uint32_t num) { From dovecot at dovecot.org Thu Jan 10 07:54:51 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 07:54:51 +0200 Subject: dovecot-2.2: replicator: Use stateful dsyncing whenever possible. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b21f3119a5ee changeset: 15625:b21f3119a5ee user: Timo Sirainen date: Thu Jan 10 07:54:39 2013 +0200 description: replicator: Use stateful dsyncing whenever possible. diffstat: src/replication/replicator/doveadm-connection.c | 32 ++++++++++++++---------- src/replication/replicator/doveadm-connection.h | 5 ++- src/replication/replicator/replicator-brain.c | 7 +++- src/replication/replicator/replicator-queue.c | 1 + src/replication/replicator/replicator-queue.h | 3 +- 5 files changed, 29 insertions(+), 19 deletions(-) diffs (206 lines): diff -r 59cef0c673b4 -r b21f3119a5ee src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Thu Jan 10 07:53:26 2013 +0200 +++ b/src/replication/replicator/doveadm-connection.c Thu Jan 10 07:54:39 2013 +0200 @@ -13,7 +13,6 @@ #define DOVEADM_FAIL_TIMEOUT_MSECS (1000*5) #define DOVEADM_HANDSHAKE "VERSION\tdoveadm-server\t1\t0\n" -#define MAX_INBUF_SIZE 1024 struct doveadm_connection { char *path; @@ -23,12 +22,12 @@ struct ostream *output; struct timeout *to; + char *state; doveadm_callback_t *callback; void *context; time_t last_connect_failure; unsigned int handshaked:1; - unsigned int end_of_print:1; unsigned int cmd_sent:1; }; @@ -43,7 +42,7 @@ } static void doveadm_callback(struct doveadm_connection *conn, - enum doveadm_reply reply) + const char *state, enum doveadm_reply reply) { doveadm_callback_t *callback = conn->callback; void *context = conn->context; @@ -53,7 +52,7 @@ conn->callback = NULL; conn->context = NULL; - callback(reply, context); + callback(reply, state, context); } static void doveadm_close(struct doveadm_connection *conn) @@ -67,7 +66,7 @@ if (close(conn->fd) < 0) i_error("close(doveadm) failed: %m"); conn->fd = -1; - conn->end_of_print = FALSE; + i_free_and_null(conn->state); conn->cmd_sent = FALSE; conn->handshaked = FALSE; } @@ -76,7 +75,7 @@ { doveadm_close(conn); if (conn->callback != NULL) - doveadm_callback(conn, DOVEADM_REPLY_FAIL); + doveadm_callback(conn, "", DOVEADM_REPLY_FAIL); } void doveadm_connection_deinit(struct doveadm_connection **_conn) @@ -92,6 +91,8 @@ static int doveadm_input_line(struct doveadm_connection *conn, const char *line) { + const char *state; + if (!conn->handshaked) { if (strcmp(line, "+") != 0) { i_error("%s: Unexpected handshake: %s", @@ -105,21 +106,21 @@ i_error("%s: Unexpected input: %s", conn->path, line); return -1; } - if (!conn->end_of_print) { - if (line[0] == '\0') - conn->end_of_print = TRUE; + if (conn->state == NULL) { + conn->state = i_strdup(t_strcut(line, '\t')); return 0; } + state = t_strdup(conn->state); line = t_strdup(line); doveadm_close(conn); if (line[0] == '+') - doveadm_callback(conn, DOVEADM_REPLY_OK); + doveadm_callback(conn, state, DOVEADM_REPLY_OK); else if (line[0] == '-') { if (strcmp(line+1, "NOUSER") == 0) - doveadm_callback(conn, DOVEADM_REPLY_NOUSER); + doveadm_callback(conn, "", DOVEADM_REPLY_NOUSER); else - doveadm_callback(conn, DOVEADM_REPLY_FAIL); + doveadm_callback(conn, "", DOVEADM_REPLY_FAIL); } else { i_error("%s: Invalid input: %s", conn->path, line); return -1; @@ -160,7 +161,7 @@ } conn->last_connect_failure = 0; conn->io = io_add(conn->fd, IO_READ, doveadm_input, conn); - conn->input = i_stream_create_fd(conn->fd, MAX_INBUF_SIZE, FALSE); + conn->input = i_stream_create_fd(conn->fd, (size_t)-1, FALSE); conn->output = o_stream_create_fd(conn->fd, (size_t)-1, FALSE); o_stream_set_no_error_handling(conn->output, TRUE); o_stream_nsend_str(conn->output, DOVEADM_HANDSHAKE); @@ -173,7 +174,7 @@ } void doveadm_connection_sync(struct doveadm_connection *conn, - const char *username, bool full, + const char *username, const char *state, bool full, doveadm_callback_t *callback, void *context) { string_t *cmd; @@ -197,6 +198,9 @@ str_append(cmd, "\tsync\t-d"); if (full) str_append(cmd, "\t-f"); + str_append(cmd, "\t-s\t"); + if (state != NULL) + str_append(cmd, state); str_append_c(cmd, '\n'); o_stream_nsend(conn->output, str_data(cmd), str_len(cmd)); } diff -r 59cef0c673b4 -r b21f3119a5ee src/replication/replicator/doveadm-connection.h --- a/src/replication/replicator/doveadm-connection.h Thu Jan 10 07:53:26 2013 +0200 +++ b/src/replication/replicator/doveadm-connection.h Thu Jan 10 07:54:39 2013 +0200 @@ -7,13 +7,14 @@ DOVEADM_REPLY_NOUSER }; -typedef void doveadm_callback_t(enum doveadm_reply reply, void *context); +typedef void doveadm_callback_t(enum doveadm_reply reply, + const char *state, void *context); struct doveadm_connection *doveadm_connection_init(const char *path); void doveadm_connection_deinit(struct doveadm_connection **conn); void doveadm_connection_sync(struct doveadm_connection *conn, - const char *username, bool full, + const char *username, const char *state, bool full, doveadm_callback_t *callback, void *context); bool doveadm_connection_is_busy(struct doveadm_connection *conn); diff -r 59cef0c673b4 -r b21f3119a5ee src/replication/replicator/replicator-brain.c --- a/src/replication/replicator/replicator-brain.c Thu Jan 10 07:53:26 2013 +0200 +++ b/src/replication/replicator/replicator-brain.c Thu Jan 10 07:54:39 2013 +0200 @@ -82,7 +82,8 @@ return conn; } -static void doveadm_sync_callback(enum doveadm_reply reply, void *context) +static void doveadm_sync_callback(enum doveadm_reply reply, const char *state, + void *context) { struct replicator_sync_context *ctx = context; @@ -90,6 +91,8 @@ /* user no longer exists, remove from replication */ replicator_queue_remove(ctx->brain->queue, &ctx->user); } else { + i_free(ctx->user->state); + ctx->user->state = i_strdup_empty(state); ctx->user->last_sync_failed = reply != DOVEADM_REPLY_OK; replicator_queue_push(ctx->brain->queue, ctx->user); @@ -125,7 +128,7 @@ ctx = i_new(struct replicator_sync_context, 1); ctx->brain = brain; ctx->user = user; - doveadm_connection_sync(conn, user->username, full, + doveadm_connection_sync(conn, user->username, user->state, full, doveadm_sync_callback, ctx); return TRUE; } diff -r 59cef0c673b4 -r b21f3119a5ee src/replication/replicator/replicator-queue.c --- a/src/replication/replicator/replicator-queue.c Thu Jan 10 07:53:26 2013 +0200 +++ b/src/replication/replicator/replicator-queue.c Thu Jan 10 07:54:39 2013 +0200 @@ -174,6 +174,7 @@ priorityq_remove(queue->user_queue, &user->item); hash_table_remove(queue->user_hash, user->username); + i_free(user->state); i_free(user->username); i_free(user); diff -r 59cef0c673b4 -r b21f3119a5ee src/replication/replicator/replicator-queue.h --- a/src/replication/replicator/replicator-queue.h Thu Jan 10 07:53:26 2013 +0200 +++ b/src/replication/replicator/replicator-queue.h Thu Jan 10 07:54:39 2013 +0200 @@ -8,12 +8,13 @@ struct priorityq_item item; char *username; - enum replication_priority priority; + char *state; /* last time this user's state was updated */ time_t last_update; /* last_fast_run is always >= last_full_run. */ time_t last_fast_sync, last_full_sync; + enum replication_priority priority; /* User isn't currently in replication queue */ unsigned int popped:1; /* Last replication sync failed */ From dovecot at dovecot.org Thu Jan 10 07:54:51 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 07:54:51 +0200 Subject: dovecot-2.2: replicator: doveadm connections weren't reset properly Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/59cef0c673b4 changeset: 15624:59cef0c673b4 user: Timo Sirainen date: Thu Jan 10 07:53:26 2013 +0200 description: replicator: doveadm connections weren't reset properly diffstat: src/replication/replicator/doveadm-connection.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 1d7d160015cb -r 59cef0c673b4 src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Thu Jan 10 07:39:02 2013 +0200 +++ b/src/replication/replicator/doveadm-connection.c Thu Jan 10 07:53:26 2013 +0200 @@ -69,6 +69,7 @@ conn->fd = -1; conn->end_of_print = FALSE; conn->cmd_sent = FALSE; + conn->handshaked = FALSE; } static void doveadm_disconnect(struct doveadm_connection *conn) From dovecot at dovecot.org Thu Jan 10 08:07:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 08:07:28 +0200 Subject: dovecot-2.2: dsync: Removed unnecessary -n parameter from dsync-... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1a47a0df6ad0 changeset: 15626:1a47a0df6ad0 user: Timo Sirainen date: Thu Jan 10 08:07:19 2013 +0200 description: dsync: Removed unnecessary -n parameter from dsync-server. The setting is sent within dsync protocol. diffstat: src/doveadm/doveadm-settings.c | 2 +- src/doveadm/dsync/doveadm-dsync.c | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diffs (50 lines): diff -r b21f3119a5ee -r 1a47a0df6ad0 src/doveadm/doveadm-settings.c --- a/src/doveadm/doveadm-settings.c Thu Jan 10 07:54:39 2013 +0200 +++ b/src/doveadm/doveadm-settings.c Thu Jan 10 08:07:19 2013 +0200 @@ -80,7 +80,7 @@ .doveadm_password = "", .doveadm_allowed_commands = "", .dsync_alt_char = "_", - .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -l%{lock_timeout} -n%{namespace}", + .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -l%{lock_timeout}", .plugin_envs = ARRAY_INIT }; diff -r b21f3119a5ee -r 1a47a0df6ad0 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 07:54:39 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 08:07:19 2013 +0200 @@ -147,7 +147,6 @@ { '\0', NULL, "login" }, { '\0', NULL, "host" }, { '\0', NULL, "lock_timeout" }, - { '\0', NULL, "namespace" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -162,7 +161,6 @@ tab[1].value = login; tab[2].value = host; tab[3].value = dec2str(ctx->lock_timeout); - tab[4].value = ctx->namespace_prefix; t_array_init(&cmd_args, 8); str = t_str_new(128); @@ -633,9 +631,6 @@ case 'r': ctx->rawlog_path = optarg; break; - case 'n': - ctx->namespace_prefix = optarg; - break; default: return FALSE; } @@ -647,7 +642,7 @@ struct dsync_cmd_context *ctx; ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); - ctx->ctx.getopt_args = "El:n:r:"; + ctx->ctx.getopt_args = "El:r:"; ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg; ctx->ctx.v.run = cmd_dsync_server_run; ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; From dovecot at dovecot.org Thu Jan 10 09:33:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 09:33:29 +0200 Subject: dovecot-2.2: dsync: Handle syncing mailboxes with same name but ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f106740d92a0 changeset: 15627:f106740d92a0 user: Timo Sirainen date: Thu Jan 10 09:33:19 2013 +0200 description: dsync: Handle syncing mailboxes with same name but different GUIDs. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 13 ++++++++----- 1 files changed, 8 insertions(+), 5 deletions(-) diffs (24 lines): diff -r 1a47a0df6ad0 -r f106740d92a0 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Thu Jan 10 08:07:19 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Thu Jan 10 09:33:19 2013 +0200 @@ -991,12 +991,15 @@ if (other_node == NULL) other_node = sorted_tree_get(other_tree, name); if (!dsync_mailbox_node_is_dir(other_node)) { - /* already exists */ + /* mailbox with same name exists both locally and + remotely, but they have different GUIDs and neither + side has the other's GUID. typically this means that + both sides had autocreated some mailboxes (e.g. + INBOX). we'll just change the GUID for one of + them. */ i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); - // FIXME: remove this assert? for conflicting GUIDs - /*i_assert(memcmp(node->mailbox_guid, - other_node->mailbox_guid, - sizeof(node->mailbox_guid)) == 0);*/ + if (other_tree == ctx->local_tree) + sync_add_create_change(ctx, node, name); } else { other_node->existence = DSYNC_MAILBOX_NODE_EXISTS; other_node->ns = node->ns; From dovecot at dovecot.org Thu Jan 10 10:02:51 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 10:02:51 +0200 Subject: dovecot-2.2: dsync: Attempt to preserve \Recent flag as well Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f6d69bc1beed changeset: 15628:f6d69bc1beed user: Timo Sirainen date: Thu Jan 10 10:02:47 2013 +0200 description: dsync: Attempt to preserve \Recent flag as well diffstat: src/doveadm/dsync/dsync-mailbox-export.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r f106740d92a0 -r f6d69bc1beed src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Thu Jan 10 09:33:19 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Thu Jan 10 10:02:47 2013 +0200 @@ -99,7 +99,7 @@ change->modseq = mail_get_modseq(mail); change->pvt_modseq = mail_get_pvt_modseq(mail); - change->final_flags = mail_get_flags(mail) & MAIL_FLAGS_NONRECENT; + change->final_flags = mail_get_flags(mail); keywords = mail_get_keywords(mail); if (!array_is_created(&change->keyword_changes) && From dovecot at dovecot.org Thu Jan 10 10:21:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 10:21:34 +0200 Subject: dovecot-2.2: dsync: Added support back for dsync locking. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ce0bd8ab1459 changeset: 15629:ce0bd8ab1459 user: Timo Sirainen date: Thu Jan 10 10:21:27 2013 +0200 description: dsync: Added support back for dsync locking. Only the source directory is now locked to avoid deadlocks when both source and destination are locked. This of course now means that both source and destination is allowed to run dsync at the same time, which isn't ideal either.. diffstat: src/doveadm/dsync/doveadm-dsync.c | 76 +++++++++++++++++++++++++++++++++----- 1 files changed, 66 insertions(+), 10 deletions(-) diffs (129 lines): diff -r f6d69bc1beed -r ce0bd8ab1459 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 10:02:47 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 10:21:27 2013 +0200 @@ -28,7 +28,8 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+adEfm:n:r:Rs:" +#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock" +#define DSYNC_COMMON_GETOPT_ARGS "+adEfl:m:n:r:Rs:" struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; @@ -348,9 +349,8 @@ } static int -cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +cmd_dsync_run_real(struct dsync_cmd_context *ctx, struct mail_user *user) { - struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; struct dsync_ibc *ibc, *ibc2 = NULL; struct dsync_brain *brain; struct mail_namespace *sync_ns = NULL; @@ -407,7 +407,7 @@ } if (dsync_brain_deinit(&brain) < 0) - _ctx->exit_code = EX_TEMPFAIL; + ctx->ctx.exit_code = EX_TEMPFAIL; dsync_ibc_deinit(&ibc); if (ibc2 != NULL) dsync_ibc_deinit(&ibc2); @@ -427,6 +427,62 @@ return ret; } +static int dsync_lock(struct mail_user *user, unsigned int lock_timeout, + const char **path_r, struct file_lock **lock_r) +{ + const char *home, *path; + int ret, fd; + + if ((ret = mail_user_get_home(user, &home)) < 0) { + i_error("Couldn't look up user's home dir"); + return -1; + } + if (ret == 0) { + i_error("User has no home directory"); + return -1; + } + + path = t_strconcat(home, "/"DSYNC_LOCK_FILENAME, NULL); + fd = creat(path, 0600); + if (fd == -1) { + i_error("Couldn't create lock %s: %m", path); + return -1; + } + + if (file_wait_lock(fd, path, F_WRLCK, FILE_LOCK_METHOD_FCNTL, + lock_timeout, lock_r) <= 0) { + i_error("Couldn't lock %s: %m", path); + (void)close(fd); + return -1; + } + *path_r = path; + return fd; +} + +static int +cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + const char *lock_path; + struct file_lock *lock; + int lock_fd, ret; + + if (!ctx->lock) + return cmd_dsync_run_real(ctx, user); + + lock_fd = dsync_lock(user, ctx->lock_timeout, &lock_path, &lock); + if (lock_fd == -1) { + _ctx->exit_code = EX_TEMPFAIL; + return -1; + } else { + ret = cmd_dsync_run_real(ctx, user); + file_lock_free(&lock); + if (close(lock_fd) < 0) + i_error("close(%s) failed: %m", lock_path); + return ret; + } +} + static int cmd_dsync_prerun(struct doveadm_mail_cmd_context *_ctx, struct mail_storage_service_user *service_user, const char **error_r) @@ -527,6 +583,11 @@ case 'f': ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; break; + case 'l': + ctx->lock = TRUE; + if (str_to_uint(optarg, &ctx->lock_timeout) < 0) + i_error("Invalid -l parameter: %s", optarg); + break; case 'm': ctx->mailbox = optarg; break; @@ -623,11 +684,6 @@ /* dsync wrapper detection flag */ legacy_dsync = TRUE; break; - case 'l': - ctx->lock = TRUE; - if (str_to_uint(optarg, &ctx->lock_timeout) < 0) - i_error("Invalid -l parameter: %s", optarg); - break; case 'r': ctx->rawlog_path = optarg; break; @@ -642,7 +698,7 @@ struct dsync_cmd_context *ctx; ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); - ctx->ctx.getopt_args = "El:r:"; + ctx->ctx.getopt_args = "Er:"; ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg; ctx->ctx.v.run = cmd_dsync_server_run; ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; From dovecot at dovecot.org Thu Jan 10 10:23:49 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 10:23:49 +0200 Subject: dovecot-2.2: lib-storage: Improved "Using permissions" debug mes... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7035da8cfaf4 changeset: 15630:7035da8cfaf4 user: Timo Sirainen date: Thu Jan 10 10:23:44 2013 +0200 description: lib-storage: Improved "Using permissions" debug message. diffstat: src/lib-storage/mailbox-list.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (18 lines): diff -r ce0bd8ab1459 -r 7035da8cfaf4 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Thu Jan 10 10:21:27 2013 +0200 +++ b/src/lib-storage/mailbox-list.c Thu Jan 10 10:23:44 2013 +0200 @@ -896,11 +896,11 @@ if (list->mail_set->mail_debug && name == NULL) { i_debug("Namespace %s: Using permissions from %s: " - "mode=0%o gid=%ld", list->ns->prefix, + "mode=0%o gid=%s", list->ns->prefix, path != NULL ? path : "", (int)permissions_r->dir_create_mode, - permissions_r->file_create_gid == (gid_t)-1 ? -1L : - (long)permissions_r->file_create_gid); + permissions_r->file_create_gid == (gid_t)-1 ? "default" : + dec2str(permissions_r->file_create_gid)); } } From dovecot at dovecot.org Thu Jan 10 11:52:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 11:52:38 +0200 Subject: dovecot-2.2: lib-storage: Maildir++ layout shouldn't disallow "c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/577c6dd315c3 changeset: 15631:577c6dd315c3 user: Timo Sirainen date: Thu Jan 10 11:52:27 2013 +0200 description: lib-storage: Maildir++ layout shouldn't disallow "cur", "new" or "tmp" mailbox names. diffstat: src/lib-storage/mailbox-list.c | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 7035da8cfaf4 -r 577c6dd315c3 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Thu Jan 10 10:23:44 2013 +0200 +++ b/src/lib-storage/mailbox-list.c Thu Jan 10 11:52:27 2013 +0200 @@ -1127,9 +1127,15 @@ /* make sure the mailbox name doesn't contain any foolishness: "../" could give access outside the mailbox directory. - "./" and "//" could fool ACL checks. */ + "./" and "//" could fool ACL checks. + + some mailbox formats have reserved directory names, such as + Maildir's cur/new/tmp. if any of those would conflict with the + mailbox directory name, it's not valid. maildir++ is kludged here as + a special case because all of its mailbox dirs begin with "." */ allow_internal_dirs = list->v.is_internal_name == NULL || - *list->set.maildir_name != '\0'; + *list->set.maildir_name != '\0' || + strcmp(list->name, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) == 0; T_BEGIN { const char *const *names; From dovecot at dovecot.org Thu Jan 10 12:12:59 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 12:12:59 +0200 Subject: dovecot-2.2: dsync: Close mail streams earlier on failures to av... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/40dd9a080b6e changeset: 15632:40dd9a080b6e user: Timo Sirainen date: Thu Jan 10 12:12:50 2013 +0200 description: dsync: Close mail streams earlier on failures to avoid assert-crashing diffstat: src/doveadm/dsync/dsync-brain.c | 1 + src/doveadm/dsync/dsync-ibc-pipe.c | 21 +++++++++++++++++++++ src/doveadm/dsync/dsync-ibc-private.h | 1 + src/doveadm/dsync/dsync-ibc-stream.c | 11 +++++++++++ src/doveadm/dsync/dsync-ibc.c | 5 +++++ src/doveadm/dsync/dsync-ibc.h | 4 ++++ 6 files changed, 43 insertions(+), 0 deletions(-) diffs (117 lines): diff -r 577c6dd315c3 -r 40dd9a080b6e src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Thu Jan 10 11:52:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Thu Jan 10 12:12:50 2013 +0200 @@ -158,6 +158,7 @@ if (dsync_ibc_has_failed(brain->ibc) || brain->state != DSYNC_STATE_DONE) brain->failed = TRUE; + dsync_ibc_close_mail_streams(brain->ibc); if (brain->box != NULL) dsync_brain_sync_mailbox_deinit(brain); diff -r 577c6dd315c3 -r 40dd9a080b6e src/doveadm/dsync/dsync-ibc-pipe.c --- a/src/doveadm/dsync/dsync-ibc-pipe.c Thu Jan 10 11:52:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-pipe.c Thu Jan 10 12:12:50 2013 +0200 @@ -434,6 +434,26 @@ return DSYNC_IBC_RECV_RET_OK; } +static void pipe_close_mail_streams(struct dsync_ibc_pipe *pipe) +{ + struct item *item; + + if (array_count(&pipe->item_queue) > 0) { + item = array_idx_modifiable(&pipe->item_queue, 0); + if (item->type == ITEM_MAIL && + item->u.mail.input != NULL) + i_stream_unref(&item->u.mail.input); + } +} + +static void dsync_ibc_pipe_close_mail_streams(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + pipe_close_mail_streams(pipe); + pipe_close_mail_streams(pipe->remote); +} + static const struct dsync_ibc_vfuncs dsync_ibc_pipe_vfuncs = { dsync_ibc_pipe_deinit, dsync_ibc_pipe_send_handshake, @@ -453,6 +473,7 @@ dsync_ibc_pipe_recv_mail_request, dsync_ibc_pipe_send_mail, dsync_ibc_pipe_recv_mail, + dsync_ibc_pipe_close_mail_streams, dsync_ibc_pipe_is_send_queue_full, dsync_ibc_pipe_has_pending_data }; diff -r 577c6dd315c3 -r 40dd9a080b6e src/doveadm/dsync/dsync-ibc-private.h --- a/src/doveadm/dsync/dsync-ibc-private.h Thu Jan 10 11:52:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-private.h Thu Jan 10 12:12:50 2013 +0200 @@ -61,6 +61,7 @@ (*recv_mail)(struct dsync_ibc *ibc, struct dsync_mail **mail_r); + void (*close_mail_streams)(struct dsync_ibc *ibc); bool (*is_send_queue_full)(struct dsync_ibc *ibc); bool (*has_pending_data)(struct dsync_ibc *ibc); }; diff -r 577c6dd315c3 -r 40dd9a080b6e src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Thu Jan 10 11:52:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Thu Jan 10 12:12:50 2013 +0200 @@ -1489,6 +1489,16 @@ return DSYNC_IBC_RECV_RET_OK; } +static void dsync_ibc_stream_close_mail_streams(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + + if (ibc->mail_output != NULL) { + i_stream_unref(&ibc->mail_output); + dsync_ibc_stream_stop(ibc); + } +} + static bool dsync_ibc_stream_is_send_queue_full(struct dsync_ibc *_ibc) { struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; @@ -1531,6 +1541,7 @@ dsync_ibc_stream_recv_mail_request, dsync_ibc_stream_send_mail, dsync_ibc_stream_recv_mail, + dsync_ibc_stream_close_mail_streams, dsync_ibc_stream_is_send_queue_full, dsync_ibc_stream_has_pending_data }; diff -r 577c6dd315c3 -r 40dd9a080b6e src/doveadm/dsync/dsync-ibc.c --- a/src/doveadm/dsync/dsync-ibc.c Thu Jan 10 11:52:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc.c Thu Jan 10 12:12:50 2013 +0200 @@ -178,6 +178,11 @@ return ibc->v.recv_mail(ibc, mail_r); } +void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc) +{ + ibc->v.close_mail_streams(ibc); +} + bool dsync_ibc_has_failed(struct dsync_ibc *ibc) { return ibc->failed; diff -r 577c6dd315c3 -r 40dd9a080b6e src/doveadm/dsync/dsync-ibc.h --- a/src/doveadm/dsync/dsync-ibc.h Thu Jan 10 11:52:27 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc.h Thu Jan 10 12:12:50 2013 +0200 @@ -110,6 +110,10 @@ enum dsync_ibc_recv_ret dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r); +/* Close any mail input streams that are kept open. This needs to be called + before the mail is attempted to be freed (usually on error conditions). */ +void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc); + bool dsync_ibc_has_failed(struct dsync_ibc *ibc); bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc); bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc); From dovecot at dovecot.org Thu Jan 10 14:00:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 14:00:29 +0200 Subject: dovecot-2.2: dsync server: Fixed crash at deinit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4da6d4fa1892 changeset: 15633:4da6d4fa1892 user: Timo Sirainen date: Thu Jan 10 13:56:59 2013 +0200 description: dsync server: Fixed crash at deinit diffstat: src/doveadm/dsync/doveadm-dsync.c | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) diffs (19 lines): diff -r 40dd9a080b6e -r 4da6d4fa1892 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 12:12:50 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 13:56:59 2013 +0200 @@ -666,12 +666,10 @@ io_loop_run(current_ioloop); + if (dsync_brain_deinit(&brain) < 0) + _ctx->exit_code = EX_TEMPFAIL; dsync_ibc_deinit(&ibc); - if (dsync_brain_deinit(&brain) < 0) { - _ctx->exit_code = EX_TEMPFAIL; - return -1; - } - return 0; + return _ctx->exit_code == 0 ? 0 : -1; } static bool From dovecot at dovecot.org Thu Jan 10 14:00:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 14:00:29 +0200 Subject: dovecot-2.2: dsync: Wait for remote command to finish before shu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/68b7c9ce713b changeset: 15634:68b7c9ce713b user: Timo Sirainen date: Thu Jan 10 14:00:10 2013 +0200 description: dsync: Wait for remote command to finish before shutting down. This makes sure that any error messages during deinit stage are logged. Also non-zero exit codes are logged. diffstat: src/doveadm/dsync/doveadm-dsync.c | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diffs (30 lines): diff -r 4da6d4fa1892 -r 68b7c9ce713b src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 13:56:59 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 14:00:10 2013 +0200 @@ -27,6 +27,7 @@ #include #include #include +#include #define DSYNC_LOCK_FILENAME ".dovecot-sync.lock" #define DSYNC_COMMON_GETOPT_ARGS "+adEfl:m:n:r:Rs:" @@ -311,8 +312,18 @@ static void cmd_dsync_run_remote(struct mail_user *user) { + int status; + i_set_failure_prefix("dsync-local(%s): ", user->username); io_loop_run(current_ioloop); + + /* wait for the remote command to finish to see any final errors */ + if (wait(&status) == -1) + i_error("wait() failed: %m"); + else if (WIFSIGNALED(status)) + i_error("Remote command died with signal %d", WTERMSIG(status)); + else if (WIFEXITED(status) && WEXITSTATUS(status) != 0) + i_error("Remote command returned error %d", WEXITSTATUS(status)); } static const char *const * From dovecot at dovecot.org Thu Jan 10 14:20:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Jan 2013 14:20:17 +0200 Subject: dovecot-2.2: dsync: Added "remoteprefix:" destination, which is ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b468c30c4522 changeset: 15635:b468c30c4522 user: Timo Sirainen date: Thu Jan 10 14:20:07 2013 +0200 description: dsync: Added "remoteprefix:" destination, which is same as "remote:" except sends "username\n" prefix. diffstat: src/doveadm/dsync/doveadm-dsync.c | 28 ++++++++++++++++++++++------ 1 files changed, 22 insertions(+), 6 deletions(-) diffs (59 lines): diff -r 68b7c9ce713b -r b468c30c4522 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 14:00:10 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Jan 10 14:20:07 2013 +0200 @@ -8,6 +8,7 @@ #include "istream.h" #include "ostream.h" #include "iostream-rawlog.h" +#include "write-full.h" #include "str.h" #include "var-expand.h" #include "settings-parser.h" @@ -53,6 +54,7 @@ unsigned int backup:1; unsigned int reverse_backup:1; unsigned int remote:1; + unsigned int remote_user_prefix:1; }; static bool legacy_dsync = FALSE; @@ -107,6 +109,13 @@ ctx->fd_out = fd_in[1]; ctx->fd_err = fd_err[0]; + if (ctx->remote_user_prefix) { + const char *prefix = + t_strdup_printf("%s\n", ctx->ctx.cur_username); + if (write_full(ctx->fd_out, prefix, strlen(prefix)) < 0) + i_fatal("write(remote out) failed: %m"); + } + fd_set_nonblock(ctx->fd_err, TRUE); ctx->err_stream = i_stream_create_fd(ctx->fd_err, IO_BLOCK_SIZE, FALSE); i_stream_set_return_partial_line(ctx->err_stream, TRUE); @@ -534,12 +543,19 @@ } } - if (remote_cmd_args == NULL && ctx->local_location != NULL && - strncmp(ctx->local_location, "remote:", 7) == 0) { - /* this is a remote (ssh) command */ - ctx->remote_name = ctx->local_location+7; - remote_cmd_args = parse_ssh_location(ctx, ctx->remote_name, - _ctx->cur_username); + if (remote_cmd_args == NULL && ctx->local_location != NULL) { + if (strncmp(ctx->local_location, "remote:", 7) == 0) { + /* this is a remote (ssh) command */ + ctx->remote_name = ctx->local_location+7; + } else if (strncmp(ctx->local_location, "remoteprefix:", 13) == 0) { + /* this is a remote (ssh) command with a "user\n" + prefix sent before dsync actually starts */ + ctx->remote_name = ctx->local_location+13; + ctx->remote_user_prefix = TRUE; + } + remote_cmd_args = ctx->remote_name == NULL ? NULL : + parse_ssh_location(ctx, ctx->remote_name, + _ctx->cur_username); } if (remote_cmd_args != NULL) { From dovecot at dovecot.org Fri Jan 11 03:59:01 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Jan 2013 03:59:01 +0200 Subject: dovecot-2.2: imap: Improved unexpected FETCH error messages. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b56e7845a9e8 changeset: 15636:b56e7845a9e8 user: Timo Sirainen date: Fri Jan 11 03:58:54 2013 +0200 description: imap: Improved unexpected FETCH error messages. diffstat: src/imap/imap-fetch-body.c | 30 ++++++++++++++++++++++++++++-- 1 files changed, 28 insertions(+), 2 deletions(-) diffs (54 lines): diff -r b468c30c4522 -r b56e7845a9e8 src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Thu Jan 10 14:20:07 2013 +0200 +++ b/src/imap/imap-fetch-body.c Fri Jan 11 03:58:54 2013 +0200 @@ -33,7 +33,8 @@ errno = state->cur_input->stream_errno; mail_storage_set_critical(state->cur_mail->box->storage, - "read(%s) failed: %m (FETCH for mailbox %s UID %u)", + "read(%s) failed: %m (FETCH %s for mailbox %s UID %u)", + state->cur_human_name, i_stream_get_name(state->cur_input), mailbox_get_vname(state->cur_mail->box), state->cur_mail->uid); } @@ -124,6 +125,31 @@ return 1; } +static const char * +get_body_human_name(pool_t pool, struct imap_fetch_body_data *body) +{ + string_t *str; + uoff_t partial_offset, partial_size; + + str = t_str_new(64); + if (body->binary) + str_append(str, "BINARY["); + else + str_append(str, "BODY["); + str_append(str, body->section); + str_append_c(str, ']'); + + partial_offset = imap_msgpart_get_partial_offset(body->msgpart); + partial_size = imap_msgpart_get_partial_size(body->msgpart); + if (partial_offset != 0 || partial_size != (uoff_t)-1) { + str_printfa(str, "<%"PRIuUOFF_T, partial_offset); + if (partial_size != (uoff_t)-1) + str_printfa(str, ".%"PRIuUOFF_T, partial_size); + str_append_c(str, '>'); + } + return p_strdup(pool, str_c(str)); +} + static int fetch_body_msgpart(struct imap_fetch_context *ctx, struct mail *mail, struct imap_fetch_body_data *body) { @@ -148,7 +174,7 @@ ctx->state.cur_input = result.input; ctx->state.cur_size = result.size; ctx->state.cur_size_field = result.size_field; - ctx->state.cur_human_name = body->section; + ctx->state.cur_human_name = get_body_human_name(ctx->ctx_pool, body); str = get_prefix(&ctx->state, body, ctx->state.cur_size, result.binary_decoded_input_has_nuls); From dovecot at dovecot.org Fri Jan 11 03:59:56 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Jan 2013 03:59:56 +0200 Subject: dovecot-2.2: lib-imap-storage: Added imap_msgpart_get_partial_si... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0e0eb30c2b9 changeset: 15637:b0e0eb30c2b9 user: Timo Sirainen date: Fri Jan 11 03:59:50 2013 +0200 description: lib-imap-storage: Added imap_msgpart_get_partial_size() missing from previous commit. diffstat: src/lib-imap-storage/imap-msgpart.c | 5 +++++ src/lib-imap-storage/imap-msgpart.h | 1 + 2 files changed, 6 insertions(+), 0 deletions(-) diffs (26 lines): diff -r b56e7845a9e8 -r b0e0eb30c2b9 src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Fri Jan 11 03:58:54 2013 +0200 +++ b/src/lib-imap-storage/imap-msgpart.c Fri Jan 11 03:59:50 2013 +0200 @@ -320,6 +320,11 @@ return msgpart->partial_offset; } +uoff_t imap_msgpart_get_partial_size(struct imap_msgpart *msgpart) +{ + return msgpart->partial_size; +} + enum mail_fetch_field imap_msgpart_get_fetch_data(struct imap_msgpart *msgpart) { return msgpart->wanted_fields; diff -r b56e7845a9e8 -r b0e0eb30c2b9 src/lib-imap-storage/imap-msgpart.h --- a/src/lib-imap-storage/imap-msgpart.h Fri Jan 11 03:58:54 2013 +0200 +++ b/src/lib-imap-storage/imap-msgpart.h Fri Jan 11 03:59:50 2013 +0200 @@ -33,6 +33,7 @@ void imap_msgpart_set_partial(struct imap_msgpart *msgpart, uoff_t offset, uoff_t size); uoff_t imap_msgpart_get_partial_offset(struct imap_msgpart *msgpart); +uoff_t imap_msgpart_get_partial_size(struct imap_msgpart *msgpart); /* Return wanted_fields mask. */ enum mail_fetch_field imap_msgpart_get_fetch_data(struct imap_msgpart *msgpart); From dovecot at dovecot.org Fri Jan 11 06:18:54 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Jan 2013 06:18:54 +0200 Subject: dovecot-2.2: lib-imap-storage: Removed broken optimization for s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d4e7a3184a82 changeset: 15638:d4e7a3184a82 user: Timo Sirainen date: Fri Jan 11 06:18:36 2013 +0200 description: lib-imap-storage: Removed broken optimization for seeking to partial offset. The optimization worked only when physical_start=0. It would have been possible to fix it by finding the message_part where its header of body offset is physical_start and only then start decreasing virtual_skip, but probably too much complexity to be worth the trouble, especially since clients rarely would benefit from this optimization. diffstat: src/lib-imap-storage/imap-msgpart.c | 45 ------------------------------------- 1 files changed, 0 insertions(+), 45 deletions(-) diffs (62 lines): diff -r b0e0eb30c2b9 -r d4e7a3184a82 src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Fri Jan 11 03:59:50 2013 +0200 +++ b/src/lib-imap-storage/imap-msgpart.c Fri Jan 11 06:18:36 2013 +0200 @@ -371,47 +371,6 @@ return 0; } -static void -skip_using_parts(struct mail *mail, struct istream *input, - uoff_t physical_start, uoff_t *virtual_skip) -{ - enum mail_lookup_abort old_lookup_abort; - struct message_part *parts, *part; - uoff_t vpos; - int ret; - - old_lookup_abort = mail->lookup_abort; - mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; - ret = mail_get_parts(mail, &parts); - mail->lookup_abort = old_lookup_abort; - if (ret < 0) - return; - - for (part = parts, vpos = 0; part != NULL; ) { - if (vpos + part->header_size.virtual_size > *virtual_skip) - break; - /* skip header */ - vpos += part->header_size.virtual_size; - *virtual_skip -= part->header_size.virtual_size; - i_stream_seek(input, physical_start + part->physical_pos + - part->header_size.physical_size); - - if (vpos + part->body_size.virtual_size <= *virtual_skip) { - /* skip body */ - vpos += part->body_size.virtual_size; - *virtual_skip -= part->body_size.virtual_size; - i_stream_seek(input, physical_start + - part->physical_pos + - part->header_size.physical_size + - part->body_size.physical_size); - part = part->next; - } else { - /* maybe we have a child and can skip using it? */ - part = part->children; - } - } -} - static struct istream * imap_msgpart_crlf_seek(struct mail *mail, struct istream *input, const struct imap_msgpart *msgpart) @@ -432,10 +391,6 @@ /* use cache */ i_stream_seek(input, physical_start + cache->physical_pos); virtual_skip -= cache->virtual_pos; - } else { - /* can't use cache, but maybe we can skip faster using the - message parts. */ - skip_using_parts(mail, input, physical_start, &virtual_skip); } if (message_skip_virtual(input, virtual_skip, &cr_skipped) < 0) { errinput = i_stream_create_error(errno); From dovecot at dovecot.org Sun Jan 13 19:15:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 13 Jan 2013 19:15:43 +0200 Subject: dovecot-2.2: doveadm: Increased doveadm client's max command lin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6654ed22af78 changeset: 15639:6654ed22af78 user: Timo Sirainen date: Sun Jan 13 19:15:37 2013 +0200 description: doveadm: Increased doveadm client's max command line length. diffstat: src/doveadm/client-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d4e7a3184a82 -r 6654ed22af78 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Fri Jan 11 06:18:36 2013 +0200 +++ b/src/doveadm/client-connection.c Sun Jan 13 19:15:37 2013 +0200 @@ -19,7 +19,7 @@ #include -#define MAX_INBUF_SIZE 1024 +#define MAX_INBUF_SIZE (1024*1024) static void client_connection_input(struct client_connection *conn); From dovecot at dovecot.org Mon Jan 14 08:01:59 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 14 Jan 2013 08:01:59 +0200 Subject: dovecot-2.2: Several fixes to handling "istream input line too l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/19403b3926f9 changeset: 15641:19403b3926f9 user: Timo Sirainen date: Mon Jan 14 08:01:47 2013 +0200 description: Several fixes to handling "istream input line too long" conditions. diffstat: src/auth/db-passwd-file.c | 2 +- src/doveadm/dsync/doveadm-dsync.c | 22 +++++++++++++++++----- src/doveadm/server-connection.c | 2 +- src/lib-lda/lmtp-client.c | 5 ++++- src/plugins/acl/acl-backend-vfile.c | 2 +- src/plugins/fts/fts-indexer.c | 6 +++++- src/plugins/fts/fts-parser-script.c | 9 ++++++--- 7 files changed, 35 insertions(+), 13 deletions(-) diffs (133 lines): diff -r 20a545f932e3 -r 19403b3926f9 src/auth/db-passwd-file.c --- a/src/auth/db-passwd-file.c Mon Jan 14 07:57:39 2013 +0200 +++ b/src/auth/db-passwd-file.c Mon Jan 14 08:01:47 2013 +0200 @@ -197,7 +197,7 @@ hash_table_create(&pw->users, pw->pool, 0, str_hash, strcmp); start_time = time(NULL); - input = i_stream_create_fd(pw->fd, 4096, FALSE); + input = i_stream_create_fd(pw->fd, (size_t)-1, FALSE); i_stream_set_return_partial_line(input, TRUE); while ((line = i_stream_read_next_line(input)) != NULL) { if (*line == '\0' || *line == ':' || *line == '#') diff -r 20a545f932e3 -r 19403b3926f9 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Mon Jan 14 07:57:39 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Mon Jan 14 08:01:47 2013 +0200 @@ -61,13 +61,25 @@ static void remote_error_input(struct dsync_cmd_context *ctx) { + const unsigned char *data; + size_t size; const char *line; - while ((line = i_stream_read_next_line(ctx->err_stream)) != NULL) - fprintf(stderr, "%s\n", line); - - if (ctx->err_stream->eof && ctx->io_err != NULL) - io_remove(&ctx->io_err); + switch (i_stream_read(ctx->err_stream)) { + case -2: + data = i_stream_get_data(ctx->err_stream, &size); + fprintf(stderr, "%.*s", (int)size, data); + i_stream_skip(ctx->err_stream, size); + break; + case -1: + if (ctx->io_err != NULL) + io_remove(&ctx->io_err); + break; + default: + while ((line = i_stream_next_line(ctx->err_stream)) != NULL) + fprintf(stderr, "%s\n", line); + break; + } } static void diff -r 20a545f932e3 -r 19403b3926f9 src/doveadm/server-connection.c --- a/src/doveadm/server-connection.c Mon Jan 14 07:57:39 2013 +0200 +++ b/src/doveadm/server-connection.c Mon Jan 14 08:01:47 2013 +0200 @@ -224,7 +224,7 @@ } } - if (i_stream_read(conn->input) == -1) { + if (i_stream_read(conn->input) < 0) { /* disconnected */ server_connection_destroy(&conn); return; diff -r 20a545f932e3 -r 19403b3926f9 src/lib-lda/lmtp-client.c --- a/src/lib-lda/lmtp-client.c Mon Jan 14 07:57:39 2013 +0200 +++ b/src/lib-lda/lmtp-client.c Mon Jan 14 08:01:47 2013 +0200 @@ -540,7 +540,10 @@ str_truncate(client->input_multiline, 0); } - if (client->input->stream_errno != 0) { + if (client->input->stream_errno == ENOBUFS) { + lmtp_client_fail(client, + "501 5.5.4 Command reply line too long"); + } else if (client->input->stream_errno != 0) { errno = client->input->stream_errno; i_error("lmtp client: read() failed: %m"); lmtp_client_fail(client, ERRSTR_TEMP_REMOTE_FAILURE diff -r 20a545f932e3 -r 19403b3926f9 src/plugins/acl/acl-backend-vfile.c --- a/src/plugins/acl/acl-backend-vfile.c Mon Jan 14 07:57:39 2013 +0200 +++ b/src/plugins/acl/acl-backend-vfile.c Mon Jan 14 08:01:47 2013 +0200 @@ -505,7 +505,7 @@ if (aclobj->aclobj.backend->debug) i_debug("acl vfile: reading file %s", path); - input = i_stream_create_fd(fd, 4096, FALSE); + input = i_stream_create_fd(fd, (size_t)-1, FALSE); i_stream_set_return_partial_line(input, TRUE); linenum = 1; while ((line = i_stream_read_next_line(input)) != NULL) { diff -r 20a545f932e3 -r 19403b3926f9 src/plugins/fts/fts-indexer.c --- a/src/plugins/fts/fts-indexer.c Mon Jan 14 07:57:39 2013 +0200 +++ b/src/plugins/fts/fts-indexer.c Mon Jan 14 08:01:47 2013 +0200 @@ -191,7 +191,11 @@ return 1; } } - if (ctx->input->eof || ctx->input->stream_errno != 0) { + if (ctx->input->stream_errno != 0) { + i_error("indexer read() failed: %m"); + return -1; + } + if (ctx->input->eof) { i_error("indexer disconnected unexpectedly"); return -1; } diff -r 20a545f932e3 -r 19403b3926f9 src/plugins/fts/fts-parser-script.c --- a/src/plugins/fts/fts-parser-script.c Mon Jan 14 07:57:39 2013 +0200 +++ b/src/plugins/fts/fts-parser-script.c Mon Jan 14 08:01:47 2013 +0200 @@ -70,7 +70,7 @@ struct istream *input; struct content *content; bool eof_seen = FALSE; - int fd; + int fd, ret = 0; fd = script_connect(user, &path); if (fd == -1) @@ -99,14 +99,17 @@ content->content_type = args[0]; content->extensions = (const void *)(args+1); } - if (!eof_seen) { + if (input->stream_errno != 0) { + i_error("parser script read() failed: %m"); + ret = -1; + } else if (!eof_seen) { if (input->v_offset == 0) i_error("parser script didn't send any data"); else i_error("parser script didn't send empty EOF line"); } i_stream_destroy(&input); - return 0; + return ret; } static bool script_support_content(struct mail_user *user, From dovecot at dovecot.org Mon Jan 14 08:01:59 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 14 Jan 2013 08:01:59 +0200 Subject: dovecot-2.2: i_stream_read_next_line() now sets stream_errno=ENO... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/20a545f932e3 changeset: 15640:20a545f932e3 user: Timo Sirainen date: Mon Jan 14 07:57:39 2013 +0200 description: i_stream_read_next_line() now sets stream_errno=ENOBUFS if input buffer gets full. Previously the caller couldn't easily separate this condition from "more data needed", potentially causing infinite loops. diffstat: src/lib/istream.c | 9 ++++++++- src/lib/istream.h | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diffs (33 lines): diff -r 6654ed22af78 -r 20a545f932e3 src/lib/istream.c --- a/src/lib/istream.c Sun Jan 13 19:15:37 2013 +0200 +++ b/src/lib/istream.c Mon Jan 14 07:57:39 2013 +0200 @@ -387,8 +387,15 @@ if (line != NULL) break; - if (i_stream_read(stream) <= 0) + switch (i_stream_read(stream)) { + case -2: + stream->stream_errno = ENOBUFS; + return NULL; + case -1: return i_stream_last_line(stream->real_stream); + case 0: + return NULL; + } } return line; } diff -r 6654ed22af78 -r 20a545f932e3 src/lib/istream.h --- a/src/lib/istream.h Sun Jan 13 19:15:37 2013 +0200 +++ b/src/lib/istream.h Mon Jan 14 07:57:39 2013 +0200 @@ -123,7 +123,8 @@ if the last line should be returned if it doesn't end with LF. */ char *i_stream_next_line(struct istream *stream); /* Like i_stream_next_line(), but reads for more data if needed. Returns NULL - if more data is needed or error occurred. */ + if more data is needed or error occurred. If the input buffer gets full, + stream_errno is set to ENOBUFS. */ char *i_stream_read_next_line(struct istream *stream); /* Returns TRUE if the last line read with i_stream_next_line() ended with CRLF (instead of LF). */ From dovecot at dovecot.org Mon Jan 14 08:38:42 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 14 Jan 2013 08:38:42 +0200 Subject: dovecot-2.2: example-config: Added mail_attribute_dict Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ca2be7b7da3e changeset: 15642:ca2be7b7da3e user: Timo Sirainen date: Mon Jan 14 08:38:36 2013 +0200 description: example-config: Added mail_attribute_dict diffstat: doc/example-config/conf.d/10-mail.conf | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 19403b3926f9 -r ca2be7b7da3e doc/example-config/conf.d/10-mail.conf --- a/doc/example-config/conf.d/10-mail.conf Mon Jan 14 08:01:47 2013 +0200 +++ b/doc/example-config/conf.d/10-mail.conf Mon Jan 14 08:38:36 2013 +0200 @@ -124,6 +124,10 @@ # or ~user/. #mail_full_filesystem_access = no +# Dictionary for key=value mailbox attributes. Currently used by URLAUTH, but +# soon intended to be used by METADATA as well. +#mail_attribute_dict = + ## ## Mail processes ## From dovecot at dovecot.org Mon Jan 14 12:56:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 14 Jan 2013 12:56:15 +0200 Subject: dovecot-2.2: doveadm user: Avoid initializing the user as much a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/70b97cecdacd changeset: 15643:70b97cecdacd user: Timo Sirainen date: Mon Jan 14 12:56:09 2013 +0200 description: doveadm user: Avoid initializing the user as much as possible. diffstat: src/doveadm/doveadm-auth.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (16 lines): diff -r ca2be7b7da3e -r 70b97cecdacd src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Mon Jan 14 08:38:36 2013 +0200 +++ b/src/doveadm/doveadm-auth.c Mon Jan 14 12:56:09 2013 +0200 @@ -415,8 +415,11 @@ if (!userdb_only) { storage_service = mail_storage_service_init(master_service, NULL, + MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP | + MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR | MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | - MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP); + MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS | + MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES); } while ((input.username = argv[optind++]) != NULL) { From dovecot at dovecot.org Mon Jan 14 13:01:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 14 Jan 2013 13:01:38 +0200 Subject: dovecot-2.1: doveadm acl: Added "recalc" command to rebuild acl_... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/6bee6d0c611f changeset: 14881:6bee6d0c611f user: Timo Sirainen date: Mon Jan 14 13:01:33 2013 +0200 description: doveadm acl: Added "recalc" command to rebuild acl_lookup_dict for the user. diffstat: src/plugins/acl/doveadm-acl.c | 29 +++++++++++++++++++++++++++++ 1 files changed, 29 insertions(+), 0 deletions(-) diffs (46 lines): diff -r 1ab8e0e699f7 -r 6bee6d0c611f src/plugins/acl/doveadm-acl.c --- a/src/plugins/acl/doveadm-acl.c Wed Jan 09 07:01:41 2013 +0200 +++ b/src/plugins/acl/doveadm-acl.c Mon Jan 14 13:01:33 2013 +0200 @@ -399,6 +399,34 @@ } static int +cmd_acl_recalc_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user) +{ + struct acl_user *auser = ACL_USER_CONTEXT(user); + + if (auser == NULL) { + i_error("ACL not enabled for %s", user->username); + doveadm_mail_failed_error(ctx, MAIL_ERROR_NOTFOUND); + return -1; + } + if (acl_lookup_dict_rebuild(auser->acl_lookup_dict) < 0) { + i_error("Failed to recalculate ACL dicts"); + doveadm_mail_failed_error(ctx, MAIL_ERROR_TEMP); + return -1; + } + return 0; +} + +static struct doveadm_mail_cmd_context * +cmd_acl_recalc_alloc(void) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); + ctx->v.run = cmd_acl_recalc_run; + return ctx; +} + +static int cmd_acl_debug_mailbox_open(struct doveadm_mail_cmd_context *ctx, struct mail_user *user, const char *mailbox, struct mailbox **box_r) @@ -591,6 +619,7 @@ { cmd_acl_add_alloc, "acl add", " [ ...]" }, { cmd_acl_remove_alloc, "acl remove", " [ ...]" }, { cmd_acl_delete_alloc, "acl delete", " " }, + { cmd_acl_recalc_alloc, "acl recalc", "" }, { cmd_acl_debug_alloc, "acl debug", "" } }; From pigeonhole at rename-it.nl Mon Jan 14 13:58:04 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:04 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: vnd.dovecot.duplicate extensi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/2b767751ff27 changeset: 1703:2b767751ff27 user: Stephan Bosch date: Wed Oct 17 02:09:55 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate extension: fixed bug in previous change. Forgot to initialize variable, causing a segfault at runtime. diffstat: src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 12a4e11ecd4c -r 2b767751ff27 src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Tue Oct 16 21:33:15 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:09:55 2012 +0200 @@ -277,7 +277,7 @@ const struct ext_duplicate_config *config = (const struct ext_duplicate_config *) ext->context; int opt_code = 0; - string_t *handle = NULL, *header = NULL, *value; + string_t *handle = NULL, *header = NULL, *value = NULL; const char *val = NULL; size_t val_len = 0; sieve_number_t seconds = config->default_period; From pigeonhole at rename-it.nl Mon Jan 14 13:58:04 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:04 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: vnd.dovecot.duplicate extensi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/762b6a885897 changeset: 1705:762b6a885897 user: Stephan Bosch date: Wed Oct 17 02:55:07 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate extension: Only track duplicate when Sieve script executes successfully. diffstat: src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c | 111 ++++++++- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h | 2 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 4 +- 3 files changed, 96 insertions(+), 21 deletions(-) diffs (221 lines): diff -r 3a972efe012b -r 762b6a885897 src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c Wed Oct 17 02:15:04 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c Wed Oct 17 02:55:07 2012 +0200 @@ -5,6 +5,7 @@ #include "md5.h" #include "ioloop.h" #include "str.h" +#include "str-sanitize.h" #include "array.h" #include "sieve-common.h" @@ -15,6 +16,7 @@ #include "sieve-code.h" #include "sieve-runtime.h" #include "sieve-actions.h" +#include "sieve-result.h" #include "ext-duplicate-common.h" @@ -62,6 +64,67 @@ } /* + * Duplicate_mark action + */ + +struct act_duplicate_mark_data { + const char *handle; + unsigned int period; + unsigned char hash[MD5_RESULTLEN]; +}; + +static void act_duplicate_mark_print + (const struct sieve_action *action, + const struct sieve_result_print_env *rpenv, bool *keep); +static bool act_duplicate_mark_commit + (const struct sieve_action *action, + const struct sieve_action_exec_env *aenv, void *tr_context, bool *keep); + +static const struct sieve_action_def act_duplicate_mark = { + "duplicate_mark", + 0, + NULL, NULL, NULL, + act_duplicate_mark_print, + NULL, NULL, + act_duplicate_mark_commit, + NULL +}; + +static void act_duplicate_mark_print +(const struct sieve_action *action, + const struct sieve_result_print_env *rpenv, bool *keep ATTR_UNUSED) +{ + struct act_duplicate_mark_data *data = + (struct act_duplicate_mark_data *) action->context; + + if (data->handle != NULL) { + sieve_result_action_printf(rpenv, "track duplicate with handle: %s", + str_sanitize(data->handle, 128)); + } else { + sieve_result_action_printf(rpenv, "track duplicate"); + } +} + +static bool act_duplicate_mark_commit +(const struct sieve_action *action, + const struct sieve_action_exec_env *aenv, + void *tr_context ATTR_UNUSED, bool *keep ATTR_UNUSED) +{ + const struct sieve_script_env *senv = aenv->scriptenv; + struct act_duplicate_mark_data *data = + (struct act_duplicate_mark_data *) action->context; + + /* Message was handled successfully until now, so track duplicate for this + * message. + */ + sieve_action_duplicate_mark + (senv, data->hash, sizeof(data->hash), ioloop_time + data->period); + + return TRUE; +} + + +/* * Duplicate checking */ @@ -77,7 +140,7 @@ unsigned int nohandle_checked:1; }; -bool ext_duplicate_check +int ext_duplicate_check (const struct sieve_runtime_env *renv, string_t *handle, const char *value, size_t value_len, sieve_number_t period) { @@ -85,13 +148,13 @@ const struct sieve_script_env *senv = renv->scriptenv; struct ext_duplicate_context *rctx; bool duplicate = FALSE; - pool_t pool = NULL; + pool_t msg_pool = NULL, result_pool = NULL; static const char *id = "sieve duplicate"; - unsigned char dupl_hash[MD5_RESULTLEN]; + struct act_duplicate_mark_data *act; struct md5_context ctx; if ( !sieve_action_duplicate_check_available(senv) || value == NULL ) - return FALSE; + return 0; /* Get context; find out whether duplicate was checked earlier */ rctx = (struct ext_duplicate_context *) @@ -99,24 +162,30 @@ if ( rctx == NULL ) { /* Create context */ - pool = sieve_message_context_pool(renv->msgctx); - rctx = p_new(pool, struct ext_duplicate_context, 1); + msg_pool = sieve_message_context_pool(renv->msgctx); + rctx = p_new(msg_pool, struct ext_duplicate_context, 1); sieve_message_context_extension_set(renv->msgctx, this_ext, (void *)rctx); } else { if ( handle == NULL ) { if ( rctx->nohandle_checked ) { /* Already checked for duplicate */ - return rctx->nohandle_duplicate; + return ( rctx->nohandle_duplicate ? 1 : 0 ); } } else if ( array_is_created(&rctx->handles) ) { const struct ext_duplicate_handle *record; array_foreach (&rctx->handles, record) { if ( strcmp(record->handle, str_c(handle)) == 0 ) - return record->duplicate; + return ( record->duplicate ? 1 : 0 ); } } } + result_pool = sieve_result_pool(renv->result); + act = p_new(result_pool, struct act_duplicate_mark_data, 1); + if (handle != NULL) + act->handle = p_strdup(result_pool, str_c(handle)); + act->period = period; + /* Create hash */ md5_init(&ctx); md5_update(&ctx, id, strlen(id)); @@ -127,31 +196,35 @@ md5_update(&ctx, "default", 7); } md5_update(&ctx, value, value_len); - md5_final(&ctx, dupl_hash); + md5_final(&ctx, act->hash); /* Check duplicate */ - duplicate = sieve_action_duplicate_check - (senv, dupl_hash, sizeof(dupl_hash)); + duplicate = sieve_action_duplicate_check(senv, act->hash, sizeof(act->hash)); - /* Create/refresh entry */ - sieve_action_duplicate_mark - (senv, dupl_hash, sizeof(dupl_hash), ioloop_time + period); + /* We may only mark the message as duplicate when Sieve script executes + * successfully; therefore defer this operation until successful result + * execution. + */ + if ( sieve_result_add_action + (renv, NULL, &act_duplicate_mark, NULL, (void *) act, 0, FALSE) < 0 ) + return -1; + /* Cache result */ if ( handle == NULL ) { rctx->nohandle_duplicate = duplicate; rctx->nohandle_checked = TRUE; } else { struct ext_duplicate_handle *record; - if ( pool == NULL ) - pool = sieve_message_context_pool(renv->msgctx); + if ( msg_pool == NULL ) + msg_pool = sieve_message_context_pool(renv->msgctx); if ( !array_is_created(&rctx->handles) ) - p_array_init(&rctx->handles, pool, 64); + p_array_init(&rctx->handles, msg_pool, 64); record = array_append_space(&rctx->handles); - record->handle = p_strdup(pool, str_c(handle)); + record->handle = p_strdup(msg_pool, str_c(handle)); record->duplicate = duplicate; } - return duplicate; + return ( duplicate ? 1 : 0 ); } diff -r 3a972efe012b -r 762b6a885897 src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h Wed Oct 17 02:15:04 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h Wed Oct 17 02:55:07 2012 +0200 @@ -38,7 +38,7 @@ * Duplicate checking */ -bool ext_duplicate_check +int ext_duplicate_check (const struct sieve_runtime_env *renv, string_t *handle, const char *value, size_t value_len, sieve_number_t period); diff -r 3a972efe012b -r 762b6a885897 src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:15:04 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:55:07 2012 +0200 @@ -343,7 +343,9 @@ if (val == NULL) { duplicate = FALSE; } else { - duplicate = ext_duplicate_check(renv, handle, val, val_len, seconds); + if ((ret=ext_duplicate_check(renv, handle, val, val_len, seconds)) < 0) + return SIEVE_EXEC_FAILURE; + duplicate = ( ret > 0 ); } /* Trace */ From pigeonhole at rename-it.nl Mon Jan 14 13:58:04 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:04 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: vnd.dovecot.duplicate: Fixed ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c3b14a8784dc changeset: 1707:c3b14a8784dc user: Stephan Bosch date: Wed Oct 17 22:10:50 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate: Fixed default max period setting. diffstat: src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (22 lines): diff -r fd7257cb40ea -r c3b14a8784dc src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c Wed Oct 17 21:49:55 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c Wed Oct 17 22:10:50 2012 +0200 @@ -24,7 +24,8 @@ * Extension configuration */ -#define EXT_DUPLICATE_DEFAULT_PERIOD (1*24*60*60) +#define EXT_DUPLICATE_DEFAULT_PERIOD (12*60*60) +#define EXT_DUPLICATE_DEFAULT_MAX_PERIOD (2*24*60*60) bool ext_duplicate_load (const struct sieve_extension *ext, void **context) @@ -43,7 +44,7 @@ if ( !sieve_setting_get_duration_value (svinst, "sieve_duplicate_max_period", &max_period) ) { - max_period = EXT_DUPLICATE_DEFAULT_PERIOD; + max_period = EXT_DUPLICATE_DEFAULT_MAX_PERIOD; } config = i_new(struct ext_duplicate_config, 1); From pigeonhole at rename-it.nl Mon Jan 14 13:58:03 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:03 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: vnd.dovecot.duplicate extensi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/12a4e11ecd4c changeset: 1702:12a4e11ecd4c user: Stephan Bosch date: Tue Oct 16 21:33:15 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate extension: Added new features to the duplicate test. It is now possible to track duplicates based on arbitrary headers or even arbitrary string values using the new :header and :value arguments respectively. The experation time can be configured using the new :seconds argument. This change is backwards compatible as long as the name argument wasn't used. This is now a :handle argument. diffstat: doc/rfc/spec-bosch-sieve-duplicate.txt | 284 +++++++-- doc/rfc/xml/reference.IMAP4FLAGS.xml | 15 + doc/rfc/xml/reference.MAILBOX.xml | 15 + doc/rfc/xml/reference.VACATION.xml | 17 + doc/rfc/xml/spec-bosch-sieve-duplicate.xml | 164 ++++- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c | 137 +++- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h | 11 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate.c | 3 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 273 ++++++++- tests/extensions/vnd.dovecot/duplicate/errors.svtest | 2 +- tests/extensions/vnd.dovecot/duplicate/errors/syntax.sieve | 5 +- tests/extensions/vnd.dovecot/duplicate/execute.svtest | 10 +- 12 files changed, 747 insertions(+), 189 deletions(-) diffs (truncated from 1305 to 300 lines): diff -r 2b5ff3818a9f -r 12a4e11ecd4c doc/rfc/spec-bosch-sieve-duplicate.txt --- a/doc/rfc/spec-bosch-sieve-duplicate.txt Sat Oct 13 10:30:57 2012 +0200 +++ b/doc/rfc/spec-bosch-sieve-duplicate.txt Tue Oct 16 21:33:15 2012 +0200 @@ -2,7 +2,7 @@ Pigeonhole Project S. Bosch - February 25, 2012 + October 16, 2012 Sieve Email Filtering: Detecting Duplicate Deliveries @@ -10,23 +10,65 @@ Abstract This document defines a new vendor-defined test command "duplicate" - for the "Sieve" email filtering language that tests whether an e-mail - message is a duplicate, i.e. whether it was seen before by the - delivery agent. Users can use this new test to remove duplicate - deliveries commonly caused by mailing list subscriptions or mail - account aliases. + for the "Sieve" email filtering language. It can be used to test + whether a particular string value is a duplicate, i.e. whether it was + seen before by the delivery agent that is executing the Sieve script. + The main application for this new test is detecting duplicate message + deliveries commonly caused by mailing list subscriptions or + redirected mail addresses. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Bosch [Page 1] + + Sieve: Detecting Duplicate Deliveries October 2012 Table of Contents - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Conventions Used in This Document . . . . . . . . . . . . . . . 2 - 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 2 - 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 3 - 5. Example . . . . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 3 - 7. Normative References . . . . . . . . . . . . . . . . . . . . . 3 - Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . . 3 + 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 4 + 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 5 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 6 + 7. References . . . . . . . . . . . . . . . . . . . . . . . . . . 6 + 7.1. Normative References . . . . . . . . . . . . . . . . . . . 6 + 7.2. Informative References . . . . . . . . . . . . . . . . . . 6 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 7 @@ -52,26 +94,52 @@ -Bosch [Page 1] + + + + + + + + + + + + + + +Bosch [Page 2] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 1. Introduction This is an extension to the Sieve filtering language defined by RFC - 5228 [SIEVE]. It adds a test to determine whether a message was seen - before by the delivery agent based on the Message-ID header. + 5228 [SIEVE]. It adds a test to determine whether a certain string + value was seen before by the delivery agent in an earlier execution + of the Sieve script. This can be used to detect and handle duplicate + message deliveries. Duplicate deliveries are a common side-effect of being subscribed to - a mailing list. If a member of the list decides to reply to both the - user and the mailing list itself, the user will get a copy of the - message directly and through mailing list. In another scenario, the - user has several aliases for his mail account and one of his contacts - sends the message to multiple addresses that eventually map to the - same account. Using the vnd.dovecot.duplicate extension, users have - the means to detect such duplicates and deal with these - appropriately, e.g. by discarding them. + a mailing list. For example, if a member of the list decides to + reply to both the user and the mailing list itself, the user will get + a copy of the message directly and through mailing list. Also, if + someone cross-posts over several mailing lists to which the user is + subscribed, the user will receive a copy from each of those lists. + In another scenario, the user has several redirected mail addresses + all pointing to his main mail account. If one of the user's contacts + sends the message to more than one of those addresses, the user will + receive more than a single copy. Using the "vnd.dovecot.duplicate" + extension, users have the means to detect and handle such duplicates, + e.g. by discarding them or putting them in a special folder. + + Duplicate messages are normally detected using the Message-ID header + field, which is required to be unique for each message. However, the + "duplicate" test is flexible enough to use different (weaker) + criteria for defining what makes a message a duplicate, for example + based on the subject line. Also, other applications of this new test + command are possible, as long as the tracked value is a string. This extension is specific to the Pigeonhole Sieve implementation for the Dovecot Secure IMAP server. It will therefore most likely not be @@ -89,31 +157,79 @@ arguments syntax. + + + + + + + +Bosch [Page 3] + + Sieve: Detecting Duplicate Deliveries October 2012 + + 3. Test "duplicate" - Usage: "duplicate" [] + Usage: "duplicate" [":seconds" ] + [":header" / + ":value" ] + [":handle" ] - The "duplicate" test keeps track of which Message-ID values were seen - before by this test in an earlier delivery operation. It evaluates - to "true" when the Message-ID header of the current message was seen - before. If it is not known, the test evaluates to "false" and the - Message-ID is added to a persistent internal tracking list. - Implementations SHOULD limit the number of messages that are tracked - and SHOULD let Message-ID entries expire after some short period of - time. + The "duplicate" test keeps track of which values were seen before by + this test in an earlier execution of this Sieve script. In its basic + form, the tested value is the content of the Message-ID header of the + message. This way, this test can be used to detect duplicate + deliveries of the same message. It can also detect duplicate + deliveries based on other message header fields if requested and it + can even use a user-provided string value, e.g. as composed from text + extracted from the message using the "variables" [VARIABLES] + extension. - Using the "name" argument, the duplicate test can be employed for - multiple independent purposes. Only when the Message-ID was seen - before in an earlier script execution by a duplicate test with the + The "duplicate" test evaluates to "true" when the provided value was + seen before. If the value is not known, the test evaluates to + "false" and the value is added to an internal value tracking list. + Implementations SHOULD limit the number of values (and thereby + messages) that are tracked. Also, implementations SHOULD let entries + in the value tracking list expire after a short period of time. + The user can explicitly control the length this expiration time by + means of the ":seconds" argument. If the ":seconds" argument is + omitted, an appropriate default MUST be used. Sites SHOULD impose a + maximum limit on the expiration time. If that limit is exceeded, the + maximum value MUST silently be substituted; exceeding the limit MUST + NOT produce an error. + By default the tracked value is the content of the message's + Message-ID header field. For more advanced purposes, the content of + another header can be chosen for tracking by specifying the ":header" + argument. The tracked string value can also be specified explicitly + using the ":value" argument. The ":header" and ":value" arguments + are mutually exclusive and specifying both for a single "duplicate" + test command MUST trigger an error at compile time. If the value is + extracted from a header, i.e. when the ":value" argument is not used, + leading and trailing whitespace (see Section 2.2 of RFC 5228 [SIEVE]) + MUST first be trimmed from the value before executing the test. -Bosch [Page 2] + Using the ":handle" argument, the duplicate test can be employed for + multiple independent purposes. Only when the tracked value was seen + before in an earlier script execution by a "duplicate" test with the + same ":handle" argument, it is recognized as a duplicate. + + NOTE: The necessary mechanism to track duplicate messages is very + + + +Bosch [Page 4] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 - same "name" argument, it is recognized as a duplicate. + similar to the mechanism that is needed for tracking duplicate + responses for the "vacation" [VACATION] action. One way to implement + the necessary mechanism for the "duplicate" test is therefore to + store a hash of the tracked value and, if provided, the ":handle" + argument. 4. Sieve Capability Strings @@ -122,29 +238,71 @@ advertise the capability string "vnd.dovecot.duplicate". -5. Example +5. Examples - In this example, duplicate deliveries are stored in a special folder - contained in the user's Trash folder. If the folder does not exist, - it is created. This way, the user has a chance to recover messages - when necessary. Messages that are not recognized as duplicates are - stored in the user's inbox as normal. + In the following basic example, message duplicates are detected by + tracking the Message-ID header. Duplicate deliveries are stored in a + special folder contained in the user's Trash folder. If the folder + does not exist, it is created automatically using the "mailbox" + [MAILBOX] extension. This way, the user has a chance to recover + messages when necessary. Messages that are not recognized as + duplicates are stored in the user's inbox as normal. require ["vnd.dovecot.duplicate", "fileinto", "mailbox"]; if duplicate { - fileinto :create "Trash/Duplicate"; + fileinto :create "Trash/Duplicate"; } + The next example shows a more complex use of the "duplicate" test. + The user gets network alerts from a set of remote automated + monitoring systems. Multiple notifications can be received about the + same event from different monitoring systems. The Message-ID of + these messages is different, because these are all distinct messages + from different senders. To avoid being notified multiple times about + the same event the user writes the following script: + + require ["vnd.dovecot.duplicate", "variables", "imap4flags", + "fileinto"]; + + if header :matches "subject" "ALERT: *" { + if duplicate :seconds 60 :value "${1}" { + setflag "\\seen"; + } + fileinto "Alerts"; + } + + The subjects of the notification message are structured with a + + + From pigeonhole at rename-it.nl Mon Jan 14 13:58:04 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:04 +0100 Subject: dovecot-2.2-pigeonhole: Improved specification of the vnd.doveco... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/fd7257cb40ea changeset: 1706:fd7257cb40ea user: Stephan Bosch date: Wed Oct 17 21:49:55 2012 +0200 description: Improved specification of the vnd.dovecot.duplicate extension. diffstat: doc/rfc/spec-bosch-sieve-duplicate.txt | 132 ++++++++++++++-------------- doc/rfc/xml/reference.INCLUDE.xml | 17 +++ doc/rfc/xml/spec-bosch-sieve-duplicate.xml | 49 +++++++--- 3 files changed, 118 insertions(+), 80 deletions(-) diffs (truncated from 324 to 300 lines): diff -r 762b6a885897 -r fd7257cb40ea doc/rfc/spec-bosch-sieve-duplicate.txt --- a/doc/rfc/spec-bosch-sieve-duplicate.txt Wed Oct 17 02:55:07 2012 +0200 +++ b/doc/rfc/spec-bosch-sieve-duplicate.txt Wed Oct 17 21:49:55 2012 +0200 @@ -2,7 +2,7 @@ Pigeonhole Project S. Bosch - October 16, 2012 + October 17, 2012 Sieve Email Filtering: Detecting Duplicate Deliveries @@ -65,9 +65,9 @@ 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 5 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 6 - 7. References . . . . . . . . . . . . . . . . . . . . . . . . . . 6 - 7.1. Normative References . . . . . . . . . . . . . . . . . . . 6 - 7.2. Informative References . . . . . . . . . . . . . . . . . . 6 + 7. References . . . . . . . . . . . . . . . . . . . . . . . . . . 7 + 7.1. Normative References . . . . . . . . . . . . . . . . . . . 7 + 7.2. Informative References . . . . . . . . . . . . . . . . . . 7 Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 7 @@ -187,20 +187,50 @@ extension. The "duplicate" test evaluates to "true" when the provided value was - seen before. If the value is not known, the test evaluates to - "false" and the value is added to an internal value tracking list. + seen before in an earlier Sieve execution for a previous message + delivery. If the value was not seen earlier, the test evaluates to + "false". + + As a side-effect, the "duplicate" test adds the evaluated value to an + internal duplicate tracking list, so that the test will evaluate to + "true" the next time the Sieve script is executed and the same value + is encountered. Note that the "duplicate" test MUST only check for + duplicates amongst values encountered in previous executions of the + Sieve script; it MUST NOT consider values encountered earlier in the + current Sieve script execution as potential duplicates. This means + that all "duplicate" tests in a Sieve script execution, including + those located in scripts included using the "include" [INCLUDE] + extension, MUST yield the same result if the arguments are identical. + + Implementations MUST prevent adding values to the internal duplicate + tracking list when the Sieve script execution fails. For example, + this can be implemented by deferring the definitive modification of + the tracking list to the end of the Sieve script execution. If + failed script executions would add values to the duplicate tracking + list, all "duplicate" tests would erroneously yield "true" for the + next delivery attempt of the same message, which can -- depending on + the action taken for a duplicate -- easily lead to discarding the + message without further notice. + Implementations SHOULD limit the number of values (and thereby messages) that are tracked. Also, implementations SHOULD let entries - in the value tracking list expire after a short period of time. + in the value tracking list expire after a short period of time. The + user can explicitly control the length of this expiration time by + means of the ":seconds" argument. If the ":seconds" argument is - The user can explicitly control the length this expiration time by - means of the ":seconds" argument. If the ":seconds" argument is + + +Bosch [Page 4] + + Sieve: Detecting Duplicate Deliveries October 2012 + + omitted, an appropriate default MUST be used. Sites SHOULD impose a maximum limit on the expiration time. If that limit is exceeded, the maximum value MUST silently be substituted; exceeding the limit MUST NOT produce an error. - By default the tracked value is the content of the message's + By default, the tracked value is the content of the message's Message-ID header field. For more advanced purposes, the content of another header can be chosen for tracking by specifying the ":header" argument. The tracked string value can also be specified explicitly @@ -209,7 +239,8 @@ test command MUST trigger an error at compile time. If the value is extracted from a header, i.e. when the ":value" argument is not used, leading and trailing whitespace (see Section 2.2 of RFC 5228 [SIEVE]) - MUST first be trimmed from the value before executing the test. + MUST first be trimmed from the value before performing the actual + duplicate verification. Using the ":handle" argument, the duplicate test can be employed for multiple independent purposes. Only when the tracked value was seen @@ -217,14 +248,6 @@ same ":handle" argument, it is recognized as a duplicate. NOTE: The necessary mechanism to track duplicate messages is very - - - -Bosch [Page 4] - - Sieve: Detecting Duplicate Deliveries October 2012 - - similar to the mechanism that is needed for tracking duplicate responses for the "vacation" [VACATION] action. One way to implement the necessary mechanism for the "duplicate" test is therefore to @@ -248,6 +271,16 @@ messages when necessary. Messages that are not recognized as duplicates are stored in the user's inbox as normal. + + + + + +Bosch [Page 5] + + Sieve: Detecting Duplicate Deliveries October 2012 + + require ["vnd.dovecot.duplicate", "fileinto", "mailbox"]; if duplicate { @@ -273,14 +306,6 @@ } The subjects of the notification message are structured with a - - - -Bosch [Page 5] - - Sieve: Detecting Duplicate Deliveries October 2012 - - predictable pattern which includes a description of the event. In the script above the "duplicate" test is used to detect duplicate alert events. The message subject is matched against a pattern and @@ -290,7 +315,7 @@ due to the specified ":seconds" argument. In the the event of a duplicate, the message is marked as seen using the "imap4flags" [IMAP4FLAGS] extension. All alert messages are put into the "Alerts" - mailbox irrespective of wether those messages are duplicates or not. + mailbox irrespective of whether those messages are duplicates or not. 6. Security Considerations @@ -302,8 +327,21 @@ 7. References + + + + + +Bosch [Page 6] + + Sieve: Detecting Duplicate Deliveries October 2012 + + 7.1. Normative References + [INCLUDE] Daboo, C. and A. Stone, "Sieve Email Filtering: Include + Extension", RFC 6609, May 2012. + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. @@ -330,13 +368,6 @@ RFC 5229, January 2008. - - -Bosch [Page 6] - - Sieve: Detecting Duplicate Deliveries October 2012 - - Author's Address Stephan Bosch @@ -357,36 +388,5 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Bosch [Page 7] diff -r 762b6a885897 -r fd7257cb40ea doc/rfc/xml/reference.INCLUDE.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/rfc/xml/reference.INCLUDE.xml Wed Oct 17 21:49:55 2012 +0200 @@ -0,0 +1,17 @@ + + + + + +Sieve Email Filtering: Include Extension + + + + + + +The Sieve Email Filtering "include" extension permits users to include one Sieve script inside another. This can make managing large scripts or multiple sets of scripts much easier, and allows a site and its users to build up libraries of scripts. Users are able to include their own personal scripts or site-wide scripts. [STANDARDS-TRACK] + + + + diff -r 762b6a885897 -r fd7257cb40ea doc/rfc/xml/spec-bosch-sieve-duplicate.xml --- a/doc/rfc/xml/spec-bosch-sieve-duplicate.xml Wed Oct 17 02:55:07 2012 +0200 +++ b/doc/rfc/xml/spec-bosch-sieve-duplicate.xml Wed Oct 17 21:49:55 2012 +0200 @@ -113,28 +113,48 @@ extension. The "duplicate" test evaluates to "true" when the provided value was seen -before. If the value is not known, the test evaluates to "false" and the value -is added to an internal value tracking list. Implementations SHOULD limit the -number of values (and thereby messages) that are tracked. Also, implementations -SHOULD let entries in the value tracking list expire after a short period of -time. +before in an earlier Sieve execution for a previous message delivery. If the +value was not seen earlier, the test evaluates to "false". -The user can explicitly control the length this expiration time by means of -the ":seconds" argument. If the ":seconds" argument is omitted, an appropriate -default MUST be used. Sites SHOULD impose a maximum limit on the expiration -time. If that limit is exceeded, the maximum value MUST silently be substituted; -exceeding the limit MUST NOT produce an error. +As a side-effect, the "duplicate" test adds the evaluated value to an +internal duplicate tracking list, so that the test will evaluate to "true" the +next time the Sieve script is executed and the same value is encountered. +Note that the "duplicate" test MUST only check for duplicates amongst values +encountered in previous executions of the Sieve script; it MUST NOT consider +values encountered earlier in the current Sieve script execution as potential +duplicates. This means that all "duplicate" tests in a Sieve script execution, +including those located in scripts included using the "include" + extension, MUST yield the same result if the arguments +are identical. -By default the tracked value is the content of the message's Message-ID +Implementations MUST prevent adding values to the internal duplicate tracking +list when the Sieve script execution fails. For example, this can be implemented +by deferring the definitive modification of the tracking list to the end of the +Sieve script execution. If failed script executions would add values to the +duplicate tracking list, all "duplicate" tests would erroneously yield "true" +for the next delivery attempt of the same message, which can -- depending on the +action taken for a duplicate -- easily lead to discarding the message without +further notice. + +Implementations SHOULD limit the number of values (and thereby messages) that +are tracked. Also, implementations SHOULD let entries in the value tracking list +expire after a short period of time. The user can explicitly control the length +of this expiration time by means of the ":seconds" argument. If the ":seconds" +argument is omitted, an appropriate default MUST be used. Sites SHOULD impose a +maximum limit on the expiration time. If that limit is exceeded, the maximum +value MUST silently be substituted; exceeding the limit MUST NOT produce an +error. + +By default, the tracked value is the content of the message's Message-ID header field. For more advanced purposes, the content of another header can be chosen for tracking by specifying the ":header" argument. The tracked string -value can also be specified explicitly using the ":value" argument. The +value can also be specified explicitly using the ":value" argument. The ":header" and ":value" arguments are mutually exclusive and specifying both for a single "duplicate" test command MUST trigger an error at compile time. If the value is extracted from a header, i.e. when the ":value" argument is not used, From pigeonhole at rename-it.nl Mon Jan 14 13:58:04 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:04 +0100 Subject: dovecot-2.2-pigeonhole: Updated installation documentation for v... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/0dc8453e07c4 changeset: 1708:0dc8453e07c4 user: Stephan Bosch date: Wed Oct 17 22:11:16 2012 +0200 description: Updated installation documentation for vnd.dovecot.duplicate extension. diffstat: doc/extensions/vnd.dovecot.duplicate.txt | 28 +++++++++++++++------------- 1 files changed, 15 insertions(+), 13 deletions(-) diffs (53 lines): diff -r c3b14a8784dc -r 0dc8453e07c4 doc/extensions/vnd.dovecot.duplicate.txt --- a/doc/extensions/vnd.dovecot.duplicate.txt Wed Oct 17 22:10:50 2012 +0200 +++ b/doc/extensions/vnd.dovecot.duplicate.txt Wed Oct 17 22:11:16 2012 +0200 @@ -8,13 +8,11 @@ Description =========== -Sieve (RFC 5228) is a highly extensible machine language specifically tailored -for internet message filtering. For the Dovecot Secure IMAP server, Sieve -support is provided by the Pigeonhole Sieve plugin. The vnd.dovecot.duplicate -extension augments the Sieve filtering implementation with a test to verify -whether a message was received earlier already based on its Message-ID. This can -be used to prevent duplicate deliveries, e.g. caused by mailinglists when people -reply both to the mailinglist and the user directly. +The vnd.dovecot.duplicate extension augments the Sieve filtering implementation +with a test to verify whether the evaluated string value was seen before in an +earlier execution of the Sieve script. The main application for this new test is +detecting and handling duplicate message deliveries, e.g. as caused by +mailinglists when people reply both to the mailinglist and the user directly. This extension is specific to the Pigeonhole Sieve implementation for the Dovecot Secure IMAP server. It will therefore most likely not be supported by @@ -35,14 +33,17 @@ ============= The "vnd.dovecot.duplicate" extension is not enabled by default and thus it -needs to be enabled explicitly. +needs to be enabled explicitly by adding it to the `sieve_extensions' or the +`sieve_global_extensions' setting. The following configuration settings are used: -sieve_duplicate_period = 1d - This option specifies after what period of time Message-IDs are purged from - the duplicate database. The period is specified in s(econds), unless followed - by a d(ay), h(our) or m(inute) specifier character. +sieve_duplicate_default_period = 12h +sieve_duplicate_max_period = 2d + These options respectively specify the default and the maximum value for the + period after which tracked values are purged from the duplicate tracking + database. The period is specified in s(econds), unless followed by a d(ay), + h(our) or m(inute) specifier character. Example ======= @@ -52,5 +53,6 @@ sieve_extensions = +vnd.dovecot.duplicate - sieve_duplicate_period = 6h + sieve_duplicate_default_period = 1h + sieve_duplicate_max_period = 1d } From pigeonhole at rename-it.nl Mon Jan 14 13:58:04 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:04 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: vacation extension: Fixed det... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b56711807edc changeset: 1709:b56711807edc user: Stephan Bosch date: Mon Nov 26 21:16:54 2012 +0100 description: lib-sieve: vacation extension: Fixed determination of From: address for when sieve_vacation_dont_check_recipient is active. diffstat: src/lib-sieve/plugins/vacation/cmd-vacation.c | 76 ++++++++++++++------------ 1 files changed, 41 insertions(+), 35 deletions(-) diffs (105 lines): diff -r 0dc8453e07c4 -r b56711807edc src/lib-sieve/plugins/vacation/cmd-vacation.c --- a/src/lib-sieve/plugins/vacation/cmd-vacation.c Wed Oct 17 22:11:16 2012 +0200 +++ b/src/lib-sieve/plugins/vacation/cmd-vacation.c Mon Nov 26 21:16:54 2012 +0100 @@ -1031,7 +1031,7 @@ const char *recipient = sieve_message_get_final_recipient(aenv->msgctx); const char *const *hdsp; const char *const *headers; - const char *reply_from = NULL; + const char *reply_from = NULL, *orig_recipient = NULL; /* Is the recipient unset? */ @@ -1142,52 +1142,58 @@ return TRUE; } + /* Fetch original recipient if necessary */ + if ( config->use_original_recipient ) + orig_recipient = sieve_message_get_orig_recipient(aenv->msgctx); + /* Is the original message directly addressed to the user or the addresses * specified using the :addresses tag? */ - if ( !config->dont_check_recipient ) { - const char *orig_recipient = NULL; + hdsp = _my_address_headers; + while ( *hdsp != NULL ) { + if ( mail_get_headers + (mail, *hdsp, &headers) >= 0 && headers[0] != NULL ) { - if ( config->use_original_recipient ) - orig_recipient = sieve_message_get_orig_recipient(aenv->msgctx); + /* Final recipient directly listed in headers? */ + if ( _contains_my_address(headers, recipient) ) { + reply_from = recipient; + break; + } - hdsp = _my_address_headers; - while ( *hdsp != NULL ) { - if ( mail_get_headers - (mail, *hdsp, &headers) >= 0 && headers[0] != NULL ) { + /* Original recipient directly listed in headers? */ + if ( orig_recipient != NULL && + _contains_my_address(headers, orig_recipient) ) { + reply_from = orig_recipient; + break; + } - if ( _contains_my_address(headers, recipient) ) { - reply_from = recipient; - break; + /* User-provided :addresses listed in headers? */ + if ( ctx->addresses != NULL ) { + bool found = FALSE; + const char * const *my_address = ctx->addresses; + + while ( !found && *my_address != NULL ) { + if ( (found=_contains_my_address(headers, *my_address)) ) + reply_from = *my_address; + my_address++; } - if ( orig_recipient != NULL && _contains_my_address(headers, orig_recipient) ) { - reply_from = orig_recipient; - break; - } + if ( found ) break; + } + } + hdsp++; + } - if ( ctx->addresses != NULL ) { - bool found = FALSE; - const char * const *my_address = ctx->addresses; + /* My address not found in the headers; we got an implicit delivery */ + if ( *hdsp == NULL ) { + if ( config->dont_check_recipient ) { + /* Send reply from envelope recipient address */ + reply_from = recipient; - while ( !found && *my_address != NULL ) { - if ( (found=_contains_my_address(headers, *my_address)) ) - reply_from = *my_address; - my_address++; - } - - if ( found ) break; - } - } - hdsp++; - } - - - /* My address not found in the headers; we got an implicit delivery */ - if ( *hdsp == NULL ) { + } else { const char *original_recipient = ""; - /* No, bail out */ + /* Bail out */ if ( config->use_original_recipient ) { original_recipient = t_strdup_printf("original-recipient=<%s>, ", From pigeonhole at rename-it.nl Mon Jan 14 13:58:04 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:04 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Prevent passing NULL sender t... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/fc0bf6b1cb6b changeset: 1710:fc0bf6b1cb6b user: Stephan Bosch date: Sat Dec 22 22:27:34 2012 +0100 description: lib-sieve: Prevent passing NULL sender to raw mail storage when active message is substituted. diffstat: src/lib-sieve/sieve-config.h | 2 ++ src/lib-sieve/sieve-message.c | 5 ++++- 2 files changed, 6 insertions(+), 1 deletions(-) diffs (33 lines): diff -r b56711807edc -r fc0bf6b1cb6b src/lib-sieve/sieve-config.h --- a/src/lib-sieve/sieve-config.h Mon Nov 26 21:16:54 2012 +0100 +++ b/src/lib-sieve/sieve-config.h Sat Dec 22 22:27:34 2012 +0100 @@ -11,4 +11,6 @@ #define SIEVE_SCRIPT_FILEEXT "sieve" #define SIEVE_BINARY_FILEEXT "svbin" +#define DEFAULT_ENVELOPE_SENDER "MAILER-DAEMON" + #endif diff -r b56711807edc -r fc0bf6b1cb6b src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Mon Nov 26 21:16:54 2012 +0100 +++ b/src/lib-sieve/sieve-message.c Sat Dec 22 22:27:34 2012 +0100 @@ -357,6 +357,7 @@ struct sieve_message_version *version; struct mailbox_header_lookup_ctx *headers_ctx; struct mailbox *box = NULL; + const char *sender; int ret; if ( msgctx->raw_mail_user == NULL ) { @@ -367,8 +368,10 @@ } i_stream_seek(input, 0); + sender = sieve_message_get_sender(msgctx); + sender = (sender == NULL ? DEFAULT_ENVELOPE_SENDER : sender ); ret = raw_mailbox_alloc_stream(msgctx->raw_mail_user, input, (time_t)-1, - sieve_message_get_sender(msgctx), &box); + sender, &box); if ( ret < 0 ) { sieve_sys_error(msgctx->svinst, "can't open substituted mail as raw: %s", From pigeonhole at rename-it.nl Mon Jan 14 13:58:04 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:04 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: vnd.dovecot.duplicate extensi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/3a972efe012b changeset: 1704:3a972efe012b user: Stephan Bosch date: Wed Oct 17 02:15:04 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate extension: Fixed segfault occurring in testsuite. diffstat: src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 2b767751ff27 -r 3a972efe012b src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:09:55 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:15:04 2012 +0200 @@ -334,7 +334,7 @@ } else if (value != NULL) { val = str_c(value); val_len = str_len(value); - } else { + } else if (renv->msgdata->id != NULL) { val = renv->msgdata->id; val_len = strlen(renv->msgdata->id); } From pigeonhole at rename-it.nl Mon Jan 14 13:58:05 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:05 +0100 Subject: dovecot-2.2-pigeonhole: Merged changes from Pigeonhole v0.3 tree. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/99eec511aa2c changeset: 1712:99eec511aa2c user: Stephan Bosch date: Mon Jan 14 12:57:58 2013 +0100 description: Merged changes from Pigeonhole v0.3 tree. diffstat: doc/extensions/vnd.dovecot.duplicate.txt | 28 +- doc/rfc/spec-bosch-sieve-duplicate.txt | 356 +++++++-- doc/rfc/xml/reference.IMAP4FLAGS.xml | 15 + doc/rfc/xml/reference.INCLUDE.xml | 17 + doc/rfc/xml/reference.MAILBOX.xml | 15 + doc/rfc/xml/reference.VACATION.xml | 17 + doc/rfc/xml/spec-bosch-sieve-duplicate.xml | 185 ++++- src/lib-sieve/Makefile.am | 4 +- src/lib-sieve/plugins/vacation/cmd-vacation.c | 76 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c | 201 ++++- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h | 13 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate.c | 3 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 275 ++++++- src/lib-sieve/sieve-config.h | 2 + src/lib-sieve/sieve-message.c | 5 +- tests/extensions/vnd.dovecot/duplicate/errors.svtest | 2 +- tests/extensions/vnd.dovecot/duplicate/errors/syntax.sieve | 5 +- tests/extensions/vnd.dovecot/duplicate/execute.svtest | 10 +- 18 files changed, 957 insertions(+), 272 deletions(-) diffs (truncated from 1695 to 300 lines): diff -r c937a4060ee2 -r 99eec511aa2c doc/extensions/vnd.dovecot.duplicate.txt --- a/doc/extensions/vnd.dovecot.duplicate.txt Wed Jan 09 00:55:49 2013 +0100 +++ b/doc/extensions/vnd.dovecot.duplicate.txt Mon Jan 14 12:57:58 2013 +0100 @@ -8,13 +8,11 @@ Description =========== -Sieve (RFC 5228) is a highly extensible machine language specifically tailored -for internet message filtering. For the Dovecot Secure IMAP server, Sieve -support is provided by the Pigeonhole Sieve plugin. The vnd.dovecot.duplicate -extension augments the Sieve filtering implementation with a test to verify -whether a message was received earlier already based on its Message-ID. This can -be used to prevent duplicate deliveries, e.g. caused by mailinglists when people -reply both to the mailinglist and the user directly. +The vnd.dovecot.duplicate extension augments the Sieve filtering implementation +with a test to verify whether the evaluated string value was seen before in an +earlier execution of the Sieve script. The main application for this new test is +detecting and handling duplicate message deliveries, e.g. as caused by +mailinglists when people reply both to the mailinglist and the user directly. This extension is specific to the Pigeonhole Sieve implementation for the Dovecot Secure IMAP server. It will therefore most likely not be supported by @@ -35,14 +33,17 @@ ============= The "vnd.dovecot.duplicate" extension is not enabled by default and thus it -needs to be enabled explicitly. +needs to be enabled explicitly by adding it to the `sieve_extensions' or the +`sieve_global_extensions' setting. The following configuration settings are used: -sieve_duplicate_period = 1d - This option specifies after what period of time Message-IDs are purged from - the duplicate database. The period is specified in s(econds), unless followed - by a d(ay), h(our) or m(inute) specifier character. +sieve_duplicate_default_period = 12h +sieve_duplicate_max_period = 2d + These options respectively specify the default and the maximum value for the + period after which tracked values are purged from the duplicate tracking + database. The period is specified in s(econds), unless followed by a d(ay), + h(our) or m(inute) specifier character. Example ======= @@ -52,5 +53,6 @@ sieve_extensions = +vnd.dovecot.duplicate - sieve_duplicate_period = 6h + sieve_duplicate_default_period = 1h + sieve_duplicate_max_period = 1d } diff -r c937a4060ee2 -r 99eec511aa2c doc/rfc/spec-bosch-sieve-duplicate.txt --- a/doc/rfc/spec-bosch-sieve-duplicate.txt Wed Jan 09 00:55:49 2013 +0100 +++ b/doc/rfc/spec-bosch-sieve-duplicate.txt Mon Jan 14 12:57:58 2013 +0100 @@ -2,7 +2,7 @@ Pigeonhole Project S. Bosch - February 25, 2012 + October 17, 2012 Sieve Email Filtering: Detecting Duplicate Deliveries @@ -10,23 +10,65 @@ Abstract This document defines a new vendor-defined test command "duplicate" - for the "Sieve" email filtering language that tests whether an e-mail - message is a duplicate, i.e. whether it was seen before by the - delivery agent. Users can use this new test to remove duplicate - deliveries commonly caused by mailing list subscriptions or mail - account aliases. + for the "Sieve" email filtering language. It can be used to test + whether a particular string value is a duplicate, i.e. whether it was + seen before by the delivery agent that is executing the Sieve script. + The main application for this new test is detecting duplicate message + deliveries commonly caused by mailing list subscriptions or + redirected mail addresses. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Bosch [Page 1] + + Sieve: Detecting Duplicate Deliveries October 2012 Table of Contents - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Conventions Used in This Document . . . . . . . . . . . . . . . 2 - 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 2 - 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 3 - 5. Example . . . . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 3 - 7. Normative References . . . . . . . . . . . . . . . . . . . . . 3 - Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . . 3 + 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 4 + 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 5 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 6 + 7. References . . . . . . . . . . . . . . . . . . . . . . . . . . 7 + 7.1. Normative References . . . . . . . . . . . . . . . . . . . 7 + 7.2. Informative References . . . . . . . . . . . . . . . . . . 7 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 7 @@ -52,26 +94,52 @@ -Bosch [Page 1] + + + + + + + + + + + + + + +Bosch [Page 2] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 1. Introduction This is an extension to the Sieve filtering language defined by RFC - 5228 [SIEVE]. It adds a test to determine whether a message was seen - before by the delivery agent based on the Message-ID header. + 5228 [SIEVE]. It adds a test to determine whether a certain string + value was seen before by the delivery agent in an earlier execution + of the Sieve script. This can be used to detect and handle duplicate + message deliveries. Duplicate deliveries are a common side-effect of being subscribed to - a mailing list. If a member of the list decides to reply to both the - user and the mailing list itself, the user will get a copy of the - message directly and through mailing list. In another scenario, the - user has several aliases for his mail account and one of his contacts - sends the message to multiple addresses that eventually map to the - same account. Using the vnd.dovecot.duplicate extension, users have - the means to detect such duplicates and deal with these - appropriately, e.g. by discarding them. + a mailing list. For example, if a member of the list decides to + reply to both the user and the mailing list itself, the user will get + a copy of the message directly and through mailing list. Also, if + someone cross-posts over several mailing lists to which the user is + subscribed, the user will receive a copy from each of those lists. + In another scenario, the user has several redirected mail addresses + all pointing to his main mail account. If one of the user's contacts + sends the message to more than one of those addresses, the user will + receive more than a single copy. Using the "vnd.dovecot.duplicate" + extension, users have the means to detect and handle such duplicates, + e.g. by discarding them or putting them in a special folder. + + Duplicate messages are normally detected using the Message-ID header + field, which is required to be unique for each message. However, the + "duplicate" test is flexible enough to use different (weaker) + criteria for defining what makes a message a duplicate, for example + based on the subject line. Also, other applications of this new test + command are possible, as long as the tracked value is a string. This extension is specific to the Pigeonhole Sieve implementation for the Dovecot Secure IMAP server. It will therefore most likely not be @@ -89,31 +157,102 @@ arguments syntax. + + + + + + + +Bosch [Page 3] + + Sieve: Detecting Duplicate Deliveries October 2012 + + 3. Test "duplicate" - Usage: "duplicate" [] + Usage: "duplicate" [":seconds" ] + [":header" / + ":value" ] + [":handle" ] - The "duplicate" test keeps track of which Message-ID values were seen - before by this test in an earlier delivery operation. It evaluates - to "true" when the Message-ID header of the current message was seen - before. If it is not known, the test evaluates to "false" and the - Message-ID is added to a persistent internal tracking list. - Implementations SHOULD limit the number of messages that are tracked - and SHOULD let Message-ID entries expire after some short period of - time. + The "duplicate" test keeps track of which values were seen before by + this test in an earlier execution of this Sieve script. In its basic + form, the tested value is the content of the Message-ID header of the + message. This way, this test can be used to detect duplicate + deliveries of the same message. It can also detect duplicate + deliveries based on other message header fields if requested and it + can even use a user-provided string value, e.g. as composed from text + extracted from the message using the "variables" [VARIABLES] + extension. - Using the "name" argument, the duplicate test can be employed for - multiple independent purposes. Only when the Message-ID was seen - before in an earlier script execution by a duplicate test with the + The "duplicate" test evaluates to "true" when the provided value was + seen before in an earlier Sieve execution for a previous message + delivery. If the value was not seen earlier, the test evaluates to + "false". + As a side-effect, the "duplicate" test adds the evaluated value to an + internal duplicate tracking list, so that the test will evaluate to + "true" the next time the Sieve script is executed and the same value + is encountered. Note that the "duplicate" test MUST only check for + duplicates amongst values encountered in previous executions of the + Sieve script; it MUST NOT consider values encountered earlier in the + current Sieve script execution as potential duplicates. This means + that all "duplicate" tests in a Sieve script execution, including + those located in scripts included using the "include" [INCLUDE] + extension, MUST yield the same result if the arguments are identical. + Implementations MUST prevent adding values to the internal duplicate + tracking list when the Sieve script execution fails. For example, + this can be implemented by deferring the definitive modification of + the tracking list to the end of the Sieve script execution. If + failed script executions would add values to the duplicate tracking + list, all "duplicate" tests would erroneously yield "true" for the + next delivery attempt of the same message, which can -- depending on + the action taken for a duplicate -- easily lead to discarding the + message without further notice. -Bosch [Page 2] + Implementations SHOULD limit the number of values (and thereby + messages) that are tracked. Also, implementations SHOULD let entries + in the value tracking list expire after a short period of time. The + user can explicitly control the length of this expiration time by + means of the ":seconds" argument. If the ":seconds" argument is + + + +Bosch [Page 4] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 - same "name" argument, it is recognized as a duplicate. + omitted, an appropriate default MUST be used. Sites SHOULD impose a + maximum limit on the expiration time. If that limit is exceeded, the + maximum value MUST silently be substituted; exceeding the limit MUST + NOT produce an error. From pigeonhole at rename-it.nl Mon Jan 14 13:58:05 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 14 Jan 2013 12:58:05 +0100 Subject: dovecot-2.2-pigeonhole: Fixed compile on Mageia Linux. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/fe5f2738d538 changeset: 1711:fe5f2738d538 user: Stephan Bosch date: Wed Dec 26 11:52:03 2012 +0100 description: Fixed compile on Mageia Linux. Added LIBDOVECOT_STORAGE to library dependencies in src/lib-sieve/Makefile.am. diffstat: src/lib-sieve/Makefile.am | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r fc0bf6b1cb6b -r fe5f2738d538 src/lib-sieve/Makefile.am --- a/src/lib-sieve/Makefile.am Sat Dec 22 22:27:34 2012 +0100 +++ b/src/lib-sieve/Makefile.am Wed Dec 26 11:52:03 2012 +0100 @@ -70,8 +70,8 @@ $(extdir)/vnd.dovecot/duplicate/libsieve_ext_duplicate.la \ $(unfinished_plugins) -libdovecot_sieve_la_DEPENDENCIES = $(plugins) $(LIBDOVECOT_LDA_DEPS) -libdovecot_sieve_la_LIBADD = $(plugins) $(LIBDOVECOT) $(LIBDOVECOT_LDA) +libdovecot_sieve_la_DEPENDENCIES = $(plugins) $(LIBDOVECOT_LDA_DEPS) $(LIBDOVECOT_STORAGE_DEPS) $(LIBDOVECOT_DEPS) +libdovecot_sieve_la_LIBADD = $(plugins) $(LIBDOVECOT_LDA) $(LIBDOVECOT_STORAGE) $(LIBDOVECOT) libdovecot_sieve_la_SOURCES = \ rfc2822.c \ From dovecot at dovecot.org Mon Jan 14 17:58:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 14 Jan 2013 17:58:14 +0200 Subject: dovecot-2.2: lib-index: Fixed marking transaction log file corru... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6f62a31bf0d4 changeset: 15644:6f62a31bf0d4 user: Timo Sirainen date: Mon Jan 14 17:58:09 2013 +0200 description: lib-index: Fixed marking transaction log file corrupted due to added O_APPEND flag. diffstat: src/lib-index/mail-transaction-log-file.c | 16 +++++++++++++++- 1 files changed, 15 insertions(+), 1 deletions(-) diffs (31 lines): diff -r 70b97cecdacd -r 6f62a31bf0d4 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Mon Jan 14 12:56:09 2013 +0200 +++ b/src/lib-index/mail-transaction-log-file.c Mon Jan 14 17:58:09 2013 +0200 @@ -32,12 +32,26 @@ { unsigned int offset = offsetof(struct mail_transaction_log_header, indexid); + int flags; if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file) || file->log->index->readonly) return; - /* indexid=0 marks the log file as corrupted */ + /* indexid=0 marks the log file as corrupted. we opened the file with + O_APPEND, and now we need to drop it for pwrite() to work (at least + in Linux) */ + flags = fcntl(file->fd, F_GETFL, 0); + if (flags < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "fcntl(F_GETFL)"); + return; + } + if (fcntl(file->fd, F_SETFL, flags & ~O_APPEND) < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "fcntl(F_SETFL)"); + return; + } if (pwrite_full(file->fd, &file->hdr.indexid, sizeof(file->hdr.indexid), offset) < 0) { mail_index_file_set_syscall_error(file->log->index, From dovecot at dovecot.org Mon Jan 14 18:24:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 14 Jan 2013 18:24:29 +0200 Subject: dovecot-2.2: lib-index: Added asserts. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/970dc6b0760b changeset: 15645:970dc6b0760b user: Timo Sirainen date: Mon Jan 14 18:24:19 2013 +0200 description: lib-index: Added asserts. diffstat: src/lib-index/mail-index-transaction-finish.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (37 lines): diff -r 6f62a31bf0d4 -r 970dc6b0760b src/lib-index/mail-index-transaction-finish.c --- a/src/lib-index/mail-index-transaction-finish.c Mon Jan 14 17:58:09 2013 +0200 +++ b/src/lib-index/mail-index-transaction-finish.c Mon Jan 14 18:24:19 2013 +0200 @@ -224,7 +224,7 @@ { struct seq_range *range, *new_range; unsigned int i, count; - uint32_t uid1, uid2; + uint32_t uid1, uid2, prev_uid = 0; if (!array_is_created(array)) return; @@ -235,10 +235,12 @@ uid1 = mail_index_transaction_get_uid(t, range->seq1); uid2 = mail_index_transaction_get_uid(t, range->seq2); + i_assert(uid1 > prev_uid); if (uid2 - uid1 == range->seq2 - range->seq1) { /* simple conversion */ range->seq1 = uid1; range->seq2 = uid2; + prev_uid = uid2; } else { /* remove expunged UIDs */ new_range = array_insert_space(array, i); @@ -249,9 +251,11 @@ new_range->seq1 = uid1; new_range->seq2 = get_nonexpunged_uid2(t, uid1, range->seq1); + i_assert(new_range->seq2 < uid2); /* continue the range without the inserted seqs */ range->seq1 += new_range->seq2 - new_range->seq1 + 1; + prev_uid = new_range->seq2; } } } From dovecot at dovecot.org Tue Jan 15 08:30:18 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Jan 2013 08:30:18 +0200 Subject: dovecot-2.1: buffer: Always keep +1 byte available for str_c()'s... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c762a9af72c1 changeset: 14882:c762a9af72c1 user: Timo Sirainen date: Tue Jan 15 08:30:03 2013 +0200 description: buffer: Always keep +1 byte available for str_c()'s NUL. diffstat: src/lib/Makefile.am | 1 + src/lib/buffer.c | 11 +++++++++-- src/lib/str.c | 3 --- src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + src/lib/test-str.c | 26 ++++++++++++++++++++++++++ 6 files changed, 38 insertions(+), 5 deletions(-) diffs (111 lines): diff -r 6bee6d0c611f -r c762a9af72c1 src/lib/Makefile.am --- a/src/lib/Makefile.am Mon Jan 14 13:01:33 2013 +0200 +++ b/src/lib/Makefile.am Tue Jan 15 08:30:03 2013 +0200 @@ -268,6 +268,7 @@ test-primes.c \ test-priorityq.c \ test-seq-range-array.c \ + test-str.c \ test-strescape.c \ test-strfuncs.c \ test-str-find.c \ diff -r 6bee6d0c611f -r c762a9af72c1 src/lib/buffer.c --- a/src/lib/buffer.c Mon Jan 14 13:01:33 2013 +0200 +++ b/src/lib/buffer.c Tue Jan 15 08:30:03 2013 +0200 @@ -39,6 +39,7 @@ static inline void buffer_check_limits(struct real_buffer *buf, size_t pos, size_t data_size) { + unsigned int extra; size_t new_size; if (unlikely((size_t)-1 - pos < data_size)) { @@ -53,7 +54,13 @@ memset(buf->w_buffer + buf->used, 0, max - buf->used); } - if (new_size > buf->alloc) { + + /* always keep +1 byte allocated available in case str_c() is called + for this buffer. this is mainly for cases where the buffer is + allocated from data stack, and str_c() is called in a separate stack + frame. */ + extra = buf->dynamic ? 1 : 0; + if (new_size + extra > buf->alloc) { if (unlikely(!buf->dynamic)) { i_panic("Buffer full (%"PRIuSIZE_T" > %"PRIuSIZE_T", " "pool %s)", pos + data_size, buf->alloc, @@ -62,7 +69,7 @@ } buffer_alloc(buf, pool_get_exp_grown_size(buf->pool, buf->alloc, - new_size)); + new_size + extra)); } #if 0 else if (new_size > buf->used && buf->alloced && diff -r 6bee6d0c611f -r c762a9af72c1 src/lib/str.c --- a/src/lib/str.c Mon Jan 14 13:01:33 2013 +0200 +++ b/src/lib/str.c Tue Jan 15 08:30:03 2013 +0200 @@ -45,9 +45,6 @@ size_t len = str_len(str); size_t alloc = buffer_get_size(str); -#ifdef DEBUG - buffer_verify_pool(str); -#endif if (len == alloc || data[len] != '\0') { buffer_write(str, len, "", 1); /* remove the \0 - we don't want to keep it */ diff -r 6bee6d0c611f -r c762a9af72c1 src/lib/test-lib.c --- a/src/lib/test-lib.c Mon Jan 14 13:01:33 2013 +0200 +++ b/src/lib/test-lib.c Tue Jan 15 08:30:03 2013 +0200 @@ -25,6 +25,7 @@ test_primes, test_priorityq, test_seq_range_array, + test_str, test_strescape, test_strfuncs, test_str_find, diff -r 6bee6d0c611f -r c762a9af72c1 src/lib/test-lib.h --- a/src/lib/test-lib.h Mon Jan 14 13:01:33 2013 +0200 +++ b/src/lib/test-lib.h Tue Jan 15 08:30:03 2013 +0200 @@ -24,6 +24,7 @@ void test_primes(void); void test_priorityq(void); void test_seq_range_array(void); +void test_str(void); void test_strescape(void); void test_strfuncs(void); void test_str_find(void); diff -r 6bee6d0c611f -r c762a9af72c1 src/lib/test-str.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-str.c Tue Jan 15 08:30:03 2013 +0200 @@ -0,0 +1,26 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "str.h" + +static void test_str_c(void) +{ + string_t *str; + unsigned int i, j; + + test_begin("str_c()"); + for (i = 0; i < 32; i++) T_BEGIN { + str = t_str_new(15); + for (j = 0; j < i; j++) + str_append_c(str, 'x'); + T_BEGIN { + (void)str_c(str); + } T_END; + } T_END; + test_end(); +} + +void test_str(void) +{ + test_str_c(); +} From dovecot at dovecot.org Tue Jan 15 08:31:05 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Jan 2013 08:31:05 +0200 Subject: dovecot-2.2: dovecot-config: Removed unnecessary/duplicate ssl l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0e12a687f5bf changeset: 15646:0e12a687f5bf user: Timo Sirainen date: Tue Jan 15 08:30:57 2013 +0200 description: dovecot-config: Removed unnecessary/duplicate ssl library links. diffstat: Makefile.am | 1 - configure.ac | 9 --------- 2 files changed, 0 insertions(+), 10 deletions(-) diffs (40 lines): diff -r 970dc6b0760b -r 0e12a687f5bf Makefile.am --- a/Makefile.am Mon Jan 14 18:24:19 2013 +0200 +++ b/Makefile.am Tue Jan 15 08:30:57 2013 +0200 @@ -66,7 +66,6 @@ -e "s|^\(LIBDOVECOT\)=.*$$|\1='-L$(pkglibdir) -ldovecot'|" \ -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1='-ldovecot-login $(SSL_LIBS)'|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ - -e "s|^\(LIBDOVECOT_SSL\)=.*$$|\1=-ldovecot-ssl|" \ -e "s|^\(LIBDOVECOT_COMPRESS\)=.*$$|\1=-ldovecot-compression|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ diff -r 970dc6b0760b -r 0e12a687f5bf configure.ac --- a/configure.ac Mon Jan 14 18:24:19 2013 +0200 +++ b/configure.ac Tue Jan 15 08:30:57 2013 +0200 @@ -2478,17 +2478,9 @@ fi if test $storage = imapc; then mailbox_list_drivers="$mailbox_list_drivers imapc" - want_ssl_libs=yes - fi - if test $storage = pop3c; then - want_ssl_libs=yes fi done LINKED_STORAGE_LDADD= -if test "$want_ssl_libs" = yes; then - LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS \$(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la" - LINKED_STORAGE_LDADD="$SSL_LIBS" -fi AC_SUBST(LINKED_STORAGE_LIBS) AC_SUBST(LINKED_STORAGE_LDADD) AC_SUBST(mailbox_list_drivers) @@ -2528,7 +2520,6 @@ AC_SUBST(LIBDOVECOT_STORAGE_DEPS) AC_SUBST(LIBDOVECOT_LOGIN) AC_SUBST(LIBDOVECOT_SQL) -AC_SUBST(LIBDOVECOT_SSL) AC_SUBST(LIBDOVECOT_COMPRESS) AC_SUBST(LIBDOVECOT_LDA) From dovecot at dovecot.org Tue Jan 15 08:39:41 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 15 Jan 2013 08:39:41 +0200 Subject: dovecot-2.2: lib-index: Newly added messages weren't always sort... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6139eac53d72 changeset: 15647:6139eac53d72 user: Timo Sirainen date: Tue Jan 15 08:39:30 2013 +0200 description: lib-index: Newly added messages weren't always sorted by UIDs. diffstat: src/lib-index/mail-index-transaction-update.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (28 lines): diff -r 0e12a687f5bf -r 6139eac53d72 src/lib-index/mail-index-transaction-update.c --- a/src/lib-index/mail-index-transaction-update.c Tue Jan 15 08:30:57 2013 +0200 +++ b/src/lib-index/mail-index-transaction-update.c Tue Jan 15 08:39:30 2013 +0200 @@ -201,6 +201,7 @@ unsigned int i, count; struct seq_range *range; uint32_t next_uid; + bool used_existing_uids; if (!array_is_created(&t->appends)) return; @@ -216,9 +217,15 @@ } /* assign missing uids */ + used_existing_uids = FALSE; for (i = 0; i < count; i++) { - if (recs[i].uid == 0 || recs[i].uid < first_uid) + if (recs[i].uid == 0 || recs[i].uid < first_uid) { recs[i].uid = next_uid++; + if (used_existing_uids) + t->appends_nonsorted = TRUE; + } else { + used_existing_uids = TRUE; + } } /* write the saved uids range */ From dovecot at dovecot.org Wed Jan 16 08:01:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Jan 2013 08:01:55 +0200 Subject: dovecot-2.2: dsync_remote_cmd: The -l parameter no longer exists. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5eedc2b9cb23 changeset: 15648:5eedc2b9cb23 user: Timo Sirainen date: Wed Jan 16 08:01:40 2013 +0200 description: dsync_remote_cmd: The -l parameter no longer exists. diffstat: src/doveadm/doveadm-settings.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 6139eac53d72 -r 5eedc2b9cb23 src/doveadm/doveadm-settings.c --- a/src/doveadm/doveadm-settings.c Tue Jan 15 08:39:30 2013 +0200 +++ b/src/doveadm/doveadm-settings.c Wed Jan 16 08:01:40 2013 +0200 @@ -80,7 +80,7 @@ .doveadm_password = "", .doveadm_allowed_commands = "", .dsync_alt_char = "_", - .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -l%{lock_timeout}", + .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u", .plugin_envs = ARRAY_INIT }; From dovecot at dovecot.org Wed Jan 16 09:15:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Jan 2013 09:15:48 +0200 Subject: dovecot-2.2: sdbox: next-uid may have been set wrong when rebuil... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/353a2a6406aa changeset: 15649:353a2a6406aa user: Timo Sirainen date: Wed Jan 16 09:15:38 2013 +0200 description: sdbox: next-uid may have been set wrong when rebuilding indexes for empty mailbox. diffstat: src/lib-storage/index/index-rebuild.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 5eedc2b9cb23 -r 353a2a6406aa src/lib-storage/index/index-rebuild.c --- a/src/lib-storage/index/index-rebuild.c Wed Jan 16 08:01:40 2013 +0200 +++ b/src/lib-storage/index/index-rebuild.c Wed Jan 16 09:15:38 2013 +0200 @@ -124,7 +124,7 @@ else if (backup_hdr != NULL && backup_hdr->next_uid != 0) next_uid = backup_hdr->next_uid; else - next_uid = gen_uidvalidity(ctx->box->list); + next_uid = 1; if (next_uid > trans_hdr->next_uid) { mail_index_update_header(ctx->trans, offsetof(struct mail_index_header, next_uid), From dovecot at dovecot.org Wed Jan 16 12:02:24 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Jan 2013 12:02:24 +0200 Subject: dovecot-2.2: sdbox: Fixed rebuilding index when UIDVALIDITY is m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ba0fae268904 changeset: 15650:ba0fae268904 user: Timo Sirainen date: Wed Jan 16 12:02:18 2013 +0200 description: sdbox: Fixed rebuilding index when UIDVALIDITY is missing. diffstat: src/lib-storage/index/dbox-single/sdbox-sync.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 353a2a6406aa -r ba0fae268904 src/lib-storage/index/dbox-single/sdbox-sync.c --- a/src/lib-storage/index/dbox-single/sdbox-sync.c Wed Jan 16 09:15:38 2013 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Wed Jan 16 12:02:18 2013 +0200 @@ -110,6 +110,7 @@ mail_storage_set_critical(box->storage, "sdbox %s: Broken index: missing UIDVALIDITY", mailbox_get_path(box)); + sdbox_set_mailbox_corrupted(box); return 0; } From dovecot at dovecot.org Thu Jan 17 09:01:04 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 17 Jan 2013 09:01:04 +0200 Subject: dovecot-2.2: doveadm: Compiling fix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/33830df49f59 changeset: 15651:33830df49f59 user: Timo Sirainen date: Thu Jan 17 09:00:54 2013 +0200 description: doveadm: Compiling fix diffstat: src/doveadm/doveadm-mail.h | 30 +++++++++++++++--------------- 1 files changed, 15 insertions(+), 15 deletions(-) diffs (39 lines): diff -r ba0fae268904 -r 33830df49f59 src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Wed Jan 16 12:02:18 2013 +0200 +++ b/src/doveadm/doveadm-mail.h Thu Jan 17 09:00:54 2013 +0200 @@ -130,20 +130,20 @@ void doveadm_mail_failed_mailbox(struct doveadm_mail_cmd_context *ctx, struct mailbox *box); -struct doveadm_mail_cmd cmd_expunge; -struct doveadm_mail_cmd cmd_search; -struct doveadm_mail_cmd cmd_fetch; -struct doveadm_mail_cmd cmd_import; -struct doveadm_mail_cmd cmd_index; -struct doveadm_mail_cmd cmd_altmove; -struct doveadm_mail_cmd cmd_copy; -struct doveadm_mail_cmd cmd_move; -struct doveadm_mail_cmd cmd_mailbox_list; -struct doveadm_mail_cmd cmd_mailbox_create; -struct doveadm_mail_cmd cmd_mailbox_delete; -struct doveadm_mail_cmd cmd_mailbox_rename; -struct doveadm_mail_cmd cmd_mailbox_subscribe; -struct doveadm_mail_cmd cmd_mailbox_unsubscribe; -struct doveadm_mail_cmd cmd_mailbox_status; +extern struct doveadm_mail_cmd cmd_expunge; +extern struct doveadm_mail_cmd cmd_search; +extern struct doveadm_mail_cmd cmd_fetch; +extern struct doveadm_mail_cmd cmd_import; +extern struct doveadm_mail_cmd cmd_index; +extern struct doveadm_mail_cmd cmd_altmove; +extern struct doveadm_mail_cmd cmd_copy; +extern struct doveadm_mail_cmd cmd_move; +extern struct doveadm_mail_cmd cmd_mailbox_list; +extern struct doveadm_mail_cmd cmd_mailbox_create; +extern struct doveadm_mail_cmd cmd_mailbox_delete; +extern struct doveadm_mail_cmd cmd_mailbox_rename; +extern struct doveadm_mail_cmd cmd_mailbox_subscribe; +extern struct doveadm_mail_cmd cmd_mailbox_unsubscribe; +extern struct doveadm_mail_cmd cmd_mailbox_status; #endif From dovecot at dovecot.org Mon Jan 21 11:40:49 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 11:40:49 +0200 Subject: dovecot-2.2: lib-fs: Added flags to iteration, and FS_ITER_FLAG_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6e5adbf30e35 changeset: 15652:6e5adbf30e35 user: Timo Sirainen date: Mon Jan 21 11:40:38 2013 +0200 description: lib-fs: Added flags to iteration, and FS_ITER_FLAG_DIRS as the first flag. diffstat: src/lib-fs/fs-api-private.h | 4 +++- src/lib-fs/fs-api.c | 5 +++-- src/lib-fs/fs-api.h | 10 ++++++++-- src/lib-fs/fs-posix.c | 30 +++++++++++++++++++++--------- src/lib-fs/fs-sis-queue.c | 5 +++-- src/lib-fs/fs-sis.c | 4 ++-- src/lib-fs/fs-test.c | 9 ++++++--- 7 files changed, 46 insertions(+), 21 deletions(-) diffs (212 lines): diff -r 33830df49f59 -r 6e5adbf30e35 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Thu Jan 17 09:00:54 2013 +0200 +++ b/src/lib-fs/fs-api-private.h Mon Jan 21 11:40:38 2013 +0200 @@ -45,7 +45,8 @@ int (*rename)(struct fs_file *src, struct fs_file *dest); int (*delete_file)(struct fs_file *file); - struct fs_iter *(*iter_init)(struct fs *fs, const char *path); + struct fs_iter *(*iter_init)(struct fs *fs, const char *path, + enum fs_iter_flags flags); const char *(*iter_next)(struct fs_iter *iter); int (*iter_deinit)(struct fs_iter *iter); }; @@ -75,6 +76,7 @@ struct fs_iter { struct fs *fs; + enum fs_iter_flags flags; }; extern const struct fs fs_class_posix; diff -r 33830df49f59 -r 6e5adbf30e35 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu Jan 17 09:00:54 2013 +0200 +++ b/src/lib-fs/fs-api.c Mon Jan 21 11:40:38 2013 +0200 @@ -359,9 +359,10 @@ return file->fs->v.delete_file(file); } -struct fs_iter *fs_iter_init(struct fs *fs, const char *path) +struct fs_iter * +fs_iter_init(struct fs *fs, const char *path, enum fs_iter_flags flags) { - return fs->v.iter_init(fs, path); + return fs->v.iter_init(fs, path, flags); } int fs_iter_deinit(struct fs_iter **_iter) diff -r 33830df49f59 -r 6e5adbf30e35 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Thu Jan 17 09:00:54 2013 +0200 +++ b/src/lib-fs/fs-api.h Mon Jan 21 11:40:38 2013 +0200 @@ -52,6 +52,11 @@ FS_OPEN_FLAG_ASYNC = 0x20 }; +enum fs_iter_flags { + /* Iterate only directories, not files */ + FS_ITER_FLAG_DIRS = 0x01 +}; + struct fs_settings { /* Dovecot instance's base_dir */ const char *base_dir; @@ -168,10 +173,11 @@ int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r); void fs_unlock(struct fs_lock **lock); -/* Iterate through all files (but not directories) in the given directory. +/* Iterate through all files or directories in the given directory. Doesn't recurse to child directories. It's not an error to iterate a nonexistent directory. */ -struct fs_iter *fs_iter_init(struct fs *fs, const char *path); +struct fs_iter * +fs_iter_init(struct fs *fs, const char *path, enum fs_iter_flags flags); /* Returns 0 if ok, -1 if iteration failed. */ int fs_iter_deinit(struct fs_iter **iter); /* Returns the next filename. */ diff -r 33830df49f59 -r 6e5adbf30e35 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Thu Jan 17 09:00:54 2013 +0200 +++ b/src/lib-fs/fs-posix.c Mon Jan 21 11:40:38 2013 +0200 @@ -644,12 +644,14 @@ return 0; } -static struct fs_iter *fs_posix_iter_init(struct fs *fs, const char *path) +static struct fs_iter * +fs_posix_iter_init(struct fs *fs, const char *path, enum fs_iter_flags flags) { struct posix_fs_iter *iter; iter = i_new(struct posix_fs_iter, 1); iter->iter.fs = fs; + iter->iter.flags = flags; iter->path = i_strdup(path); iter->dir = opendir(path); if (iter->dir == NULL && errno != ENOENT) { @@ -659,18 +661,20 @@ return &iter->iter; } -static bool fs_posix_iter_want(const char *dir, const char *fname) +static bool fs_posix_iter_want(struct posix_fs_iter *iter, const char *fname) { bool ret; T_BEGIN { - const char *path = t_strdup_printf("%s/%s", dir, fname); + const char *path = t_strdup_printf("%s/%s", iter->path, fname); struct stat st; if (stat(path, &st) < 0) ret = FALSE; + else if (!S_ISDIR(st.st_mode)) + ret = (iter->iter.flags & FS_ITER_FLAG_DIRS) == 0; else - ret = !S_ISDIR(st.st_mode); + ret = (iter->iter.flags & FS_ITER_FLAG_DIRS) != 0; } T_END; return ret; } @@ -684,24 +688,32 @@ return NULL; errno = 0; - while ((d = readdir(iter->dir)) != NULL) { + for (; (d = readdir(iter->dir)) != NULL; errno = 0) { + if (strcmp(d->d_name, ".") == 0 || + strcmp(d->d_name, "..") == 0) + continue; #ifdef HAVE_DIRENT_D_TYPE switch (d->d_type) { case DT_UNKNOWN: - if (!fs_posix_iter_want(iter->path, d->d_name)) + if (!fs_posix_iter_want(iter, d->d_name)) break; /* fall through */ case DT_REG: case DT_LNK: - return d->d_name; + if ((iter->iter.flags & FS_ITER_FLAG_DIRS) == 0) + return d->d_name; + break; + case DT_DIR: + if ((iter->iter.flags & FS_ITER_FLAG_DIRS) != 0) + return d->d_name; + break; default: break; } #else - if (fs_posix_iter_want(iter->path, d->d_name)) + if (fs_posix_iter_want(iter, d->d_name)) return d->d_name; #endif - errno = 0; } if (errno != 0) { iter->err = errno; diff -r 33830df49f59 -r 6e5adbf30e35 src/lib-fs/fs-sis-queue.c --- a/src/lib-fs/fs-sis-queue.c Thu Jan 17 09:00:54 2013 +0200 +++ b/src/lib-fs/fs-sis-queue.c Mon Jan 21 11:40:38 2013 +0200 @@ -328,11 +328,12 @@ } static struct fs_iter * -fs_sis_queue_iter_init(struct fs *_fs, const char *path) +fs_sis_queue_iter_init(struct fs *_fs, const char *path, + enum fs_iter_flags flags) { struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs; - return fs_iter_init(fs->super, path); + return fs_iter_init(fs->super, path, flags); } const struct fs fs_class_sis_queue = { diff -r 33830df49f59 -r 6e5adbf30e35 src/lib-fs/fs-sis.c --- a/src/lib-fs/fs-sis.c Thu Jan 17 09:00:54 2013 +0200 +++ b/src/lib-fs/fs-sis.c Mon Jan 21 11:40:38 2013 +0200 @@ -463,11 +463,11 @@ } static struct fs_iter * -fs_sis_iter_init(struct fs *_fs, const char *path) +fs_sis_iter_init(struct fs *_fs, const char *path, enum fs_iter_flags flags) { struct sis_fs *fs = (struct sis_fs *)_fs; - return fs_iter_init(fs->super, path); + return fs_iter_init(fs->super, path, flags); } const struct fs fs_class_sis = { diff -r 33830df49f59 -r 6e5adbf30e35 src/lib-fs/fs-test.c --- a/src/lib-fs/fs-test.c Thu Jan 17 09:00:54 2013 +0200 +++ b/src/lib-fs/fs-test.c Mon Jan 21 11:40:38 2013 +0200 @@ -105,12 +105,13 @@ fs_file_deinit(&file); } -static void fs_test_file_iter(struct fs *fs, const char *path) +static void +fs_test_file_iter(struct fs *fs, const char *path, enum fs_iter_flags flags) { struct fs_iter *iter; const char *fname; - iter = fs_iter_init(fs, path); + iter = fs_iter_init(fs, path, flags); while ((fname = fs_iter_next(iter)) != NULL) printf("%s\n", fname); if (fs_iter_deinit(&iter) < 0) { @@ -144,7 +145,9 @@ else if (strcmp(argv[3], "delete") == 0) fs_test_file_delete(fs, argv[4]); else if (strcmp(argv[3], "iter") == 0) - fs_test_file_iter(fs, argv[4]); + fs_test_file_iter(fs, argv[4], 0); + else if (strcmp(argv[3], "iter-dir") == 0) + fs_test_file_iter(fs, argv[4], FS_ITER_FLAG_DIRS); else i_fatal("Unknown command: %s", argv[3]); From dovecot at dovecot.org Mon Jan 21 14:19:32 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 14:19:32 +0200 Subject: dovecot-2.2: lib-fs: fs_file_path() is now a virtual function. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df8fd01f355c changeset: 15653:df8fd01f355c user: Timo Sirainen date: Mon Jan 21 14:19:21 2013 +0200 description: lib-fs: fs_file_path() is now a virtual function. This fixes it returning the correct filename when file was created using FS_OPEN_MODE_CREATE_UNIQUE_128 and a wrapper fs. diffstat: src/lib-fs/fs-api-private.h | 1 + src/lib-fs/fs-api.c | 3 ++- src/lib-fs/fs-posix.c | 1 + src/lib-fs/fs-sis-queue.c | 8 ++++++++ src/lib-fs/fs-sis.c | 8 ++++++++ 5 files changed, 20 insertions(+), 1 deletions(-) diffs (85 lines): diff -r 6e5adbf30e35 -r df8fd01f355c src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Mon Jan 21 11:40:38 2013 +0200 +++ b/src/lib-fs/fs-api-private.h Mon Jan 21 14:19:21 2013 +0200 @@ -15,6 +15,7 @@ enum fs_open_mode mode, enum fs_open_flags flags); void (*file_deinit)(struct fs_file *file); + const char *(*get_path)(struct fs_file *file); void (*set_async_callback)(struct fs_file *file, fs_file_async_callback_t *callback, diff -r 6e5adbf30e35 -r df8fd01f355c src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Mon Jan 21 11:40:38 2013 +0200 +++ b/src/lib-fs/fs-api.c Mon Jan 21 14:19:21 2013 +0200 @@ -178,7 +178,8 @@ const char *fs_file_path(struct fs_file *file) { - return file->path; + return file->fs->v.get_path == NULL ? file->path : + file->fs->v.get_path(file); } const char *fs_last_error(struct fs *fs) diff -r 6e5adbf30e35 -r df8fd01f355c src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Mon Jan 21 11:40:38 2013 +0200 +++ b/src/lib-fs/fs-posix.c Mon Jan 21 14:19:21 2013 +0200 @@ -745,6 +745,7 @@ fs_posix_get_properties, fs_posix_file_init, fs_posix_file_deinit, + NULL, NULL, NULL, NULL, NULL, fs_posix_prefetch, diff -r 6e5adbf30e35 -r df8fd01f355c src/lib-fs/fs-sis-queue.c --- a/src/lib-fs/fs-sis-queue.c Mon Jan 21 11:40:38 2013 +0200 +++ b/src/lib-fs/fs-sis-queue.c Mon Jan 21 14:19:21 2013 +0200 @@ -117,6 +117,13 @@ i_free(file); } +static const char *fs_sis_queue_file_get_path(struct fs_file *_file) +{ + struct sis_queue_fs_file *file = (struct sis_queue_fs_file *)_file; + + return fs_file_path(file->super); +} + static void fs_sis_queue_set_async_callback(struct fs_file *_file, fs_file_async_callback_t *callback, @@ -345,6 +352,7 @@ fs_sis_queue_get_properties, fs_sis_queue_file_init, fs_sis_queue_file_deinit, + fs_sis_queue_file_get_path, fs_sis_queue_set_async_callback, fs_sis_queue_wait_async, fs_sis_queue_set_metadata, diff -r 6e5adbf30e35 -r df8fd01f355c src/lib-fs/fs-sis.c --- a/src/lib-fs/fs-sis.c Mon Jan 21 11:40:38 2013 +0200 +++ b/src/lib-fs/fs-sis.c Mon Jan 21 14:19:21 2013 +0200 @@ -155,6 +155,13 @@ i_free(file); } +static const char *fs_sis_file_get_path(struct fs_file *_file) +{ + struct sis_fs_file *file = (struct sis_fs_file *)_file; + + return fs_file_path(file->super); +} + static void fs_sis_set_async_callback(struct fs_file *_file, fs_file_async_callback_t *callback, void *context) @@ -479,6 +486,7 @@ fs_sis_get_properties, fs_sis_file_init, fs_sis_file_deinit, + fs_sis_file_get_path, fs_sis_set_async_callback, fs_sis_wait_async, fs_sis_set_metadata, From dovecot at dovecot.org Mon Jan 21 14:43:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 14:43:14 +0200 Subject: dovecot-2.2: i_stream_read_next_line() should have also set eof=... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ee177df07178 changeset: 15654:ee177df07178 user: Timo Sirainen date: Mon Jan 21 14:42:10 2013 +0200 description: i_stream_read_next_line() should have also set eof=TRUE on ENOBUFS errors. diffstat: src/lib/istream.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r df8fd01f355c -r ee177df07178 src/lib/istream.c --- a/src/lib/istream.c Mon Jan 21 14:19:21 2013 +0200 +++ b/src/lib/istream.c Mon Jan 21 14:42:10 2013 +0200 @@ -390,6 +390,7 @@ switch (i_stream_read(stream)) { case -2: stream->stream_errno = ENOBUFS; + stream->eof = TRUE; return NULL; case -1: return i_stream_last_line(stream->real_stream); From dovecot at dovecot.org Mon Jan 21 14:43:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 14:43:14 +0200 Subject: dovecot-2.2: lib-fs: Added "metawrap" wrapper to implement metad... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5818d8f488c1 changeset: 15655:5818d8f488c1 user: Timo Sirainen date: Mon Jan 21 14:43:09 2013 +0200 description: lib-fs: Added "metawrap" wrapper to implement metadata as headers in the file content. diffstat: src/lib-fs/Makefile.am | 3 + src/lib-fs/fs-api-private.h | 15 + src/lib-fs/fs-api.c | 96 ++++++++- src/lib-fs/fs-metawrap.c | 429 ++++++++++++++++++++++++++++++++++++++++++ src/lib-fs/istream-metawrap.c | 79 +++++++ src/lib-fs/istream-metawrap.h | 14 + 6 files changed, 625 insertions(+), 11 deletions(-) diffs (truncated from 765 to 300 lines): diff -r ee177df07178 -r 5818d8f488c1 src/lib-fs/Makefile.am --- a/src/lib-fs/Makefile.am Mon Jan 21 14:42:10 2013 +0200 +++ b/src/lib-fs/Makefile.am Mon Jan 21 14:43:09 2013 +0200 @@ -8,16 +8,19 @@ libfs_la_SOURCES = \ fs-api.c \ + fs-metawrap.c \ fs-posix.c \ fs-sis.c \ fs-sis-common.c \ fs-sis-queue.c \ + istream-metawrap.c \ ostream-cmp.c headers = \ fs-api.h \ fs-api-private.h \ fs-sis-common.h \ + istream-metawrap.h \ ostream-cmp.h fs_test_SOURCES = fs-test.c diff -r ee177df07178 -r 5818d8f488c1 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Mon Jan 21 14:42:10 2013 +0200 +++ b/src/lib-fs/fs-api-private.h Mon Jan 21 14:43:09 2013 +0200 @@ -69,6 +69,13 @@ struct istream *pending_read_input; bool write_pending; + + pool_t metadata_pool; + ARRAY_TYPE(fs_metadata) metadata; + + struct fs_file *copy_src; + struct istream *copy_input; + struct ostream *copy_output; }; struct fs_lock { @@ -81,6 +88,7 @@ }; extern const struct fs fs_class_posix; +extern const struct fs fs_class_metawrap; extern const struct fs fs_class_sis; extern const struct fs fs_class_sis_queue; @@ -89,4 +97,11 @@ void fs_set_error_async(struct fs *fs); +ssize_t fs_read_via_stream(struct fs_file *file, void *buf, size_t size); +int fs_write_via_stream(struct fs_file *file, const void *data, size_t size); +void fs_metadata_init(struct fs_file *file); +void fs_default_set_metadata(struct fs_file *file, + const char *key, const char *value); +int fs_default_copy(struct fs_file *src, struct fs_file *dest); + #endif diff -r ee177df07178 -r 5818d8f488c1 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Mon Jan 21 14:42:10 2013 +0200 +++ b/src/lib-fs/fs-api.c Mon Jan 21 14:43:09 2013 +0200 @@ -46,6 +46,7 @@ { i_array_init(&fs_classes, 8); fs_class_register(&fs_class_posix); + fs_class_register(&fs_class_metawrap); fs_class_register(&fs_class_sis); fs_class_register(&fs_class_sis_queue); } @@ -146,6 +147,7 @@ void fs_file_deinit(struct fs_file **_file) { struct fs_file *file = *_file; + pool_t metadata_pool = file->metadata_pool; i_assert(file->fs->files_open_count > 0); @@ -153,6 +155,9 @@ file->fs->files_open_count--; file->fs->v.file_deinit(file); + + if (metadata_pool != NULL) + pool_unref(&metadata_pool); } enum fs_properties fs_get_properties(struct fs *fs) @@ -160,6 +165,25 @@ return fs->v.get_properties(fs); } +void fs_metadata_init(struct fs_file *file) +{ + if (file->metadata_pool == NULL) { + file->metadata_pool = pool_alloconly_create("fs metadata", 1024); + p_array_init(&file->metadata, file->metadata_pool, 8); + } +} + +void fs_default_set_metadata(struct fs_file *file, + const char *key, const char *value) +{ + struct fs_metadata *metadata; + + fs_metadata_init(file); + metadata = array_append_space(&file->metadata); + metadata->key = p_strdup(file->metadata_pool, key); + metadata->value = p_strdup(file->metadata_pool, value); +} + void fs_set_metadata(struct fs_file *file, const char *key, const char *value) { if (file->fs->v.set_metadata != NULL) @@ -199,17 +223,14 @@ return file->fs->v.prefetch(file, length); } -ssize_t fs_read(struct fs_file *file, void *buf, size_t size) +ssize_t fs_read_via_stream(struct fs_file *file, void *buf, size_t size) { const unsigned char *data; size_t data_size; ssize_t ret; - if (file->fs->v.read != NULL) - return file->fs->v.read(file, buf, size); + i_assert(size > 0); - /* backend didn't bother to implement read(), but we can do it with - streams. */ if (file->pending_read_input == NULL) file->pending_read_input = fs_read_stream(file, size+1); ret = i_stream_read_data(file->pending_read_input, @@ -229,22 +250,27 @@ return ret; } +ssize_t fs_read(struct fs_file *file, void *buf, size_t size) +{ + if (file->fs->v.read != NULL) + return file->fs->v.read(file, buf, size); + + /* backend didn't bother to implement read(), but we can do it with + streams. */ + return fs_read_via_stream(file, buf, size); +} + struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size) { return file->fs->v.read_stream(file, max_buffer_size); } -int fs_write(struct fs_file *file, const void *data, size_t size) +int fs_write_via_stream(struct fs_file *file, const void *data, size_t size) { struct ostream *output; ssize_t ret; int err; - if (file->fs->v.write != NULL) - return file->fs->v.write(file, data, size); - - /* backend didn't bother to implement write(), but we can do it with - streams. */ if (!file->write_pending) { output = fs_write_stream(file); if ((ret = o_stream_send(output, data, size)) < 0) { @@ -269,6 +295,16 @@ return ret < 0 ? -1 : 0; } +int fs_write(struct fs_file *file, const void *data, size_t size) +{ + if (file->fs->v.write != NULL) + return file->fs->v.write(file, data, size); + + /* backend didn't bother to implement write(), but we can do it with + streams. */ + return fs_write_via_stream(file, data, size); +} + struct ostream *fs_write_stream(struct fs_file *file) { file->fs->v.write_stream(file); @@ -338,6 +374,44 @@ return file->fs->v.stat(file, st_r); } +int fs_default_copy(struct fs_file *src, struct fs_file *dest) +{ + if (dest->copy_src != NULL) { + i_assert(src == NULL || src == dest->copy_src); + if (dest->copy_output == NULL) { + i_assert(dest->copy_input == NULL); + if (fs_write_stream_finish_async(dest) < 0) + return -1; + dest->copy_src = NULL; + return 0; + } + } else { + dest->copy_src = src; + dest->copy_input = fs_read_stream(src, IO_BLOCK_SIZE); + dest->copy_output = fs_write_stream(dest); + } + while (o_stream_send_istream(dest->copy_output, dest->copy_input) > 0) ; + if (dest->copy_input->stream_errno != 0) { + fs_set_error(dest->fs, "read(%s) failed: %m", + i_stream_get_name(dest->copy_input)); + return -1; + } + if (dest->copy_output->stream_errno != 0) { + fs_set_error(dest->fs, "write(%s) failed: %m", + o_stream_get_name(dest->copy_output)); + return -1; + } + if (!dest->copy_input->eof) { + fs_set_error_async(dest->fs); + return -1; + } + i_stream_unref(&dest->copy_input); + if (fs_write_stream_finish(dest, &dest->copy_output) < 0) + return -1; + dest->copy_src = NULL; + return 0; +} + int fs_copy(struct fs_file *src, struct fs_file *dest) { i_assert(src->fs == dest->fs); diff -r ee177df07178 -r 5818d8f488c1 src/lib-fs/fs-metawrap.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fs/fs-metawrap.c Mon Jan 21 14:43:09 2013 +0200 @@ -0,0 +1,429 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "istream.h" +#include "istream-private.h" +#include "istream-metawrap.h" +#include "ostream.h" +#include "fs-api-private.h" + +struct metawrap_fs { + struct fs fs; + struct fs *super; + bool wrap_metadata; +}; + +struct metawrap_fs_file { + struct fs_file file; + struct metawrap_fs *fs; + struct fs_file *super; + enum fs_open_mode open_mode; +}; + +static void fs_metawrap_copy_error(struct metawrap_fs *fs) +{ + fs_set_error(&fs->fs, "%s", fs_last_error(fs->super)); +} + +static void fs_metawrap_file_copy_error(struct metawrap_fs_file *file) +{ + struct metawrap_fs *fs = (struct metawrap_fs *)file->file.fs; + + fs_metawrap_copy_error(fs); +} + +static struct fs *fs_metawrap_alloc(void) +{ + struct metawrap_fs *fs; + + fs = i_new(struct metawrap_fs, 1); + fs->fs = fs_class_metawrap; + return &fs->fs; +} + +static int +fs_metawrap_init(struct fs *_fs, const char *args, const + struct fs_settings *set) +{ + struct metawrap_fs *fs = (struct metawrap_fs *)_fs; + const char *parent_name, *parent_args, *error; + + if (*args == '\0') { + fs_set_error(_fs, "Parent filesystem not given as parameter"); + return -1; + } + + parent_args = strchr(args, ':'); + if (parent_args == NULL) { + parent_name = args; + parent_args = ""; + } else { + parent_name = t_strdup_until(args, parent_args); + parent_args++; From dovecot at dovecot.org Mon Jan 21 16:44:39 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 16:44:39 +0200 Subject: dovecot-2.2: dsync: Fixed syncing to non-remote locations. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4260de42d93c changeset: 15656:4260de42d93c user: Timo Sirainen date: Mon Jan 21 16:44:33 2013 +0200 description: dsync: Fixed syncing to non-remote locations. diffstat: src/doveadm/dsync/doveadm-dsync.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 5818d8f488c1 -r 4260de42d93c src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Mon Jan 21 14:43:09 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Mon Jan 21 16:44:33 2013 +0200 @@ -564,6 +564,8 @@ prefix sent before dsync actually starts */ ctx->remote_name = ctx->local_location+13; ctx->remote_user_prefix = TRUE; + } else { + ctx->remote_name = NULL; } remote_cmd_args = ctx->remote_name == NULL ? NULL : parse_ssh_location(ctx, ctx->remote_name, From dovecot at dovecot.org Mon Jan 21 16:57:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 16:57:55 +0200 Subject: dovecot-2.2: dsync: Verify messages' GUIDs better. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f0dbe8fc8905 changeset: 15657:f0dbe8fc8905 user: Timo Sirainen date: Mon Jan 21 16:52:49 2013 +0200 description: dsync: Verify messages' GUIDs better. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 25 ++++++++++++++++++++++--- 1 files changed, 22 insertions(+), 3 deletions(-) diffs (50 lines): diff -r 4260de42d93c -r f0dbe8fc8905 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 16:44:33 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 16:52:49 2013 +0200 @@ -432,6 +432,22 @@ return TRUE; } +static bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + if (change->guid == NULL || *change->guid == '\0') + return TRUE; + if (strcmp(importer->cur_guid, change->guid) != 0) { + i_error("Mailbox %s: Unexpected GUID mismatch for " + "UID=%u: %s != %s", mailbox_get_vname(importer->box), + change->uid, importer->cur_guid, change->guid); + importer->last_common_uid = 1; + importer->failed = TRUE; + return FALSE; + } + return TRUE; +} + static void merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove, uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove, @@ -718,9 +734,11 @@ i_assert((change->add_flags & change->remove_flags) == 0); if (importer->cur_mail != NULL && - importer->cur_mail->uid == change->uid) + importer->cur_mail->uid == change->uid) { + if (!dsync_check_cur_guid(importer, change)) + return; mail = importer->cur_mail; - else { + } else { if (!dsync_import_set_mail(importer, change)) return; mail = importer->mail; @@ -833,7 +851,8 @@ } else if (change->uid == importer->last_common_uid) { /* already verified that the GUID matches */ i_assert(importer->cur_mail->uid == change->uid); - mail_expunge(importer->cur_mail); + if (dsync_check_cur_guid(importer, change)) + mail_expunge(importer->cur_mail); } else { /* we don't know yet if we should expunge this message or not. queue it until we do. */ From dovecot at dovecot.org Mon Jan 21 16:57:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 16:57:55 +0200 Subject: dovecot-2.2: dsync: Fixed crashes when sending already expunged ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3a395a61f109 changeset: 15658:3a395a61f109 user: Timo Sirainen date: Mon Jan 21 16:57:49 2013 +0200 description: dsync: Fixed crashes when sending already expunged messages. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r f0dbe8fc8905 -r 3a395a61f109 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 16:52:49 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 16:57:49 2013 +0200 @@ -1263,7 +1263,7 @@ if (*mail->guid != '\0') mailbox_save_set_guid(save_ctx, mail->guid); dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change); - if (*mail->pop3_uidl != '\0') + if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0') mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); if (mail->pop3_order > 0) mailbox_save_set_pop3_order(save_ctx, mail->pop3_order); @@ -1341,7 +1341,7 @@ { struct importer_new_mail *newmail, *allmails; - i_assert(mail->input->seekable); + i_assert(mail->input == NULL || mail->input->seekable); i_assert(importer->new_uids_assigned); newmail = *mail->guid != '\0' ? From dovecot at dovecot.org Mon Jan 21 17:44:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 17:44:27 +0200 Subject: dovecot-2.2: dsync: GUIDs weren't set properly to expunge record... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9d461d3b2fe7 changeset: 15659:9d461d3b2fe7 user: Timo Sirainen date: Mon Jan 21 17:44:16 2013 +0200 description: dsync: GUIDs weren't set properly to expunge records, causing non-optimal behavior. diffstat: src/doveadm/dsync/dsync-transaction-log-scan.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (31 lines): diff -r 3a395a61f109 -r 9d461d3b2fe7 src/doveadm/dsync/dsync-transaction-log-scan.c --- a/src/doveadm/dsync/dsync-transaction-log-scan.c Mon Jan 21 16:57:49 2013 +0200 +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c Mon Jan 21 17:44:16 2013 +0200 @@ -118,7 +118,7 @@ end = CONST_PTR_OFFSET(data, hdr->size); for (; rec != end; rec++) { - if (!external && !mail_index_lookup_seq(view, rec->uid, &seq)) { + if (!external && mail_index_lookup_seq(view, rec->uid, &seq)) { /* expunge request that hasn't been actually done yet. we check non-external ones because they might have the GUID while external ones don't. */ @@ -126,7 +126,8 @@ } if (export_change_get(ctx, rec->uid, DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, - &change)) T_BEGIN { + &change) && + !guid_128_is_empty(rec->guid_128)) T_BEGIN { change->guid = p_strdup(ctx->pool, guid_128_to_string(rec->guid_128)); } T_END; @@ -150,7 +151,7 @@ DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, &change)) i_unreached(); - T_BEGIN { + if (!guid_128_is_empty(rec->guid_128)) T_BEGIN { change->guid = p_strdup(ctx->pool, guid_128_to_string(rec->guid_128)); } T_END; From dovecot at dovecot.org Mon Jan 21 17:45:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 17:45:25 +0200 Subject: dovecot-2.2: lib-index: mail_transaction_log_view_set_all() didn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22a1f4cfe9fb changeset: 15660:22a1f4cfe9fb user: Timo Sirainen date: Mon Jan 21 17:45:18 2013 +0200 description: lib-index: mail_transaction_log_view_set_all() didn't necessarily scan .log.2 file. diffstat: src/lib-index/mail-transaction-log-view.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (21 lines): diff -r 9d461d3b2fe7 -r 22a1f4cfe9fb src/lib-index/mail-transaction-log-view.c --- a/src/lib-index/mail-transaction-log-view.c Mon Jan 21 17:44:16 2013 +0200 +++ b/src/lib-index/mail-transaction-log-view.c Mon Jan 21 17:45:18 2013 +0200 @@ -282,13 +282,14 @@ int mail_transaction_log_view_set_all(struct mail_transaction_log_view *view) { - struct mail_transaction_log_file *file, *first = view->log->files; - - i_assert(first != NULL); + struct mail_transaction_log_file *file, *first; /* make sure .log.2 file is opened */ (void)mail_transaction_log_find_file(view->log, 1, FALSE, &file); + first = view->log->files; + i_assert(first != NULL); + for (file = view->log->files; file != NULL; file = file->next) { if (mail_transaction_log_file_map(file, file->hdr.hdr_size, (uoff_t)-1) < 0) From dovecot at dovecot.org Mon Jan 21 17:57:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 17:57:30 +0200 Subject: dovecot-2.2: dsync backup: Old unwanted messages weren't deleted... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/940414cea18d changeset: 15661:940414cea18d user: Timo Sirainen date: Mon Jan 21 17:56:36 2013 +0200 description: dsync backup: Old unwanted messages weren't deleted as they should have. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 21 ++++++++++++++------- 1 files changed, 14 insertions(+), 7 deletions(-) diffs (84 lines): diff -r 22a1f4cfe9fb -r 940414cea18d src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 17:45:18 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 17:56:36 2013 +0200 @@ -76,7 +76,7 @@ unsigned int debug:1; unsigned int last_common_uid_found:1; unsigned int cur_uid_has_change:1; - unsigned int cur_mail_saved:1; + unsigned int cur_mail_skip:1; unsigned int local_expunged_guids_set:1; unsigned int new_uids_assigned:1; unsigned int want_mail_requests:1; @@ -105,7 +105,7 @@ if (mailbox_search_next(importer->search_ctx, &importer->cur_mail)) importer->next_local_seq = importer->cur_mail->seq; /* this flag causes cur_guid to be looked up later */ - importer->cur_mail_saved = TRUE; + importer->cur_mail_skip = TRUE; } struct dsync_mailbox_importer * @@ -212,7 +212,7 @@ importer->cur_mail->uid); } - importer->cur_mail_saved = FALSE; + importer->cur_mail_skip = FALSE; if (!mailbox_search_next(importer->search_ctx, &importer->cur_mail)) { importer->cur_mail = NULL; @@ -335,12 +335,17 @@ m2.uid = save_change->uid; } - newmail = p_new(importer->pool, struct importer_new_mail, 1); - diff = importer_mail_cmp(&m1, &m2); if (diff < 0) { /* add a record for local mail */ i_assert(importer->cur_mail != NULL); + if (importer->revert_local_changes) { + mail_expunge(importer->cur_mail); + importer->cur_mail_skip = TRUE; + importer->next_local_seq++; + return FALSE; + } + newmail = p_new(importer->pool, struct importer_new_mail, 1); newmail->guid = p_strdup(importer->pool, importer->cur_guid); newmail->uid = importer->cur_mail->uid; newmail->uid_in_local = TRUE; @@ -349,6 +354,7 @@ remote_saved = FALSE; } else if (diff > 0) { i_assert(save_change != NULL); + newmail = p_new(importer->pool, struct importer_new_mail, 1); newmail->guid = save_change->guid; newmail->uid = save_change->uid; newmail->uid_in_local = FALSE; @@ -359,6 +365,7 @@ /* identical */ i_assert(importer->cur_mail != NULL); i_assert(save_change != NULL); + newmail = p_new(importer->pool, struct importer_new_mail, 1); newmail->guid = save_change->guid; newmail->uid = importer->cur_mail->uid; newmail->uid_in_local = TRUE; @@ -368,7 +375,7 @@ } if (newmail->uid_in_local) { - importer->cur_mail_saved = TRUE; + importer->cur_mail_skip = TRUE; importer->next_local_seq++; } else { /* NOTE: assumes save_change is allocated from importer pool */ @@ -384,7 +391,7 @@ dsync_mailbox_try_save(struct dsync_mailbox_importer *importer, struct dsync_mail_change *save_change) { - if (importer->cur_mail_saved) { + if (importer->cur_mail_skip) { if (!importer_next_mail(importer, 0) && save_change == NULL) return FALSE; } From dovecot at dovecot.org Mon Jan 21 17:57:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 17:57:30 +0200 Subject: dovecot-2.2: dsync: If remote doesn't send some message bodies, ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fc5c3281d6d3 changeset: 15662:fc5c3281d6d3 user: Timo Sirainen date: Mon Jan 21 17:57:18 2013 +0200 description: dsync: If remote doesn't send some message bodies, log an error about each one separately. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 42 +++++++++---------------------- 1 files changed, 12 insertions(+), 30 deletions(-) diffs (96 lines): diff -r 940414cea18d -r fc5c3281d6d3 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 17:56:36 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 17:57:18 2013 +0200 @@ -1560,13 +1560,12 @@ return ret; } -static unsigned int -dsync_mailbox_import_count_missing_guid_imports(struct dsync_mailbox_importer *importer) +static void +dsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer) { struct hash_iterate_context *iter; const char *key; struct importer_new_mail *mail; - unsigned int msgs_left = 0; iter = hash_table_iterate_init(importer->import_guids); while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) { @@ -1574,26 +1573,20 @@ if (mail->uid_in_local || mail->skip) continue; - if (importer->debug) { - i_debug("Mailbox %s: Missing mail GUID=%s (UID=%u)", - mailbox_get_vname(importer->box), - mail->guid, mail->uid); - } - msgs_left++; - break; + i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)", + mailbox_get_vname(importer->box), + mail->guid, mail->uid); } } hash_table_iterate_deinit(&iter); - return msgs_left; } -static unsigned int -dsync_mailbox_import_count_missing_uid_imports(struct dsync_mailbox_importer *importer) +static void +dsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer) { struct hash_iterate_context *iter; void *key; struct importer_new_mail *mail; - unsigned int msgs_left = 0; iter = hash_table_iterate_init(importer->import_uids); while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) { @@ -1601,17 +1594,12 @@ if (mail->uid_in_local || mail->skip) continue; - if (importer->debug) { - i_debug("Mailbox %s: Missing mail UID=%u", - mailbox_get_vname(importer->box), - mail->uid); - } - msgs_left++; - break; + i_error("Mailbox %s: Remote didn't send mail UID=%u", + mailbox_get_vname(importer->box), + mail->uid); } } hash_table_iterate_deinit(&iter); - return msgs_left; } int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer, @@ -1621,7 +1609,6 @@ bool *changes_during_sync_r) { struct dsync_mailbox_importer *importer = *_importer; - unsigned int msgs_left; int ret; *_importer = NULL; @@ -1630,13 +1617,8 @@ if (!importer->new_uids_assigned) dsync_mailbox_import_assign_new_uids(importer); - msgs_left = importer->failed ? 0 : - dsync_mailbox_import_count_missing_guid_imports(importer) + - dsync_mailbox_import_count_missing_uid_imports(importer); - if (msgs_left > 0) { - i_error("%s: Remote didn't send %u expected message bodies", - mailbox_get_vname(importer->box), msgs_left); - } + dsync_mailbox_import_check_missing_guid_imports(importer); + dsync_mailbox_import_check_missing_uid_imports(importer); if (importer->search_ctx != NULL) { if (mailbox_search_deinit(&importer->search_ctx) < 0) From dovecot at dovecot.org Mon Jan 21 18:12:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 18:12:28 +0200 Subject: dovecot-2.2: lib-index: Fixed sorting appended messages when som... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df2e46c38c92 changeset: 15663:df2e46c38c92 user: Timo Sirainen date: Mon Jan 21 18:12:12 2013 +0200 description: lib-index: Fixed sorting appended messages when some were assigned new UIDs. diffstat: src/lib-index/mail-index-transaction-sort-appends.c | 11 ++++++++++- src/lib-index/mail-index-transaction-update.c | 9 +++------ 2 files changed, 13 insertions(+), 7 deletions(-) diffs (51 lines): diff -r fc5c3281d6d3 -r df2e46c38c92 src/lib-index/mail-index-transaction-sort-appends.c --- a/src/lib-index/mail-index-transaction-sort-appends.c Mon Jan 21 17:57:18 2013 +0200 +++ b/src/lib-index/mail-index-transaction-sort-appends.c Mon Jan 21 18:12:12 2013 +0200 @@ -130,8 +130,17 @@ uint32_t *old_to_newseq_map; unsigned int i, count; - if (!t->appends_nonsorted || !array_is_created(&t->appends)) + if (!array_is_created(&t->appends)) return; + if (!t->appends_nonsorted) { +#ifdef DEBUG + recs = array_get_modifiable(&t->appends, &count); + i_assert(count > 0); + for (i = 1; i < count; i++) + i_assert(recs[i-1].uid < recs[i].uid); +#endif + return; + } /* first make a copy of the UIDs and map them to sequences */ recs = array_get_modifiable(&t->appends, &count); diff -r fc5c3281d6d3 -r df2e46c38c92 src/lib-index/mail-index-transaction-update.c --- a/src/lib-index/mail-index-transaction-update.c Mon Jan 21 17:57:18 2013 +0200 +++ b/src/lib-index/mail-index-transaction-update.c Mon Jan 21 18:12:12 2013 +0200 @@ -201,7 +201,6 @@ unsigned int i, count; struct seq_range *range; uint32_t next_uid; - bool used_existing_uids; if (!array_is_created(&t->appends)) return; @@ -217,14 +216,12 @@ } /* assign missing uids */ - used_existing_uids = FALSE; for (i = 0; i < count; i++) { - if (recs[i].uid == 0 || recs[i].uid < first_uid) { + if (recs[i].uid == 0 || recs[i].uid < first_uid) recs[i].uid = next_uid++; - if (used_existing_uids) + else { + if (next_uid != first_uid) t->appends_nonsorted = TRUE; - } else { - used_existing_uids = TRUE; } } From dovecot at dovecot.org Mon Jan 21 18:12:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 18:12:48 +0200 Subject: dovecot-2.2: dsync: Fixed infinite loop when message suddenly go... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9aba8584e5eb changeset: 15664:9aba8584e5eb user: Timo Sirainen date: Mon Jan 21 18:12:41 2013 +0200 description: dsync: Fixed infinite loop when message suddenly got expunged. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r df2e46c38c92 -r 9aba8584e5eb src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 18:12:12 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 18:12:41 2013 +0200 @@ -226,6 +226,7 @@ if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID, &importer->cur_guid) < 0) { dsync_mail_error(importer, importer->cur_mail, "GUID"); + importer->next_local_seq = importer->cur_mail->seq + 1; return importer_next_mail(importer, wanted_uid); } /* make sure next_local_seq gets updated in case we came here From dovecot at dovecot.org Mon Jan 21 18:31:20 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 18:31:20 +0200 Subject: dovecot-2.2: dict: Added support for cdb backend. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2c249941f9c2 changeset: 15665:2c249941f9c2 user: Timo Sirainen date: Mon Jan 21 18:31:08 2013 +0200 description: dict: Added support for cdb backend. Based on patch by Hleb Valoshka. diffstat: configure.ac | 23 +++++++ src/dict/main.c | 3 + src/lib-dict/Makefile.am | 1 + src/lib-dict/dict-cdb.c | 138 ++++++++++++++++++++++++++++++++++++++++++++ src/lib-dict/dict-private.h | 1 + 5 files changed, 166 insertions(+), 0 deletions(-) diffs (223 lines): diff -r 9aba8584e5eb -r 2c249941f9c2 configure.ac --- a/configure.ac Mon Jan 21 18:12:41 2013 +0200 +++ b/configure.ac Mon Jan 21 18:31:08 2013 +0200 @@ -126,6 +126,11 @@ # want_db=no) want_db=no +AC_ARG_WITH(cdb, +AS_HELP_STRING([--with-cdb], [Build with CDB support]), + TEST_WITH(cdb, $withval), + want_cdb=no) + dnl The --with-sql is useful only if Dovecot is being built with all the SQL dnl drivers as modules. If any SQL driver is built-in, this option is ignored. AC_ARG_WITH(sql, @@ -2113,6 +2118,23 @@ fi fi +if test $want_cdb != no; then + AC_CHECK_LIB(cdb, cdb_init, [ + AC_CHECK_HEADER(cdb.h, [ + DICT_LIBS="$DICT_LIBS -lcdb" + AC_DEFINE(BUILD_CDB,, Build with CDB support) + ], [ + if test $want_cdb = yes; then + AC_ERROR([Can't build with CDB support: cdb.h not found]) + fi + ]) + ], [ + if test $want_cdb = yes; then + AC_ERROR([Can't build with CDB support: libcdb not found]) + fi + ]) +fi + if test $want_pgsql != no; then AC_CHECK_PROG(PG_CONFIG, pg_config, pg_config, NO) if test $PG_CONFIG = NO; then @@ -2397,6 +2419,7 @@ AC_SUBST(SQLITE_LIBS) AC_SUBST(DICT_LIBS) +AC_SUBST(CDB_LIBS) AC_SUBST(dict_drivers) dnl ** diff -r 9aba8584e5eb -r 2c249941f9c2 src/dict/main.c --- a/src/dict/main.c Mon Jan 21 18:12:41 2013 +0200 +++ b/src/dict/main.c Mon Jan 21 18:31:08 2013 +0200 @@ -35,6 +35,9 @@ /* Load built-in SQL drivers (if any) */ sql_drivers_init(); sql_drivers_register_all(); +#ifdef HAVE_CDB + dict_driver_register(&dict_driver_cdb); +#endif restrict_access_by_env(NULL, FALSE); restrict_access_allow_coredumps(TRUE); diff -r 9aba8584e5eb -r 2c249941f9c2 src/lib-dict/Makefile.am --- a/src/lib-dict/Makefile.am Mon Jan 21 18:12:41 2013 +0200 +++ b/src/lib-dict/Makefile.am Mon Jan 21 18:31:08 2013 +0200 @@ -14,6 +14,7 @@ dict.c \ dict-client.c \ dict-file.c \ + dict-cdb.c \ dict-memcached.c \ dict-memcached-ascii.c \ dict-redis.c \ diff -r 9aba8584e5eb -r 2c249941f9c2 src/lib-dict/dict-cdb.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-dict/dict-cdb.c Mon Jan 21 18:31:08 2013 +0200 @@ -0,0 +1,138 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" + +#ifdef BUILD_CDB +#include "dict-private.h" + +#include +#include +#include +#include + +#define CDB_WITH_NULL 1 +#define CDB_WITHOUT_NULL 2 + +struct cdb_dict { + struct dict dict; + struct cdb cdb; + char *path; + int fd, flag; +}; + +static void cdb_dict_deinit(struct dict *_dict); + +static int +cdb_dict_init(struct dict *driver, const char *uri, + enum dict_data_type value_type ATTR_UNUSED, + const char *username ATTR_UNUSED, + const char *base_dir ATTR_UNUSED, + struct dict **dict_r, const char **error_r) +{ + struct cdb_dict *dict; + + dict = i_new(struct cdb_dict, 1); + dict->dict = *driver; + dict->path = i_strdup(uri); + dict->flag = CDB_WITH_NULL | CDB_WITHOUT_NULL; + + /* initialize cdb to 0 (unallocated) */ + memset(&dict->cdb, 0, sizeof(struct cdb)); + + dict->fd = open(dict->path, O_RDONLY); + if (dict->fd == -1) { + *error_r = t_strdup_printf("open(%s) failed: %m", dict->path); + cdb_dict_deinit(&dict->dict); + return -1; + } + +#ifdef TINYCDB_VERSION + if (cdb_init(&dict->cdb, dict->fd) < 0) { + *error_r = t_strdup_printf("cdb_init(%s) failed: %m", dict->path); + cdb_dict_deinit(&dict->dict); + return -1; + } +#else + cdb_init(&dict->cdb, dict->fd); +#endif + + *dict_r = &dict->dict; + return 0; +} + +static void cdb_dict_deinit(struct dict *_dict) +{ + struct cdb_dict *dict = (struct cdb_dict *)_dict; + + /* we can safely deinit unallocated cdb */ + cdb_free(&dict->cdb); + + if (dict->fd != -1) { + if (close(dict->fd) < 0) + i_error("close(%s) failed: %m", dict->path); + } + + i_free(dict->path); + i_free(dict); +} + +static int cdb_dict_lookup(struct dict *_dict, pool_t pool, + const char *key, const char **value_r) +{ + struct cdb_dict *dict = (struct cdb_dict *)_dict; + unsigned datalen; + int ret = 0; + char *data; + + /* keys and values may be null terminated... */ + if ((dict->flag & CDB_WITH_NULL) != 0) { + ret = cdb_find(&dict->cdb, key, (unsigned)strlen(key)+1); + if (ret > 0) + dict->flag &= ~CDB_WITHOUT_NULL; + } + + /* ...or not */ + if (ret == 0 && (dict->flag & CDB_WITHOUT_NULL) != 0) { + ret = cdb_find(&dict->cdb, key, (unsigned)strlen(key)); + if (ret > 0) + dict->flag &= ~CDB_WITH_NULL; + } + + if (ret <= 0) { + *value_r = NULL; + /* something bad with db */ + if (ret < 0) { + i_error("cdb_lookup(%s) failed: %m", dict->path); + return -1; + } + /* found nothing */ + return 0; + } + + datalen = cdb_datalen(&dict->cdb); + data = p_new(pool, char, datalen + 1); + cdb_read(&dict->cdb, data, datalen, cdb_datapos(&dict->cdb)); + *value_r = data; + return 1; +} + +struct dict dict_driver_cdb = { + .name = "cdb", + { + cdb_dict_init, + cdb_dict_deinit, + NULL, + cdb_dict_lookup, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + } +}; +#endif diff -r 9aba8584e5eb -r 2c249941f9c2 src/lib-dict/dict-private.h --- a/src/lib-dict/dict-private.h Mon Jan 21 18:12:41 2013 +0200 +++ b/src/lib-dict/dict-private.h Mon Jan 21 18:31:08 2013 +0200 @@ -59,5 +59,6 @@ extern struct dict dict_driver_memcached; extern struct dict dict_driver_memcached_ascii; extern struct dict dict_driver_redis; +extern struct dict dict_driver_cdb; #endif From dovecot at dovecot.org Mon Jan 21 18:34:00 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 18:34:00 +0200 Subject: dovecot-2.2: dict-cdb: Handle cdb_read() errors. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/87a709eb7b28 changeset: 15666:87a709eb7b28 user: Timo Sirainen date: Mon Jan 21 18:33:55 2013 +0200 description: dict-cdb: Handle cdb_read() errors. diffstat: src/lib-dict/dict-cdb.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (26 lines): diff -r 2c249941f9c2 -r 87a709eb7b28 src/lib-dict/dict-cdb.c --- a/src/lib-dict/dict-cdb.c Mon Jan 21 18:31:08 2013 +0200 +++ b/src/lib-dict/dict-cdb.c Mon Jan 21 18:33:55 2013 +0200 @@ -102,7 +102,7 @@ *value_r = NULL; /* something bad with db */ if (ret < 0) { - i_error("cdb_lookup(%s) failed: %m", dict->path); + i_error("cdb_find(%s) failed: %m", dict->path); return -1; } /* found nothing */ @@ -110,8 +110,11 @@ } datalen = cdb_datalen(&dict->cdb); - data = p_new(pool, char, datalen + 1); - cdb_read(&dict->cdb, data, datalen, cdb_datapos(&dict->cdb)); + data = p_malloc(pool, datalen + 1); + if (cdb_read(&dict->cdb, data, datalen, cdb_datapos(&dict->cdb)) < 0) { + i_error("cdb_read(%s) failed: %m", dict->path); + return -1; + } *value_r = data; return 1; } From dovecot at dovecot.org Mon Jan 21 19:44:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 19:44:55 +0200 Subject: dovecot-2.2: dsync: Fixes to handling storage formats that don't... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1b18ff11effc changeset: 15667:1b18ff11effc user: Timo Sirainen date: Mon Jan 21 19:41:24 2013 +0200 description: dsync: Fixes to handling storage formats that don't use 128bit GUIDs. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 31 +++++++++++++++++++++++-------- 1 files changed, 23 insertions(+), 8 deletions(-) diffs (84 lines): diff -r 87a709eb7b28 -r 1b18ff11effc src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 18:33:55 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 19:41:24 2013 +0200 @@ -195,6 +195,22 @@ } static bool +dsync_mail_change_guid_equals(const struct dsync_mail_change *change, + const char *guid) +{ + guid_128_t guid_128, change_guid_128; + + if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) + return strcmp(change->guid, guid) == 0; + + if (guid_128_from_string(change->guid, change_guid_128) < 0) + i_unreached(); + + mail_generate_guid_128_hash(guid, guid_128); + return memcmp(change_guid_128, guid_128, GUID_128_SIZE) == 0; +} + +static bool importer_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid) { if (importer->cur_mail == NULL) { @@ -334,6 +350,7 @@ if (save_change != NULL) { m2.guid = save_change->guid; m2.uid = save_change->uid; + i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); } diff = importer_mail_cmp(&m1, &m2); @@ -429,7 +446,7 @@ dsync_mail_error(importer, importer->mail, "GUID"); return FALSE; } - if (strcmp(guid, change->guid) != 0) { + if (!dsync_mail_change_guid_equals(change, guid)) { i_error("Mailbox %s: Unexpected GUID mismatch for " "UID=%u: %s != %s", mailbox_get_vname(importer->box), change->uid, guid, change->guid); @@ -445,7 +462,7 @@ { if (change->guid == NULL || *change->guid == '\0') return TRUE; - if (strcmp(importer->cur_guid, change->guid) != 0) { + if (!dsync_mail_change_guid_equals(change, importer->cur_guid)) { i_error("Mailbox %s: Unexpected GUID mismatch for " "UID=%u: %s != %s", mailbox_get_vname(importer->box), change->uid, importer->cur_guid, change->guid); @@ -926,7 +943,8 @@ if (*change->guid != '\0' && *importer->cur_guid != '\0') { /* we have GUIDs, verify them */ - return strcmp(change->guid, importer->cur_guid) == 0 ? 1 : 0; + return dsync_mail_change_guid_equals(change, + importer->cur_guid) ? 1 : 0; } /* verify hdr_hash if it exists */ @@ -951,7 +969,6 @@ const struct dsync_mail_change *change) { const struct dsync_mail_change *local_change; - guid_128_t guid_128, change_guid_128; if (*change->guid == '\0') { /* remote doesn't support GUIDs, can't verify expunge */ @@ -966,11 +983,9 @@ POINTER_CAST(change->uid)); if (local_change == NULL || local_change->guid == NULL) return FALSE; - if (guid_128_from_string(local_change->guid, guid_128) < 0) - i_unreached(); - mail_generate_guid_128_hash(change->guid, change_guid_128); - if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) { + i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); + if (!dsync_mail_change_guid_equals(local_change, change->guid)) { /* mismatch - found the first non-common UID */ dsync_mailbox_common_uid_found(importer); } else { From dovecot at dovecot.org Mon Jan 21 19:44:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 19:44:55 +0200 Subject: dovecot-2.2: dsync: Added a quick way to debug unexpected changes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/583ea6b63c5d changeset: 15668:583ea6b63c5d user: Timo Sirainen date: Mon Jan 21 19:43:39 2013 +0200 description: dsync: Added a quick way to debug unexpected changes. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diffs (47 lines): diff -r 1b18ff11effc -r 583ea6b63c5d src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 19:41:24 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 21 19:43:39 2013 +0200 @@ -33,6 +33,10 @@ unsigned int copy_failed:1; }; +/* for quickly testing that two-way sync doesn't actually do any unexpected + modifications. */ +#define IMPORTER_DEBUG_CHANGE(importer) /*i_assert(!importer->master_brain)*/ + HASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *); HASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *); @@ -279,6 +283,7 @@ struct dsync_mail_request *request; if (importer->want_mail_requests && !newmail->uid_in_local) { + IMPORTER_DEBUG_CHANGE(importer); request = array_append_space(&importer->mail_requests); request->guid = newmail->guid; request->uid = newmail->uid; @@ -1108,6 +1113,8 @@ { struct mail_save_context *save_ctx; + IMPORTER_DEBUG_CHANGE(importer); + if (!mail_set_uid(importer->mail, old_uid)) return; @@ -1132,6 +1139,7 @@ if (newmail->skip) { /* already assigned */ if (newmail->uid_in_local) { + IMPORTER_DEBUG_CHANGE(importer); if (mail_set_uid(importer->mail, newmail->uid)) mail_expunge(importer->mail); } @@ -1483,6 +1491,7 @@ if (highest_wanted_uid < saved_uid) highest_wanted_uid = saved_uid; } else { + IMPORTER_DEBUG_CHANGE(importer); if (lowest_unwanted_uid > saved_uid) lowest_unwanted_uid = saved_uid; } From dovecot at dovecot.org Mon Jan 21 19:45:19 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Jan 2013 19:45:19 +0200 Subject: dovecot-2.2: Increased initial memory pool sizes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/be5ce6c06b08 changeset: 15669:be5ce6c06b08 user: Timo Sirainen date: Mon Jan 21 19:45:13 2013 +0200 description: Increased initial memory pool sizes. diffstat: src/doveadm/dsync/dsync-ibc-pipe.c | 2 +- src/lib-storage/index/index-status.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 583ea6b63c5d -r be5ce6c06b08 src/doveadm/dsync/dsync-ibc-pipe.c --- a/src/doveadm/dsync/dsync-ibc-pipe.c Mon Jan 21 19:43:39 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-pipe.c Mon Jan 21 19:45:13 2013 +0200 @@ -60,7 +60,7 @@ pools = array_get_modifiable(&pipe->pools, &count); if (count == 0) - return pool_alloconly_create("pipe item pool", 128); + return pool_alloconly_create("pipe item pool", 1024); ret = pools[count-1]; array_delete(&pipe->pools, count-1, 1); diff -r 583ea6b63c5d -r be5ce6c06b08 src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Mon Jan 21 19:43:39 2013 +0200 +++ b/src/lib-storage/index/index-status.c Mon Jan 21 19:45:13 2013 +0200 @@ -190,7 +190,7 @@ if (box->metadata_pool == NULL) { box->metadata_pool = - pool_alloconly_create("mailbox metadata", 2048); + pool_alloconly_create("mailbox metadata", 1024*3); } fields = mail_cache_register_get_list(box->cache, From dovecot at dovecot.org Tue Jan 22 12:49:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Jan 2013 12:49:22 +0200 Subject: dovecot-2.2: imapc: Avoid unnecessarily selecting a mailbox when... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1706b3490966 changeset: 15670:1706b3490966 user: Timo Sirainen date: Tue Jan 22 12:49:04 2013 +0200 description: imapc: Avoid unnecessarily selecting a mailbox when looking up its GUID. diffstat: src/lib-storage/index/imapc/imapc-storage.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (29 lines): diff -r be5ce6c06b08 -r 1706b3490966 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Mon Jan 21 19:45:13 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Tue Jan 22 12:49:04 2013 +0200 @@ -755,12 +755,11 @@ struct imapc_mailbox *mbox = (struct imapc_mailbox *)box; const struct imapc_namespace *ns; - if (index_mailbox_get_metadata(box, items, metadata_r) < 0) - return -1; if ((items & MAILBOX_METADATA_GUID) != 0) { /* a bit ugly way to do this, but better than nothing for now. FIXME: if indexes are enabled, keep this there. */ mail_generate_guid_128_hash(box->name, metadata_r->guid); + items &= ~MAILBOX_METADATA_GUID; } if ((items & MAILBOX_METADATA_BACKEND_NAMESPACE) != 0) { if (imapc_mailbox_get_namespaces(mbox->storage) < 0) @@ -771,6 +770,11 @@ metadata_r->backend_ns_prefix = ns->prefix; metadata_r->backend_ns_type = ns->type; } + items &= ~MAILBOX_METADATA_BACKEND_NAMESPACE; + } + if (items != 0) { + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; } return 0; } From dovecot at dovecot.org Tue Jan 22 12:59:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Jan 2013 12:59:43 +0200 Subject: dovecot-2.2: imapc: Return INTERNALDATE as save-date (better tha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa60f457a23f changeset: 15671:aa60f457a23f user: Timo Sirainen date: Tue Jan 22 12:58:56 2013 +0200 description: imapc: Return INTERNALDATE as save-date (better than error). diffstat: src/lib-storage/index/imapc/imapc-mail.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 1706b3490966 -r aa60f457a23f src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Tue Jan 22 12:49:04 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Tue Jan 22 12:58:56 2013 +0200 @@ -100,8 +100,8 @@ struct index_mail_data *data = &mail->data; if (data->save_date == (time_t)-1) { - /* FIXME */ - return -1; + /* FIXME: we could use a value stored in cache */ + return imapc_mail_get_received_date(_mail, date_r); } *date_r = data->save_date; return 0; From dovecot at dovecot.org Tue Jan 22 12:59:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Jan 2013 12:59:43 +0200 Subject: dovecot-2.2: imapc: Handle mailbox_update() when possible, inste... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/83803323f357 changeset: 15672:83803323f357 user: Timo Sirainen date: Tue Jan 22 12:59:22 2013 +0200 description: imapc: Handle mailbox_update() when possible, instead of always just failing. diffstat: src/lib-storage/index/imapc/imapc-storage.c | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-) diffs (23 lines): diff -r aa60f457a23f -r 83803323f357 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Tue Jan 22 12:58:56 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Tue Jan 22 12:59:22 2013 +0200 @@ -561,11 +561,15 @@ } static int imapc_mailbox_update(struct mailbox *box, - const struct mailbox_update *update ATTR_UNUSED) + const struct mailbox_update *update) { - mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, - "Not supported"); - return -1; + if (!guid_128_is_empty(update->mailbox_guid) || + update->uid_validity != 0 || update->min_next_uid != 0 || + update->min_first_recent_uid != 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "Not supported"); + } + return index_storage_mailbox_update(box, update); } static void imapc_untagged_status(const struct imapc_untagged_reply *reply, From dovecot at dovecot.org Tue Jan 22 12:59:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Jan 2013 12:59:43 +0200 Subject: dovecot-2.2: dsync: Error handling crashfix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4673af53e7e6 changeset: 15673:4673af53e7e6 user: Timo Sirainen date: Tue Jan 22 12:59:32 2013 +0200 description: dsync: Error handling crashfix diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 83803323f357 -r 4673af53e7e6 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Tue Jan 22 12:59:22 2013 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Tue Jan 22 12:59:32 2013 +0200 @@ -545,8 +545,8 @@ if (mailbox_update(box, &update) < 0) { i_error("Couldn't update mailbox %s metadata: %s", - mailbox_get_vname(brain->box), - mailbox_get_last_error(brain->box, NULL)); + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); brain->failed = TRUE; } } From dovecot at dovecot.org Tue Jan 22 16:31:31 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Jan 2013 16:31:31 +0200 Subject: dovecot-2.1: lib-storage: Catch input stream errors when parsing... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/2b76d357a56a changeset: 14883:2b76d357a56a user: Timo Sirainen date: Tue Jan 22 16:31:17 2013 +0200 description: lib-storage: Catch input stream errors when parsing mail. diffstat: src/lib-storage/index/index-mail-headers.c | 6 ++++- src/lib-storage/index/index-mail.c | 34 ++++++++++++++++++++++++++--- src/lib-storage/index/index-mail.h | 1 + 3 files changed, 36 insertions(+), 5 deletions(-) diffs (91 lines): diff -r c762a9af72c1 -r 2b76d357a56a src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Tue Jan 15 08:30:03 2013 +0200 +++ b/src/lib-storage/index/index-mail-headers.c Tue Jan 22 16:31:17 2013 +0200 @@ -374,6 +374,7 @@ input2 = tee_i_stream_create_child(mail->data.tee_stream); index_mail_parse_header_init(mail, NULL); + mail->data.parser_input = input; mail->data.parser_ctx = message_parser_init(mail->data_pool, input, hdr_parser_flags, msg_parser_flags); @@ -386,10 +387,13 @@ struct index_mail_data *data = &mail->data; struct message_part *parts; - if (data->parser_ctx != NULL) + if (data->parser_ctx != NULL) { + data->parser_input = NULL; (void)message_parser_deinit(&data->parser_ctx, &parts); + } if (data->parts == NULL) { + data->parser_input = data->stream; data->parser_ctx = message_parser_init(mail->data_pool, data->stream, hdr_parser_flags, diff -r c762a9af72c1 -r 2b76d357a56a src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Tue Jan 15 08:30:03 2013 +0200 +++ b/src/lib-storage/index/index-mail.c Tue Jan 22 16:31:17 2013 +0200 @@ -751,13 +751,38 @@ static int index_mail_parse_body_finish(struct index_mail *mail, enum index_cache_field field) { - if (message_parser_deinit(&mail->data.parser_ctx, - &mail->data.parts) < 0) { - mail_set_cache_corrupted(&mail->mail.mail, - MAIL_FETCH_MESSAGE_PARTS); + struct istream *parser_input = mail->data.parser_input; + int ret; + + if (parser_input == NULL) { + ret = message_parser_deinit(&mail->data.parser_ctx, + &mail->data.parts) < 0 ? 0 : 1; + } else { + mail->data.parser_input = NULL; + i_stream_ref(parser_input); + ret = message_parser_deinit(&mail->data.parser_ctx, + &mail->data.parts) < 0 ? 0 : 1; + if (parser_input->stream_errno != 0) { + errno = parser_input->stream_errno; + mail_storage_set_critical(mail->mail.mail.box->storage, + "read(%s) failed: %m", + i_stream_get_name(parser_input)); + ret = -1; + } + i_assert(i_stream_read(parser_input) == -1 && + !i_stream_have_bytes_left(parser_input)); + i_stream_unref(&parser_input); + } + if (ret <= 0) { + if (ret == 0) { + mail_set_cache_corrupted(&mail->mail.mail, + MAIL_FETCH_MESSAGE_PARTS); + } + mail->data.parts = NULL; mail->data.parsed_bodystructure = FALSE; return -1; } + if (mail->data.no_caching) { /* if we're here because we aborted parsing, don't get any further or we may crash while generating output from @@ -1150,6 +1175,7 @@ mail_set_cache_corrupted(&mail->mail.mail, MAIL_FETCH_MESSAGE_PARTS); } + mail->data.parser_input = NULL; } if (data->filter_stream != NULL) i_stream_unref(&data->filter_stream); diff -r c762a9af72c1 -r 2b76d357a56a src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Tue Jan 15 08:30:03 2013 +0200 +++ b/src/lib-storage/index/index-mail.h Tue Jan 22 16:31:17 2013 +0200 @@ -96,6 +96,7 @@ struct istream *stream, *filter_stream; struct tee_istream *tee_stream; struct message_size hdr_size, body_size; + struct istream *parser_input; struct message_parser_ctx *parser_ctx; int parsing_count; ARRAY_TYPE(keywords) keywords; From dovecot at dovecot.org Tue Jan 22 16:54:05 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Jan 2013 16:54:05 +0200 Subject: dovecot-2.1: lib-index: Fixes to handling broken cache records t... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/99b7be36631b changeset: 14884:99b7be36631b user: Timo Sirainen date: Tue Jan 22 16:53:52 2013 +0200 description: lib-index: Fixes to handling broken cache records that point outside file. Especially try to avoid failing by trying to allocate gigabytes of memory. diffstat: src/lib-index/mail-cache-lookup.c | 13 ++++++------- src/lib-index/mail-cache.c | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 9 deletions(-) diffs (76 lines): diff -r 2b76d357a56a -r 99b7be36631b src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Tue Jan 22 16:31:17 2013 +0200 +++ b/src/lib-index/mail-cache-lookup.c Tue Jan 22 16:53:52 2013 +0200 @@ -15,6 +15,7 @@ { const struct mail_cache_record *rec; const void *data; + int ret; i_assert(offset != 0); @@ -41,17 +42,15 @@ } if (rec->size > CACHE_PREFETCH) { /* larger than we guessed. map the rest of the record. */ - if (mail_cache_map(cache, offset, rec->size, &data) < 0) + if ((ret = mail_cache_map(cache, offset, rec->size, &data)) < 0) return -1; + if (ret == 0) { + mail_cache_set_corrupted(cache, "record points outside file"); + return -1; + } rec = data; } - if (rec->size > cache->mmap_length || - offset + rec->size > cache->mmap_length) { - mail_cache_set_corrupted(cache, "record points outside file"); - return -1; - } - *rec_r = rec; return 0; } diff -r 2b76d357a56a -r 99b7be36631b src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Jan 22 16:31:17 2013 +0200 +++ b/src/lib-index/mail-cache.c Tue Jan 22 16:53:52 2013 +0200 @@ -357,12 +357,28 @@ int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, const void **data_r) { + struct stat st; const void *data; ssize_t ret; if (size == 0) size = sizeof(struct mail_cache_header); + /* verify offset + size before trying to allocate a huge amount of + memory due to them. note that we may be prefetching more than we + actually need, so don't fail too early. */ + if (size > cache->mmap_length || offset + size > cache->mmap_length) { + if (fstat(cache->fd, &st) < 0) { + i_error("fstat(%s) failed: %m", cache->filepath); + return -1; + } + if (offset >= (uoff_t)st.st_size) { + *data_r = NULL; + return 0; + } + size = st.st_size - offset; + } + cache->remap_counter++; if (cache->map_with_read) return mail_cache_map_with_read(cache, offset, size, data_r); @@ -451,8 +467,7 @@ mail_cache_init_file_cache(cache); - if (mail_cache_map(cache, 0, sizeof(struct mail_cache_header), - &data) < 0) + if (mail_cache_map(cache, 0, 0, &data) < 0) return -1; return 1; } From dovecot at dovecot.org Thu Jan 24 16:30:49 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Jan 2013 16:30:49 +0200 Subject: dovecot-2.2: lib-storage: Fixed mailbox_exists() for LAYOUT=index. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6fa779e1d663 changeset: 15674:6fa779e1d663 user: Timo Sirainen date: Thu Jan 24 16:30:33 2013 +0200 description: lib-storage: Fixed mailbox_exists() for LAYOUT=index. diffstat: src/lib-storage/index/index-storage.c | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diffs (25 lines): diff -r 4673af53e7e6 -r 6fa779e1d663 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Tue Jan 22 12:59:32 2013 +0200 +++ b/src/lib-storage/index/index-storage.c Thu Jan 24 16:30:33 2013 +0200 @@ -163,13 +163,19 @@ enum mailbox_existence *existence_r) { struct stat st; + enum mail_error error; const char *path, *path2; int ret; /* see if it's selectable */ ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, &path); - if (ret < 0) - return -1; + if (ret < 0) { + mailbox_list_get_last_error(box->list, &error); + if (error != MAIL_ERROR_NOTFOUND) + return -1; + *existence_r = MAILBOX_EXISTENCE_NONE; + return 0; + } if (ret == 0) { /* no mailboxes in this storage? */ *existence_r = MAILBOX_EXISTENCE_NONE; From pigeonhole at rename-it.nl Sat Jan 26 10:10:33 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 26 Jan 2013 09:10:33 +0100 Subject: dovecot-2.1-pigeonhole: Changed behavior of redirect in case of ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/ab15c5eabc09 changeset: 1670:ab15c5eabc09 user: Stephan Bosch date: Sat Jan 26 08:52:27 2013 +0100 description: Changed behavior of redirect in case of a duplicate message delivery or a mail loop. If a duplicate is detected the implicit keep is canceled, as though the redirect was successful. This prevents getting local deliveries. The original SMTP recipient is used when it is available to augment the entry in the LDA duplicate database. This way, duplicates are only detected when (initially) addressed to the same recipient. The main goal of the duplicate detection is mail loop prevention, but this also has the effect that mere duplicate deliveries are handled specially whithout a good reason. We should fix this in a future version. diffstat: src/lib-sieve/cmd-redirect.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (24 lines): diff -r fe5f2738d538 -r ab15c5eabc09 src/lib-sieve/cmd-redirect.c --- a/src/lib-sieve/cmd-redirect.c Wed Dec 26 11:52:03 2012 +0100 +++ b/src/lib-sieve/cmd-redirect.c Sat Jan 26 08:52:27 2013 +0100 @@ -385,16 +385,18 @@ action->mail : sieve_message_get_mail(aenv->msgctx) ); const struct sieve_message_data *msgdata = aenv->msgdata; const struct sieve_script_env *senv = aenv->scriptenv; + const char *orig_recipient = sieve_message_get_orig_recipient(aenv->msgctx); const char *dupeid; /* Prevent mail loops if possible */ - dupeid = msgdata->id == NULL ? - NULL : t_strdup_printf("%s-%s", msgdata->id, ctx->to_address); + dupeid = msgdata->id == NULL ? NULL : t_strdup_printf + ("%s-%s-%s", msgdata->id, orig_recipient, ctx->to_address); if (dupeid != NULL) { /* Check whether we've seen this message before */ if (sieve_action_duplicate_check(senv, dupeid, strlen(dupeid))) { sieve_result_global_log(aenv, "discarded duplicate forward to <%s>", str_sanitize(ctx->to_address, 128)); + *keep = FALSE; return TRUE; } } From pigeonhole at rename-it.nl Sat Jan 26 10:10:33 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 26 Jan 2013 09:10:33 +0100 Subject: dovecot-2.1-pigeonhole: lib-sieve: Increased a few initial memor... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/f8fad363984d changeset: 1671:f8fad363984d user: Stephan Bosch date: Sat Jan 26 09:10:26 2013 +0100 description: lib-sieve: Increased a few initial memory pool sizes. diffstat: src/lib-sieve/plugins/editheader/ext-editheader-common.c | 2 +- src/lib-sieve/sieve-ast.c | 2 +- src/lib-sieve/sieve-binary.c | 2 +- src/lib-sieve/sieve-error.c | 4 ++-- src/lib-sieve/sieve-validator.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diffs (69 lines): diff -r ab15c5eabc09 -r f8fad363984d src/lib-sieve/plugins/editheader/ext-editheader-common.c --- a/src/lib-sieve/plugins/editheader/ext-editheader-common.c Sat Jan 26 08:52:27 2013 +0100 +++ b/src/lib-sieve/plugins/editheader/ext-editheader-common.c Sat Jan 26 09:10:26 2013 +0100 @@ -64,7 +64,7 @@ } T_BEGIN { - pool = pool_alloconly_create("editheader_config", 512); + pool = pool_alloconly_create("editheader_config", 1024); ext_config = p_new(pool, struct ext_editheader_config, 1); ext_config->pool = pool; ext_config->max_header_size = EXT_EDITHEADER_DEFAULT_MAX_HEADER_SIZE; diff -r ab15c5eabc09 -r f8fad363984d src/lib-sieve/sieve-ast.c --- a/src/lib-sieve/sieve-ast.c Sat Jan 26 08:52:27 2013 +0100 +++ b/src/lib-sieve/sieve-ast.c Sat Jan 26 09:10:26 2013 +0100 @@ -59,7 +59,7 @@ struct sieve_ast *ast; unsigned int ext_count; - pool = pool_alloconly_create("sieve_ast", 16384); + pool = pool_alloconly_create("sieve_ast", 32768); ast = p_new(pool, struct sieve_ast, 1); ast->pool = pool; ast->refcount = 1; diff -r ab15c5eabc09 -r f8fad363984d src/lib-sieve/sieve-binary.c --- a/src/lib-sieve/sieve-binary.c Sat Jan 26 08:52:27 2013 +0100 +++ b/src/lib-sieve/sieve-binary.c Sat Jan 26 09:10:26 2013 +0100 @@ -43,7 +43,7 @@ const struct sieve_extension *const *ext_preloaded; unsigned int i, ext_count; - pool = pool_alloconly_create("sieve_binary", 8192); + pool = pool_alloconly_create("sieve_binary", 16384); sbin = p_new(pool, struct sieve_binary, 1); sbin->pool = pool; sbin->refcount = 1; diff -r ab15c5eabc09 -r f8fad363984d src/lib-sieve/sieve-error.c --- a/src/lib-sieve/sieve-error.c Sat Jan 26 08:52:27 2013 +0100 +++ b/src/lib-sieve/sieve-error.c Sat Jan 26 09:10:26 2013 +0100 @@ -1221,7 +1221,7 @@ if ( parent == NULL ) return NULL; - pool = pool_alloconly_create("sieve_prefix_error_handler", 256); + pool = pool_alloconly_create("sieve_prefix_error_handler", 512); ehandler = p_new(pool, struct sieve_prefix_ehandler, 1); sieve_error_handler_init_from_parent(&ehandler->handler, pool, parent); @@ -1320,7 +1320,7 @@ return parent; } - pool = pool_alloconly_create("sieve_varexpand_error_handler", 1024); + pool = pool_alloconly_create("sieve_varexpand_error_handler", 2048); ehandler = p_new(pool, struct sieve_varexpand_ehandler, 1); sieve_error_handler_init_from_parent(&ehandler->handler, pool, parent); diff -r ab15c5eabc09 -r f8fad363984d src/lib-sieve/sieve-validator.c --- a/src/lib-sieve/sieve-validator.c Sat Jan 26 08:52:27 2013 +0100 +++ b/src/lib-sieve/sieve-validator.c Sat Jan 26 09:10:26 2013 +0100 @@ -150,7 +150,7 @@ const struct sieve_extension *const *ext_preloaded; unsigned int i, ext_count; - pool = pool_alloconly_create("sieve_validator", 8192); + pool = pool_alloconly_create("sieve_validator", 16384); valdtr = p_new(pool, struct sieve_validator, 1); valdtr->pool = pool; From dovecot at dovecot.org Mon Jan 28 13:53:11 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Jan 2013 13:53:11 +0200 Subject: dovecot-2.2: dsync: Fixed syncing locally when GUIDs weren't ava... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ceb75581cfcb changeset: 15675:ceb75581cfcb user: Timo Sirainen date: Mon Jan 28 13:48:09 2013 +0200 description: dsync: Fixed syncing locally when GUIDs weren't available. diffstat: src/doveadm/dsync/dsync-ibc-pipe.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 6fa779e1d663 -r ceb75581cfcb src/doveadm/dsync/dsync-ibc-pipe.c --- a/src/doveadm/dsync/dsync-ibc-pipe.c Thu Jan 24 16:30:33 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-pipe.c Mon Jan 28 13:48:09 2013 +0200 @@ -408,6 +408,7 @@ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL); item->u.mail.guid = p_strdup(item->pool, mail->guid); + item->u.mail.uid = mail->uid; item->u.mail.pop3_uidl = p_strdup(item->pool, mail->pop3_uidl); item->u.mail.pop3_order = mail->pop3_order; item->u.mail.received_date = mail->received_date; From dovecot at dovecot.org Mon Jan 28 13:56:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Jan 2013 13:56:30 +0200 Subject: dovecot-2.2: dsync: When syncing locally, mailbox_copy() the mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d3cda5a567be changeset: 15676:d3cda5a567be user: Timo Sirainen date: Mon Jan 28 13:56:24 2013 +0200 description: dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. diffstat: src/doveadm/dsync/dsync-ibc-pipe.c | 2 ++ src/doveadm/dsync/dsync-mail.h | 6 ++++++ src/doveadm/dsync/dsync-mailbox-export.c | 2 ++ src/doveadm/dsync/dsync-mailbox-import.c | 11 +++++++++++ 4 files changed, 21 insertions(+), 0 deletions(-) diffs (61 lines): diff -r ceb75581cfcb -r d3cda5a567be src/doveadm/dsync/dsync-ibc-pipe.c --- a/src/doveadm/dsync/dsync-ibc-pipe.c Mon Jan 28 13:48:09 2013 +0200 +++ b/src/doveadm/dsync/dsync-ibc-pipe.c Mon Jan 28 13:56:24 2013 +0200 @@ -416,6 +416,8 @@ item->u.mail.input = mail->input; i_stream_ref(mail->input); } + item->u.mail.input_mail = mail->input_mail; + item->u.mail.input_mail_uid = mail->input_mail_uid; } static enum dsync_ibc_recv_ret diff -r ceb75581cfcb -r d3cda5a567be src/doveadm/dsync/dsync-mail.h --- a/src/doveadm/dsync/dsync-mail.h Mon Jan 28 13:48:09 2013 +0200 +++ b/src/doveadm/dsync/dsync-mail.h Mon Jan 28 13:56:24 2013 +0200 @@ -17,6 +17,12 @@ /* Input stream containing the message text, or NULL if all instances of the message were already expunged from this mailbox. */ struct istream *input; + + /* If non-NULL, we're syncing within the dsync process using ibc-pipe. + This mail can be used to mailbox_copy() the mail. */ + struct mail *input_mail; + /* Verify that this equals to input_mail->uid */ + uint32_t input_mail_uid; }; struct dsync_mail_request { diff -r ceb75581cfcb -r d3cda5a567be src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Mon Jan 28 13:48:09 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Mon Jan 28 13:56:24 2013 +0200 @@ -581,6 +581,8 @@ return -1; } + dmail->input_mail = mail; + dmail->input_mail_uid = mail->uid; if (mail_get_stream(mail, NULL, NULL, &dmail->input) < 0) return dsync_mail_error(exporter, mail, "body"); diff -r ceb75581cfcb -r d3cda5a567be src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 28 13:48:09 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 28 13:56:24 2013 +0200 @@ -1317,6 +1317,17 @@ if (save_ctx == NULL) save_ctx = dsync_mailbox_save_init(importer, mail, newmail); } + if (ret <= 0 && mail->input_mail != NULL) { + /* copy using the source mail */ + i_assert(mail->input_mail->uid == mail->input_mail_uid); + if (mailbox_copy(&save_ctx, mail->input_mail) == 0) + ret = 1; + else { + ret = -1; + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + } + + } if (ret > 0) { array_append(&importer->wanted_uids, &newmail->uid, 1); return; From dovecot at dovecot.org Mon Jan 28 14:38:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Jan 2013 14:38:13 +0200 Subject: dovecot-2.2: lib-storage: Allow mail_storage_copy_error() with t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6724d4ae26ec changeset: 15677:6724d4ae26ec user: Timo Sirainen date: Mon Jan 28 14:37:59 2013 +0200 description: lib-storage: Allow mail_storage_copy_error() with the same source and dest. If it was used without checking it would try to free the error string and after that strdup() it. diffstat: src/lib-storage/mail-storage.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r d3cda5a567be -r 6724d4ae26ec src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Mon Jan 28 13:56:24 2013 +0200 +++ b/src/lib-storage/mail-storage.c Mon Jan 28 14:37:59 2013 +0200 @@ -492,6 +492,9 @@ const char *str; enum mail_error error; + if (src == dest) + return; + str = mail_storage_get_last_error(src, &error); mail_storage_set_error(dest, error, str); } From dovecot at dovecot.org Tue Jan 29 13:31:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Jan 2013 13:31:23 +0200 Subject: dovecot-2.2: README: Added RFC 6851 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/449c56aabbb6 changeset: 15678:449c56aabbb6 user: Timo Sirainen date: Tue Jan 29 13:31:05 2013 +0200 description: README: Added RFC 6851 diffstat: README | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 6724d4ae26ec -r 449c56aabbb6 README --- a/README Mon Jan 28 14:37:59 2013 +0200 +++ b/README Tue Jan 29 13:31:05 2013 +0200 @@ -62,6 +62,7 @@ 5957 - Display-Based Address Sorting for the IMAP4 SORT Extension 6154 - IMAP LIST Extension for Special-Use Mailboxes 6203 - IMAP4 Extension for Fuzzy Search + 6851 - Internet Message Access Protocol (IMAP) - MOVE Extension Contact info ------------ From dovecot at dovecot.org Tue Jan 29 17:55:26 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Jan 2013 17:55:26 +0200 Subject: dovecot-2.1: lib-index: Fixed invalidating buffered cache file w... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/0b0399f1b6aa changeset: 14885:0b0399f1b6aa user: Timo Sirainen date: Tue Jan 29 17:55:20 2013 +0200 description: lib-index: Fixed invalidating buffered cache file with map_with_read. diffstat: src/lib-index/mail-cache-fields.c | 2 ++ src/lib-index/mail-cache-sync-update.c | 5 +++++ src/lib-index/mail-cache.c | 2 ++ 3 files changed, 9 insertions(+), 0 deletions(-) diffs (51 lines): diff -r 99b7be36631b -r 0b0399f1b6aa src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Tue Jan 22 16:53:52 2013 +0200 +++ b/src/lib-index/mail-cache-fields.c Tue Jan 29 17:55:20 2013 +0200 @@ -285,6 +285,8 @@ file_cache_invalidate(cache->file_cache, offset, field_hdr_size); } + if (cache->read_buf != NULL && invalidate) + buffer_set_used_size(cache->read_buf, 0); ret = mail_cache_map(cache, offset, field_hdr_size, &data); if (ret < 0) return -1; diff -r 99b7be36631b -r 0b0399f1b6aa src/lib-index/mail-cache-sync-update.c --- a/src/lib-index/mail-cache-sync-update.c Tue Jan 22 16:53:52 2013 +0200 +++ b/src/lib-index/mail-cache-sync-update.c Tue Jan 29 17:55:20 2013 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "buffer.h" #include "file-cache.h" #include "mail-cache-private.h" #include "mail-index-view-private.h" @@ -144,6 +145,8 @@ ctx->invalidate_highwater - *new_cache_offset); ctx->invalidate_highwater = *new_cache_offset; + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); } } @@ -188,4 +191,6 @@ cache->fd, cache->locked); } file_cache_invalidate(cache->file_cache, 0, (uoff_t)-1); + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); } diff -r 99b7be36631b -r 0b0399f1b6aa src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Jan 22 16:53:52 2013 +0200 +++ b/src/lib-index/mail-cache.c Tue Jan 29 17:55:20 2013 +0200 @@ -705,6 +705,8 @@ file_cache_invalidate(cache->file_cache, 0, sizeof(struct mail_cache_header)); } + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); if (mail_cache_map(cache, 0, 0, &data) > 0) cache->hdr_copy = *cache->hdr; else { From dovecot at dovecot.org Wed Jan 30 18:12:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 18:12:30 +0200 Subject: dovecot-2.2: auth: Code cleanup - removed auth_stream_split() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0897fc01fb93 changeset: 15679:0897fc01fb93 user: Timo Sirainen date: Wed Jan 30 18:12:23 2013 +0200 description: auth: Code cleanup - removed auth_stream_split() diffstat: src/auth/auth-request.c | 9 ++------- src/auth/auth-stream.c | 5 ----- src/auth/auth-stream.h | 1 - 3 files changed, 2 insertions(+), 13 deletions(-) diffs (50 lines): diff -r 449c56aabbb6 -r 0897fc01fb93 src/auth/auth-request.c --- a/src/auth/auth-request.c Tue Jan 29 13:31:05 2013 +0200 +++ b/src/auth/auth-request.c Wed Jan 30 18:12:23 2013 +0200 @@ -1454,18 +1454,13 @@ static bool auth_request_proxy_is_self(struct auth_request *request) { - const char *const *tmp, *port = NULL; + const char *port = NULL; if (!request->proxy_host_is_self) return FALSE; /* check if the port is the same */ - tmp = auth_stream_split(request->extra_fields); - for (; *tmp != NULL; tmp++) { - if (strncmp(*tmp, "port=", 5) == 0) - port = *tmp + 5; - } - + port = auth_stream_reply_find(request->extra_fields, "port"); if (port != NULL && !str_uint_equals(port, request->local_port)) return FALSE; /* don't check destuser. in some systems destuser is intentionally diff -r 449c56aabbb6 -r 0897fc01fb93 src/auth/auth-stream.c --- a/src/auth/auth-stream.c Tue Jan 29 13:31:05 2013 +0200 +++ b/src/auth/auth-stream.c Wed Jan 30 18:12:23 2013 +0200 @@ -138,11 +138,6 @@ return reply == NULL || str_len(reply->str) == 0; } -const char *const *auth_stream_split(struct auth_stream_reply *reply) -{ - return t_strsplit_tab(str_c(reply->str)); -} - string_t *auth_stream_reply_get_str(struct auth_stream_reply *reply) { return reply->str; diff -r 449c56aabbb6 -r 0897fc01fb93 src/auth/auth-stream.h --- a/src/auth/auth-stream.h Tue Jan 29 13:31:05 2013 +0200 +++ b/src/auth/auth-stream.h Wed Jan 30 18:12:23 2013 +0200 @@ -17,7 +17,6 @@ const char *auth_stream_reply_export(struct auth_stream_reply *reply); bool auth_stream_is_empty(struct auth_stream_reply *reply); -const char *const *auth_stream_split(struct auth_stream_reply *reply); string_t *auth_stream_reply_get_str(struct auth_stream_reply *reply); #endif From dovecot at dovecot.org Wed Jan 30 19:08:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 19:08:08 +0200 Subject: dovecot-2.2: auth: Code cleanup: Removed unnecessary auth_stream... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/55d20120b348 changeset: 15680:55d20120b348 user: Timo Sirainen date: Wed Jan 30 18:35:02 2013 +0200 description: auth: Code cleanup: Removed unnecessary auth_stream_reply usage from auth-worker communication. diffstat: src/auth/auth-request.c | 52 +++++++++++++++------------ src/auth/auth-request.h | 3 +- src/auth/auth-stream.c | 6 +++ src/auth/auth-stream.h | 2 + src/auth/auth-worker-client.c | 81 +++++++++++++++++------------------------- src/auth/auth-worker-server.c | 10 ++-- src/auth/auth-worker-server.h | 2 +- src/auth/passdb-blocking.c | 44 ++++++++++++----------- src/auth/userdb-blocking.c | 23 +++++------ 9 files changed, 110 insertions(+), 113 deletions(-) diffs (truncated from 456 to 300 lines): diff -r 0897fc01fb93 -r 55d20120b348 src/auth/auth-request.c --- a/src/auth/auth-request.c Wed Jan 30 18:12:23 2013 +0200 +++ b/src/auth/auth-request.c Wed Jan 30 18:35:02 2013 +0200 @@ -187,49 +187,55 @@ pool_unref(&request->pool); } -void auth_request_export(struct auth_request *request, - struct auth_stream_reply *reply) +static void +auth_str_add_keyvalue(string_t *dest, const char *key, const char *value) { - auth_stream_reply_add(reply, "user", request->user); - auth_stream_reply_add(reply, "service", request->service); + str_append_c(dest, '\t'); + str_append(dest, key); + str_append_c(dest, '='); + str_append_tabescaped(dest, value); +} + +void auth_request_export(struct auth_request *request, string_t *dest) +{ + str_append(dest, "user="); + str_append_tabescaped(dest, request->user); + + auth_str_add_keyvalue(dest, "service", request->service); if (request->master_user != NULL) { - auth_stream_reply_add(reply, "master_user", + auth_str_add_keyvalue(dest, "master_user", request->master_user); } - auth_stream_reply_add(reply, "original_username", + auth_str_add_keyvalue(dest, "original_username", request->original_username); - auth_stream_reply_add(reply, "requested_login_user", + auth_str_add_keyvalue(dest, "requested_login_user", request->requested_login_user); if (request->local_ip.family != 0) { - auth_stream_reply_add(reply, "lip", + auth_str_add_keyvalue(dest, "lip", net_ip2addr(&request->local_ip)); } if (request->remote_ip.family != 0) { - auth_stream_reply_add(reply, "rip", + auth_str_add_keyvalue(dest, "rip", net_ip2addr(&request->remote_ip)); } - if (request->local_port != 0) { - auth_stream_reply_add(reply, "lport", - dec2str(request->local_port)); - } - if (request->remote_port != 0) { - auth_stream_reply_add(reply, "rport", - dec2str(request->remote_port)); - } + if (request->local_port != 0) + str_printfa(dest, "\tlport=%u", request->local_port); + if (request->remote_port != 0) + str_printfa(dest, "\trport=%u", request->remote_port); if (request->secured) - auth_stream_reply_add(reply, "secured", "1"); + str_append(dest, "\tsecured"); if (request->skip_password_check) - auth_stream_reply_add(reply, "skip_password_check", "1"); + str_append(dest, "\tskip_password_check"); if (request->valid_client_cert) - auth_stream_reply_add(reply, "valid-client-cert", "1"); + str_append(dest, "\tvalid-client-cert"); if (request->no_penalty) - auth_stream_reply_add(reply, "no-penalty", "1"); + str_append(dest, "\tno-penalty"); if (request->successful) - auth_stream_reply_add(reply, "successful", "1"); + str_append(dest, "\tsuccessful"); if (request->mech_name != NULL) - auth_stream_reply_add(reply, "mech", request->mech_name); + auth_str_add_keyvalue(dest, "mech", request->mech_name); } bool auth_request_import_info(struct auth_request *request, diff -r 0897fc01fb93 -r 55d20120b348 src/auth/auth-request.h --- a/src/auth/auth-request.h Wed Jan 30 18:12:23 2013 +0200 +++ b/src/auth/auth-request.h Wed Jan 30 18:35:02 2013 +0200 @@ -151,8 +151,7 @@ void auth_request_fail(struct auth_request *request); void auth_request_internal_failure(struct auth_request *request); -void auth_request_export(struct auth_request *request, - struct auth_stream_reply *reply); +void auth_request_export(struct auth_request *request, string_t *dest); bool auth_request_import(struct auth_request *request, const char *key, const char *value); bool auth_request_import_info(struct auth_request *request, diff -r 0897fc01fb93 -r 55d20120b348 src/auth/auth-stream.c --- a/src/auth/auth-stream.c Wed Jan 30 18:12:23 2013 +0200 +++ b/src/auth/auth-stream.c Wed Jan 30 18:35:02 2013 +0200 @@ -133,6 +133,12 @@ return str_c(reply->str); } +void auth_stream_reply_append(struct auth_stream_reply *reply, + string_t *dest) +{ + str_append_str(dest, reply->str); +} + bool auth_stream_is_empty(struct auth_stream_reply *reply) { return reply == NULL || str_len(reply->str) == 0; diff -r 0897fc01fb93 -r 55d20120b348 src/auth/auth-stream.h --- a/src/auth/auth-stream.h Wed Jan 30 18:12:23 2013 +0200 +++ b/src/auth/auth-stream.h Wed Jan 30 18:35:02 2013 +0200 @@ -15,6 +15,8 @@ void auth_stream_reply_import(struct auth_stream_reply *reply, const char *str); const char *auth_stream_reply_export(struct auth_stream_reply *reply); +void auth_stream_reply_append(struct auth_stream_reply *reply, + string_t *dest); bool auth_stream_is_empty(struct auth_stream_reply *reply); string_t *auth_stream_reply_get_str(struct auth_stream_reply *reply); diff -r 0897fc01fb93 -r 55d20120b348 src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Wed Jan 30 18:12:23 2013 +0200 +++ b/src/auth/auth-worker-client.c Wed Jan 30 18:35:02 2013 +0200 @@ -8,6 +8,7 @@ #include "ostream.h" #include "hex-binary.h" #include "str.h" +#include "strescape.h" #include "process-title.h" #include "master-service.h" #include "auth-request.h" @@ -110,39 +111,33 @@ struct auth_request *request) { struct auth_worker_client *client = request->context; - struct auth_stream_reply *reply; string_t *str; if (request->passdb_failure && result == PASSDB_RESULT_OK) result = PASSDB_RESULT_PASSWORD_MISMATCH; - reply = auth_stream_reply_init(pool_datastack_create()); - auth_stream_reply_add(reply, NULL, dec2str(request->id)); + str = t_str_new(128); + str_printfa(str, "%u\t", request->id); if (result == PASSDB_RESULT_OK) - auth_stream_reply_add(reply, "OK", NULL); - else { - auth_stream_reply_add(reply, "FAIL", NULL); - auth_stream_reply_add(reply, NULL, - t_strdup_printf("%d", result)); - } + str_append(str, "OK"); + else + str_printfa(str, "FAIL\t%d", result); if (result != PASSDB_RESULT_INTERNAL_FAILURE) { - auth_stream_reply_add(reply, NULL, request->user); - auth_stream_reply_add(reply, NULL, - request->passdb_password == NULL ? "" : - request->passdb_password); - if (request->extra_fields != NULL) { - const char *fields = - auth_stream_reply_export(request->extra_fields); - auth_stream_reply_import(reply, fields); + str_append_c(str, '\t'); + str_append_tabescaped(str, request->user); + str_append_c(str, '\t'); + if (request->passdb_password != NULL) + str_append_tabescaped(str, request->passdb_password); + if (!auth_stream_is_empty(request->extra_fields)) { + str_append_c(str, '\t'); + auth_stream_reply_append(request->extra_fields, str); } - if (request->extra_cache_fields != NULL) { - const char *fields = - auth_stream_reply_export(request->extra_cache_fields); - auth_stream_reply_import(reply, fields); + if (!auth_stream_is_empty(request->extra_cache_fields)) { + str_append_c(str, '\t'); + auth_stream_reply_append(request->extra_cache_fields, str); } } - str = auth_stream_reply_get_str(reply); str_append_c(str, '\n'); auth_worker_send_reply(client, str); @@ -207,40 +202,31 @@ struct auth_request *request) { struct auth_worker_client *client = request->context; - struct auth_stream_reply *reply; string_t *str; if (request->passdb_failure && result == PASSDB_RESULT_OK) result = PASSDB_RESULT_PASSWORD_MISMATCH; - reply = auth_stream_reply_init(pool_datastack_create()); - auth_stream_reply_add(reply, NULL, dec2str(request->id)); + str = t_str_new(128); + str_printfa(str, "%u\t", request->id); - if (result != PASSDB_RESULT_OK) { - auth_stream_reply_add(reply, "FAIL", NULL); - auth_stream_reply_add(reply, NULL, - t_strdup_printf("%d", result)); - } else { - auth_stream_reply_add(reply, "OK", NULL); - auth_stream_reply_add(reply, NULL, request->user); + if (result != PASSDB_RESULT_OK) + str_printfa(str, "FAIL\t%d", result); + else { + str_append(str, "OK\t"); + str_append_tabescaped(str, request->user); + str_printfa(str, "\t{%s.b64}", request->credentials_scheme); + base64_encode(credentials, size, str); - str = t_str_new(64); - str_printfa(str, "{%s.b64}", request->credentials_scheme); - base64_encode(credentials, size, str); - auth_stream_reply_add(reply, NULL, str_c(str)); - - if (request->extra_fields != NULL) { - const char *fields = - auth_stream_reply_export(request->extra_fields); - auth_stream_reply_import(reply, fields); + if (!auth_stream_is_empty(request->extra_fields)) { + str_append_c(str, '\t'); + auth_stream_reply_append(request->extra_fields, str); } - if (request->extra_cache_fields != NULL) { - const char *fields = - auth_stream_reply_export(request->extra_cache_fields); - auth_stream_reply_import(reply, fields); + if (!auth_stream_is_empty(request->extra_cache_fields)) { + str_append_c(str, '\t'); + auth_stream_reply_append(request->extra_cache_fields, str); } } - str = auth_stream_reply_get_str(reply); str_append_c(str, '\n'); auth_worker_send_reply(client, str); @@ -352,7 +338,6 @@ struct auth_request *auth_request) { struct auth_worker_client *client = auth_request->context; - struct auth_stream_reply *reply = auth_request->userdb_reply; string_t *str; str = t_str_new(128); @@ -366,7 +351,7 @@ break; case USERDB_RESULT_OK: str_append(str, "OK\t"); - str_append(str, auth_stream_reply_export(reply)); + auth_stream_reply_append(auth_request->userdb_reply, str); if (auth_request->userdb_lookup_failed) str_append(str, "\ttempfail"); break; diff -r 0897fc01fb93 -r 55d20120b348 src/auth/auth-worker-server.c --- a/src/auth/auth-worker-server.c Wed Jan 30 18:12:23 2013 +0200 +++ b/src/auth/auth-worker-server.c Wed Jan 30 18:35:02 2013 +0200 @@ -25,7 +25,7 @@ struct auth_worker_request { unsigned int id; time_t created; - const char *data_str; + const char *data; auth_worker_callback_t *callback; void *context; }; @@ -96,8 +96,8 @@ iov[0].iov_base = t_strdup_printf("%d\t", request->id); iov[0].iov_len = strlen(iov[0].iov_base); - iov[1].iov_base = request->data_str; - iov[1].iov_len = strlen(request->data_str); + iov[1].iov_base = request->data; + iov[1].iov_len = strlen(request->data); iov[2].iov_base = "\n"; iov[2].iov_len = 1; @@ -387,7 +387,7 @@ } struct auth_worker_connection * -auth_worker_call(pool_t pool, struct auth_stream_reply *data, +auth_worker_call(pool_t pool, const char *data, auth_worker_callback_t *callback, void *context) { struct auth_worker_connection *conn; @@ -395,7 +395,7 @@ request = p_new(pool, struct auth_worker_request, 1); request->created = ioloop_time; - request->data_str = p_strdup(pool, auth_stream_reply_export(data)); From dovecot at dovecot.org Wed Jan 30 19:08:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 19:08:08 +0200 Subject: dovecot-2.2: auth: Code cleanup: Removed unnecessary userdb_ pre... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3fac9306be3e changeset: 15681:3fac9306be3e user: Timo Sirainen date: Wed Jan 30 18:40:49 2013 +0200 description: auth: Code cleanup: Removed unnecessary userdb_ prefix checks. The userdb_ fields are nowadays placed to userdb_reply immediately. diffstat: src/auth/auth-request-handler.c | 24 ++++-------------------- src/auth/auth-request.h | 3 +-- 2 files changed, 5 insertions(+), 22 deletions(-) diffs (55 lines): diff -r 55d20120b348 -r 3fac9306be3e src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Jan 30 18:35:02 2013 +0200 +++ b/src/auth/auth-request-handler.c Wed Jan 30 18:40:49 2013 +0200 @@ -156,34 +156,18 @@ static void get_client_extra_fields(struct auth_request *request, struct auth_stream_reply *reply) { - const char **fields, *extra_fields; - unsigned int src; - bool seen_pass = FALSE; + const char *extra_fields; if (auth_stream_is_empty(request->extra_fields)) return; extra_fields = auth_stream_reply_export(request->extra_fields); - - if (!request->proxy && strstr(extra_fields, "userdb_") == NULL) { - /* optimization: there are no userdb_* fields, we can just - import */ - auth_stream_reply_import(reply, extra_fields); - return; - } - - fields = t_strsplit_tab(extra_fields); - for (src = 0; fields[src] != NULL; src++) { - if (strncmp(fields[src], "userdb_", 7) != 0) { - if (!seen_pass && strncmp(fields[src], "pass=", 5) == 0) - seen_pass = TRUE; - auth_stream_reply_import(reply, fields[src]); - } - } + auth_stream_reply_import(reply, extra_fields); if (request->proxy && !request->auth_only) { /* we're proxying */ - if (!seen_pass && request->mech_password != NULL) { + if (!auth_stream_reply_exists(reply, "pass") && + request->mech_password != NULL) { /* send back the password that was sent by user (not the password in passdb). */ auth_stream_reply_add(reply, "pass", diff -r 55d20120b348 -r 3fac9306be3e src/auth/auth-request.h --- a/src/auth/auth-request.h Wed Jan 30 18:35:02 2013 +0200 +++ b/src/auth/auth-request.h Wed Jan 30 18:40:49 2013 +0200 @@ -48,8 +48,7 @@ char *mech_password; /* set if verify_plain() is called */ char *passdb_password; /* set after password lookup if successful */ /* extra_fields are returned in authentication reply. Fields prefixed - with "userdb_" are skipped. If prefetch userdb is used, it uses - the "userdb_" prefixed fields. */ + with "userdb_" are automatically placed to userdb_reply instead. */ struct auth_stream_reply *extra_fields; /* extra_fields that aren't supposed to be sent to the client, but are supposed to be stored to auth cache. */ From dovecot at dovecot.org Wed Jan 30 19:08:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 19:08:08 +0200 Subject: dovecot-2.2: auth: Code cleanup: Avoid using auth_stream_reply a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/526aa986d534 changeset: 15682:526aa986d534 user: Timo Sirainen date: Wed Jan 30 19:08:00 2013 +0200 description: auth: Code cleanup: Avoid using auth_stream_reply as temporary strings. diffstat: src/auth/auth-client-connection.c | 7 +- src/auth/auth-master-connection.c | 23 ++--- src/auth/auth-master-connection.h | 3 +- src/auth/auth-request-handler.c | 151 ++++++++++++++++--------------------- src/auth/auth-request-handler.h | 4 +- src/auth/auth-request.c | 19 +--- src/auth/auth-request.h | 1 + 7 files changed, 86 insertions(+), 122 deletions(-) diffs (truncated from 540 to 300 lines): diff -r 3fac9306be3e -r 526aa986d534 src/auth/auth-client-connection.c --- a/src/auth/auth-client-connection.c Wed Jan 30 18:40:49 2013 +0200 +++ b/src/auth/auth-client-connection.c Wed Jan 30 19:08:00 2013 +0200 @@ -71,16 +71,15 @@ } } -static void auth_callback(struct auth_stream_reply *reply, +static void auth_callback(const char *reply, struct auth_client_connection *conn) { if (reply == NULL) { /* handler destroyed */ auth_client_connection_unref(&conn); - return; + } else { + auth_client_send(conn, reply); } - - auth_client_send(conn, auth_stream_reply_export(reply)); } static bool diff -r 3fac9306be3e -r 526aa986d534 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Jan 30 18:40:49 2013 +0200 +++ b/src/auth/auth-master-connection.c Wed Jan 30 19:08:00 2013 +0200 @@ -69,22 +69,18 @@ return t_strarray_join((void *)args, "\t"); } -void auth_master_request_callback(struct auth_stream_reply *reply, - void *context) +void auth_master_request_callback(const char *reply, void *context) { struct auth_master_connection *conn = context; struct const_iovec iov[2]; - const char *reply_str; - - reply_str = auth_stream_reply_export(reply); if (conn->auth->set->debug) { i_debug("master userdb out: %s", - auth_master_reply_hide_passwords(conn, reply_str)); + auth_master_reply_hide_passwords(conn, reply)); } - iov[0].iov_base = reply_str; - iov[0].iov_len = strlen(reply_str); + iov[0].iov_base = reply; + iov[0].iov_len = strlen(reply); iov[1].iov_base = "\n"; iov[1].iov_len = 1; @@ -250,7 +246,6 @@ struct auth_request *auth_request) { struct auth_master_connection *conn = auth_request->master; - struct auth_stream_reply *reply = auth_request->userdb_reply; string_t *str; const char *value; @@ -267,7 +262,8 @@ case USERDB_RESULT_INTERNAL_FAILURE: str_printfa(str, "FAIL\t%u", auth_request->id); if (auth_request->userdb_lookup_failed) { - value = auth_stream_reply_find(reply, "reason"); + value = auth_stream_reply_find(auth_request->userdb_reply, + "reason"); if (value != NULL) str_printfa(str, "\treason=%s", value); } @@ -279,7 +275,7 @@ str_printfa(str, "USER\t%u\t", auth_request->id); str_append_tabescaped(str, auth_request->user); str_append_c(str, '\t'); - str_append(str, auth_stream_reply_export(reply)); + auth_stream_reply_append(auth_request->userdb_reply, str); break; } @@ -320,7 +316,6 @@ enum passdb_result result) { struct auth_master_connection *conn = auth_request->master; - struct auth_stream_reply *reply = auth_request->extra_fields; string_t *str; str = t_str_new(128); @@ -328,9 +323,9 @@ case PASSDB_RESULT_OK: str_printfa(str, "PASS\t%u\tuser=", auth_request->id); str_append_tabescaped(str, auth_request->user); - if (reply != NULL) { + if (!auth_stream_is_empty(auth_request->extra_fields)) { str_append_c(str, '\t'); - str_append(str, auth_stream_reply_export(reply)); + auth_stream_reply_append(auth_request->extra_fields, str); } break; case PASSDB_RESULT_USER_UNKNOWN: diff -r 3fac9306be3e -r 526aa986d534 src/auth/auth-master-connection.h --- a/src/auth/auth-master-connection.h Wed Jan 30 18:40:49 2013 +0200 +++ b/src/auth/auth-master-connection.h Wed Jan 30 19:08:00 2013 +0200 @@ -34,8 +34,7 @@ void auth_master_connection_ref(struct auth_master_connection *conn); void auth_master_connection_unref(struct auth_master_connection **conn); -void auth_master_request_callback(struct auth_stream_reply *reply, - void *context); +void auth_master_request_callback(const char *reply, void *context); void auth_master_connections_destroy_all(void); diff -r 3fac9306be3e -r 526aa986d534 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Jan 30 18:40:49 2013 +0200 +++ b/src/auth/auth-request-handler.c Wed Jan 30 19:08:00 2013 +0200 @@ -8,6 +8,7 @@ #include "hash.h" #include "net.h" #include "str.h" +#include "strescape.h" #include "str-sanitize.h" #include "master-interface.h" #include "auth-penalty.h" @@ -153,38 +154,44 @@ auth_request_unref(&request); } -static void get_client_extra_fields(struct auth_request *request, - struct auth_stream_reply *reply) +static void +auth_str_add_keyvalue(string_t *dest, const char *key, const char *value) { - const char *extra_fields; + str_append_c(dest, '\t'); + str_append(dest, key); + str_append_c(dest, '='); + str_append_tabescaped(dest, value); +} +static void +auth_str_append_extra_fields(struct auth_request *request, string_t *dest) +{ if (auth_stream_is_empty(request->extra_fields)) return; - extra_fields = auth_stream_reply_export(request->extra_fields); - auth_stream_reply_import(reply, extra_fields); + str_append_c(dest, '\t'); + auth_stream_reply_append(request->extra_fields, dest); if (request->proxy && !request->auth_only) { /* we're proxying */ - if (!auth_stream_reply_exists(reply, "pass") && + if (!auth_stream_reply_exists(request->extra_fields, "pass") && request->mech_password != NULL) { /* send back the password that was sent by user (not the password in passdb). */ - auth_stream_reply_add(reply, "pass", + auth_str_add_keyvalue(dest, "pass", request->mech_password); } if (request->master_user != NULL && - auth_stream_reply_find(reply, "master") == NULL) { + !auth_stream_reply_exists(request->extra_fields, "master")) { /* the master username needs to be forwarded */ - auth_stream_reply_add(reply, "master", + auth_str_add_keyvalue(dest, "master", request->master_user); } } } static void -auth_request_handle_failure(struct auth_request *request, - struct auth_stream_reply *reply) +auth_request_handle_failure(struct auth_request *request, const char *reply) { struct auth_request_handler *handler = request->handler; @@ -228,52 +235,47 @@ auth_request_handler_reply_success_finish(struct auth_request *request) { struct auth_request_handler *handler = request->handler; - struct auth_stream_reply *reply; - - reply = auth_stream_reply_init(pool_datastack_create()); + string_t *str = t_str_new(128); if (request->last_penalty != 0 && auth_penalty != NULL) { /* reset penalty */ auth_penalty_update(auth_penalty, request, 0); } - auth_stream_reply_add(reply, "OK", NULL); - auth_stream_reply_add(reply, NULL, dec2str(request->id)); - auth_stream_reply_add(reply, "user", request->user); - get_client_extra_fields(request, reply); + str_printfa(str, "OK\t%u\tuser=", request->id); + str_append_tabescaped(str, request->user); + auth_str_append_extra_fields(request, str); if (request->no_login || handler->master_callback == NULL) { /* this request doesn't have to wait for master process to pick it up. delete it */ auth_request_handler_remove(handler, request); } - handler->callback(reply, handler->context); + handler->callback(str_c(str), handler->context); } static void auth_request_handler_reply_failure_finish(struct auth_request *request) { - struct auth_stream_reply *reply; + string_t *str = t_str_new(128); - reply = auth_stream_reply_init(pool_datastack_create()); - auth_stream_reply_add(reply, "FAIL", NULL); - auth_stream_reply_add(reply, NULL, dec2str(request->id)); + str_printfa(str, "FAIL\t%u", request->id); if (request->user != NULL) - auth_stream_reply_add(reply, "user", request->user); + auth_str_add_keyvalue(str, "user", request->user); else if (request->original_username != NULL) { - auth_stream_reply_add(reply, "user", + auth_str_add_keyvalue(str, "user", request->original_username); } if (request->internal_failure) - auth_stream_reply_add(reply, "temp", NULL); + str_append(str, "\ttemp"); else if (request->master_user != NULL) { /* authentication succeeded, but we can't log in as the wanted user */ - auth_stream_reply_add(reply, "authz", NULL); + str_append(str, "\tauthz"); } if (request->no_failure_delay) - auth_stream_reply_add(reply, "nodelay", NULL); - get_client_extra_fields(request, reply); + str_append(str, "\tnodelay"); + auth_str_append_extra_fields(request, str); switch (request->passdb_result) { case PASSDB_RESULT_INTERNAL_FAILURE: @@ -283,14 +285,14 @@ case PASSDB_RESULT_OK: break; case PASSDB_RESULT_USER_DISABLED: - auth_stream_reply_add(reply, "user_disabled", NULL); + str_append(str, "\tuser_disabled"); break; case PASSDB_RESULT_PASS_EXPIRED: - auth_stream_reply_add(reply, "pass_expired", NULL); + str_append(str, "\tpass_expired"); break; } - auth_request_handle_failure(request, reply); + auth_request_handle_failure(request, str_c(str)); } static void @@ -310,7 +312,6 @@ const void *auth_reply, size_t reply_size) { struct auth_request_handler *handler = request->handler; - struct auth_stream_reply *reply; string_t *str; int ret; @@ -326,16 +327,12 @@ switch (result) { case AUTH_CLIENT_RESULT_CONTINUE: - reply = auth_stream_reply_init(pool_datastack_create()); - auth_stream_reply_add(reply, "CONT", NULL); - auth_stream_reply_add(reply, NULL, dec2str(request->id)); - - str = t_str_new(MAX_BASE64_ENCODED_SIZE(reply_size)); + str = t_str_new(16 + MAX_BASE64_ENCODED_SIZE(reply_size)); + str_printfa(str, "CONT\t%u\t", request->id); base64_encode(auth_reply, reply_size, str); - auth_stream_reply_add(reply, NULL, str_c(str)); request->accept_input = TRUE; - handler->callback(reply, handler->context); + handler->callback(str_c(str), handler->context); break; case AUTH_CLIENT_RESULT_SUCCESS: if (reply_size > 0) { @@ -373,16 +370,14 @@ struct auth_request *request, const char *reason) { - struct auth_stream_reply *reply; + string_t *str = t_str_new(128); auth_request_log_info(request, request->mech->mech_name, "%s", reason); From dovecot at dovecot.org Wed Jan 30 19:11:42 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 19:11:42 +0200 Subject: dovecot-2.2: auth: Code cleanup: Minor auth_stream API simplific... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f11ae64365b5 changeset: 15683:f11ae64365b5 user: Timo Sirainen date: Wed Jan 30 19:11:30 2013 +0200 description: auth: Code cleanup: Minor auth_stream API simplifications. diffstat: src/auth/auth-stream.c | 19 ++++++------------- src/auth/auth-stream.h | 4 +--- 2 files changed, 7 insertions(+), 16 deletions(-) diffs (56 lines): diff -r 526aa986d534 -r f11ae64365b5 src/auth/auth-stream.c --- a/src/auth/auth-stream.c Wed Jan 30 19:08:00 2013 +0200 +++ b/src/auth/auth-stream.c Wed Jan 30 19:11:30 2013 +0200 @@ -23,18 +23,16 @@ void auth_stream_reply_add(struct auth_stream_reply *reply, const char *key, const char *value) { + i_assert(*key != '\0'); + i_assert(strchr(key, '\t') == NULL && + strchr(key, '\n') == NULL); + if (str_len(reply->str) > 0) str_append_c(reply->str, '\t'); - if (key != NULL) { - i_assert(*key != '\0'); - i_assert(strchr(key, '\t') == NULL && - strchr(key, '\n') == NULL); - str_append(reply->str, key); - if (value != NULL) - str_append_c(reply->str, '='); - } + str_append(reply->str, key); if (value != NULL) { + str_append_c(reply->str, '='); /* escape dangerous characters in the value */ str_append_tabescaped(reply->str, value); } @@ -143,8 +141,3 @@ { return reply == NULL || str_len(reply->str) == 0; } - -string_t *auth_stream_reply_get_str(struct auth_stream_reply *reply) -{ - return reply->str; -} diff -r 526aa986d534 -r f11ae64365b5 src/auth/auth-stream.h --- a/src/auth/auth-stream.h Wed Jan 30 19:08:00 2013 +0200 +++ b/src/auth/auth-stream.h Wed Jan 30 19:11:30 2013 +0200 @@ -5,7 +5,7 @@ struct auth_stream_reply *auth_stream_reply_init(pool_t pool); void auth_stream_reply_add(struct auth_stream_reply *reply, - const char *key, const char *value) ATTR_NULL(2, 3); + const char *key, const char *value) ATTR_NULL(3); void auth_stream_reply_reset(struct auth_stream_reply *reply); void auth_stream_reply_remove(struct auth_stream_reply *reply, const char *key); @@ -19,6 +19,4 @@ string_t *dest); bool auth_stream_is_empty(struct auth_stream_reply *reply); -string_t *auth_stream_reply_get_str(struct auth_stream_reply *reply); - #endif From dovecot at dovecot.org Wed Jan 30 22:17:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 22:17:28 +0200 Subject: dovecot-2.2: auth: Code cleanup: Merged extra_cache_fields into ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cc4472f02f70 changeset: 15684:cc4472f02f70 user: Timo Sirainen date: Wed Jan 30 19:46:58 2013 +0200 description: auth: Code cleanup: Merged extra_cache_fields into extra_fields. They are separated using a hidden-flag in the extra field. This required a new implementation for auth-streams. diffstat: src/auth/auth-master-connection.c | 5 +- src/auth/auth-request-handler.c | 7 +- src/auth/auth-request.c | 53 +++++------ src/auth/auth-request.h | 3 - src/auth/auth-stream.c | 170 +++++++++++++++++++------------------ src/auth/auth-stream.h | 24 ++++- src/auth/auth-worker-client.c | 14 +-- src/auth/db-checkpassword.c | 21 ++-- src/auth/userdb-blocking.c | 2 +- 9 files changed, 153 insertions(+), 146 deletions(-) diffs (truncated from 596 to 300 lines): diff -r f11ae64365b5 -r cc4472f02f70 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Jan 30 19:11:30 2013 +0200 +++ b/src/auth/auth-master-connection.c Wed Jan 30 19:46:58 2013 +0200 @@ -275,7 +275,7 @@ str_printfa(str, "USER\t%u\t", auth_request->id); str_append_tabescaped(str, auth_request->user); str_append_c(str, '\t'); - auth_stream_reply_append(auth_request->userdb_reply, str); + auth_stream_reply_append(auth_request->userdb_reply, str, FALSE); break; } @@ -325,7 +325,8 @@ str_append_tabescaped(str, auth_request->user); if (!auth_stream_is_empty(auth_request->extra_fields)) { str_append_c(str, '\t'); - auth_stream_reply_append(auth_request->extra_fields, str); + auth_stream_reply_append(auth_request->extra_fields, + str, FALSE); } break; case PASSDB_RESULT_USER_UNKNOWN: diff -r f11ae64365b5 -r cc4472f02f70 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Jan 30 19:11:30 2013 +0200 +++ b/src/auth/auth-request-handler.c Wed Jan 30 19:46:58 2013 +0200 @@ -170,7 +170,7 @@ return; str_append_c(dest, '\t'); - auth_stream_reply_append(request->extra_fields, dest); + auth_stream_reply_append(request->extra_fields, dest, FALSE); if (request->proxy && !request->auth_only) { /* we're proxying */ @@ -338,7 +338,8 @@ if (reply_size > 0) { str = t_str_new(MAX_BASE64_ENCODED_SIZE(reply_size)); base64_encode(auth_reply, reply_size, str); - auth_stream_reply_add(request->extra_fields, "resp", str_c(str)); + auth_stream_reply_add(request->extra_fields, "resp", + str_c(str), 0); } ret = auth_request_proxy_finish(request, auth_request_handler_proxy_callback); @@ -635,7 +636,7 @@ case USERDB_RESULT_OK: str_printfa(str, "USER\t%u\t", request->id); str_append_tabescaped(str, request->user); - auth_stream_reply_append(request->userdb_reply, str); + auth_stream_reply_append(request->userdb_reply, str, FALSE); if (request->master_user != NULL && !auth_stream_reply_exists(request->userdb_reply, diff -r f11ae64365b5 -r cc4472f02f70 src/auth/auth-request.c --- a/src/auth/auth-request.c Wed Jan 30 19:11:30 2013 +0200 +++ b/src/auth/auth-request.c Wed Jan 30 19:46:58 2013 +0200 @@ -427,11 +427,7 @@ if (!auth_stream_is_empty(request->extra_fields)) { str_append_c(str, '\t'); - auth_stream_reply_append(request->extra_fields, str); - } - if (!auth_stream_is_empty(request->extra_cache_fields)) { - str_append_c(str, '\t'); - auth_stream_reply_append(request->extra_cache_fields, str); + auth_stream_reply_append(request->extra_fields, str, TRUE); } auth_cache_insert(passdb_cache, request, passdb->cache_key, str_c(str), result == PASSDB_RESULT_OK); @@ -516,7 +512,7 @@ auth_stream_reply_init(request->pool); } auth_stream_reply_add(request->extra_fields, "reason", - "Password expired"); + "Password expired", 0); } else if (request->passdb->next != NULL && *result != PASSDB_RESULT_USER_DISABLED) { /* try next passdb. */ @@ -826,16 +822,23 @@ enum userdb_result result) { struct userdb_module *userdb = request->userdb->userdb; - const char *str; + string_t *str; + const char *cache_value; if (passdb_cache == NULL || userdb->cache_key == NULL || request->master_user != NULL) return; - str = result == USERDB_RESULT_USER_UNKNOWN ? "" : - auth_stream_reply_export(request->userdb_reply); + if (result == USERDB_RESULT_USER_UNKNOWN) + cache_value = ""; + else { + str = t_str_new(128); + auth_stream_reply_append(request->userdb_reply, str, TRUE); + cache_value = str_c(str); + } /* last_success has no meaning with userdb */ - auth_cache_insert(passdb_cache, request, userdb->cache_key, str, FALSE); + auth_cache_insert(passdb_cache, request, userdb->cache_key, + cache_value, FALSE); } static bool auth_request_lookup_user_cache(struct auth_request *request, @@ -869,7 +872,7 @@ *result_r = USERDB_RESULT_OK; *reply_r = auth_stream_reply_init(request->pool); - auth_stream_reply_import(*reply_r, value); + auth_stream_reply_import(*reply_r, value, 0); return TRUE; } @@ -1190,8 +1193,7 @@ if (request->extra_fields == NULL) request->extra_fields = auth_stream_reply_init(request->pool); - auth_stream_reply_remove(request->extra_fields, name); - auth_stream_reply_add(request->extra_fields, name, value); + auth_stream_reply_add(request->extra_fields, name, value, 0); } static const char * @@ -1305,11 +1307,8 @@ /* we'll need to get this field stored into cache, or we're a worker and we'll need to send this to the main auth process that can store it in the cache. */ - if (request->extra_cache_fields == NULL) { - request->extra_cache_fields = - auth_stream_reply_init(request->pool); - } - auth_stream_reply_add(request->extra_cache_fields, name, value); + auth_stream_reply_add(request->extra_fields, name, value, + AUTH_STREAM_FIELD_FLAG_HIDDEN); } } @@ -1363,9 +1362,9 @@ "stat(%s) failed: %m", str_c(path)); } else { auth_stream_reply_add(request->userdb_reply, - "uid", dec2str(st.st_uid)); + "uid", dec2str(st.st_uid), 0); auth_stream_reply_add(request->userdb_reply, - "gid", dec2str(st.st_gid)); + "gid", dec2str(st.st_gid), 0); } } @@ -1398,7 +1397,7 @@ auth_request_set_uidgid_file(request, value); return; } else if (strcmp(name, "userdb_import") == 0) { - auth_stream_reply_import(request->userdb_reply, value); + auth_stream_reply_import(request->userdb_reply, value, 0); return; } else if (strcmp(name, "system_user") == 0) { /* FIXME: the system_user is for backwards compatibility */ @@ -1410,8 +1409,7 @@ name = "system_groups_user"; } - auth_stream_reply_remove(request->userdb_reply, name); - auth_stream_reply_add(request->userdb_reply, name, value); + auth_stream_reply_add(request->userdb_reply, name, value, 0); } void auth_request_set_userdb_field_values(struct auth_request *request, @@ -1439,7 +1437,7 @@ str_append(value, dec2str(gid)); } auth_stream_reply_add(request->userdb_reply, name, - str_c(value)); + str_c(value), 0); } else { /* add only one */ if (values[1] != NULL) { @@ -1496,7 +1494,7 @@ } else if (!auth_request_proxy_is_self(request)) { /* proxy destination isn't ourself - proxy */ auth_stream_reply_remove(request->extra_fields, "proxy_maybe"); - auth_stream_reply_add(request->extra_fields, "proxy", NULL); + auth_stream_reply_add(request->extra_fields, "proxy", NULL, 0); request->no_login = TRUE; } else { /* proxying to ourself - log in without proxying by dropping @@ -1507,7 +1505,7 @@ if (proxy_always) { /* director adds the host */ auth_stream_reply_add(request->extra_fields, - "proxy", NULL); + "proxy", NULL, 0); request->proxy = TRUE; } } @@ -1538,9 +1536,8 @@ "DNS lookup for %s took %u.%03u s", host, result->msecs/1000, result->msecs % 1000); } - auth_stream_reply_remove(request->extra_fields, "hostip"); auth_stream_reply_add(request->extra_fields, "hostip", - net_ip2addr(&result->ips[0])); + net_ip2addr(&result->ips[0]), 0); for (i = 0; i < result->ips_count; i++) { if (auth_request_proxy_ip_is_self(request, &result->ips[i])) { diff -r f11ae64365b5 -r cc4472f02f70 src/auth/auth-request.h --- a/src/auth/auth-request.h Wed Jan 30 19:11:30 2013 +0200 +++ b/src/auth/auth-request.h Wed Jan 30 19:46:58 2013 +0200 @@ -50,9 +50,6 @@ /* extra_fields are returned in authentication reply. Fields prefixed with "userdb_" are automatically placed to userdb_reply instead. */ struct auth_stream_reply *extra_fields; - /* extra_fields that aren't supposed to be sent to the client, but - are supposed to be stored to auth cache. */ - struct auth_stream_reply *extra_cache_fields; /* the whole userdb result reply */ struct auth_stream_reply *userdb_reply; struct auth_request_proxy_dns_lookup_ctx *dns_lookup_ctx; diff -r f11ae64365b5 -r cc4472f02f70 src/auth/auth-stream.c --- a/src/auth/auth-stream.c Wed Jan 30 19:11:30 2013 +0200 +++ b/src/auth/auth-stream.c Wed Jan 30 19:46:58 2013 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2005-2012 Dovecot authors, see the included COPYING file */ #include "auth-common.h" +#include "array.h" #include "str.h" #include "strescape.h" #include "ostream.h" @@ -8,7 +9,8 @@ #include "auth-stream.h" struct auth_stream_reply { - string_t *str; + pool_t pool; + ARRAY_TYPE(auth_stream_field) fields; }; struct auth_stream_reply *auth_stream_reply_init(pool_t pool) @@ -16,128 +18,132 @@ struct auth_stream_reply *reply; reply = p_new(pool, struct auth_stream_reply, 1); - reply->str = str_new(pool, 128); + reply->pool = pool; + p_array_init(&reply->fields, pool, 16); return reply; } +static bool +auth_stream_reply_find_idx(struct auth_stream_reply *reply, const char *key, + unsigned int *idx_r) +{ + const struct auth_stream_field *fields; + unsigned int i, count; + + fields = array_get(&reply->fields, &count); + for (i = 0; i < count; i++) { + if (strcmp(fields[i].key, key) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + void auth_stream_reply_add(struct auth_stream_reply *reply, - const char *key, const char *value) + const char *key, const char *value, + enum auth_stream_field_flags flags) { + struct auth_stream_field *field; + unsigned int idx; + i_assert(*key != '\0'); i_assert(strchr(key, '\t') == NULL && strchr(key, '\n') == NULL); - if (str_len(reply->str) > 0) - str_append_c(reply->str, '\t'); - - str_append(reply->str, key); - if (value != NULL) { - str_append_c(reply->str, '='); - /* escape dangerous characters in the value */ - str_append_tabescaped(reply->str, value); + if (!auth_stream_reply_find_idx(reply, key, &idx)) { + field = array_append_space(&reply->fields); + field->key = p_strdup(reply->pool, key); + } else { + field = array_idx_modifiable(&reply->fields, idx); } -} - -static bool -auth_stream_reply_find_area(struct auth_stream_reply *reply, const char *key, - unsigned int *idx_r, unsigned int *len_r) From dovecot at dovecot.org Wed Jan 30 22:17:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 22:17:28 +0200 Subject: dovecot-2.2: auth: Code cleanup: Renamed auth-stream to auth-fie... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/17f5257d60c1 changeset: 15685:17f5257d60c1 user: Timo Sirainen date: Wed Jan 30 19:57:20 2013 +0200 description: auth: Code cleanup: Renamed auth-stream to auth-fields. diffstat: src/auth/Makefile.am | 4 +- src/auth/auth-client-connection.c | 2 +- src/auth/auth-fields.c | 147 ++++++++++++++++++++++++++++++++++++ src/auth/auth-fields.h | 34 ++++++++ src/auth/auth-master-connection.c | 16 +- src/auth/auth-postfix-connection.c | 3 +- src/auth/auth-request-handler.c | 20 ++-- src/auth/auth-request.c | 85 ++++++++++---------- src/auth/auth-request.h | 4 +- src/auth/auth-stream.c | 149 ------------------------------------- src/auth/auth-stream.h | 36 -------- src/auth/auth-worker-client.c | 10 +- src/auth/db-checkpassword.c | 8 +- src/auth/passdb-blocking.c | 6 +- src/auth/userdb-blocking.c | 6 +- src/auth/userdb.h | 2 +- 16 files changed, 261 insertions(+), 271 deletions(-) diffs (truncated from 944 to 300 lines): diff -r cc4472f02f70 -r 17f5257d60c1 src/auth/Makefile.am --- a/src/auth/Makefile.am Wed Jan 30 19:46:58 2013 +0200 +++ b/src/auth/Makefile.am Wed Jan 30 19:57:20 2013 +0200 @@ -71,7 +71,7 @@ auth-request.c \ auth-request-handler.c \ auth-settings.c \ - auth-stream.c \ + auth-fields.c \ auth-token.c \ auth-worker-client.c \ auth-worker-server.c \ @@ -138,7 +138,7 @@ auth-request.h \ auth-request-handler.h \ auth-settings.h \ - auth-stream.h \ + auth-fields.h \ auth-token.h \ auth-worker-client.h \ auth-worker-server.h \ diff -r cc4472f02f70 -r 17f5257d60c1 src/auth/auth-client-connection.c --- a/src/auth/auth-client-connection.c Wed Jan 30 19:46:58 2013 +0200 +++ b/src/auth/auth-client-connection.c Wed Jan 30 19:57:20 2013 +0200 @@ -14,7 +14,7 @@ #include "safe-memset.h" #include "master-service.h" #include "mech.h" -#include "auth-stream.h" +#include "auth-fields.h" #include "auth-request-handler.h" #include "auth-client-interface.h" #include "auth-client-connection.h" diff -r cc4472f02f70 -r 17f5257d60c1 src/auth/auth-fields.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/auth/auth-fields.c Wed Jan 30 19:57:20 2013 +0200 @@ -0,0 +1,147 @@ +/* Copyright (c) 2005-2012 Dovecot authors, see the included COPYING file */ + +#include "auth-common.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "ostream.h" +#include "auth-request.h" +#include "auth-fields.h" + +struct auth_fields { + pool_t pool; + ARRAY_TYPE(auth_field) fields; +}; + +struct auth_fields *auth_fields_init(pool_t pool) +{ + struct auth_fields *reply; + + reply = p_new(pool, struct auth_fields, 1); + reply->pool = pool; + p_array_init(&reply->fields, pool, 16); + return reply; +} + +static bool +auth_fields_find_idx(struct auth_fields *reply, const char *key, + unsigned int *idx_r) +{ + const struct auth_field *fields; + unsigned int i, count; + + fields = array_get(&reply->fields, &count); + for (i = 0; i < count; i++) { + if (strcmp(fields[i].key, key) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +void auth_fields_add(struct auth_fields *reply, + const char *key, const char *value, + enum auth_field_flags flags) +{ + struct auth_field *field; + unsigned int idx; + + i_assert(*key != '\0'); + i_assert(strchr(key, '\t') == NULL && + strchr(key, '\n') == NULL); + + if (!auth_fields_find_idx(reply, key, &idx)) { + field = array_append_space(&reply->fields); + field->key = p_strdup(reply->pool, key); + } else { + field = array_idx_modifiable(&reply->fields, idx); + } + field->value = p_strdup_empty(reply->pool, value); + field->flags = flags; +} + +void auth_fields_remove(struct auth_fields *reply, const char *key) +{ + unsigned int idx; + + if (auth_fields_find_idx(reply, key, &idx)) + array_delete(&reply->fields, idx, 1); +} + +const char *auth_fields_find(struct auth_fields *reply, const char *key) +{ + const struct auth_field *field; + unsigned int idx; + + if (!auth_fields_find_idx(reply, key, &idx)) + return NULL; + + field = array_idx(&reply->fields, idx); + return field->value == NULL ? "" : field->value; +} + +bool auth_fields_exists(struct auth_fields *reply, const char *key) +{ + return auth_fields_find(reply, key) != NULL; +} + +void auth_fields_reset(struct auth_fields *reply) +{ + array_clear(&reply->fields); +} + +void auth_fields_import(struct auth_fields *reply, const char *str, + enum auth_field_flags flags) +{ + T_BEGIN { + const char *const *arg = t_strsplit_tab(str); + const char *key, *value; + + for (; *arg != NULL; arg++) { + value = strchr(*arg, '='); + if (value == NULL) { + key = *arg; + value = NULL; + } else { + key = t_strdup_until(*arg, value++); + } + auth_fields_add(reply, key, value, flags); + } + } T_END; +} + +const ARRAY_TYPE(auth_field) *auth_fields_export(struct auth_fields *reply) +{ + return &reply->fields; +} + +void auth_fields_append(struct auth_fields *reply, string_t *dest, + bool include_hidden) +{ + const struct auth_field *fields; + unsigned int i, count; + bool first = TRUE; + + fields = array_get(&reply->fields, &count); + for (i = 0; i < count; i++) { + if (!include_hidden && + (fields[i].flags & AUTH_FIELD_FLAG_HIDDEN) != 0) + continue; + + if (first) + first = FALSE; + else + str_append_c(dest, '\t'); + str_append(dest, fields[i].key); + if (fields[i].value != NULL) { + str_append_c(dest, '='); + str_append_tabescaped(dest, fields[i].value); + } + } +} + +bool auth_fields_is_empty(struct auth_fields *reply) +{ + return reply == NULL || array_count(&reply->fields) == 0; +} diff -r cc4472f02f70 -r 17f5257d60c1 src/auth/auth-fields.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/auth/auth-fields.h Wed Jan 30 19:57:20 2013 +0200 @@ -0,0 +1,34 @@ +#ifndef AUTH_FIELDS_H +#define AUTH_FIELDS_H + +struct auth_request; + +enum auth_field_flags { + /* This field is internal to auth process and won't be sent to client */ + AUTH_FIELD_FLAG_HIDDEN = 0x01 +}; + +struct auth_field { + const char *key, *value; + enum auth_field_flags flags; +}; +ARRAY_DEFINE_TYPE(auth_field, struct auth_field); + +struct auth_fields *auth_fields_init(pool_t pool); +void auth_fields_add(struct auth_fields *fields, + const char *key, const char *value, + enum auth_field_flags flags) ATTR_NULL(3); +void auth_fields_reset(struct auth_fields *reply); +void auth_fields_remove(struct auth_fields *reply, const char *key); + +const char *auth_fields_find(struct auth_fields *reply, const char *key); +bool auth_fields_exists(struct auth_fields *reply, const char *key); + +void auth_fields_import(struct auth_fields *reply, const char *str, + enum auth_field_flags flags); +const ARRAY_TYPE(auth_field) *auth_fields_export(struct auth_fields *reply); +void auth_fields_append(struct auth_fields *reply, string_t *dest, + bool include_hidden); +bool auth_fields_is_empty(struct auth_fields *reply); + +#endif diff -r cc4472f02f70 -r 17f5257d60c1 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Jan 30 19:46:58 2013 +0200 +++ b/src/auth/auth-master-connection.c Wed Jan 30 19:57:20 2013 +0200 @@ -214,14 +214,14 @@ user_verify_restricted_uid(struct auth_request *auth_request) { struct auth_master_connection *conn = auth_request->master; - struct auth_stream_reply *reply = auth_request->userdb_reply; + struct auth_fields *reply = auth_request->userdb_reply; const char *value, *reason; uid_t uid; if (conn->userdb_restricted_uid == 0) return 0; - value = auth_stream_reply_find(reply, "uid"); + value = auth_fields_find(reply, "uid"); if (value == NULL) reason = "userdb reply doesn't contain uid"; else if (str_to_uid(value, &uid) < 0) @@ -262,8 +262,8 @@ case USERDB_RESULT_INTERNAL_FAILURE: str_printfa(str, "FAIL\t%u", auth_request->id); if (auth_request->userdb_lookup_failed) { - value = auth_stream_reply_find(auth_request->userdb_reply, - "reason"); + value = auth_fields_find(auth_request->userdb_reply, + "reason"); if (value != NULL) str_printfa(str, "\treason=%s", value); } @@ -275,7 +275,7 @@ str_printfa(str, "USER\t%u\t", auth_request->id); str_append_tabescaped(str, auth_request->user); str_append_c(str, '\t'); - auth_stream_reply_append(auth_request->userdb_reply, str, FALSE); + auth_fields_append(auth_request->userdb_reply, str, FALSE); break; } @@ -323,10 +323,10 @@ case PASSDB_RESULT_OK: str_printfa(str, "PASS\t%u\tuser=", auth_request->id); str_append_tabescaped(str, auth_request->user); - if (!auth_stream_is_empty(auth_request->extra_fields)) { + if (!auth_fields_is_empty(auth_request->extra_fields)) { str_append_c(str, '\t'); - auth_stream_reply_append(auth_request->extra_fields, - str, FALSE); + auth_fields_append(auth_request->extra_fields, + str, FALSE); } break; case PASSDB_RESULT_USER_UNKNOWN: diff -r cc4472f02f70 -r 17f5257d60c1 src/auth/auth-postfix-connection.c --- a/src/auth/auth-postfix-connection.c Wed Jan 30 19:46:58 2013 +0200 +++ b/src/auth/auth-postfix-connection.c Wed Jan 30 19:57:20 2013 +0200 @@ -66,7 +66,6 @@ user_callback(enum userdb_result result, struct auth_request *auth_request) { struct auth_postfix_connection *conn = auth_request->context; - struct auth_stream_reply *reply = auth_request->userdb_reply; string_t *str; const char *value; @@ -77,7 +76,7 @@ switch (result) { case USERDB_RESULT_INTERNAL_FAILURE: if (auth_request->userdb_lookup_failed) - value = auth_stream_reply_find(reply, "reason"); + value = auth_fields_find(auth_request->userdb_reply, "reason"); else value = NULL; str_printfa(str, "400 %s", diff -r cc4472f02f70 -r 17f5257d60c1 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Jan 30 19:46:58 2013 +0200 +++ b/src/auth/auth-request-handler.c Wed Jan 30 19:57:20 2013 +0200 @@ -166,15 +166,15 @@ From dovecot at dovecot.org Wed Jan 30 22:17:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 22:17:28 +0200 Subject: dovecot-2.2: auth: Allocate auth-fields array lazily. Also some ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/06b41206ed4e changeset: 15686:06b41206ed4e user: Timo Sirainen date: Wed Jan 30 20:06:43 2013 +0200 description: auth: Allocate auth-fields array lazily. Also some naming cleanups. diffstat: src/auth/auth-fields.c | 88 ++++++++++++++++++++++++++++--------------------- src/auth/auth-fields.h | 16 ++++---- 2 files changed, 58 insertions(+), 46 deletions(-) diffs (206 lines): diff -r 17f5257d60c1 -r 06b41206ed4e src/auth/auth-fields.c --- a/src/auth/auth-fields.c Wed Jan 30 19:57:20 2013 +0200 +++ b/src/auth/auth-fields.c Wed Jan 30 20:06:43 2013 +0200 @@ -15,24 +15,26 @@ struct auth_fields *auth_fields_init(pool_t pool) { - struct auth_fields *reply; + struct auth_fields *fields; - reply = p_new(pool, struct auth_fields, 1); - reply->pool = pool; - p_array_init(&reply->fields, pool, 16); - return reply; + fields = p_new(pool, struct auth_fields, 1); + fields->pool = pool; + return fields; } static bool -auth_fields_find_idx(struct auth_fields *reply, const char *key, +auth_fields_find_idx(struct auth_fields *fields, const char *key, unsigned int *idx_r) { - const struct auth_field *fields; + const struct auth_field *f; unsigned int i, count; - fields = array_get(&reply->fields, &count); + if (!array_is_created(&fields->fields)) + return FALSE; + + f = array_get(&fields->fields, &count); for (i = 0; i < count; i++) { - if (strcmp(fields[i].key, key) == 0) { + if (strcmp(f[i].key, key) == 0) { *idx_r = i; return TRUE; } @@ -40,7 +42,7 @@ return FALSE; } -void auth_fields_add(struct auth_fields *reply, +void auth_fields_add(struct auth_fields *fields, const char *key, const char *value, enum auth_field_flags flags) { @@ -51,47 +53,51 @@ i_assert(strchr(key, '\t') == NULL && strchr(key, '\n') == NULL); - if (!auth_fields_find_idx(reply, key, &idx)) { - field = array_append_space(&reply->fields); - field->key = p_strdup(reply->pool, key); + if (!auth_fields_find_idx(fields, key, &idx)) { + if (!array_is_created(&fields->fields)) + p_array_init(&fields->fields, fields->pool, 16); + + field = array_append_space(&fields->fields); + field->key = p_strdup(fields->pool, key); } else { - field = array_idx_modifiable(&reply->fields, idx); + field = array_idx_modifiable(&fields->fields, idx); } - field->value = p_strdup_empty(reply->pool, value); + field->value = p_strdup_empty(fields->pool, value); field->flags = flags; } -void auth_fields_remove(struct auth_fields *reply, const char *key) +void auth_fields_remove(struct auth_fields *fields, const char *key) { unsigned int idx; - if (auth_fields_find_idx(reply, key, &idx)) - array_delete(&reply->fields, idx, 1); + if (auth_fields_find_idx(fields, key, &idx)) + array_delete(&fields->fields, idx, 1); } -const char *auth_fields_find(struct auth_fields *reply, const char *key) +const char *auth_fields_find(struct auth_fields *fields, const char *key) { const struct auth_field *field; unsigned int idx; - if (!auth_fields_find_idx(reply, key, &idx)) + if (!auth_fields_find_idx(fields, key, &idx)) return NULL; - field = array_idx(&reply->fields, idx); + field = array_idx(&fields->fields, idx); return field->value == NULL ? "" : field->value; } -bool auth_fields_exists(struct auth_fields *reply, const char *key) +bool auth_fields_exists(struct auth_fields *fields, const char *key) { - return auth_fields_find(reply, key) != NULL; + return auth_fields_find(fields, key) != NULL; } -void auth_fields_reset(struct auth_fields *reply) +void auth_fields_reset(struct auth_fields *fields) { - array_clear(&reply->fields); + if (array_is_created(&fields->fields)) + array_clear(&fields->fields); } -void auth_fields_import(struct auth_fields *reply, const char *str, +void auth_fields_import(struct auth_fields *fields, const char *str, enum auth_field_flags flags) { T_BEGIN { @@ -106,42 +112,48 @@ } else { key = t_strdup_until(*arg, value++); } - auth_fields_add(reply, key, value, flags); + auth_fields_add(fields, key, value, flags); } } T_END; } -const ARRAY_TYPE(auth_field) *auth_fields_export(struct auth_fields *reply) +const ARRAY_TYPE(auth_field) *auth_fields_export(struct auth_fields *fields) { - return &reply->fields; + if (!array_is_created(&fields->fields)) + p_array_init(&fields->fields, fields->pool, 1); + return &fields->fields; } -void auth_fields_append(struct auth_fields *reply, string_t *dest, +void auth_fields_append(struct auth_fields *fields, string_t *dest, bool include_hidden) { - const struct auth_field *fields; + const struct auth_field *f; unsigned int i, count; bool first = TRUE; - fields = array_get(&reply->fields, &count); + if (!array_is_created(&fields->fields)) + return; + + f = array_get(&fields->fields, &count); for (i = 0; i < count; i++) { if (!include_hidden && - (fields[i].flags & AUTH_FIELD_FLAG_HIDDEN) != 0) + (f[i].flags & AUTH_FIELD_FLAG_HIDDEN) != 0) continue; if (first) first = FALSE; else str_append_c(dest, '\t'); - str_append(dest, fields[i].key); - if (fields[i].value != NULL) { + str_append(dest, f[i].key); + if (f[i].value != NULL) { str_append_c(dest, '='); - str_append_tabescaped(dest, fields[i].value); + str_append_tabescaped(dest, f[i].value); } } } -bool auth_fields_is_empty(struct auth_fields *reply) +bool auth_fields_is_empty(struct auth_fields *fields) { - return reply == NULL || array_count(&reply->fields) == 0; + return fields == NULL || !array_is_created(&fields->fields) || + array_count(&fields->fields) == 0; } diff -r 17f5257d60c1 -r 06b41206ed4e src/auth/auth-fields.h --- a/src/auth/auth-fields.h Wed Jan 30 19:57:20 2013 +0200 +++ b/src/auth/auth-fields.h Wed Jan 30 20:06:43 2013 +0200 @@ -18,17 +18,17 @@ void auth_fields_add(struct auth_fields *fields, const char *key, const char *value, enum auth_field_flags flags) ATTR_NULL(3); -void auth_fields_reset(struct auth_fields *reply); -void auth_fields_remove(struct auth_fields *reply, const char *key); +void auth_fields_reset(struct auth_fields *fields); +void auth_fields_remove(struct auth_fields *fields, const char *key); -const char *auth_fields_find(struct auth_fields *reply, const char *key); -bool auth_fields_exists(struct auth_fields *reply, const char *key); +const char *auth_fields_find(struct auth_fields *fields, const char *key); +bool auth_fields_exists(struct auth_fields *fields, const char *key); -void auth_fields_import(struct auth_fields *reply, const char *str, +void auth_fields_import(struct auth_fields *fields, const char *str, enum auth_field_flags flags); -const ARRAY_TYPE(auth_field) *auth_fields_export(struct auth_fields *reply); -void auth_fields_append(struct auth_fields *reply, string_t *dest, +const ARRAY_TYPE(auth_field) *auth_fields_export(struct auth_fields *fields); +void auth_fields_append(struct auth_fields *fields, string_t *dest, bool include_hidden); -bool auth_fields_is_empty(struct auth_fields *reply); +bool auth_fields_is_empty(struct auth_fields *fields); #endif From dovecot at dovecot.org Wed Jan 30 22:17:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 22:17:28 +0200 Subject: dovecot-2.2: auth: Cleaned up flags in auth request. Removed tho... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/686f32406220 changeset: 15687:686f32406220 user: Timo Sirainen date: Wed Jan 30 21:08:58 2013 +0200 description: auth: Cleaned up flags in auth request. Removed those that already exist in extra_fields. It's now slightly slower to check for those flags in extra_fields, but it's going to be easier to make commit/rollback feature to extra_fields. diffstat: src/auth/auth-request-handler.c | 23 +++-- src/auth/auth-request.c | 148 ++++++++++++++------------------------- src/auth/auth-request.h | 54 ++++++++------ src/auth/auth-worker-client.c | 4 +- src/auth/db-checkpassword.c | 2 +- src/auth/mech-gssapi.c | 3 - src/auth/passdb-dict.c | 2 +- src/auth/passdb-ldap.c | 2 +- src/auth/passdb-sql.c | 2 +- 9 files changed, 106 insertions(+), 134 deletions(-) diffs (truncated from 606 to 300 lines): diff -r 06b41206ed4e -r 686f32406220 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Jan 30 20:06:43 2013 +0200 +++ b/src/auth/auth-request-handler.c Wed Jan 30 21:08:58 2013 +0200 @@ -172,7 +172,8 @@ str_append_c(dest, '\t'); auth_fields_append(request->extra_fields, dest, FALSE); - if (request->proxy && !request->auth_only) { + if (!request->auth_only && + auth_fields_exists(request->extra_fields, "proxy")) { /* we're proxying */ if (!auth_fields_exists(request->extra_fields, "pass") && request->mech_password != NULL) { @@ -195,7 +196,7 @@ { struct auth_request_handler *handler = request->handler; - if (request->delayed_failure) { + if (request->in_delayed_failure_queue) { /* we came here from flush_failures() */ handler->callback(reply, handler->context); return; @@ -205,7 +206,7 @@ auth_request_ref(request); auth_request_handler_remove(handler, request); - if (request->no_failure_delay) { + if (auth_fields_exists(request->extra_fields, "nodelay")) { /* passdb specifically requested not to delay the reply. */ handler->callback(reply, handler->context); auth_request_unref(&request); @@ -214,7 +215,7 @@ /* failure. don't announce it immediately to avoid a) timing attacks, b) flooding */ - request->delayed_failure = TRUE; + request->in_delayed_failure_queue = TRUE; handler->refcount++; if (auth_penalty != NULL) { @@ -245,7 +246,9 @@ str_printfa(str, "OK\t%u\tuser=", request->id); str_append_tabescaped(str, request->user); auth_str_append_extra_fields(request, str); - if (request->no_login || handler->master_callback == NULL) { + if (handler->master_callback == NULL || + auth_fields_exists(request->extra_fields, "nologin") || + auth_fields_exists(request->extra_fields, "proxy")) { /* this request doesn't have to wait for master process to pick it up. delete it */ auth_request_handler_remove(handler, request); @@ -273,8 +276,10 @@ as the wanted user */ str_append(str, "\tauthz"); } - if (request->no_failure_delay) + if (auth_fields_exists(request->extra_fields, "nodelay")) { + /* this is normally a hidden field, need to add it explicitly */ str_append(str, "\tnodelay"); + } auth_str_append_extra_fields(request, str); switch (request->passdb_result) { @@ -331,7 +336,7 @@ str_printfa(str, "CONT\t%u\t", request->id); base64_encode(auth_reply, reply_size, str); - request->accept_input = TRUE; + request->accept_cont_input = TRUE; handler->callback(str_c(str), handler->context); break; case AUTH_CLIENT_RESULT_SUCCESS: @@ -582,12 +587,12 @@ } /* accept input only once after mechanism has sent a CONT reply */ - if (!request->accept_input) { + if (!request->accept_cont_input) { auth_request_handler_auth_fail(handler, request, "Unexpected continuation"); return TRUE; } - request->accept_input = FALSE; + request->accept_cont_input = FALSE; data_len = strlen(data); buf = buffer_create_dynamic(pool_datastack_create(), diff -r 06b41206ed4e -r 686f32406220 src/auth/auth-request.c --- a/src/auth/auth-request.c Wed Jan 30 20:06:43 2013 +0200 +++ b/src/auth/auth-request.c Wed Jan 30 21:08:58 2013 +0200 @@ -61,6 +61,7 @@ request->set = global_auth_settings; request->mech = mech; request->mech_name = mech == NULL ? NULL : mech->mech_name; + request->extra_fields = auth_fields_init(request->pool); return request; } @@ -117,7 +118,7 @@ { i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); - if (request->passdb_failure) { + if (request->failed) { /* password was valid, but some other check failed. */ auth_request_fail(request); return; @@ -316,8 +317,6 @@ request->original_username = p_strdup(request->pool, value); else if (strcmp(key, "requested_login_user") == 0) request->requested_login_user = p_strdup(request->pool, value); - else if (strcmp(key, "nologin") == 0) - request->no_login = TRUE; else if (strcmp(key, "successful") == 0) request->successful = TRUE; else if (strcmp(key, "skip_password_check") == 0) { @@ -390,7 +389,8 @@ return; } - if (!request->no_password && request->passdb_password == NULL) { + if (request->passdb_password == NULL && + !auth_fields_exists(request->extra_fields, "nopassword")) { /* passdb didn't provide the correct password */ if (result != PASSDB_RESULT_OK || request->mech_password == NULL) @@ -437,7 +437,7 @@ { struct auth_passdb *passdb; - if (request->passdb_failure) + if (request->failed) return TRUE; /* master login successful. update user and master_user variables. */ @@ -507,8 +507,6 @@ } } } else if (*result == PASSDB_RESULT_PASS_EXPIRED) { - if (request->extra_fields == NULL) - request->extra_fields = auth_fields_init(request->pool); auth_fields_add(request->extra_fields, "reason", "Password expired", 0); } else if (request->passdb->next != NULL && @@ -517,25 +515,20 @@ request->passdb = request->passdb->next; request->passdb_password = NULL; - request->proxy = FALSE; - request->proxy_maybe = FALSE; - request->proxy_always = FALSE; - if (*result == PASSDB_RESULT_USER_UNKNOWN) { /* remember that we did at least one successful passdb lookup */ - request->passdb_user_unknown = TRUE; + request->passdbs_seen_user_unknown = TRUE; } else if (*result == PASSDB_RESULT_INTERNAL_FAILURE) { /* remember that we have had an internal failure. at the end return internal failure if we couldn't successfully login. */ - request->passdb_internal_failure = TRUE; + request->passdbs_seen_internal_failure = TRUE; } - if (request->extra_fields != NULL) - auth_fields_reset(request->extra_fields); + auth_fields_reset(request->extra_fields); return FALSE; - } else if (request->passdb_internal_failure) { + } else if (request->passdbs_seen_internal_failure) { /* last passdb lookup returned internal failure. it may have had the correct password, so return internal failure instead of plain failure. */ @@ -693,7 +686,7 @@ binary_to_hex(credentials, size)); } if (result == PASSDB_RESULT_SCHEME_NOT_AVAILABLE && - request->passdb_user_unknown) { + request->passdbs_seen_user_unknown) { /* one of the passdbs accepted the scheme, but the user was unknown there */ result = PASSDB_RESULT_USER_UNKNOWN; @@ -882,7 +875,7 @@ if (result != USERDB_RESULT_OK && request->userdb->next != NULL) { /* try next userdb. */ if (result == USERDB_RESULT_INTERNAL_FAILURE) - request->userdb_internal_failure = TRUE; + request->userdbs_seen_internal_failure = TRUE; request->userdb = request->userdb->next; auth_request_lookup_user(request, @@ -892,7 +885,7 @@ if (result == USERDB_RESULT_OK) userdb_template_export(userdb->override_fields_tmpl, request); - else if (request->userdb_internal_failure) { + else if (request->userdbs_seen_internal_failure) { /* one of the userdb lookups failed. the user might have been in there, so this is an internal failure */ result = USERDB_RESULT_INTERNAL_FAILURE; @@ -1111,7 +1104,7 @@ /* IP not known */ auth_request_log_info(request, "passdb", "allow_nets check failed: Remote IP not known"); - request->passdb_failure = TRUE; + request->failed = TRUE; return; } @@ -1134,7 +1127,7 @@ auth_request_log_info(request, "passdb", "allow_nets check failed: IP not in allowed networks"); } - request->passdb_failure = !found; + request->failed = !found; } static void @@ -1162,38 +1155,6 @@ } } -static void auth_request_set_reply_field(struct auth_request *request, - const char *name, const char *value) -{ - if (strcmp(name, "nologin") == 0) { - /* user can't actually login - don't keep this - reply for master */ - request->no_login = TRUE; - value = NULL; - } else if (strcmp(name, "proxy") == 0) { - /* we're proxying authentication for this user. send - password back if using plaintext authentication. */ - request->proxy = TRUE; - value = NULL; - } else if (strcmp(name, "proxy_always") == 0) { - /* when proxy_maybe=yes and proxying wouldn't normally be done, - with this enabled proxy=y is still returned without host. - this can be used to make director set the host. */ - request->proxy_always = TRUE; - value = NULL; - } else if (strcmp(name, "proxy_maybe") == 0) { - /* like "proxy", but log in normally if we're proxying to - ourself */ - request->proxy = TRUE; - request->proxy_maybe = TRUE; - value = NULL; - } - - if (request->extra_fields == NULL) - request->extra_fields = auth_fields_init(request->pool); - auth_fields_add(request->extra_fields, name, value, 0); -} - static const char * get_updated_username(const char *old_username, const char *name, const char *value) @@ -1270,7 +1231,6 @@ to cache. */ } else if (strcmp(name, "nodelay") == 0) { /* don't delay replying to client of the failure */ - request->no_failure_delay = TRUE; } else if (strcmp(name, "nopassword") == 0) { /* NULL password - anything goes */ const char *password = request->passdb_password; @@ -1285,7 +1245,6 @@ return; } } - request->no_password = TRUE; request->passdb_password = NULL; } else if (strcmp(name, "allow_nets") == 0) { auth_request_validate_networks(request, value); @@ -1296,7 +1255,7 @@ auth_request_set_userdb_field(request, name + 7, value); } else { /* these fields are returned to client */ - auth_request_set_reply_field(request, name, value); + auth_fields_add(request->extra_fields, name, value, 0); return; } @@ -1450,9 +1409,6 @@ { const char *port = NULL; - if (!request->proxy_host_is_self) - return FALSE; - /* check if the port is the same */ port = auth_fields_find(request->extra_fields, "port"); if (port != NULL && !str_uint_equals(port, request->local_port)) @@ -1483,27 +1439,32 @@ return FALSE; } -static void auth_request_proxy_finish_ip(struct auth_request *request) +static void +auth_request_proxy_finish_ip(struct auth_request *request, From dovecot at dovecot.org Wed Jan 30 22:18:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 22:18:14 +0200 Subject: dovecot-2.2: auth: Fixed login USER reply broken by recent changes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf63624b2566 changeset: 15688:cf63624b2566 user: Timo Sirainen date: Wed Jan 30 22:18:09 2013 +0200 description: auth: Fixed login USER reply broken by recent changes. diffstat: src/auth/auth-request-handler.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 686f32406220 -r cf63624b2566 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Jan 30 21:08:58 2013 +0200 +++ b/src/auth/auth-request-handler.c Wed Jan 30 22:18:09 2013 +0200 @@ -640,6 +640,7 @@ case USERDB_RESULT_OK: str_printfa(str, "USER\t%u\t", request->id); str_append_tabescaped(str, request->user); + str_append_c(str, '\t'); auth_fields_append(request->userdb_reply, str, FALSE); if (request->master_user != NULL && From dovecot at dovecot.org Wed Jan 30 22:19:19 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 22:19:19 +0200 Subject: dovecot-2.2: auth: Auth requests' flag fields were never passed ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/105c40084c5c changeset: 15689:105c40084c5c user: Timo Sirainen date: Wed Jan 30 22:19:13 2013 +0200 description: auth: Auth requests' flag fields were never passed to worker processes. diffstat: src/auth/auth-worker-client.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r cf63624b2566 -r 105c40084c5c src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Wed Jan 30 22:18:09 2013 +0200 +++ b/src/auth/auth-worker-client.c Wed Jan 30 22:19:13 2013 +0200 @@ -89,7 +89,9 @@ for (; *args != NULL; args++) { value = strchr(*args, '='); - if (value != NULL) { + if (value == NULL) + (void)auth_request_import(auth_request, *args, NULL); + else { key = t_strdup_until(*args, value++); (void)auth_request_import(auth_request, key, value); } From dovecot at dovecot.org Wed Jan 30 22:23:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 22:23:25 +0200 Subject: dovecot-2.2: auth: Code cleanup: Renamed some fields used in int... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9e93a5743c95 changeset: 15690:9e93a5743c95 user: Timo Sirainen date: Wed Jan 30 22:23:09 2013 +0200 description: auth: Code cleanup: Renamed some fields used in internal communication with worker. diffstat: src/auth/auth-request.c | 18 ++++++++++-------- 1 files changed, 10 insertions(+), 8 deletions(-) diffs (51 lines): diff -r 105c40084c5c -r 9e93a5743c95 src/auth/auth-request.c --- a/src/auth/auth-request.c Wed Jan 30 22:19:13 2013 +0200 +++ b/src/auth/auth-request.c Wed Jan 30 22:23:09 2013 +0200 @@ -205,13 +205,15 @@ auth_str_add_keyvalue(dest, "service", request->service); if (request->master_user != NULL) { - auth_str_add_keyvalue(dest, "master_user", + auth_str_add_keyvalue(dest, "master-user", request->master_user); } auth_str_add_keyvalue(dest, "original_username", request->original_username); - auth_str_add_keyvalue(dest, "requested_login_user", - request->requested_login_user); + if (request->requested_login_user != NULL) { + auth_str_add_keyvalue(dest, "requested-login-user", + request->requested_login_user); + } if (request->local_ip.family != 0) { auth_str_add_keyvalue(dest, "lip", @@ -228,7 +230,7 @@ if (request->secured) str_append(dest, "\tsecured"); if (request->skip_password_check) - str_append(dest, "\tskip_password_check"); + str_append(dest, "\tskip-password-check"); if (request->valid_client_cert) str_append(dest, "\tvalid-client-cert"); if (request->no_penalty) @@ -311,15 +313,15 @@ /* for communication between auth master and worker processes */ if (strcmp(key, "user") == 0) request->user = p_strdup(request->pool, value); - else if (strcmp(key, "master_user") == 0) + else if (strcmp(key, "master-user") == 0) request->master_user = p_strdup(request->pool, value); - else if (strcmp(key, "original_username") == 0) + else if (strcmp(key, "original-username") == 0) request->original_username = p_strdup(request->pool, value); - else if (strcmp(key, "requested_login_user") == 0) + else if (strcmp(key, "requested-login-user") == 0) request->requested_login_user = p_strdup(request->pool, value); else if (strcmp(key, "successful") == 0) request->successful = TRUE; - else if (strcmp(key, "skip_password_check") == 0) { + else if (strcmp(key, "skip-password-check") == 0) { i_assert(request->master_user != NULL); request->skip_password_check = TRUE; } else if (strcmp(key, "mech") == 0) From dovecot at dovecot.org Wed Jan 30 22:45:01 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 22:45:01 +0200 Subject: dovecot-2.2: auth: Added passdb { result_* and skip } settings. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d60aa734c72d changeset: 15691:d60aa734c72d user: Timo Sirainen date: Wed Jan 30 22:44:48 2013 +0200 description: auth: Added passdb { result_* and skip } settings. passdb { skip = never | authenticated | unauthenticated } can be used to skip over a passdb lookup based on previous passdb lookups. passdb { result_success, result_failure, result_internalfail } can be used to specify what to do on those conditions. Choices as continue, continue-ok, continue-fail, return, return-ok, return-fail. The -ok and -fail variants update the current "success" flag, while continue/return uses the current flag. The authentication succeeds only if the success flag is set after the last lookup. The continue variants continue to the next passdb, while return variants finish the lookup immediately. diffstat: src/auth/auth-fields.c | 62 +++++++++++- src/auth/auth-fields.h | 13 ++- src/auth/auth-master-connection.c | 5 +- src/auth/auth-request-handler.c | 6 +- src/auth/auth-request.c | 181 +++++++++++++++++++++++++------------ src/auth/auth-request.h | 5 + src/auth/auth-settings.c | 12 ++ src/auth/auth-settings.h | 4 + src/auth/auth-worker-client.c | 6 +- src/auth/auth.c | 39 ++++++++ src/auth/auth.h | 20 ++++ src/auth/mech-gssapi.c | 1 + src/auth/passdb-blocking.c | 11 +-- 13 files changed, 281 insertions(+), 84 deletions(-) diffs (truncated from 711 to 300 lines): diff -r 9e93a5743c95 -r d60aa734c72d src/auth/auth-fields.c --- a/src/auth/auth-fields.c Wed Jan 30 22:23:09 2013 +0200 +++ b/src/auth/auth-fields.c Wed Jan 30 22:44:48 2013 +0200 @@ -10,7 +10,9 @@ struct auth_fields { pool_t pool; - ARRAY_TYPE(auth_field) fields; + ARRAY_TYPE(auth_field) fields, snapshot_fields; + unsigned int snapshot_idx; + bool snapshotted; }; struct auth_fields *auth_fields_init(pool_t pool) @@ -22,6 +24,16 @@ return fields; } +static void auth_fields_snapshot_preserve(struct auth_fields *fields) +{ + if (!fields->snapshotted || array_is_created(&fields->snapshot_fields)) + return; + + p_array_init(&fields->snapshot_fields, fields->pool, + array_count(&fields->fields)); + array_append_array(&fields->snapshot_fields, &fields->fields); +} + static bool auth_fields_find_idx(struct auth_fields *fields, const char *key, unsigned int *idx_r) @@ -60,18 +72,21 @@ field = array_append_space(&fields->fields); field->key = p_strdup(fields->pool, key); } else { + auth_fields_snapshot_preserve(fields); field = array_idx_modifiable(&fields->fields, idx); } field->value = p_strdup_empty(fields->pool, value); - field->flags = flags; + field->flags = flags | AUTH_FIELD_FLAG_CHANGED; } void auth_fields_remove(struct auth_fields *fields, const char *key) { unsigned int idx; - if (auth_fields_find_idx(fields, key, &idx)) + if (auth_fields_find_idx(fields, key, &idx)) { + auth_fields_snapshot_preserve(fields); array_delete(&fields->fields, idx, 1); + } } const char *auth_fields_find(struct auth_fields *fields, const char *key) @@ -93,8 +108,10 @@ void auth_fields_reset(struct auth_fields *fields) { - if (array_is_created(&fields->fields)) + if (array_is_created(&fields->fields)) { + auth_fields_snapshot_preserve(fields); array_clear(&fields->fields); + } } void auth_fields_import(struct auth_fields *fields, const char *str, @@ -125,7 +142,8 @@ } void auth_fields_append(struct auth_fields *fields, string_t *dest, - bool include_hidden) + enum auth_field_flags flags_mask, + enum auth_field_flags flags_result) { const struct auth_field *f; unsigned int i, count; @@ -136,8 +154,7 @@ f = array_get(&fields->fields, &count); for (i = 0; i < count; i++) { - if (!include_hidden && - (f[i].flags & AUTH_FIELD_FLAG_HIDDEN) != 0) + if ((f[i].flags & flags_mask) != flags_result) continue; if (first) @@ -157,3 +174,34 @@ return fields == NULL || !array_is_created(&fields->fields) || array_count(&fields->fields) == 0; } + +void auth_fields_snapshot(struct auth_fields *fields) +{ + struct auth_field *field; + + fields->snapshotted = TRUE; + if (!array_is_created(&fields->fields)) + return; + + if (!array_is_created(&fields->snapshot_fields)) { + /* try to avoid creating this array */ + fields->snapshot_idx = array_count(&fields->fields); + } else { + array_clear(&fields->snapshot_fields); + array_append_array(&fields->snapshot_fields, &fields->fields); + } + array_foreach_modifiable(&fields->fields, field) + field->flags &= ~AUTH_FIELD_FLAG_CHANGED; +} + +void auth_fields_rollback(struct auth_fields *fields) +{ + if (array_is_created(&fields->snapshot_fields)) { + array_clear(&fields->fields); + array_append_array(&fields->fields, &fields->snapshot_fields); + } else if (array_is_created(&fields->fields)) { + array_delete(&fields->fields, fields->snapshot_idx, + array_count(&fields->fields) - + fields->snapshot_idx); + } +} diff -r 9e93a5743c95 -r d60aa734c72d src/auth/auth-fields.h --- a/src/auth/auth-fields.h Wed Jan 30 22:23:09 2013 +0200 +++ b/src/auth/auth-fields.h Wed Jan 30 22:44:48 2013 +0200 @@ -5,7 +5,9 @@ enum auth_field_flags { /* This field is internal to auth process and won't be sent to client */ - AUTH_FIELD_FLAG_HIDDEN = 0x01 + AUTH_FIELD_FLAG_HIDDEN = 0x01, + /* Changed since last snapshot. Set/cleared automatically. */ + AUTH_FIELD_FLAG_CHANGED = 0x02 }; struct auth_field { @@ -27,8 +29,15 @@ void auth_fields_import(struct auth_fields *fields, const char *str, enum auth_field_flags flags); const ARRAY_TYPE(auth_field) *auth_fields_export(struct auth_fields *fields); +/* Append fields where (flag & flags_mask) == flags_result. */ void auth_fields_append(struct auth_fields *fields, string_t *dest, - bool include_hidden); + enum auth_field_flags flags_mask, + enum auth_field_flags flags_result); bool auth_fields_is_empty(struct auth_fields *fields); +/* Remember the current fields. */ +void auth_fields_snapshot(struct auth_fields *fields); +/* Rollback to previous snapshot, or clear the fields if there isn't any. */ +void auth_fields_rollback(struct auth_fields *fields); + #endif diff -r 9e93a5743c95 -r d60aa734c72d src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Jan 30 22:23:09 2013 +0200 +++ b/src/auth/auth-master-connection.c Wed Jan 30 22:44:48 2013 +0200 @@ -275,7 +275,8 @@ str_printfa(str, "USER\t%u\t", auth_request->id); str_append_tabescaped(str, auth_request->user); str_append_c(str, '\t'); - auth_fields_append(auth_request->userdb_reply, str, FALSE); + auth_fields_append(auth_request->userdb_reply, str, + AUTH_FIELD_FLAG_HIDDEN, 0); break; } @@ -326,7 +327,7 @@ if (!auth_fields_is_empty(auth_request->extra_fields)) { str_append_c(str, '\t'); auth_fields_append(auth_request->extra_fields, - str, FALSE); + str, AUTH_FIELD_FLAG_HIDDEN, 0); } break; case PASSDB_RESULT_USER_UNKNOWN: diff -r 9e93a5743c95 -r d60aa734c72d src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Jan 30 22:23:09 2013 +0200 +++ b/src/auth/auth-request-handler.c Wed Jan 30 22:44:48 2013 +0200 @@ -170,7 +170,8 @@ return; str_append_c(dest, '\t'); - auth_fields_append(request->extra_fields, dest, FALSE); + auth_fields_append(request->extra_fields, dest, + AUTH_FIELD_FLAG_HIDDEN, 0); if (!request->auth_only && auth_fields_exists(request->extra_fields, "proxy")) { @@ -641,7 +642,8 @@ str_printfa(str, "USER\t%u\t", request->id); str_append_tabescaped(str, request->user); str_append_c(str, '\t'); - auth_fields_append(request->userdb_reply, str, FALSE); + auth_fields_append(request->userdb_reply, str, + AUTH_FIELD_FLAG_HIDDEN, 0); if (request->master_user != NULL && !auth_fields_exists(request->userdb_reply, "master_user")) { diff -r 9e93a5743c95 -r d60aa734c72d src/auth/auth-request.c --- a/src/auth/auth-request.c Wed Jan 30 22:23:09 2013 +0200 +++ b/src/auth/auth-request.c Wed Jan 30 22:44:48 2013 +0200 @@ -81,6 +81,7 @@ request->last_access = ioloop_time; request->session_pid = (pid_t)-1; request->set = global_auth_settings; + request->extra_fields = auth_fields_init(request->pool); return request; } @@ -118,7 +119,7 @@ { i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); - if (request->failed) { + if (request->failed || !request->passdb_success) { /* password was valid, but some other check failed. */ auth_request_fail(request); return; @@ -193,8 +194,10 @@ { str_append_c(dest, '\t'); str_append(dest, key); - str_append_c(dest, '='); - str_append_tabescaped(dest, value); + if (value != NULL) { + str_append_c(dest, '='); + str_append_tabescaped(dest, value); + } } void auth_request_export(struct auth_request *request, string_t *dest) @@ -321,10 +324,9 @@ request->requested_login_user = p_strdup(request->pool, value); else if (strcmp(key, "successful") == 0) request->successful = TRUE; - else if (strcmp(key, "skip-password-check") == 0) { - i_assert(request->master_user != NULL); + else if (strcmp(key, "skip-password-check") == 0) request->skip_password_check = TRUE; - } else if (strcmp(key, "mech") == 0) + else if (strcmp(key, "mech") == 0) request->mech_name = p_strdup(request->pool, value); else return FALSE; @@ -429,18 +431,22 @@ if (!auth_fields_is_empty(request->extra_fields)) { str_append_c(str, '\t'); - auth_fields_append(request->extra_fields, str, TRUE); + /* add only those extra fields to cache that were set by this + passdb lookup. the CHANGED flag does this, because we + snapshotted the extra_fields before the current passdb + lookup. */ + auth_fields_append(request->extra_fields, str, + AUTH_FIELD_FLAG_CHANGED, + AUTH_FIELD_FLAG_CHANGED); } auth_cache_insert(passdb_cache, request, passdb->cache_key, str_c(str), result == PASSDB_RESULT_OK); } -static bool auth_request_master_lookup_finish(struct auth_request *request) +static void auth_request_master_lookup_finish(struct auth_request *request) { - struct auth_passdb *passdb; - if (request->failed) - return TRUE; + return; /* master login successful. update user and master_user variables. */ auth_request_log_info(request, "passdb", "Master user logging in as %s", @@ -449,35 +455,35 @@ request->master_user = request->user; request->user = request->requested_login_user; request->requested_login_user = NULL; +} - request->skip_password_check = TRUE; - request->passdb_password = NULL; +static bool +auth_request_want_skip_passdb(struct auth_request *request, + struct auth_passdb *passdb) +{ + /* skip_password_check basically specifies if authentication is + finished */ + bool authenticated = request->skip_password_check; - if (!request->passdb->set->pass) { - /* skip the passdb lookup, we're authenticated now. */ - return TRUE; + switch (passdb->skip) { + case AUTH_PASSDB_SKIP_NEVER: + return FALSE; + case AUTH_PASSDB_SKIP_AUTHENTICATED: + return authenticated; + case AUTH_PASSDB_SKIP_UNAUTHENTICATED: + return !authenticated; } - From dovecot at dovecot.org Wed Jan 30 23:24:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 23:24:30 +0200 Subject: dovecot-2.2: auth: Fixed compiling GSSAPI after recent changes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9afe5c6c1c1f changeset: 15692:9afe5c6c1c1f user: Timo Sirainen date: Wed Jan 30 23:24:20 2013 +0200 description: auth: Fixed compiling GSSAPI after recent changes. diffstat: src/auth/mech-gssapi.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r d60aa734c72d -r 9afe5c6c1c1f src/auth/mech-gssapi.c --- a/src/auth/mech-gssapi.c Wed Jan 30 22:44:48 2013 +0200 +++ b/src/auth/mech-gssapi.c Wed Jan 30 23:24:20 2013 +0200 @@ -408,7 +408,7 @@ { const char *value, *const *authorized_names, *const *tmp; - value = auth_stream_reply_find(request->extra_fields, "k5principals"); + value = auth_fields_find(request->extra_fields, "k5principals"); if (value == NULL) return FALSE; @@ -639,7 +639,7 @@ /* Continue in callback once auth_request is populated with passdb information. */ - request->passdb_success = TRUE; /* default to success */ + auth_request->passdb_success = TRUE; /* default to success */ auth_request_lookup_credentials(&request->auth_request, "", gssapi_credentials_callback); return 0; From dovecot at dovecot.org Wed Jan 30 23:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 23:50:34 +0200 Subject: dovecot-2.1: Released v2.1.14. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b314c97d4bbf changeset: 14886:b314c97d4bbf user: Timo Sirainen date: Wed Jan 30 23:45:06 2013 +0200 description: Released v2.1.14. diffstat: NEWS | 8 ++++++++ configure.in | 2 +- 2 files changed, 9 insertions(+), 1 deletions(-) diffs (25 lines): diff -r 0b0399f1b6aa -r b314c97d4bbf NEWS --- a/NEWS Tue Jan 29 17:55:20 2013 +0200 +++ b/NEWS Wed Jan 30 23:45:06 2013 +0200 @@ -1,3 +1,11 @@ +v2.1.14 2013-01-30 Timo Sirainen + + + doveadm acl recalc command added + - v2.1.11+ had a race condition where it sometimes overwrote data in + dovecot.index.cache file. This could have caused Dovecot to return + the same cached data to two different messages. + - mdbox: Fixes to handling duplicate GUIDs during index rebuild + v2.1.13 2013-01-06 Timo Sirainen - Some fixes to cache file changes in v2.1.11. diff -r 0b0399f1b6aa -r b314c97d4bbf configure.in --- a/configure.in Tue Jan 29 17:55:20 2013 +0200 +++ b/configure.in Wed Jan 30 23:45:06 2013 +0200 @@ -1,5 +1,5 @@ AC_PREREQ([2.59]) -AC_INIT([Dovecot],[2.1.13],[dovecot at dovecot.org]) +AC_INIT([Dovecot],[2.1.14],[dovecot at dovecot.org]) AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) From dovecot at dovecot.org Wed Jan 30 23:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 23:50:34 +0200 Subject: dovecot-2.1: Added tag 2.1.14 for changeset b314c97d4bbf Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/49e85db52ae4 changeset: 14887:49e85db52ae4 user: Timo Sirainen date: Wed Jan 30 23:45:06 2013 +0200 description: Added tag 2.1.14 for changeset b314c97d4bbf diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r b314c97d4bbf -r 49e85db52ae4 .hgtags --- a/.hgtags Wed Jan 30 23:45:06 2013 +0200 +++ b/.hgtags Wed Jan 30 23:45:06 2013 +0200 @@ -90,3 +90,4 @@ 741d800a192fa23572bb14196df2a8917cf20614 2.1.11 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 cf9d62fd0b143efa8e49fac998eb78a648cdd8a9 2.1.13 +b314c97d4bbffd01b20f8492592aa422c13e3d55 2.1.14 From dovecot at dovecot.org Wed Jan 30 23:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Jan 2013 23:50:34 +0200 Subject: dovecot-2.1: Added signature for changeset b314c97d4bbf Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/57eb9a6ac6c0 changeset: 14888:57eb9a6ac6c0 user: Timo Sirainen date: Wed Jan 30 23:45:10 2013 +0200 description: Added signature for changeset b314c97d4bbf diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 49e85db52ae4 -r 57eb9a6ac6c0 .hgsigs --- a/.hgsigs Wed Jan 30 23:45:06 2013 +0200 +++ b/.hgsigs Wed Jan 30 23:45:10 2013 +0200 @@ -53,3 +53,4 @@ 741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy 75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99 cf9d62fd0b143efa8e49fac998eb78a648cdd8a9 0 iEYEABECAAYFAlDqjXUACgkQyUhSUUBViskUEwCfYTWHeDmPr8HfxSBQN17SD5IwDygAnROhb3IVTm9niDun4gxPxbHLo/Pe +b314c97d4bbffd01b20f8492592aa422c13e3d55 0 iEYEABECAAYFAlEJlGMACgkQyUhSUUBVismNdQCgggPP/dt1duU1CMYfkpE4Kyc9Ju0An0kphokRqrtppkeqg7pF1JR01Mgq From dovecot at dovecot.org Thu Jan 31 00:14:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 00:14:34 +0200 Subject: dovecot-2.2: dsync: If remote command child process doesn't die ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/070ca24e5846 changeset: 15693:070ca24e5846 user: Timo Sirainen date: Thu Jan 31 00:14:28 2013 +0200 description: dsync: If remote command child process doesn't die at exit, kill -9 it. diffstat: src/doveadm/dsync/doveadm-dsync.c | 24 +++++++++++++++++------- 1 files changed, 17 insertions(+), 7 deletions(-) diffs (61 lines): diff -r 9afe5c6c1c1f -r 070ca24e5846 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Wed Jan 30 23:24:20 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Jan 31 00:14:28 2013 +0200 @@ -41,6 +41,7 @@ const char *remote_name; const char *local_location; + pid_t remote_pid; int fd_in, fd_out, fd_err; struct io *io_err; @@ -90,7 +91,8 @@ if (pipe(fd_in) < 0 || pipe(fd_out) < 0 || pipe(fd_err) < 0) i_fatal("pipe() failed: %m"); - switch (fork()) { + ctx->remote_pid = fork(); + switch (ctx->remote_pid) { case -1: i_fatal("fork() failed: %m"); case 0: @@ -331,17 +333,25 @@ } static void -cmd_dsync_run_remote(struct mail_user *user) +cmd_dsync_run_remote(struct dsync_cmd_context *ctx, struct mail_user *user) { int status; i_set_failure_prefix("dsync-local(%s): ", user->username); io_loop_run(current_ioloop); - /* wait for the remote command to finish to see any final errors */ - if (wait(&status) == -1) - i_error("wait() failed: %m"); - else if (WIFSIGNALED(status)) + /* wait for the remote command to finish to see any final errors. + don't wait very long though. */ + alarm(5); + if (wait(&status) == -1) { + if (errno != EINTR) + i_error("wait() failed: %m"); + else { + i_error("Remote command process isn't dying, killing it"); + if (kill(ctx->remote_pid, SIGKILL) < 0 && errno != ESRCH) + i_error("kill() failed: %m"); + } + } else if (WIFSIGNALED(status)) i_error("Remote command died with signal %d", WTERMSIG(status)); else if (WIFEXITED(status) && WEXITSTATUS(status) != 0) i_error("Remote command returned error %d", WEXITSTATUS(status)); @@ -429,7 +439,7 @@ if (cmd_dsync_run_local(ctx, user, brain, ibc2) < 0) ret = -1; } else { - cmd_dsync_run_remote(user); + cmd_dsync_run_remote(ctx, user); } if (ctx->state_input != NULL) { From dovecot at dovecot.org Thu Jan 31 00:27:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 00:27:28 +0200 Subject: dovecot-2.1: auth: Removed assert-checks, allowing plugins to se... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/5751963a3be8 changeset: 14889:5751963a3be8 user: Timo Sirainen date: Thu Jan 31 00:27:17 2013 +0200 description: auth: Removed assert-checks, allowing plugins to set skip_password_check. diffstat: src/auth/auth-request.c | 6 ++---- 1 files changed, 2 insertions(+), 4 deletions(-) diffs (24 lines): diff -r 57eb9a6ac6c0 -r 5751963a3be8 src/auth/auth-request.c --- a/src/auth/auth-request.c Wed Jan 30 23:45:10 2013 +0200 +++ b/src/auth/auth-request.c Thu Jan 31 00:27:17 2013 +0200 @@ -291,10 +291,9 @@ request->no_login = TRUE; else if (strcmp(key, "successful") == 0) request->successful = TRUE; - else if (strcmp(key, "skip_password_check") == 0) { - i_assert(request->master_user != NULL); + else if (strcmp(key, "skip_password_check") == 0) request->skip_password_check = TRUE; - } else if (strcmp(key, "mech") == 0) + else if (strcmp(key, "mech") == 0) request->mech_name = p_strdup(request->pool, value); else return FALSE; @@ -1705,7 +1704,6 @@ if (request->skip_password_check) { /* currently this can happen only with master logins */ - i_assert(request->master_user != NULL); return 1; } From dovecot at dovecot.org Thu Jan 31 17:01:59 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 17:01:59 +0200 Subject: dovecot-2.2: dsync: Ignore GUID mismatch on expunged message pair. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1d88f01ba2aa changeset: 15694:1d88f01ba2aa user: Timo Sirainen date: Thu Jan 31 17:01:46 2013 +0200 description: dsync: Ignore GUID mismatch on expunged message pair. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 070ca24e5846 -r 1d88f01ba2aa src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 00:14:28 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 17:01:46 2013 +0200 @@ -991,8 +991,10 @@ i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); if (!dsync_mail_change_guid_equals(local_change, change->guid)) { - /* mismatch - found the first non-common UID */ - dsync_mailbox_common_uid_found(importer); + /* GUID mismatch for two expunged mails. dsync can't update + GUIDs for already expunged messages, so we can't immediately + determine that the rest of the messages are a mismatch. so + for now we'll just skip over this pair. */ } else { importer->last_common_uid = change->uid; } From dovecot at dovecot.org Thu Jan 31 17:03:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 17:03:22 +0200 Subject: dovecot-2.2: dsync: Increased "remote cmd exit wait" from 5 to 3... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/47ad56ea1788 changeset: 15695:47ad56ea1788 user: Timo Sirainen date: Thu Jan 31 17:03:16 2013 +0200 description: dsync: Increased "remote cmd exit wait" from 5 to 30 seconds. diffstat: src/doveadm/dsync/doveadm-dsync.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (20 lines): diff -r 1d88f01ba2aa -r 47ad56ea1788 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Thu Jan 31 17:01:46 2013 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Jan 31 17:03:16 2013 +0200 @@ -32,6 +32,7 @@ #define DSYNC_LOCK_FILENAME ".dovecot-sync.lock" #define DSYNC_COMMON_GETOPT_ARGS "+adEfl:m:n:r:Rs:" +#define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30 struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; @@ -342,7 +343,7 @@ /* wait for the remote command to finish to see any final errors. don't wait very long though. */ - alarm(5); + alarm(DSYNC_REMOTE_CMD_EXIT_WAIT_SECS); if (wait(&status) == -1) { if (errno != EINTR) i_error("wait() failed: %m"); From dovecot at dovecot.org Thu Jan 31 18:17:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 18:17:21 +0200 Subject: dovecot-2.1: iostream-rawlog: Ignore 0 byte writes instead of as... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/f35a8649634c changeset: 14890:f35a8649634c user: Timo Sirainen date: Thu Jan 31 18:17:10 2013 +0200 description: iostream-rawlog: Ignore 0 byte writes instead of assert-crashing. diffstat: src/lib/iostream-rawlog.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 5751963a3be8 -r f35a8649634c src/lib/iostream-rawlog.c --- a/src/lib/iostream-rawlog.c Thu Jan 31 00:27:17 2013 +0200 +++ b/src/lib/iostream-rawlog.c Thu Jan 31 18:17:10 2013 +0200 @@ -44,7 +44,8 @@ { size_t i, start; - i_assert(size > 0); + if (size == 0) + return; io_loop_time_refresh(); if (rstream->write_timestamp) From dovecot at dovecot.org Thu Jan 31 18:27:32 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 18:27:32 +0200 Subject: dovecot-2.1: lib-ssl-iostream: Pass the plain istream's fd to ss... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/bf80034a547d changeset: 14891:bf80034a547d user: Timo Sirainen date: Thu Jan 31 18:27:22 2013 +0200 description: lib-ssl-iostream: Pass the plain istream's fd to ssl_istream. diffstat: src/lib-ssl-iostream/istream-openssl.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (11 lines): diff -r f35a8649634c -r bf80034a547d src/lib-ssl-iostream/istream-openssl.c --- a/src/lib-ssl-iostream/istream-openssl.c Thu Jan 31 18:17:10 2013 +0200 +++ b/src/lib-ssl-iostream/istream-openssl.c Thu Jan 31 18:27:22 2013 +0200 @@ -82,5 +82,6 @@ sstream->istream.read = i_stream_ssl_read; sstream->istream.istream.readable_fd = FALSE; - return i_stream_create(&sstream->istream, NULL, -1); + return i_stream_create(&sstream->istream, NULL, + i_stream_get_fd(ssl_io->plain_input)); } From dovecot at dovecot.org Thu Jan 31 18:39:54 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 18:39:54 +0200 Subject: dovecot-2.2: dsync: Fixed syncing keyword removals. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2801de211b78 changeset: 15696:2801de211b78 user: Timo Sirainen date: Thu Jan 31 18:39:44 2013 +0200 description: dsync: Fixed syncing keyword removals. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 47ad56ea1788 -r 2801de211b78 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 17:03:16 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 18:39:44 2013 +0200 @@ -688,7 +688,7 @@ } if (change_remove[i] != 0) { keywords_append(&remove_keywords, &all_keywords, - change_add[i], i*32); + change_remove[i], i*32); } } From dovecot at dovecot.org Thu Jan 31 18:47:32 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 18:47:32 +0200 Subject: dovecot-2.2: dsync: If mailbox UIDVALIDITY is 0, sync it to assi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/86629f621fe4 changeset: 15697:86629f621fe4 user: Timo Sirainen date: Thu Jan 31 18:47:20 2013 +0200 description: dsync: If mailbox UIDVALIDITY is 0, sync it to assign it instead of assert-crashing. diffstat: src/doveadm/dsync/dsync-mailbox-tree-fill.c | 28 +++++++++++++++++++++++++--- 1 files changed, 25 insertions(+), 3 deletions(-) diffs (52 lines): diff -r 2801de211b78 -r 86629f621fe4 src/doveadm/dsync/dsync-mailbox-tree-fill.c --- a/src/doveadm/dsync/dsync-mailbox-tree-fill.c Thu Jan 31 18:39:44 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-fill.c Thu Jan 31 18:47:20 2013 +0200 @@ -30,6 +30,30 @@ return 0; } +static int +dsync_mailbox_tree_get_selectable(struct mailbox *box, + struct mailbox_metadata *metadata_r, + struct mailbox_status *status_r) +{ + /* try the fast path */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, metadata_r) < 0) + return -1; + if (mailbox_get_status(box, STATUS_UIDVALIDITY, status_r) < 0) + return -1; + + i_assert(!guid_128_is_empty(metadata_r->guid)); + if (status_r->uidvalidity != 0) + return 0; + + /* no UIDVALIDITY assigned yet. syncing a mailbox should add it. */ + if (mailbox_sync(box, 0) < 0) + return -1; + if (mailbox_get_status(box, STATUS_UIDVALIDITY, status_r) < 0) + return -1; + i_assert(status_r->uidvalidity != 0); + return 0; +} + static int dsync_mailbox_tree_add(struct dsync_mailbox_tree *tree, const struct mailbox_info *info) { @@ -52,8 +76,7 @@ /* get GUID and UIDVALIDITY for selectable mailbox */ box = mailbox_alloc(info->ns->list, info->vname, 0); - if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0 || - mailbox_get_status(box, STATUS_UIDVALIDITY, &status) < 0) { + if (dsync_mailbox_tree_get_selectable(box, &metadata, &status) < 0) { errstr = mailbox_get_last_error(box, &error); switch (error) { case MAIL_ERROR_NOTFOUND: @@ -69,7 +92,6 @@ return -1; } } else { - i_assert(status.uidvalidity != 0); memcpy(node->mailbox_guid, metadata.guid, sizeof(node->mailbox_guid)); node->uid_validity = status.uidvalidity; From dovecot at dovecot.org Thu Jan 31 19:06:01 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 19:06:01 +0200 Subject: dovecot-2.2: dsync: When we know that remote does a flag update,... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/798a335d99b0 changeset: 15698:798a335d99b0 user: Timo Sirainen date: Thu Jan 31 19:05:55 2013 +0200 description: dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 35 ++++++++++++++++++++++--------- 1 files changed, 25 insertions(+), 10 deletions(-) diffs (89 lines): diff -r 86629f621fe4 -r 798a335d99b0 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 18:47:20 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 19:05:55 2013 +0200 @@ -482,7 +482,8 @@ merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove, uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove, uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote, - uint32_t *change_add_r, uint32_t *change_remove_r) + uint32_t *change_add_r, uint32_t *change_remove_r, + bool *remote_changed) { uint32_t combined_add, combined_remove, conflict_flags; uint32_t local_wanted, remote_wanted, conflict_pvt_flags; @@ -538,6 +539,8 @@ *change_add_r = local_wanted & ~local_final; *change_remove_r = local_final & ~local_wanted; + if (local_wanted != remote_final) + *remote_changed = TRUE; } static bool @@ -576,7 +579,7 @@ static void merge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes, const ARRAY_TYPE(const_string) *remote_changes, - bool prefer_remote) + bool prefer_remote, bool *remote_changed) { /* local_changes and remote_changes are assumed to have no duplicates names */ @@ -681,7 +684,7 @@ merge_flags(local_final[i], local_add[i], local_remove[i], remote_final[i], remote_add[i], remote_remove[i], 0, prefer_remote, prefer_remote, - &change_add[i], &change_remove[i]); + &change_add[i], &change_remove[i], remote_changed); if (change_add[i] != 0) { keywords_append(&add_keywords, &all_keywords, change_add[i], i*32); @@ -757,9 +760,10 @@ const struct dsync_mail_change *local_change; enum mail_flags local_add, local_remove; uint32_t change_add, change_remove; + uint64_t new_modseq; ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT; struct mail *mail; - bool prefer_remote, prefer_pvt_remote; + bool prefer_remote, prefer_pvt_remote, remote_changed = FALSE; i_assert((change->add_flags & change->remove_flags) == 0); @@ -812,7 +816,7 @@ change->final_flags, change->add_flags, change->remove_flags, mailbox_get_private_flags_mask(mail->box), prefer_remote, prefer_pvt_remote, - &change_add, &change_remove); + &change_add, &change_remove, &remote_changed); if (change_add != 0) mail_update_flags(mail, MODIFY_ADD, change_add); @@ -821,11 +825,22 @@ /* merge keywords */ merge_keywords(mail, &local_keyword_changes, &change->keyword_changes, - prefer_remote); - if (mail_get_modseq(mail) < change->modseq) - mail_update_modseq(mail, change->modseq); - if (mail_get_pvt_modseq(mail) < change->pvt_modseq) - mail_update_pvt_modseq(mail, change->pvt_modseq); + prefer_remote, &remote_changed); + + /* update modseqs. try to anticipate when we have to increase modseq + to get it closer to what remote has (although we can't guess it + exactly correctly) */ + new_modseq = change->modseq; + if (remote_changed && new_modseq <= importer->remote_highest_modseq) + new_modseq = importer->remote_highest_modseq+1; + if (mail_get_modseq(mail) < new_modseq) + mail_update_modseq(mail, new_modseq); + + new_modseq = change->pvt_modseq; + if (remote_changed && new_modseq <= importer->remote_highest_pvt_modseq) + new_modseq = importer->remote_highest_pvt_modseq+1; + if (mail_get_pvt_modseq(mail) < new_modseq) + mail_update_pvt_modseq(mail, new_modseq); } static void From dovecot at dovecot.org Thu Jan 31 19:45:41 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 19:45:41 +0200 Subject: dovecot-2.2: dsync: If both local and remote have the same messa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9b7d80a8db44 changeset: 15699:9b7d80a8db44 user: Timo Sirainen date: Thu Jan 31 19:44:28 2013 +0200 description: dsync: If both local and remote have the same message flags, don't go changing them. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 798a335d99b0 -r 9b7d80a8db44 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 19:05:55 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 19:44:28 2013 +0200 @@ -520,6 +520,11 @@ combined_remove = local_remove|remote_remove; i_assert((combined_add & combined_remove) == 0); + /* don't change flags that are currently identical in both sides */ + conflict_flags = local_final ^ remote_final; + combined_add &= conflict_flags; + combined_remove &= conflict_flags; + /* see if there are conflicting final flags */ local_wanted = (local_final|combined_add) & ~combined_remove; remote_wanted = (remote_final|combined_add) & ~combined_remove; From dovecot at dovecot.org Thu Jan 31 19:45:41 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 19:45:41 +0200 Subject: dovecot-2.2: dsync: Fixed syncing message keywords. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/067179cbabc2 changeset: 15700:067179cbabc2 user: Timo Sirainen date: Thu Jan 31 19:45:33 2013 +0200 description: dsync: Fixed syncing message keywords. "add" and "final" can't be mixed together always without causing keywords to be added back unintentionally. diffstat: src/doveadm/dsync/dsync-mail.h | 4 +- src/doveadm/dsync/dsync-mailbox-export.c | 35 +++++++++++++++++++++---------- src/doveadm/dsync/dsync-mailbox-import.c | 14 +++++++++--- 3 files changed, 36 insertions(+), 17 deletions(-) diffs (136 lines): diff -r 9b7d80a8db44 -r 067179cbabc2 src/doveadm/dsync/dsync-mail.h --- a/src/doveadm/dsync/dsync-mail.h Thu Jan 31 19:44:28 2013 +0200 +++ b/src/doveadm/dsync/dsync-mail.h Thu Jan 31 19:45:33 2013 +0200 @@ -40,6 +40,7 @@ #define KEYWORD_CHANGE_ADD '+' #define KEYWORD_CHANGE_REMOVE '-' #define KEYWORD_CHANGE_FINAL '=' +#define KEYWORD_CHANGE_ADD_AND_FINAL '&' struct dsync_mail_change { enum dsync_mail_change_type type; @@ -71,8 +72,7 @@ old transaction logs, new ones never reset keywords (just explicitly remove unwanted keywords) */ bool keywords_reset; - /* +add, -remove, =final. If the flag is both +added and in =final, - it's not not duplicated as =flag to avoid wasting space. */ + /* +add, -remove, =final, &add_and_final. */ ARRAY_TYPE(const_string) keyword_changes; }; diff -r 9b7d80a8db44 -r 067179cbabc2 src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Thu Jan 31 19:44:28 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Thu Jan 31 19:45:33 2013 +0200 @@ -68,20 +68,33 @@ } static bool -final_keyword_check(struct dsync_mail_change *change, const char *name) +final_keyword_check(struct dsync_mail_change *change, const char *name, + char *type_r) { const char *const *changes; unsigned int i, count; + *type_r = KEYWORD_CHANGE_ADD; + changes = array_get(&change->keyword_changes, &count); for (i = 0; i < count; i++) { - if (strcmp(changes[i]+1, name) == 0) { - if (changes[i][0] == KEYWORD_CHANGE_REMOVE) { - /* a final keyword is marked as removed. - this shouldn't normally happen. */ - array_delete(&change->keyword_changes, i, 1); - break; - } + if (strcmp(changes[i]+1, name) != 0) + continue; + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + /* replace with ADD_AND_FINAL */ + array_delete(&change->keyword_changes, i, 1); + *type_r = KEYWORD_CHANGE_ADD_AND_FINAL; + return FALSE; + case KEYWORD_CHANGE_REMOVE: + /* a final keyword is marked as removed. + this shouldn't normally happen. */ + array_delete(&change->keyword_changes, i, 1); + return FALSE; + case KEYWORD_CHANGE_ADD_AND_FINAL: + case KEYWORD_CHANGE_FINAL: + /* no change */ return TRUE; } } @@ -94,6 +107,7 @@ { const char *const *keywords; unsigned int i; + char type; i_assert((change->add_flags & change->remove_flags) == 0); @@ -110,11 +124,10 @@ for (i = 0; keywords[i] != NULL; i++) { /* add the final keyword if it's not already there as +keyword */ - if (!final_keyword_check(change, keywords[i])) { + if (!final_keyword_check(change, keywords[i], &type)) { const char *keyword_change = p_strdup_printf(exporter->pool, "%c%s", - KEYWORD_CHANGE_ADD, - keywords[i]); + type, keywords[i]); array_append(&change->keyword_changes, &keyword_change, 1); } diff -r 9b7d80a8db44 -r 067179cbabc2 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 19:44:28 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 19:45:33 2013 +0200 @@ -638,12 +638,16 @@ switch (changes[i][0]) { case KEYWORD_CHANGE_ADD: remote_add[name_idx/32] |= 1U << (name_idx%32); - /* fall through */ + break; + case KEYWORD_CHANGE_REMOVE: + remote_remove[name_idx/32] |= 1U << (name_idx%32); + break; case KEYWORD_CHANGE_FINAL: remote_final[name_idx/32] |= 1U << (name_idx%32); break; - case KEYWORD_CHANGE_REMOVE: - remote_remove[name_idx/32] |= 1U << (name_idx%32); + case KEYWORD_CHANGE_ADD_AND_FINAL: + remote_add[name_idx/32] |= 1U << (name_idx%32); + remote_final[name_idx/32] |= 1U << (name_idx%32); break; } } @@ -664,6 +668,7 @@ switch (changes[i][0]) { case KEYWORD_CHANGE_ADD: + case KEYWORD_CHANGE_ADD_AND_FINAL: local_add[name_idx/32] |= 1U << (name_idx%32); break; case KEYWORD_CHANGE_REMOVE: @@ -737,6 +742,7 @@ switch (changes[i][0]) { case KEYWORD_CHANGE_ADD: case KEYWORD_CHANGE_FINAL: + case KEYWORD_CHANGE_ADD_AND_FINAL: name = changes[i]+1; array_append(&keywords, &name, 1); break; @@ -1245,7 +1251,7 @@ t_array_init(&keywords, count); for (i = 0; i < count; i++) { if (changes[i][0] == KEYWORD_CHANGE_ADD || - changes[i][0] == KEYWORD_CHANGE_FINAL) { + changes[i][0] == KEYWORD_CHANGE_ADD_AND_FINAL) { const char *name = changes[i]+1; array_append(&keywords, &name, 1); From dovecot at dovecot.org Thu Jan 31 19:45:57 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 19:45:57 +0200 Subject: dovecot-2.2: doveadm fetch: Added "modseq" field. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/61aaf4102761 changeset: 15701:61aaf4102761 user: Timo Sirainen date: Thu Jan 31 19:45:50 2013 +0200 description: doveadm fetch: Added "modseq" field. diffstat: src/doveadm/doveadm-mail-fetch.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diffs (24 lines): diff -r 067179cbabc2 -r 61aaf4102761 src/doveadm/doveadm-mail-fetch.c --- a/src/doveadm/doveadm-mail-fetch.c Thu Jan 31 19:45:33 2013 +0200 +++ b/src/doveadm/doveadm-mail-fetch.c Thu Jan 31 19:45:50 2013 +0200 @@ -101,6 +101,12 @@ return 0; } +static int fetch_modseq(struct fetch_cmd_context *ctx) +{ + doveadm_print_num(mail_get_modseq(ctx->mail)); + return 0; +} + static int fetch_hdr(struct fetch_cmd_context *ctx) { struct istream *input; @@ -404,6 +410,7 @@ { "uid", 0, fetch_uid }, { "guid", 0, fetch_guid }, { "flags", MAIL_FETCH_FLAGS, fetch_flags }, + { "modseq", 0, fetch_modseq }, { "hdr", MAIL_FETCH_STREAM_HEADER, fetch_hdr }, { "body", MAIL_FETCH_STREAM_BODY, fetch_body }, { "text", MAIL_FETCH_STREAM_HEADER | From dovecot at dovecot.org Thu Jan 31 19:58:44 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 19:58:44 +0200 Subject: dovecot-2.2: dsync: More fixes to merging flag/keyword changes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/85e464d088da changeset: 15702:85e464d088da user: Timo Sirainen date: Thu Jan 31 19:58:34 2013 +0200 description: dsync: More fixes to merging flag/keyword changes. diffstat: src/doveadm/dsync/dsync-mailbox-export.c | 2 +- src/doveadm/dsync/dsync-mailbox-import.c | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diffs (103 lines): diff -r 61aaf4102761 -r 85e464d088da src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Thu Jan 31 19:45:50 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Thu Jan 31 19:58:34 2013 +0200 @@ -74,7 +74,7 @@ const char *const *changes; unsigned int i, count; - *type_r = KEYWORD_CHANGE_ADD; + *type_r = KEYWORD_CHANGE_FINAL; changes = array_get(&change->keyword_changes, &count); for (i = 0; i < count; i++) { diff -r 61aaf4102761 -r 85e464d088da src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 19:45:50 2013 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Jan 31 19:58:34 2013 +0200 @@ -483,7 +483,7 @@ uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove, uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote, uint32_t *change_add_r, uint32_t *change_remove_r, - bool *remote_changed) + bool *remote_changed, bool *remote_pvt_changed) { uint32_t combined_add, combined_remove, conflict_flags; uint32_t local_wanted, remote_wanted, conflict_pvt_flags; @@ -544,8 +544,10 @@ *change_add_r = local_wanted & ~local_final; *change_remove_r = local_final & ~local_wanted; - if (local_wanted != remote_final) + if ((local_wanted & ~pvt_mask) != (remote_final & ~pvt_mask)) *remote_changed = TRUE; + if ((local_wanted & pvt_mask) != (remote_final & pvt_mask)) + *remote_pvt_changed = TRUE; } static bool @@ -584,7 +586,8 @@ static void merge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes, const ARRAY_TYPE(const_string) *remote_changes, - bool prefer_remote, bool *remote_changed) + bool prefer_remote, + bool *remote_changed, bool *remote_pvt_changed) { /* local_changes and remote_changes are assumed to have no duplicates names */ @@ -675,7 +678,7 @@ local_remove[name_idx/32] |= 1U << (name_idx%32); break; case KEYWORD_CHANGE_FINAL: - i_unreached(); + break; } } for (i = 0; local_keywords[i] != NULL; i++) { @@ -694,7 +697,8 @@ merge_flags(local_final[i], local_add[i], local_remove[i], remote_final[i], remote_add[i], remote_remove[i], 0, prefer_remote, prefer_remote, - &change_add[i], &change_remove[i], remote_changed); + &change_add[i], &change_remove[i], + remote_changed, remote_pvt_changed); if (change_add[i] != 0) { keywords_append(&add_keywords, &all_keywords, change_add[i], i*32); @@ -774,7 +778,8 @@ uint64_t new_modseq; ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT; struct mail *mail; - bool prefer_remote, prefer_pvt_remote, remote_changed = FALSE; + bool prefer_remote, prefer_pvt_remote; + bool remote_changed = FALSE, remote_pvt_changed = FALSE; i_assert((change->add_flags & change->remove_flags) == 0); @@ -827,7 +832,8 @@ change->final_flags, change->add_flags, change->remove_flags, mailbox_get_private_flags_mask(mail->box), prefer_remote, prefer_pvt_remote, - &change_add, &change_remove, &remote_changed); + &change_add, &change_remove, + &remote_changed, &remote_pvt_changed); if (change_add != 0) mail_update_flags(mail, MODIFY_ADD, change_add); @@ -836,7 +842,7 @@ /* merge keywords */ merge_keywords(mail, &local_keyword_changes, &change->keyword_changes, - prefer_remote, &remote_changed); + prefer_remote, &remote_changed, &remote_pvt_changed); /* update modseqs. try to anticipate when we have to increase modseq to get it closer to what remote has (although we can't guess it @@ -848,7 +854,7 @@ mail_update_modseq(mail, new_modseq); new_modseq = change->pvt_modseq; - if (remote_changed && new_modseq <= importer->remote_highest_pvt_modseq) + if (remote_pvt_changed && new_modseq <= importer->remote_highest_pvt_modseq) new_modseq = importer->remote_highest_pvt_modseq+1; if (mail_get_pvt_modseq(mail) < new_modseq) mail_update_pvt_modseq(mail, new_modseq); From dovecot at dovecot.org Thu Jan 31 21:49:12 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 21:49:12 +0200 Subject: dovecot-2.2: replicator: Crashfix at deinit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1bd70d505ef7 changeset: 15703:1bd70d505ef7 user: Timo Sirainen date: Thu Jan 31 21:01:09 2013 +0200 description: replicator: Crashfix at deinit diffstat: src/replication/replicator/replicator-brain.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (30 lines): diff -r 85e464d088da -r 1bd70d505ef7 src/replication/replicator/replicator-brain.c --- a/src/replication/replicator/replicator-brain.c Thu Jan 31 19:58:34 2013 +0200 +++ b/src/replication/replicator/replicator-brain.c Thu Jan 31 21:01:09 2013 +0200 @@ -20,6 +20,8 @@ struct timeout *to; ARRAY(struct doveadm_connection *) doveadm_conns; + + unsigned int deinitializing:1; }; static void replicator_brain_fill(struct replicator_brain *brain); @@ -57,6 +59,7 @@ *_brain = NULL; + brain->deinitializing = TRUE; array_foreach_modifiable(&brain->doveadm_conns, connp) doveadm_connection_deinit(connp); if (brain->to != NULL) @@ -97,7 +100,8 @@ reply != DOVEADM_REPLY_OK; replicator_queue_push(ctx->brain->queue, ctx->user); } - replicator_brain_fill(ctx->brain); + if (!ctx->brain->deinitializing) + replicator_brain_fill(ctx->brain); i_free(ctx); } From dovecot at dovecot.org Thu Jan 31 21:49:12 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 21:49:12 +0200 Subject: dovecot-2.2: replicator: Use locking when running dsync. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/138f1c76c0ec changeset: 15704:138f1c76c0ec user: Timo Sirainen date: Thu Jan 31 21:49:00 2013 +0200 description: replicator: Use locking when running dsync. diffstat: src/replication/replicator/doveadm-connection.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diffs (24 lines): diff -r 1bd70d505ef7 -r 138f1c76c0ec src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Thu Jan 31 21:01:09 2013 +0200 +++ b/src/replication/replicator/doveadm-connection.c Thu Jan 31 21:49:00 2013 +0200 @@ -14,6 +14,11 @@ #define DOVEADM_FAIL_TIMEOUT_MSECS (1000*5) #define DOVEADM_HANDSHAKE "VERSION\tdoveadm-server\t1\t0\n" +/* normally there shouldn't be any need for locking, since replicator doesn't + start dsync in parallel for the same user. we'll do locking just in case + anyway */ +#define DSYNC_LOCK_TIMEOUT_SECS 30 + struct doveadm_connection { char *path; int fd; @@ -195,7 +200,7 @@ cmd = t_str_new(256); str_append_c(cmd, '\t'); str_append_tabescaped(cmd, username); - str_append(cmd, "\tsync\t-d"); + str_printfa(cmd, "\tsync\t-d\t-l\t%u", DSYNC_LOCK_TIMEOUT_SECS); if (full) str_append(cmd, "\t-f"); str_append(cmd, "\t-s\t"); From dovecot at dovecot.org Thu Jan 31 22:14:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 22:14:29 +0200 Subject: dovecot-2.2: replicator plugin: Use non-blocking open for the re... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0a5a19067d95 changeset: 15705:0a5a19067d95 user: Timo Sirainen date: Thu Jan 31 22:13:36 2013 +0200 description: replicator plugin: Use non-blocking open for the replicator fifo. This fixes an infinite hang if the replicator process isn't running. diffstat: src/plugins/replication/replication-plugin.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 138f1c76c0ec -r 0a5a19067d95 src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Thu Jan 31 21:49:00 2013 +0200 +++ b/src/plugins/replication/replication-plugin.c Thu Jan 31 22:13:36 2013 +0200 @@ -56,13 +56,12 @@ if (fifo_failed) return -1; if (fifo_fd == -1) { - fifo_fd = open(fifo_path, O_WRONLY); + fifo_fd = open(fifo_path, O_WRONLY | O_NONBLOCK); if (fifo_fd == -1) { i_error("open(%s) failed: %m", fifo_path); fifo_failed = TRUE; return -1; } - fd_set_nonblock(fifo_fd, TRUE); } /* \t */ str = t_str_new(256); From dovecot at dovecot.org Thu Jan 31 22:14:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 22:14:30 +0200 Subject: dovecot-2.2: replicator: Don't try to reuse doveadm connections ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e9e6a95cea21 changeset: 15706:e9e6a95cea21 user: Timo Sirainen date: Thu Jan 31 22:14:12 2013 +0200 description: replicator: Don't try to reuse doveadm connections (and have them immediately fail) diffstat: src/replication/replicator/doveadm-connection.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (17 lines): diff -r 0a5a19067d95 -r e9e6a95cea21 src/replication/replicator/doveadm-connection.c --- a/src/replication/replicator/doveadm-connection.c Thu Jan 31 22:13:36 2013 +0200 +++ b/src/replication/replicator/doveadm-connection.c Thu Jan 31 22:14:12 2013 +0200 @@ -57,7 +57,13 @@ conn->callback = NULL; conn->context = NULL; + + /* make sure callback doesn't try to reuse this connection, since + we can't currently handle it */ + i_assert(!conn->cmd_sent); + conn->cmd_sent = TRUE; callback(reply, state, context); + conn->cmd_sent = FALSE; } static void doveadm_close(struct doveadm_connection *conn) From dovecot at dovecot.org Thu Jan 31 22:58:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 22:58:36 +0200 Subject: dovecot-2.2: ostream: Default seek&write_at implementations shou... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/07ee9842a00b changeset: 15707:07ee9842a00b user: Timo Sirainen date: Thu Jan 31 22:58:24 2013 +0200 description: ostream: Default seek&write_at implementations should fail with ESPIPE, not EPIPE. diffstat: src/lib/ostream.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r e9e6a95cea21 -r 07ee9842a00b src/lib/ostream.c --- a/src/lib/ostream.c Thu Jan 31 22:14:12 2013 +0200 +++ b/src/lib/ostream.c Thu Jan 31 22:58:24 2013 +0200 @@ -438,7 +438,7 @@ o_stream_default_seek(struct ostream_private *_stream, uoff_t offset ATTR_UNUSED) { - _stream->ostream.stream_errno = EPIPE; + _stream->ostream.stream_errno = ESPIPE; return -1; } @@ -447,7 +447,7 @@ const void *data ATTR_UNUSED, size_t size ATTR_UNUSED, uoff_t offset ATTR_UNUSED) { - _stream->ostream.stream_errno = EPIPE; + _stream->ostream.stream_errno = ESPIPE; return -1; } From dovecot at dovecot.org Thu Jan 31 22:59:18 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 31 Jan 2013 22:59:18 +0200 Subject: dovecot-2.2: ostream: If stream is closed with stream_errno=0, s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4a59737dd2ca changeset: 15708:4a59737dd2ca user: Timo Sirainen date: Thu Jan 31 22:59:13 2013 +0200 description: ostream: If stream is closed with stream_errno=0, set it to EPIPE. diffstat: src/lib/ostream.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 07ee9842a00b -r 4a59737dd2ca src/lib/ostream.c --- a/src/lib/ostream.c Thu Jan 31 22:58:24 2013 +0200 +++ b/src/lib/ostream.c Thu Jan 31 22:59:13 2013 +0200 @@ -55,6 +55,9 @@ { io_stream_close(&stream->real_stream->iostream); stream->closed = TRUE; + + if (stream->stream_errno == 0) + stream->stream_errno = EPIPE; } #undef o_stream_set_flush_callback