Replication stalled by failed attemts to read attachments (mail_attachment_dir)
Ralf Becker
rb at egroupware.org
Thu Aug 5 19:03:32 EEST 2021
I'm migrating an older Dovecot 2.2.19 installation with dbox and
mail_attachment_dir to 2.3.15 with replication to a second server.
The storage from the old server was rsync'ed to a new server running
Dovecot 2.3.15 in a container using similar configuration with dbox and
mail_attachments. That servers alone seems to run fine.
As a next step I tried to establish replication to the second empty
server, which is configured to use mdbox and no mail_attachment_dir, as
the rest of my Dovecot servers. Looking at "doveadm replicator status
'*'" some of the mailboxes replicate correct, but roughly half of them
fail :(
Looking a the Dovecot log with mail_debug=yes shows the replication
fails on the source (server with dbox and mail_attachment_dir) typically
with one of two errors:
a) attachment has different size:
Aug 05 15:41:58 doveadm: Debug: Mailbox INBOX: UID 30: Looked up field
date.received from mail cache
Aug 05 15:41:58 doveadm: Error: dsync(cdaf50f7580e):
read(attachments-connector(/var/dovecot/imap/<domain>/<user>/mailboxes/INBOX/dbox-Mails/u.30))
failed:
read(/var/dovecot/imap/attachments/64/4e/644ecfbc09223b8ccafc6f36aa86603cde91e76716b9b5ed0cf875bc0e183b6b4581d671f8321647be99f26d2d22088e999714529e189b25a6e7d5b428fbf5d5-1a6b701e68c9c050b10a0000838cbfe1-f86a701e68c9c050b10a0000838cbfe1-30[base64:18
b/l]) failed: Stream is larger than expected (262658 > 262657, eof=1)
(last sent=mail, last recv=mail_request (EOL))
showing on the other log as:
Aug 05 15:49:18 doveadm: Debug: Mailbox INBOX: UID 29: Looked up field
guid from mail cache
Aug 05 15:49:19 doveadm: Error: dsync(142878d8d6bf): read() failed:
read(10.44.99.1) failed: dot-input stream ends without '.' line (last
sent=mail_request (EOL), last recv=mail)
b) attachment is not found at all:
Aug 05 15:43:58 doveadm(<user>@<domain> 1441): Debug: fs-sis: fs-sis:
fs-posix:
open(/var/dovecot/imap/attachments/e2/0a/e20ab165f3a2e7164520341cddb06cd053dc1a87fdd5c6360b45c20a2c9d33576c8ea2daa42062f9bb0fd94f9c5f9919ad86fb11e29effe3bdf4e853fa7c0e1d-20a0b63613e6c050620b0000838cbfe1-c8d5221c39a40b61250900007dc04144-3)
failed: No such file or directory
Aug 05 15:43:58 doveadm(<user>@<domain> 1441): Error:
dsync(cdaf50f7580e):
read(attachments-connector(/var/dovecot/imap/<domain>/<user>/mailboxes/INBOX/dbox-Mails/u.3))
failed:
read(/var/dovecot/imap/attachments/e2/0a/e20ab165f3a2e7164520341cddb06cd053dc1a87fdd5c6360b45c20a2c9d33576c8ea2daa42062f9bb0fd94f9c5f9919ad86fb11e29effe3bdf4e853fa7c0e1d-20a0b63613e6c050620b0000838cbfe1-c8d5221c39a40b61250900007dc04144-3[base64:19
b/l]) failed:
open(/var/dovecot/imap/attachments/e2/0a/e20ab165f3a2e7164520341cddb06cd053dc1a87fdd5c6360b45c20a2c9d33576c8ea2daa42062f9bb0fd94f9c5f9919ad86fb11e29effe3bdf4e853fa7c0e1d-20a0b63613e6c050620b0000838cbfe1-c8d5221c39a40b61250900007dc04144-3)
failed: No such file or directory (last sent=mail, last
recv=mail_request (EOL))
That caused the replication to fail and not newer mails to be replicated
for the given user and mailbox, which can be verified eg. with "doveadm
mailbox status -u <user>@<domain> all inbox".
The concerned mails are pretty old / the attachment itself can not be
recovered, and nothing can be done about them. But how can I repair the
mailbox, so all unaffected mails / attachments get replicated?
I already tried multiple "doveadm force-resync -u <user>@<domain> INBOX"
in case a) with no visible effect / still same error.
Both Dovecot configurations are attached, in case they matter ...
Ralf
--
Ralf Becker
EGroupware GmbH [www.egroupware.org]
Handelsregister HRB Kaiserslautern 3587
Geschäftsführer Birgit und Ralf Becker
Leibnizstr. 17, 67663 Kaiserslautern, Germany
Telefon +49 631 31657-0
-------------- next part --------------
# 2.3.15 (0503334ab1): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.15 (e6a84e31)
# OS: Linux 4.15.0-140-generic x86_64
# Hostname: cdaf50f7580e
auth_cache_negative_ttl = 2 mins
auth_cache_size = 10 M
auth_cache_ttl = 5 mins
auth_master_user_separator = *
auth_mechanisms = plain login
auth_username_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@#"
default_client_limit = 3500
default_process_limit = 512
disable_plaintext_auth = no
doveadm_password = # hidden, use -P to show it
doveadm_port = 12345
first_valid_uid = 90
listen = *
log_path = /dev/stderr
login_greeting = Dovecot FRA ready
mail_access_groups = dovecot
mail_attribute_dict = file:%h/dovecot-metadata
mail_debug = yes
mail_gid = dovecot
mail_location = mdbox:~/mdbox
mail_log_prefix = "%s(%u %p): "
mail_max_userip_connections = 200
mail_plugins = acl quota notify replication mail_log
mail_uid = dovecot
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date ihave vnd.dovecot.debug
mbox_min_index_size = 1000 B
mdbox_rotate_size = 50 M
namespace inboxes {
inbox = yes
location =
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
auto = subscribe
special_use = \Junk
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox Templates {
auto = subscribe
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
prefix = INBOX/
separator = /
subscriptions = no
}
namespace subs {
hidden = yes
list = no
location =
prefix =
separator = /
}
namespace users {
location = mdbox:%%h/mdbox
prefix = user/%%n/
separator = /
subscriptions = no
type = shared
}
passdb {
args = /etc/dovecot/dovecot-dict-master-auth.conf
driver = dict
master = yes
}
passdb {
args = /etc/dovecot/dovecot-dict-auth.conf
driver = dict
}
plugin {
acl = vfile
acl_shared_dict = file:/var/dovecot/imap/%d/shared-mailboxes.db
mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
mail_log_fields = uid box msgid size
mail_replica = tcp:10.44.99.3
quota = dict:User quota::ns=INBOX/:file:%h/dovecot-quota
quota_rule = *:storage=200GB
sieve = ~/sieve/dovecot.sieve
sieve_after = /var/dovecot/sieve/after.d/
sieve_before = /var/dovecot/sieve/before.d/
sieve_dir = ~/sieve
sieve_extensions = +editheader
sieve_user_log = ~/.sieve.log
}
postmaster_address = admins at egroupware.org
protocols = imap pop3 lmtp sieve
quota_full_tempfail = yes
replication_dsync_parameters = -d -n INBOX -l 30 -U
service aggregator {
fifo_listener replication-notify-fifo {
user = dovecot
}
unix_listener replication-notify {
user = dovecot
}
}
service auth-worker {
user = $default_internal_user
}
service auth {
drop_priv_before_exec = no
inet_listener {
port = 113
}
}
service doveadm {
inet_listener {
port = 12345
}
inet_listener {
port = 26
}
vsz_limit = 640 M
}
service imap-login {
inet_listener imap {
port = 143
}
inet_listener imaps {
port = 993
ssl = yes
}
process_min_avail = 5
service_count = 1
vsz_limit = 64 M
}
service imap {
executable = imap
process_limit = 2048
vsz_limit = 640 M
}
service lmtp {
inet_listener lmtp {
port = 24
}
unix_listener lmtp {
mode = 0666
}
vsz_limit = 512 M
}
service managesieve-login {
inet_listener sieve {
port = 4190
}
inet_listener sieve_deprecated {
port = 2000
}
}
service pop3-login {
inet_listener pop3 {
port = 110
}
inet_listener pop3s {
port = 995
ssl = yes
}
}
service pop3 {
executable = pop3
}
service postlogin {
executable = script-login -d rawlog -b -t
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
group = dovecot
mode = 0660
user = dovecot
}
}
ssl_cert = </etc/certs/mail.egroupware.org.pem
ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
userdb {
args = /etc/dovecot/dovecot-sql.conf
driver = sql
}
userdb {
args = /etc/dovecot/dovecot-dict-auth.conf
driver = dict
}
verbose_proctitle = yes
protocol lda {
mail_plugins = acl quota notify replication mail_log acl sieve quota
}
protocol imap {
imap_metadata = yes
mail_max_userip_connections = 200
mail_plugins = acl quota notify replication mail_log acl imap_acl quota imap_quota
}
protocol lmtp {
mail_max_lock_timeout = 25 secs
mail_plugins = acl quota notify replication mail_log acl sieve quota
}
-------------- next part --------------
# 2.3.15 (0503334ab1): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.15 (e6a84e31)
# OS: Linux 4.15.0-143-generic x86_64 Ubuntu 20.04.2 LTS
# Hostname: 142878d8d6bf
auth_cache_negative_ttl = 2 mins
auth_cache_size = 10 M
auth_cache_ttl = 5 mins
auth_master_user_separator = *
auth_mechanisms = plain login
auth_username_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@#"
default_client_limit = 3500
default_process_limit = 512
disable_plaintext_auth = no
doveadm_password = # hidden, use -P to show it
doveadm_port = 12345
first_valid_uid = 90
listen = *
log_path = /dev/stderr
login_greeting = Dovecot KA ready
mail_access_groups = dovecot
mail_attachment_dir = /var/dovecot/imap/attachments
mail_attachment_hash = %{sha512}
mail_attribute_dict = file:%h/dovecot-metadata
mail_debug = yes
mail_gid = dovecot
mail_location = sdbox:%h
mail_log_prefix = "%s(%u %p): "
mail_max_userip_connections = 200
mail_plugins = acl quota notify replication mail_log mail_lua notify push_notification push_notification_lua
mail_uid = dovecot
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date ihave vnd.dovecot.debug
mbox_min_index_size = 1000 B
mbox_write_locks = fcntl
mdbox_rotate_size = 50 M
namespace inboxes {
inbox = yes
location =
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
auto = subscribe
special_use = \Junk
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox Templates {
auto = subscribe
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
prefix = INBOX/
separator = /
subscriptions = no
}
namespace subs {
hidden = yes
list = no
location =
prefix =
separator = /
}
namespace users {
location = sdbox:%%h
prefix = user/%%n/
separator = /
subscriptions = no
type = shared
}
passdb {
args = /etc/dovecot/dovecot-dict-master-auth.conf
driver = dict
master = yes
}
passdb {
args = /etc/dovecot/dovecot-dict-auth.conf
driver = dict
}
plugin {
acl = vfile
acl_shared_dict = file:/var/dovecot/imap/%d/shared-mailboxes.db
mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
mail_log_fields = uid box msgid size
mail_replica = tcp:10.44.88.3
push_lua_url = http://172.17.0.1/
push_notification_driver = lua:file=/etc/dovecot/dovecot-push.lua
quota = dict:User quota::ns=INBOX/:file:%h/dovecot-quota
quota_rule = *:storage=200GB
sieve = ~/sieve/dovecot.sieve
sieve_after = /var/dovecot/sieve/after.d/
sieve_before = /var/dovecot/sieve/before.d/
sieve_dir = ~/sieve
sieve_extensions = +editheader
sieve_user_log = ~/.sieve.log
}
postmaster_address = admins at egroupware.org
protocols = imap pop3 lmtp sieve
quota_full_tempfail = yes
replication_dsync_parameters = -d -n INBOX -l 30 -U
service aggregator {
fifo_listener replication-notify-fifo {
user = dovecot
}
unix_listener replication-notify {
user = dovecot
}
}
service auth-worker {
user = $default_internal_user
}
service auth {
drop_priv_before_exec = no
inet_listener {
port = 113
}
}
service doveadm {
inet_listener {
port = 12345
}
inet_listener {
port = 26
}
vsz_limit = 640 M
}
service imap-login {
inet_listener imap {
port = 143
}
inet_listener imaps {
port = 993
ssl = yes
}
process_min_avail = 5
service_count = 1
vsz_limit = 64 M
}
service imap {
executable = imap
process_limit = 2048
vsz_limit = 640 M
}
service lmtp {
inet_listener lmtp {
port = 24
}
unix_listener lmtp {
mode = 0666
}
vsz_limit = 512 M
}
service managesieve-login {
inet_listener sieve {
port = 4190
}
inet_listener sieve_deprecated {
port = 2000
}
}
service pop3-login {
inet_listener pop3 {
port = 110
}
inet_listener pop3s {
port = 995
ssl = yes
}
}
service pop3 {
executable = pop3
}
service postlogin {
executable = script-login -d rawlog -b -t
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
group = dovecot
mode = 0660
user = dovecot
}
}
ssl_cert = </etc/certs/mail.egroupware.org.pem
ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
userdb {
args = /etc/dovecot/dovecot-sql.conf
driver = sql
}
userdb {
args = /etc/dovecot/dovecot-dict-auth.conf
driver = dict
}
verbose_proctitle = yes
protocol lda {
mail_plugins = acl quota notify replication mail_log mail_lua notify push_notification push_notification_lua acl sieve quota
}
protocol imap {
imap_metadata = yes
mail_max_userip_connections = 200
mail_plugins = acl quota notify replication mail_log mail_lua notify push_notification push_notification_lua acl imap_acl quota imap_quota
}
protocol lmtp {
mail_max_lock_timeout = 25 secs
mail_plugins = acl quota notify replication mail_log mail_lua notify push_notification push_notification_lua acl sieve quota notify push_notification
}
More information about the dovecot
mailing list