replication fails and corrupts index with zlib enabled
fauno
fauno at partidopirata.com.ar
Sat Aug 4 18:44:09 EEST 2018
Hi, I have two Debian Jessie servers with Dovecot 2.2.13 TCP replication
on that have worked fine for years, but now one of them is running low
on disk space, so I wanted to try enabling zlib.
I crafted a script following the description given in
https://wiki.dovecot.org/Plugins/Zlib and xz'ed some inboxes on the
stand-by server, the one with low disk space. So every email in those
inboxes is xz'ed but the file name hasn't changed and contains the
original size.
This server is on stand-by so most of the email is replicated
unidirectionally to it. But administrative emails like cronjobs and
monitoring are delivered locally, so it replicates those to the hot server.
The issue appeared when this stand-by server receives such an email and
tries to replicate them to the other server.
I'm attaching the full snippet of the log from the hot server, because
it throws a longer backtrace. The short version is like this:
dovecot[8438]: imap(redacted at address.org): Error: Cached message size
larger than expected (478 > 289)
dovecot[8438]: imap(redacted at address.org): Error: Maildir filename has
wrong S value, renamed the file from
/srv/email/address.org/redacted/Maildir/cur/1533393328.M502775P20341.standby_server,S=478:2,
to
/srv/email/address.org/redacted/Maildir/cur/1533393328.M502775P20341.standby_server,S=289:2,
dovecot[8438]: imap(redacted at address.org): Error: Corrupted index cache
file /srv/email/address.org/redacted/Maildir/dovecot.index.cache: Broken
physical size for mail UID 45123
After this there's an error and the replication fails. The file is
there, it's gzipped and can be zcat'ed but it appears as a blank email
on clients.
I've recovered a backup but the issue persists. I also changed from xz
to gz because the Debian package docs only mention gz and bzip2, but the
issue is the same.
From what I understand and tested, the stand-by server is receiving the
email and compressing it but maintaining the original size on the file
name. So that's ok, but when the hot server receives the copy, it
believes the size is wrong and changes it to the compressed size. Then
for some reason the index gets corrupted.
I'm attaching the doveconf for both servers. They're mostly the same,
and the only changes introduced were the zlib plugin and its options.
Also the script(s) that I used to compress the inboxes.
Am I correct? Is it an issue of replicator not understanding the emails
are compressed? I couldn't find anything related to zlib with
replication. Maybe it's something fixed in newer versions and I should
go that rabbit hole?
Thanks! :)
-------------- next part --------------
A non-text attachment was scrubbed...
Name: dovecot.log
Type: text/x-log
Size: 2656 bytes
Desc: not available
URL: <https://dovecot.org/pipermail/dovecot/attachments/20180804/e00c115d/attachment.bin>
-------------- next part --------------
# 2.2.13: /etc/dovecot/dovecot.conf
# OS: Linux 4.14.12-gnu x86_64 Debian 8.9
auth_mechanisms = plain login
auth_socket_path = /var/run/dovecot/auth-userdb
auth_verbose = yes
auth_verbose_passwords = sha1
disable_plaintext_auth = no
doveadm_password = a replication password
doveadm_port = 4691
hostname = address.org
imap_client_workarounds = delay-newmail
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
lmtp_save_to_detail_mailbox = yes
login_log_format_elements = user=<%u> method=%m rip=127.0.0.1 lip=127.0.0.1 mpid=%e %c
mail_location = maildir:~/Maildir
mail_plugins = " notify replication notify replication stats zlib"
mailbox_list_index = yes
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date ihave imapflags notify
namespace inbox {
inbox = yes
location =
mailbox Drafts {
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
special_use = \Sent
}
mailbox Trash {
special_use = \Trash
}
mailbox virtual/All {
special_use = \All
}
mailbox virtual/Flagged {
special_use = \Flagged
}
prefix =
type = private
}
passdb {
args = /etc/dovecot/dovecot-ldap.conf.ext
driver = ldap
}
plugin {
antispam_backend = pipe
antispam_mail_notspam = learn_ham
antispam_mail_sendmail = /usr/bin/rspamc
antispam_mail_sendmail_args = -h;localhost:11334;-P;q1
antispam_mail_spam = learn_spam
antispam_spam = Junk
antispam_trash = Trash
mail_replica = tcp:stand.by.server.ip.address
sieve = ~/.dovecot.sieve
sieve_before = /etc/dovecot/sieve/before.d/
sieve_dir = ~/sieve
sieve_extensions = +notify +imapflags +imap4flags
sieve_global_path = /etc/dovecot/sieve/main.sieve
stats_refresh = 30 secs
stats_track_cmds = yes
zlib_save = gz
zlib_save_level = 6
}
postmaster_address = root at address.org
protocols = imap pop3 lmtp sieve
service aggregator {
fifo_listener replication-notify-fifo {
mode = 0666
}
unix_listener replication-notify {
mode = 0666
}
}
service auth {
inet_listener {
port = 12345
}
unix_listener /var/spool/postfix/private/auth {
mode = 0666
}
unix_listener auth-userdb {
group = sasl
mode = 0660
}
}
service doveadm {
inet_listener {
port = 4691
}
}
service imap-login {
inet_listener imap {
port = 143
}
inet_listener imaps {
port = 993
ssl = yes
}
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
}
service managesieve-login {
inet_listener sieve {
port = 4190
}
service_count = 1
}
service managesieve {
process_limit = 1024
}
service pop3-login {
inet_listener pop3 {
port = 110
}
inet_listener pop3s {
port = 995
ssl = yes
}
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
group = email
mode = 0660
}
}
service stats {
fifo_listener stats-mail {
mode = 0666
user = dovecot
}
}
ssl_cert = </var/lib/acme/live/address.org/fullchain
ssl_cipher_list = ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
ssl_client_ca_dir = /etc/ssl/certs
ssl_client_ca_file = /etc/ssl/certs/ca-certificates.crt
ssl_dh_parameters_length = 2048
ssl_key = </var/lib/acme/live/address.org/privkey
ssl_prefer_server_ciphers = yes
ssl_protocols = TLSv1 TLSv1.1 TLSv1.2
userdb {
args = /etc/dovecot/dovecot-ldap.conf.ext
driver = ldap
}
protocol lmtp {
mail_plugins = " notify replication sieve"
postmaster_address = hola at address.org
}
protocol imap {
mail_max_userip_connections = 100
mail_plugins = " notify replication antispam stats"
}
-------------- next part --------------
# 2.2.13: /etc/dovecot/dovecot.conf
# OS: Linux 4.14.12-gnu x86_64 Debian 8.10
auth_mechanisms = plain login
auth_socket_path = /var/run/dovecot/auth-userdb
auth_verbose = yes
auth_verbose_passwords = sha1
disable_plaintext_auth = no
doveadm_password = a replication password
doveadm_port = 4691
hostname = address.org
imap_client_workarounds = delay-newmail
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
lmtp_save_to_detail_mailbox = yes
login_log_format_elements = user=<%u> method=%m rip=127.0.0.1 lip=127.0.0.1 mpid=%e %c
mail_location = maildir:~/Maildir
mail_plugins = " notify replication notify replication stats zlib"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date ihave imapflags notify
namespace inbox {
inbox = yes
location =
mailbox Drafts {
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
special_use = \Sent
}
mailbox Trash {
special_use = \Trash
}
mailbox virtual/All {
special_use = \All
}
mailbox virtual/Flagged {
special_use = \Flagged
}
prefix =
type = private
}
passdb {
args = /etc/dovecot/dovecot-ldap.conf.ext
driver = ldap
}
plugin {
antispam_backend = pipe
antispam_mail_notspam = learn_ham
antispam_mail_sendmail = /usr/bin/rspamc
antispam_mail_sendmail_args = -h;localhost:11334;-P;q1
antispam_mail_spam = learn_spam
antispam_spam = Junk
antispam_trash = Trash
mail_replica = tcp:hot.server.ip.address
sieve = ~/.dovecot.sieve
sieve_before = /etc/dovecot/sieve/before.d/
sieve_dir = ~/sieve
sieve_extensions = +notify +imapflags +imap4flags
sieve_global_path = /etc/dovecot/sieve/main.sieve
stats_refresh = 30 secs
stats_track_cmds = yes
zlib_save = gz
zlib_save_level = 6
}
postmaster_address = root at address.org
protocols = imap pop3 lmtp sieve
service aggregator {
fifo_listener replication-notify-fifo {
mode = 0666
}
unix_listener replication-notify {
mode = 0666
}
}
service auth {
inet_listener {
port = 12345
}
unix_listener /var/spool/postfix/private/auth {
mode = 0666
}
unix_listener auth-userdb {
group = sasl
mode = 0660
}
}
service doveadm {
inet_listener {
port = 4691
}
}
service imap-login {
inet_listener imap {
port = 143
}
inet_listener imaps {
port = 993
ssl = yes
}
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
}
service managesieve-login {
inet_listener sieve {
port = 4190
}
service_count = 1
}
service managesieve {
process_limit = 1024
}
service pop3-login {
inet_listener pop3 {
port = 110
}
inet_listener pop3s {
port = 995
ssl = yes
}
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
group = email
mode = 0660
}
}
service stats {
fifo_listener stats-mail {
mode = 0666
user = dovecot
}
}
ssl_cert = </var/lib/acme/live/address.org/fullchain
ssl_cipher_list = ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
ssl_client_ca_dir = /etc/ssl/certs
ssl_client_ca_file = /etc/ssl/certs/ca-certificates.crt
ssl_dh_parameters_length = 2048
ssl_key = </var/lib/acme/live/address.org/privkey
ssl_prefer_server_ciphers = yes
ssl_protocols = TLSv1 TLSv1.1 TLSv1.2
userdb {
args = /etc/dovecot/dovecot-ldap.conf.ext
driver = ldap
}
protocol lmtp {
mail_plugins = " notify replication sieve"
postmaster_address = hola at address.org
}
protocol imap {
mail_max_userip_connections = 100
mail_plugins = " notify replication antispam stats"
}
-------------- next part --------------
#!/bin/bash
for mail in $@; do
# Asegurarse que estamos trabajando con un mail
file --mime-type "${mail}" | cut -d " " -f 2 | grep -qE "message/rfc822|text/plain" || continue
# Agregar una Z al final para indicar que ya fue comprimido
tmpfile="${mail%/cur/*}/tmp/${mail##*/}Z"
# Comprimir a tmp
xz --stdout "${mail}" > "${tmpfile}"
# Copiar metadatos
chown --reference="${mail}" "${tmpfile}"
chmod --reference="${mail}" "${tmpfile}"
touch --reference="${mail}" "${tmpfile}"
mv --force "${tmpfile}" "${mail}" || continue
# Agregar la Z al final
mv "${mail}" "${mail}Z"
done
-------------- next part --------------
#!/bin/bash
# https://wiki.dovecot.org/Plugins/Zlib
USE_TMP=${USE_TMP:-false}
dir="${1}"
test -d "${dir}" || exit 1
test -d "${dir}/cur" || exit 1
# Trabajar en memoria (si estamos sincronizando al mismo tiempo, dovecot
# empieza a fallar)
${USE_TMP} && mount -t tmpfs none "${dir}/tmp"
# Bloquear el directorio por un rato
/usr/lib/dovecot/maildirlock "${dir}" 3600 > "${dir}/tmp/lock"
test $? -eq 0 || exit 1
du -hd1 "${dir}/cur"
# Solo buscamos en cur porque new ya va a estar comprimido
find "${dir}"/cur -type f \
| grep -v dovecot \
| grep -v "Z$" \
| grep "S=" \
| xargs -r -d "\n" -P 5 compress-mail
du -hd1 "${dir}/cur"
# Matar el lock
cat "${dir}/tmp/lock" | xargs kill
# Desmontar
${USE_TMP} && umount "${dir}/tmp"
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 618 bytes
Desc: OpenPGP digital signature
URL: <https://dovecot.org/pipermail/dovecot/attachments/20180804/e00c115d/attachment.sig>
More information about the dovecot
mailing list