Mail replication fails between v2.2.27 and v2.3.4.1

[Apostolis Hardalias]

Hello,

I have two installations of dovecot configured to replicate mailboxes
between them. recently, i upgraded the operating system on one of them
(mx2.example.com) and now i'm running one installation on version 2.2.27
(debian stretch) and another on version 2.3.4.1 (mx1.example.com debian
buster).

My setup includes 3 shared namespaces that point to the mailboxes of 3
accounts. these accounts are setup to use maildir format.

After upgrading one of the hosts, i started having issues with failed
replications on those mailboxes.

I tried running the replication manually and repeatedly by issuing the
following on mx1.example.com:

doveadv -d sync -u exception_bucket at example.com -d -l 30 -n -u

and the logs on mx2.example.com included the following lines consistently:

Jul 02 14:21:58 dsync-server(exception_bucket at example.com): Debug: brain 
S: Change during sync: Mailbox GUID 2d32711ddfd27250390e0000a712b6e3 was 
lost
Jul 02 14:21:58 dsync-server(exception_bucket at example.com): Debug: brain 
S: We don't have mailbox 2d32711ddfd27250390e0000a712b6e3

I double checked for the existance of the mailbox with that guid (that
guid points to the inbox) and the mailbox existed on both servers but,
the contents of the mailbox on the two servers where different (missing
e-mails on both servers).

I attempted running a force-resync command as follows but with no luck.

doveadm -d force-resync -u exception_bucket at example.com inbox

but the issue persisted. admittedly, i didn't run the command on both
servers because i read it doesn't replicate after figuring a workaround.

The next thing i tried was attempting replication without the -n flag,
given that you mention that there are some issues replicating shared
namespaces. That didn't fix the replication status either.

After looking around, i came onto the solution on this thread:
https://dovecot.org/pipermail/dovecot/2017-october/109620.html

which proposed setting mailbox_list_index = no.

This was the default setting on version 2.2.27, so i changed that
setting on mx2.example.com which was running version 2.3.4.1.

This allowed the replication to work but i don't think it is a setting i
want in my configuration.

Is there something i can do to re-enable dovecot to check index files
instead of mailboxes? Could that be an issue when replicating between
those two different versions?

I'm attaching the configuration files for both the mailservers, a
separate configuration file is included since mail_replica settings are
set from there, and a link to the compressed log file i pulled during
the operation.

https://filebin.net/d9yygnq8jg74unwb/replication_debug.log.bz2?t=vv2o81ka

Thank you in advance,
Apostolis Hardalias

-------------- next part --------------
driver = mysql
connect = host=127.0.0.1 dbname=mailer user=mailer password=***reducted***
default_pass_scheme = PLAIN-MD5

user_query = SELECT '/home/vmail/%d/%n' as home, 'vmail' as uid, 'vmail' as gid, mailbox as mail, 'tcps:mx1.example.com:12345' as mail_replica FROM virtual_users LEFT JOIN virtual_domains ON virtual_users.domain_id = virtual_domains.id WHERE virtual_users.user='%n' AND virtual_domains.name='%d';

password_query = SELECT password FROM virtual_users LEFT JOIN virtual_domains ON virtual_users.domain_id = virtual_domains.id WHERE virtual_users.user='%n' AND virtual_domains.name='%d';

iterate_query = SELECT CONCAT(u.user, '@', d.name) AS user FROM virtual_users as u LEFT JOIN virtual_domains AS d on u.domain_id = d.id;
-------------- next part --------------
# 2.2.27 (c0f36b0): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.4.16 (fed8554)
# OS: Linux 4.9.0-11-amd64 x86_64 Debian 9.11 
auth_mechanisms = plain login
default_vsz_limit = 512 M
doveadm_password =  # hidden, use -P to show it
first_valid_uid = 100
haproxy_trusted_networks = 192.168.0.0/24
mail_location = mdbox:~/mdbox:ALT=/srv/mail/alt/%d/%n
mail_log_prefix = "%Ls[%p]: user=<%u>, "
mail_plugins = " acl notify replication acl stats"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext spamtest spamtestplus
namespace {
  location = maildir:/home/vmail/example.com/exception_bucket/Maildir:INDEXPVT=~/exceptions
  prefix = Exceptions.
  subscriptions = no
  type = public
}
namespace {
  location = maildir:/home/vmail/example.com/support/Maildir:INDEXPVT=~/support at example.com
  prefix = Support.
  subscriptions = no
  type = public
}
namespace {
  location = maildir:/home/vmail/example.com/staging/Maildir:INDEXPVT=~/staging at example.com
  prefix = Staging.
  subscriptions = no
  type = public
}
namespace inbox {
  inbox = yes
  location = 
  mailbox Drafts {
    auto = subscribe
    special_use = \Drafts
  }
  mailbox Sent {
    auto = subscribe
    special_use = \Sent
  }
  mailbox Spam {
    auto = subscribe
    special_use = \Junk
  }
  mailbox Trash {
    auto = subscribe
    special_use = \Trash
  }
  prefix = 
  separator = .
  type = private
}
passdb {
  args = /etc/dovecot/dovecot-sql.conf
  driver = sql
}
plugin {
  acl = vfile
  antispam_backend = spool2dir
  antispam_spam = spam;Spam;junk;Junk
  antispam_spool2dir_notspam = /var/spool/spamtrainer/%%lu-%u-%%05lu-ham
  antispam_spool2dir_spam = /var/spool/spamtrainer/%%lu-%u-%%05lu-spam
  antispam_trash = trash;Trash;Deleted Items;Deleted Messages
  mail_log_events = delete
  mail_log_fields = uid box msgid from subject
  sieve = ~/.dovecot.sieve
  sieve_before = /home/vmail/global.sieve
  sieve_default = /home/vmail/global.sieve
  sieve_dir = ~/sieve
  sieve_duplicate_default_period = 1h
  sieve_duplicate_max_period = 1d
  sieve_execute_bin_dir = /usr/local/lib/dovecot/sieve-execute
  sieve_extensions = +spamtest +spamtestplus
  sieve_global_extensions = +vnd.dovecot.execute
  sieve_plugins = sieve_extprograms
  sieve_spamtest_max_header = X-Spam-Status: required=([0-9.]+)
  sieve_spamtest_status_header = X-Spam-Status: score=(-?[0-9.]+)
  sieve_spamtest_status_type = score
  stats_refresh = 30secs
  stats_track_cmds = yes
}
protocols = " imap sieve"
service aggregator {
  fifo_listener replication-notify-fifo {
    user = vmail
  }
  unix_listener replication-notify {
    user = vmail
  }
}
service anvil {
  client_limit = 2000
}
service auth {
  client_limit = 2000
  unix_listener /var/spool/postfix/private/auth {
    group = postfix
    mode = 0660
    user = postfix
  }
  unix_listener auth-userdb {
    group = vmail
    mode = 0777
  }
}
service doveadm {
  inet_listener {
    port = 12345
    ssl = yes
  }
  vsz_limit = 1500 M
}
service imap-login {
  inet_listener imap_haproxy {
    haproxy = yes
    port = 20143
  }
  inet_listener imaps_haproxy {
    haproxy = yes
    port = 20993
    ssl = yes
  }
  process_limit = 2048
  process_min_avail = 5
}
service imap-postlogin {
  executable = script-login /usr/local/sbin/ldap-postlogin
  user = $default_internal_user
}
service imap {
  executable = imap imap-postlogin
  process_limit = 2048
}
service replicator {
  process_min_avail = 1
  unix_listener replicator-doveadm {
    mode = 0600
    user = vmail
  }
}
service stats {
  fifo_listener stats-mail {
    mode = 0600
    user = vmail
  }
  fifo_listener stats-user {
    mode = 0600
    user = vmail
  }
}
ssl_cert = </etc/ssl/certs/star.example.com.crt
ssl_client_ca_dir = /etc/ssl/certs
ssl_key =  # hidden, use -P to show it
stats_carbon_name = metamail
stats_carbon_server = 192.168.0.12:2003
userdb {
  args = /etc/dovecot/dovecot-sql.conf
  driver = sql
}
protocol lda {
  log_path = /var/log/dovecot/deliver.log
  mail_plugins = sieve replication notify
  postmaster_address = postmaster at example.com
}
protocol imap {
  mail_max_userip_connections = 500
  mail_plugins = " acl notify replication acl stats antispam mail_log imap_stats"
}
-------------- next part --------------
# 2.3.4.1 (f79e8e7e4): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.4 ()
# OS: Linux 4.19.0-9-amd64 x86_64 Debian 10.4 
# Hostname: mx2.example.com
auth_mechanisms = plain login
default_vsz_limit = 512 M
doveadm_password = # hidden, use -P to show it
first_valid_uid = 100
haproxy_trusted_networks = 192.168.0.0/24
login_greeting = Dovecot ready.
mail_location = mdbox:~/mdbox:ALT=false/%d/%n
mail_plugins = " notify replication acl old_stats"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext spamtest spamtestplus
namespace {
  location = maildir:/home/vmail/example.com/exception_bucket/Maildir:INDEXPVT=~/exceptions
  prefix = Exceptions.
  subscriptions = no
  type = public
}
namespace {
  location = maildir:/home/vmail/example.com/support/Maildir:INDEXPVT=~/support at example.com
  prefix = Support.
  subscriptions = no
  type = public
}
namespace {
  location = maildir:/home/vmail/example.com/staging/Maildir:INDEXPVT=~/staging at example.com
  prefix = Staging.
  subscriptions = no
  type = public
}
namespace inbox {
  inbox = yes
  location = 
  mailbox Drafts {
    auto = subscribe
    special_use = \Drafts
  }
  mailbox Sent {
    auto = subscribe
    special_use = \Sent
  }
  mailbox Spam {
    auto = subscribe
    special_use = \Junk
  }
  mailbox Trash {
    auto = subscribe
    special_use = \Trash
  }
  prefix = 
  separator = .
  type = private
}
old_stats_carbon_name = koromail
old_stats_carbon_server = 192.168.0.12:2003
passdb {
  args = /etc/dovecot/dovecot-sql.conf
  driver = sql
}
plugin {
  acl = vfile
  antispam_backend = spool2dir
  antispam_spam = spam;Spam;junk;Junk
  antispam_spool2dir_notspam = /var/spool/spamtrainer/%%lu-%u-%%05lu-ham
  antispam_spool2dir_spam = /var/spool/spamtrainer/%%lu-%u-%%05lu-spam
  antispam_trash = trash;Trash;Deleted Items;Deleted Messages
  mail_log_events = delete
  mail_log_fields = uid box msgid from subject
  old_stats_refresh = 30s
  old_stats_track_cmds = yes
  sieve = ~/.dovecot.sieve
  sieve_before = /home/vmail/global.sieve
  sieve_default = /home/vmail/global.sieve
  sieve_dir = ~/sieve
  sieve_duplicate_default_period = 1h
  sieve_duplicate_max_period = 1d
  sieve_execute_bin_dir = /usr/local/lib/dovecot/sieve-execute
  sieve_extensions = +spamtest +spamtestplus
  sieve_global_extensions = +vnd.dovecot.execute
  sieve_plugins = sieve_extprograms
  sieve_spamtest_max_header = X-Spam-Status: required=([0-9.]+)
  sieve_spamtest_status_header = X-Spam-Status: score=(-?[0-9.]+)
  sieve_spamtest_status_type = score
}
protocols = " imap sieve"
service aggregator {
  fifo_listener replication-notify-fifo {
    user = vmail
  }
  unix_listener replication-notify {
    user = vmail
  }
}
service anvil {
  client_limit = 2000
}
service auth {
  client_limit = 2000
  unix_listener /var/spool/postfix/private/auth {
    group = postfix
    mode = 0660
    user = postfix
  }
  unix_listener auth-userdb {
    group = vmail
    mode = 0777
  }
}
service doveadm {
  inet_listener {
    port = 12345
    ssl = yes
  }
  vsz_limit = 1500 M
}
service imap-login {
  inet_listener imap_haproxy {
    haproxy = yes
    port = 20143
  }
  inet_listener imaps_haproxy {
    haproxy = yes
    port = 20993
    ssl = yes
  }
  process_limit = 2048
  process_min_avail = 5
  service_count = 0
}
service imap-postlogin {
  executable = script-login /usr/local/sbin/ldap-postlogin
  user = $default_internal_user
}
service imap {
  executable = imap imap-postlogin
  process_limit = 2048
}
service old-stats {
  fifo_listener old-stats-mail {
    mode = 0600
    user = vmail
  }
  fifo_listener old-stats-user {
    mode = 0600
    user = vmail
  }
}
service replicator {
  process_min_avail = 1
  unix_listener replicator-doveadm {
    mode = 0600
    user = vmail
  }
}
ssl_cert = </etc/ssl/certs/star.example.com.crt
ssl_client_ca_dir = /etc/ssl/certs
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
userdb {
  args = /etc/dovecot/dovecot-sql.conf
  driver = sql
}
protocol lda {
  log_path = /var/log/dovecot/deliver.log
  mail_plugins = sieve replication notify
  postmaster_address = postmaster at example.com
}
protocol imap {
  mail_plugins = " notify replication acl old_stats antispam mail_log imap_old_stats"
}