Master-Master replication question

6 Jul 2016


      Dear list,
I have setup a master-master replication setup. My primairy MX's send email over on a DNS loadbalanced way, so DNS is doing some kind of round-robin way of sending mail to both master servers.
I found out, that on one of the two machines, the email synchronisation is heavily delayed.
Lets assume server A receives a mail from the MX; it synchronises almost instantly with the other server.
Whenever server B receives the email, it could take up to several hours to synchronise the email, it seems that it is not detected prior.
It is also interesting to see, that the mailboxes on server A (Where users login to retrieve their email via webmail/clients) are significantly smaller then the mailboxes on server B. When investigating, it seems that "older" mailboxes (or storage rather since we use mdbox) are still there on server B, which already had been removed on server A.
My personal mailbox was 170MB on server A, while it was still 2.5GB on server B. (which was around that size before cleaning up the mailsboxes).
I enabled debugging on the servers, and I see rather quick : "Replication requests" on server A, but when getting an email on server B, I do not see the request at all.
My servers are both running the same version, same configuration (utilizing puppet), both running on ZFS and FreeBSD. Where server B is more loaded in it's memory because of some bhyve VM's and the server A does not run any VM.
Does someone have any pointers on where to look?
Thanks in advance ;-)
Remko
Included below the configurations from server A and B:
Server A:
2.2.25 (7be1766): /usr/local/etc/dovecot/dovecot.conf
Pigeonhole version 0.4.14 (099a97c)
OS: FreeBSD 10.3-RELEASE-p2 amd64
auth_mechanisms = plain login
disable_plaintext_auth = no
doveadm_password =  # hidden, use -P to show it
haproxy_trusted_networks = YYYY
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
lmtp_save_to_detail_mailbox = yes
mail_debug = yes
mail_fsync = always
mail_location = mdbox:~/mdbox
mail_plugins = " quota notify replication"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext
namespace {
inbox = yes
location =
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Spam {
auto = subscribe
special_use = \Junk
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
prefix =
separator = .
}
passdb {
driver = pam
}
plugin {
antispam_backend = mailtrain
antispam_mail_notspam = --ham
antispam_mail_sendmail = /usr/local/bin/sa-learn.sh
antispam_mail_spam = --spam
antispam_spam_pattern_ignorecase = spam;junk
antispam_trash_pattern_ignorecase = trash;deleted items;deleted messages
antispam_verbose_debug = 1
mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
mail_log_fields = uid box msgid size
mail_replica = tcps:the other server:12346
sieve = ~/.dovecot.sieve
sieve_dir = ~/sieve
sieve_global_dir = /usr/local/etc/dovecot/sieve/global/
sieve_global_path = /usr/local/etc/dovecot/sieve/default.sieve
}
postmaster_address = postmaster@xxx
protocols = imap pop3 lmtp sieve
replication_dsync_parameters = -d -N -l 60 -U
replication_max_conns = 100
service aggregator {
fifo_listener replication-notify-fifo {
mode = 0666
}
unix_listener replication-notify {
mode = 0666
}
}
service auth {
unix_listener /var/spool/postfix/private/auth {
mode = 0666
}
}
service doveadm {
inet_listener {
port = 12346
ssl = yes
}
}
service imap-login {
inet_listener imap_haproxy {
haproxy = yes
port = 10143
}
inet_listener imaps_haproxy {
haproxy = yes
port = 10144
ssl = yes
}
service_count = 1
}
service imap {
process_limit = 1024
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
}
service pop3 {
process_limit = 1024
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
mode = 0666
}
}
ssl_ca = </usr/local/etc/letsencrypt/live/XXX
ssl_cert = </usr/local/etc/letsencrypt/live/XXX
ssl_client_ca_file = /usr/local/certificates/letsencrypt-ca.pem
ssl_key = </usr/local/etc/letsencrypt/live/XXX
ssl_protocols = !SSLv2 !SSLv3
userdb {
driver = passwd
}
verbose_proctitle = yes
protocol lmtp {
auth_username_format = %n
mail_plugins = quota sieve
postmaster_address = postmaster@xxxx
}
protocol lda {
mail_plugins = " quota notify replication sieve"
}
protocol imap {
mail_max_userip_connections = 50
mail_plugins = " quota notify replication imap_quota antispam"
}
Server B:
2.2.25 (7be1766): /usr/local/etc/dovecot/dovecot.conf
Pigeonhole version 0.4.14 (099a97c)
OS: FreeBSD 10.3-RELEASE amd64
auth_mechanisms = plain login
disable_plaintext_auth = no
doveadm_password =  # hidden, use -P to show it
haproxy_trusted_networks = YYYY
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
lmtp_save_to_detail_mailbox = yes
mail_debug = yes
mail_fsync = always
mail_location = mdbox:~/mdbox
mail_plugins = " quota notify replication"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext
namespace {
inbox = yes
location =
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Spam {
auto = subscribe
special_use = \Junk
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
prefix =
separator = .
}
passdb {
driver = pam
}
plugin {
antispam_backend = mailtrain
antispam_mail_notspam = --ham
antispam_mail_sendmail = /usr/local/bin/sa-learn.sh
antispam_mail_spam = --spam
antispam_spam_pattern_ignorecase = spam;junk
antispam_trash_pattern_ignorecase = trash;deleted items;deleted messages
antispam_verbose_debug = 1
mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
mail_log_fields = uid box msgid size
mail_replica = tcps:the other host:12346
sieve = ~/.dovecot.sieve
sieve_dir = ~/sieve
sieve_global_dir = /usr/local/etc/dovecot/sieve/global/
sieve_global_path = /usr/local/etc/dovecot/sieve/default.sieve
}
postmaster_address = postmaster@xxx
protocols = imap pop3 lmtp sieve
replication_dsync_parameters = -d -N -l 60 -U
replication_max_conns = 100
service aggregator {
fifo_listener replication-notify-fifo {
mode = 0666
}
unix_listener replication-notify {
mode = 0666
}
}
service auth {
unix_listener /var/spool/postfix/private/auth {
mode = 0666
}
}
service doveadm {
inet_listener {
port = 12346
ssl = yes
}
}
service imap-login {
inet_listener imap_haproxy {
haproxy = yes
port = 10143
}
inet_listener imaps_haproxy {
haproxy = yes
port = 10144
ssl = yes
}
service_count = 1
}
service imap {
process_limit = 1024
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
}
service pop3 {
process_limit = 1024
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
mode = 0666
}
}
ssl_ca = </usr/local/etc/letsencrypt/live/xxx
ssl_cert = </usr/local/etc/letsencrypt/live/xxx
ssl_client_ca_file = /usr/local/certificates/letsencrypt-ca.pem
ssl_key = </usr/local/etc/letsencrypt/live/xxx
ssl_protocols = !SSLv2 !SSLv3
userdb {
driver = passwd
}
verbose_proctitle = yes
protocol lmtp {
auth_username_format = %n
mail_plugins = quota sieve
postmaster_address = postmaster@XXXXX
}
protocol lda {
mail_plugins = " quota notify replication sieve"
}
protocol imap {
mail_max_userip_connections = 50
mail_plugins = " quota notify replication imap_quota antispam"
}

Master-Master replication question

Remko Lodder

2.2.25 (7be1766): /usr/local/etc/dovecot/dovecot.conf

Pigeonhole version 0.4.14 (099a97c)

OS: FreeBSD 10.3-RELEASE-p2 amd64

2.2.25 (7be1766): /usr/local/etc/dovecot/dovecot.conf

Pigeonhole version 0.4.14 (099a97c)

OS: FreeBSD 10.3-RELEASE amd64