dovecot replication crashing

Mon Mar 27 22:24:41 UTC 2023

Hi,

I'm a but clueless, having issues with replication. `doveadm dsync -u hans` works.
But using the following replication setup, I see coredumps.

Where to go next?

Interestingly not for all users. (For testing purposes I've only 2
users. One having about 20 messages: here even the replication works,
but the 2nd user (having about 14k messages) fails. The failure seems to
happen immediatly after starting the replication attempt).

I saw the replication crashing using Dovecot packages from the current
Debian distro. In order to debug this, I'm now using 2.3.20, built from
Git.

The replicator config boils down to this (complete config is attached)

    replication_full_sync_interval = 90d
    replication_max_conns = 16
    mail_plugins = $mail_plugins notify replication
    plugin {
      mail_replica = tcps:smtp-mz.example.com:9090
    }
    service replicator {
      process_min_avail = 1
      unix_listener replicator-doveadm {
        mode = 0666
      }
    }
    service aggregator {
      fifo_listener replication-notify-fifo {
        user = dovecot
        mode = 0666
      }
      unix_listener replication-notify {
        user = dovecot
        mode = 0666
      }
    }

The stacktrace I get looks like this:

Stack trace of thread 729589:
#0  0x00007f9e4f2c8ce1 __GI_raise (libc.so.6 + 0x38ce1)
#1  0x00007f9e4f2b2537 __GI_abort (libc.so.6 + 0x22537)
#2  0x00007f9e4f5fe8c6 default_fatal_finish (libdovecot.so.0 + 0x558c6)
#3  0x00007f9e4f6ab601 i_internal_fatal_handler (libdovecot.so.0 + 0x102601)
#4  0x00007f9e4f5fe589 i_panic (libdovecot.so.0 + 0x55589)
#5  0x00007f9e4f5fe99e fd_set_nonblock (libdovecot.so.0 + 0x5599e)
#6  0x00005564cbe6a08d cmd_dsync_ibc_stream_init (doveadm-server + 0x3008d)
#7  0x00005564cbe6b772 cmd_dsync_run (doveadm-server + 0x31772)
#8  0x00005564cbe6d284 doveadm_mail_next_user (doveadm-server + 0x33284)
#9  0x00005564cbe6e4ba doveadm_mail_cmd_exec (doveadm-server + 0x344ba)
#10 0x00005564cbe7eb71 doveadm_cmd_run_ver2 (doveadm-server + 0x44b71)
#11 0x00005564cbe8300a doveadm_cmd_server_run_ver2 (doveadm-server + 0x4900a)
#12 0x00007f9e4f6c1799 io_loop_call_io (libdovecot.so.0 + 0x118799)
#13 0x00007f9e4f6c2e82 io_loop_handler_run_internal (libdovecot.so.0 + 0x119e82)
#14 0x00007f9e4f6c1840 io_loop_handler_run (libdovecot.so.0 + 0x118840)
#15 0x00007f9e4f6c1a00 io_loop_run (libdovecot.so.0 + 0x118a00)
#16 0x00007f9e4f6343a3 master_service_run (libdovecot.so.0 + 0x8b3a3)
#17 0x00005564cbe5d9a2 main (doveadm-server + 0x239a2)
#18 0x00007f9e4f2b3d0a __libc_start_main (libc.so.6 + 0x23d0a)
#19 0x00005564cbe5da2a _start (doveadm-server + 0x23a2a)

2023 Mar 28 00:09:28 clone doveadm(hans)<729589><TdSzNhcUImT1IQsAKQCViw>: Fatal: master: service(doveadm): child 729589 killed with signal 6 (core dumped)

I'm a bit clueless now, even using the coredump and gdb.

-- 
Heiko

-------------- next part --------------
# 2.3.20 (80a5ac675d): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.20 (149edcf2)
# OS: Linux 5.10.0-10-amd64 x86_64 Debian 11.6 
# Hostname: clone
auth_cache_negative_ttl = 5 mins
auth_cache_size = 64 M
auth_cache_ttl = 5 mins
auth_master_user_separator = *
auth_username_format = %Ln
default_client_limit = 8192
default_process_limit = 2048
doveadm_password = # hidden, use -P to show it
first_valid_uid = 996
haproxy_trusted_networks = <redacted>
imap_id_log = *
imap_id_send = *
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
login_trusted_networks = 10.0.0.0/8
mail_gid = vmail
mail_location = maildir:~/Maildir:INBOX=/var/vmail/inbox/%u:VOLATILEDIR=/var/vmail/cache/%u:INDEX=/var/vmail/index/%u
mail_plugins = quota notify notify replication
mail_uid = vmail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext
namespace inbox {
  inbox = yes
  location = 
  mailbox Archive {
    auto = subscribe
  }
  mailbox Drafts {
    special_use = \Drafts
  }
  mailbox Junk {
    special_use = \Junk
  }
  mailbox Sent {
    special_use = \Sent
  }
  mailbox "Sent Messages" {
    special_use = \Sent
  }
  mailbox Trash {
    special_use = \Trash
  }
  prefix = 
}
passdb {
  args = /etc/dovecot/master-users
  driver = passwd-file
  master = yes
}
passdb {
  args = cache_key=#hidden_use-P_to_show#
  driver = pam
}
plugin {
  autocreate = Trash
  autocreate2 = Junk
  autocreate3 = Drafts
  autocreate4 = Sent
  autosubscribe = Trash
  autosubscribe2 = Junk
  autosubscribe3 = Drafts
  autosubscribe4 = Sent
  mail_compress_save = bz2
  mail_compress_save_level = 9
  mail_replica = tcps:smtp-mz.example.com:9090
  quota = maildir:user
  quota_rule = *:storage=11GB
  quota_rule2 = Archive*:ignore
  sieve = file:~/sieve;active=~/.dovecot.sieve
  sieve_after = /etc/dovecot/sieve-after.d/
  sieve_before = /etc/dovecot/sieve-before.d/
}
protocols = imap pop3 lmtp sieve
replication_full_sync_interval = 90 days
replication_max_conns = 16
service aggregator {
  fifo_listener replication-notify-fifo {
    mode = 0666
    user = dovecot
  }
  unix_listener replication-notify {
    mode = 0666
    user = dovecot
  }
}
service auth-worker {
  process_limit = 256
  user = root
}
service auth {
  user = root
}
service doveadm {
  inet_listener {
    port = 9090
    ssl = yes
  }
}
service imap-login {
  inet_listener haproxy_imaps {
    haproxy = yes
    port = 1993
    ssl = yes
  }
  inet_listener haproxy_imaps2 {
    haproxy = yes
    port = 52993
    ssl = yes
  }
  inet_listener imaps2 {
    port = 51993
    ssl = yes
  }
}
service imap {
  process_limit = 4096
}
service lmtp {
  inet_listener lmtp {
    address = 127.0.0.1
    port = 2525
  }
}
service managesieve-login {
  inet_listener sieve {
    port = 4190
  }
  process_min_avail = 1
}
service replicator {
  process_min_avail = 1
  unix_listener replicator-doveadm {
    mode = 0666
  }
}
ssl_cert = </etc/dovecot/ssl.crt
ssl_client_ca_dir = /etc/ssl/certs
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
submission_host = localhost:25
userdb {
  args = /etc/dovecot/users
  default_fields = noreplicate:protected=true
  driver = passwd-file
  override_fields = uid=vmail gid=vmail home=/var/vmail/home/%u
}
protocol lmtp {
  auth_username_format = %Ln
  mail_plugins = quota notify notify replication sieve
}
protocol lda {
  mail_plugins = quota notify notify replication sieve
}
protocol imap {
  mail_max_userip_connections = 20
  rawlog_dir = /var/log/dovecot/%u
}
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: not available
URL: <https://dovecot.org/pipermail/dovecot/attachments/20230328/3042c3c1/attachment-0001.sig>