Hello,

We run a 5 instances dovecot cluster on debian 11 virtual machines, a dovecot proxy, 2 directors and 2 dovecot backends.
All dovecot instances run version 2.3.20. We use a glusterfs mounted volume on the backends for the mail storage.  Our users all use thunderbird mail client and they do not report any issue with the imap service.
Since we have configured the dovecot cluster we see multiple random imap crashes with signal 11 and timeout leaks in the logs on both the backends for multiple users.

Is this is a configuration problem or a bug with imapc or imap;  I would appreciate if you could assist us to debug this further.

A coredump is attached if it helps.

The logs look similar to this:

Jun 01 16:21:13 doveb0-rmt0-cn1 dovecot[75106]: imap(user@domie02.com)<105800><BxSqJw39DLgKZQD+>: Warning: Timeout leak: 0x7f9a92c0c1c0 (imapc-client.c:475)
Jun 01 16:21:13 doveb0-rmt0-cn1 dovecot[75106]: imap(user@domie02.com)<105800><BxSqJw39DLgKZQD+>: Warning: Raw backtrace for leaks: /usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x42) [0x7f9a92a7e872] -> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7f9a92a7e98e] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_destroy+0x2b5) [0x7f9a92aa21e5] -> /usr/lib/dovecot/libdovecot-storage.so.0(imapc_client_run+0x115) [0x7f9a92c0bcb5] -> /usr/lib/dovecot/libdovecot-storage.so.0(imapc_mailbox_run_nofetch+0x20) [0x7f9a92c089f0] -> /usr/lib/dovecot/libdovecot-storage.so.0(imapc_mailbox_sync_init+0x868) [0x7f9a92c0b198] -> /usr/lib/dovecot/libdovecot-storage.so.0(mailbox_sync_init+0x58) [0x7f9a92bbfa68] -> dovecot/imap(imap_sync_init+0x95) [0x5605e4ebbe55] -> dovecot/imap(+0x17343) [0x5605e4ea5343] -> dovecot/imap(cmd_idle+0xb4) [0x5605e4ea5674] -> dovecot/imap(command_exec+0xa4) [0x5605e4eb04d4] -> dovecot/imap(+0x2044f) [0x5605e4eae44f] -> dovecot/imap(+0x204fa) [0x5605e4eae4fa] -> dovecot/imap(client_handle_input+0x1b5) [0x5605e4eae9d5] -> dovecot/imap(client_input+0x70) [0x5605e4eaef40] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7f9a92aa1819] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x132) [0x7f9a92aa2f02] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x50) [0x7f9a92aa18c0] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x40) [0x7f9a92aa1a80] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f9a92a143c3] -> dovecot/imap(main+0x4fa) [0x5605e4e9ff0a] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xea) [0x7f9a927d2d0a] -> dovecot/imap(_start+0x2a) [0x5605e4e9ffca]
Jun 01 16:23:49 doveb0-rmt0-cn1 dovecot[75106]: imap(user@domie02.com)<105800><BxSqJw39DLgKZQD+>: Fatal: master: service(imap): child 105800 killed with signal 11 (core dumped)

Below i post the dovecot configuration of one of the two dovecot backends:
# 2.3.20 (80a5ac675d): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.20 (149edcf2)
# OS: Linux 5.10.0-23-amd64 x86_64 Debian 11.7 fuse.glusterfs
# Hostname: doveb0-rmt0-cn1
auth_cache_negative_ttl = 5 mins
auth_cache_size = 50 M
auth_default_realm = domie02.com
auth_master_user_separator = *
auth_mechanisms = plain login
auth_verbose = yes
auth_worker_max_count = 16
disable_plaintext_auth = no
first_valid_uid = 499
hostname = smtp.domie02.com
imapc_features = fetch-bodystructure fetch-headers rfc822.size search modseq acl delay-login
imapc_host = 10.101.0.250
imapc_max_idle_time = 5 mins
imapc_password = # hidden, use -P to show it
imapc_sasl_mechanisms = plain login
imapc_ssl = starttls
imapc_ssl_verify = no
last_valid_uid = 499
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
login_greeting = You have successfully logged in to domie02.com IMAP server
login_trusted_networks = 10.101.0.74 10.101.0.73
mail_always_cache_fields = flags hdr.* date.received date.sent
mail_cache_fields = flags date.received guid size.physical size.virtual imap.bodystructure body.snippet
mail_debug = yes
mail_fsync = always
mail_location = mdbox:/var/vmail/%d/%n/mdbox:VOLATILEDIR=/dev/shm/dovecot/%2.256Nu/%d/%n
mail_plugins = " notify mail_log zlib acl"
mail_privileged_group = mail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext
mdbox_rotate_size = 200 M
mmap_disable = yes
namespace Domie02 {
  list = children
  location = imapc:/var/vmail/%d/%n/shared/indexes/%%n:INDEXPVT=/var/vmail/%d/%n/shared-pvt/%%n:VOLATILEDIR=/dev/shm/dovecot/%2.256Nu/%d/%n/%%n
  prefix = shared.%%n.
  separator = .
  subscriptions = no
  type = shared
}
namespace inbox {
  inbox = yes
  location = 
  mailbox Drafts {
    special_use = \Drafts
  }
  mailbox Junk {
    special_use = \Junk
  }
  mailbox Sent {
    special_use = \Sent
  }
  mailbox "Sent Messages" {
    special_use = \Sent
  }
  mailbox Spam.Virus {
    auto = subscribe
  }
  mailbox Spam {
    auto = subscribe
  }
  mailbox Trash {
    special_use = \Trash
  }
  prefix = 
  separator = .
  subscriptions = yes
  type = private
}
passdb {
  args = password=#hidden_use-P_to_show#
  driver = static
  master = yes
  override_fields = allow_real_nets=127.0.0.0/8,local,10.101.0.74,10.101.0.73 userdb_acl_user=%{auth_user} userdb_namespace/Domie02/disabled=yes
  result_success = continue
}
passdb {
  args = password=#hidden_use-P_to_show#
  driver = static
  override_fields = allow_real_nets=127.0.0.0/8,local,10.101.0.74,10.101.0.73
  result_success = continue
}
passdb {
  args = /etc/dovecot/dovecot-ldap-users.conf.ext
  driver = ldap
}
passdb {
  args = /etc/dovecot/dovecot-ldap-groups.conf.ext
  driver = ldap
}
plugin {
  acl = vfile:/etc/dovecot/global-acls/%d/master-dovecot-acl:cache_secs=300
  acl_defaults_from_inbox = yes
  acl_globals_only = no
  acl_shared_dict = file:/var/dovecot-acl/%d/shared-mailboxes.db
  imap_compress_deflate_level = 6
  mail_log_events = flag_change
  mail_log_fields = uid box flags
  sieve = file:~/sieve;active=~/sieve/%n.sieve
  sieve_before = /var/vmail/global-sieve/global.sieve
  sieve_env_email_address = %u
  sieve_env_mailbox_prefix = %{userdb:mailbox_prefix}
  sieve_global = /var/vmail/sieve
  sieve_global_extensions = +editheader +vnd.dovecot.environment
  zlib_save = zstd
  zlib_save_level = 5
}
protocols = " imap lmtp sieve submission"
service auth-worker {
  user = dovecot
}
service auth {
  unix_listener auth-userdb {
    group = mail
    mode = 0666
    user = vmail
  }
}
service doveadm {
  inet_listener dovadm-dom-mgmt {
    address = 10.100.1.71
    port = 8080
    ssl = no
  }
  inet_listener dovadm-dom {
    address = 10.101.0.71
    port = 8080
    ssl = no
  }
  vsz_limit = 256 M
}
service imap-login {
  inet_listener imap-dom {
    address = 10.101.0.71
    port = 143
    ssl = no
  }
  inet_listener imap {
    port = 0
  }
  inet_listener imaps-dom {
    address = 10.101.0.71
    port = 993
    ssl = yes
  }
  inet_listener imaps {
    port = 0
  }
  process_limit = 400
  process_min_avail = 2
  service_count = 20
  vsz_limit = 512 M
}
service imap {
  process_limit = 2048
  process_min_avail = 0
  service_count = 1
  vsz_limit = 4 G
}
service lmtp {
  inet_listener lmtp-dom {
    address = 10.101.0.71
    port = 24
    ssl = no
  }
  process_limit = 0
  process_min_avail = 0
  service_count = 0
  unix_listener lmtp {
    mode = 0666
  }
  vsz_limit = 256 M
}
service managesieve-login {
  inet_listener sieve-dom {
    address = 10.101.0.71
    port = 4190
    ssl = no
  }
  inet_listener sieve {
    port = 0
  }
  process_limit = 0
  process_min_avail = 0
  service_count = 0
  vsz_limit = 64 M
}
service managesieve {
  process_limit = 1024
  process_min_avail = 0
  service_count = 1
  vsz_limit = 256 M
}
service submission-login {
  inet_listener submission-dom {
    address = 10.101.0.71
    port = 587
    ssl = no
  }
  inet_listener submission {
    port = 0
  }
  inet_listener submissions-dom {
    address = 10.101.0.71
    port = 465
    ssl = yes
  }
  process_limit = 250
  process_min_avail = 2
  service_count = 1
  vsz_limit = 256 M
}
service submission {
  process_limit = 1024
  process_min_avail = 0
  service_count = 1
  vsz_limit = 256 M
}
ssl = required
ssl_cert = </etc/ssl/certs/doveb0-rmt0-cn1.mail.rmt.domie02.com.pem
ssl_client_ca_dir = /etc/ssl/certs
ssl_key = # hidden, use -P to show it
submission_max_mail_size = 30000 k
submission_relay_host = 10.101.0.7
submission_relay_port = 587
submission_relay_ssl = starttls
submission_relay_ssl_verify = no
submission_relay_trusted = yes
userdb {
  driver = prefetch
}
userdb {
  args = /etc/dovecot/dovecot-ldap-users.conf.ext
  driver = ldap
}
userdb {
  args = /etc/dovecot/dovecot-ldap-groups.conf.ext
  driver = ldap
}
protocol lmtp {
  mail_plugins = " notify mail_log zlib acl sieve acl"
}
protocol lda {
  mail_plugins = " notify mail_log zlib acl sieve acl"
}
protocol imap {
  mail_max_userip_connections = 100
  mail_plugins = " notify mail_log zlib acl"
}
protocol sieve {
  mail_max_userip_connections = 40
}
protocol submission {
  mail_max_userip_connections = 70
}
local 10.101.0.71 {
  doveadm_allowed_commands = 
  doveadm_password = # hidden, use -P to show it
}
local 10.100.1.71 {
  doveadm_allowed_commands = 
  doveadm_password = # hidden, use -P to show it
}
}
Thank you,
Nikolaos Pyrgiotis