dovecot-2.3.17: "Panic: file ioloop.c: line 865"
Hello,
since the upgrade from dovecot-2.3.16 to 2.3.17 I get 'Panic'-errors in the system-log of my replicator hosts. See below for log messages, doveconf-output and gdb-backtrace. Storage filesystem is ZFS.
The error seems to be correlated with exactly one user-mailbox, this user is repeatedly noted as 'failed' in replication status, although replication of this mailbox seems to work, as the mailbox status shows always the identical size on both replicators. The error can be produced by forcing replication of this individual mailbox.
Is there anything I can do to repair this mailbox?
Thanks for any hints, Andreas
Example log-messages:
Nov 11 07:17:50 vhrz311 dovecot: doveadm(119455): Error: write(<local>) failed: Timed out after 60 seconds Nov 11 07:17:50 vhrz311 dovecot: doveadm(119455): Panic: file ioloop.c: line 865 (io_loop_destroy): assertion failed: (ioloop == cur rent_ioloop) Nov 11 07:17:50 vhrz311 dovecot: doveadm(119455): Error: Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x42) [0x7 f4e59419182] -> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7f4e5941929e] -> /usr/lib/dovecot/libdovecot.so.0(+0xf8a5e) [0x7f4e59427a5e] -> /usr/lib/dovecot/libdovecot.so.0(+0xf8b01) [0x7f4e59427b01] -> /usr/lib/dovecot/libdovecot.so.0(i_fatal+0) [0x7f 4e593744a7] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_destroy+0x2dc) [0x7f4e5944121c] -> dovecot/doveadm-server(+0x46fb6) [0x563d0 150ffb6] -> dovecot/doveadm-server(doveadm_print+0x27) [0x563d01508fb7] -> dovecot/doveadm-server(+0x2b427) [0x563d014f4427] -> dove cot/doveadm-server(+0x2da44) [0x563d014f6a44] -> dovecot/doveadm-server(doveadm_cmd_ver2_to_mail_cmd_wrapper+0x2cf) [0x563d014f7c6f] -> dovecot/doveadm-server(doveadm_cmd_run_ver2+0x43b) [0x563d01508c1b] -> dovecot/doveadm-server(+0x44612) [0x563d0150d612] -> /usr /lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7f4e594406d9] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal +0x134) [0x7f4e59442034] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x50) [0x7f4e594407e0] -> /usr/lib/dovecot/libdovec ot.so.0(io_loop_run+0x38) [0x7f4e59440a18] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f4e593a6a23] -> dovecot/ doveadm-server(main+0x102) [0x563d014e6922] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xe7) [0x7f4e58bc1bf7] -> dovecot/d oveadm-server(_start+0x2a) [0x563d014e69aa] Nov 11 07:17:50 vhrz311 dovecot: doveadm(119455): Fatal: master: service(doveadm): child 119455 killed with signal 6 (core dumped)
'doveconf -n':
# 2.3.17 (e2aa53df5b): /etc/dovecot/dovecot.conf # Pigeonhole version 0.5.17 (054dddfa) # OS: Linux 4.15.0-162-generic x86_64 Ubuntu 18.04.6 LTS # Hostname: vhrz311.HRZ.Uni-Marburg.DE auth_cache_negative_ttl = 0 auth_cache_size = 10 M auth_master_user_separator = * auth_worker_max_count = 1024 base_dir = /var/run/dovecot/ default_client_limit = 10000 default_vsz_limit = 6 G doveadm_password = # hidden, use -P to show it doveadm_port = 12345 first_valid_gid = 10000 first_valid_uid = 10000 imap_max_line_length = 640 k last_valid_gid = 10000 last_valid_uid = 10000 mail_attribute_dict = file:%h/mdbox/dovecot-attributes mail_gid = 10000 mail_location = mdbox:%h/mdbox mail_plugins = " mail_log notify zlib notify replication" mail_privileged_group = mail mail_uid = 10000 managesieve_notify_capability = mailto managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext namespace inbox { hidden = no inbox = yes list = yes location = prefix = separator = / subscriptions = yes type = private } passdb { args = /etc/dovecot.deny deny = yes driver = passwd-file } passdb { args = /etc/dovecot/private/passwd.masterusers driver = passwd-file master = yes } passdb { args = /etc/dovecot/dovecot-ldap.hrz.conf.ext driver = ldap name = passdb_ldap_hrz } passdb { args = /etc/dovecot/dovecot-ldap.staff.conf.ext driver = ldap name = passdb_ldap_staff } passdb { args = /etc/dovecot/dovecot-ldap.students.conf.ext driver = ldap name = passdb_ldap_students } plugin { mail_replica = tcp:172.25.1.95 sieve = file:~/sieve;active=~/.dovecot.sieve sieve_default = /var/lib/dovecot/default.sieve sieve_max_actions = 55 sieve_max_redirects = 50 } pop3_uidl_format = %08Xv%08Xu postmaster_address = postmaster@hrz.uni-marburg.de protocols = imap pop3 lmtp sieve replication_dsync_parameters = -d -n INBOX -l 30 -U replication_max_conns = 20 service aggregator { fifo_listener replication-notify-fifo { user = vmail } unix_listener replication-notify { user = vmail } } service auth-worker { user = $default_internal_user } service auth { client_limit = 10000 } service config { process_min_avail = 8 } service doveadm { inet_listener { port = 12345 } vsz_limit = 6 G } service imap-login { process_min_avail = 64 service_count = 0 } service imap { process_limit = 8192 } service lmtp { inet_listener lmtp { port = 24 } } service managesieve-login { inet_listener sieve { port = 4190 } process_min_avail = 8 service_count = 0 } service pop3-login { process_min_avail = 8 service_count = 0 } service replicator { process_min_avail = 1 unix_listener replicator-doveadm { mode = 0600 user = vmail } } service submission-login { service_count = 0 } ssl = required ssl_ca =
gdb:
gdb /usr/lib/dovecot/doveadm-server /var/vmail/system-cores/core GNU gdb (Ubuntu 8.1.1-0ubuntu1) 8.1.1 Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: http://www.gnu.org/software/gdb/bugs/. Find the GDB manual and other documentation resources online at: http://www.gnu.org/software/gdb/documentation/. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/lib/dovecot/doveadm-server...(no debugging symbols found)...done. [New LWP 119455] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `dovecot/doveadm-server'. Program terminated with signal SIGABRT, Aborted. #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. (gdb) bt #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x00007f4e58be0921 in __GI_abort () at abort.c:79 #2 0x00007f4e59427a11 in ?? () from /usr/lib/dovecot/libdovecot.so.0 #3 0x00007f4e59427b01 in ?? () from /usr/lib/dovecot/libdovecot.so.0 #4 0x00007f4e593744a7 in i_panic () from /usr/lib/dovecot/libdovecot.so.0 #5 0x00007f4e5944121c in io_loop_destroy () from /usr/lib/dovecot/libdovecot.so.0 #6 0x0000563d0150ffb6 in ?? () #7 0x0000563d01508fb7 in doveadm_print () #8 0x0000563d014f4427 in ?? () #9 0x0000563d014f6a44 in ?? () #10 0x0000563d014f7c6f in doveadm_cmd_ver2_to_mail_cmd_wrapper () #11 0x0000563d01508c1b in doveadm_cmd_run_ver2 () #12 0x0000563d0150d612 in ?? () #13 0x00007f4e594406d9 in io_loop_call_io () from /usr/lib/dovecot/libdovecot.so.0 #14 0x00007f4e59442034 in io_loop_handler_run_internal () from /usr/lib/dovecot/libdovecot.so.0 #15 0x00007f4e594407e0 in io_loop_handler_run () from /usr/lib/dovecot/libdovecot.so.0 #16 0x00007f4e59440a18 in io_loop_run () from /usr/lib/dovecot/libdovecot.so.0 #17 0x00007f4e593a6a23 in master_service_run () from /usr/lib/dovecot/libdovecot.so.0 #18 0x0000563d014e6922 in main ()
On 11/11/2021 15:53 Piper Andreas piper@hrz.uni-marburg.de wrote:
Hello,
since the upgrade from dovecot-2.3.16 to 2.3.17 I get 'Panic'-errors in the system-log of my replicator hosts. See below for log messages, doveconf-output and gdb-backtrace. Storage filesystem is ZFS.
The error seems to be correlated with exactly one user-mailbox, this user is repeatedly noted as 'failed' in replication status, although replication of this mailbox seems to work, as the mailbox status shows always the identical size on both replicators. The error can be produced by forcing replication of this individual mailbox.
Is there anything I can do to repair this mailbox?
Thanks for any hints, Andreas
Can you install dovecot debug symbols and redo the gdb backtrace? Package name is likely dovecot-dbg.
Aki
Hello,
Can you install dovecot debug symbols and redo the gdb backtrace? Package name is likely
dovecot-dbg.
done, thanks for the hint.
Andreas
(gdb) bt
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1 0x00007f4e58be0921 in __GI_abort () at abort.c:79
#2 0x00007f4e59427a11 in default_fatal_finish (status=0,
type=LOG_TYPE_PANIC) at failures.c:459
#3 fatal_handler_real (ctx=<optimized out>, format=<optimized out>,
args=<optimized out>) at failures.c:471
#4 0x00007f4e59427b01 in i_internal_fatal_handler (ctx=<optimized out>,
format=<optimized out>, args=<optimized out>)
at failures.c:872
#5 0x00007f4e593744a7 in i_panic (format=format@entry=0x7f4e5946ab88
"file %s: line %d (%s): assertion failed: (%s)")
at failures.c:524
#6 0x00007f4e5944121c in io_loop_destroy
(_ioloop=_ioloop@entry=0x7ffd0e203e68) at ioloop.c:865
#7 0x0000563d0150ffb6 in doveadm_print_server_flush () at
doveadm-print-server.c:106
#8 0x0000563d01508fb7 in doveadm_print (
value=0x563d0269ced8 "AQAAAFoYRziwaStNyDEAAEKX94joZK5JAwAAAAQ", 'A'
--
Dr. Andreas Piper, Hochschulrechenzentrum der Philipps-Univ. Marburg Hans-Meerwein-Straße 6, 35032 Marburg, Germany Phone: +49 6421 28-23521 Fax: -26994 E-Mail: piper@HRZ.Uni-Marburg.DE
Hello,
as these 'Panic'-Events continue to appear every few minutes in my replicator-log, I would like to ask, if you have any hints how to repair the correlated mailbox?
Andreas
Am 11.11.21 um 15:04 schrieb Piper Andreas:
Hello,
Can you install dovecot debug symbols and redo the gdb backtrace? Package name is likely
dovecot-dbg.done, thanks for the hint.
Andreas
(gdb) bt #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x00007f4e58be0921 in __GI_abort () at abort.c:79 #2 0x00007f4e59427a11 in default_fatal_finish (status=0, type=LOG_TYPE_PANIC) at failures.c:459 #3 fatal_handler_real (ctx=<optimized out>, format=<optimized out>, args=<optimized out>) at failures.c:471 #4 0x00007f4e59427b01 in i_internal_fatal_handler (ctx=<optimized out>, format=<optimized out>, args=<optimized out>) at failures.c:872 #5 0x00007f4e593744a7 in i_panic (format=format@entry=0x7f4e5946ab88 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:524 #6 0x00007f4e5944121c in io_loop_destroy (_ioloop=_ioloop@entry=0x7ffd0e203e68) at ioloop.c:865 #7 0x0000563d0150ffb6 in doveadm_print_server_flush () at doveadm-print-server.c:106 #8 0x0000563d01508fb7 in doveadm_print ( value=0x563d0269ced8 "AQAAAFoYRziwaStNyDEAAEKX94joZK5JAwAAAAQ", 'A'
, "DAAAAMu73BjjBalfFPQAANPIMJnB2rkkBAAAAAg", 'A' , "EAAAD3Gkc4sGkrTcgxAABCl/eI02OuSQIAAAAD", 'A' , "gAAAD4SNQYx691evBkAANXM"...) at doveadm-print.c:87 #9 0x0000563d014f4427 in cmd_dsync_run (_ctx=0x563d01fc7cc8, user=<optimized out>) at doveadm-dsync.c:713 #10 0x0000563d014f6a44 in doveadm_mail_next_user (ctx=ctx@entry=0x563d01fc7cc8, error_r=error_r@entry=0x7ffd0e204108) at doveadm-mail.c:462 #11 0x0000563d014f76be in doveadm_mail_single_user (ctx=ctx@entry=0x563d01fc7cc8, error_r=error_r@entry=0x7ffd0e204108) at doveadm-mail.c:493 #12 0x0000563d014f7c6f in doveadm_mail_cmd_exec (wildcard_user=0x0, ctx=0x563d01fc7cc8) at doveadm-mail.c:654 #13 doveadm_cmd_ver2_to_mail_cmd_wrapper (cctx=<optimized out>) at doveadm-mail.c:983 #14 0x0000563d01508c1b in doveadm_cmd_run_ver2 (argc=argc@entry=9, argv=argv@entry=0x563d01f6b3d0, cctx=cctx@entry=0x7ffd0e204260) at doveadm-cmd.c:465 #15 0x0000563d0150d612 in doveadm_cmd_server_run_ver2 (cctx=0x7ffd0e204260, argv=0x563d01f6b3d0, argc=9, conn=0x563d01fc1988) at client-connection-tcp.c:187 #16 doveadm_cmd_handle (cctx=0x7ffd0e204260, argv=<optimized out>, argc=<optimized out>, cmd_name=<optimized out>, conn=0x563d01fc1988) at client-connection-tcp.c:214 #17 client_handle_command (args=<optimized out>, conn=0x563d01fc1988) at client-connection-tcp.c:284 #18 client_connection_tcp_input (conn=0x563d01fc1988) at client-connection-tcp.c:436 #19 0x00007f4e594406d9 in io_loop_call_io (io=0x563d01fc5950) at ioloop.c:737 #20 0x00007f4e59442034 in io_loop_handler_run_internal (ioloop=ioloop@entry=0x563d01f73ed0) at ioloop-epoll.c:222 #21 0x00007f4e594407e0 in io_loop_handler_run (ioloop=<optimized out>) at ioloop.c:789 #22 0x00007f4e59440a18 in io_loop_run (ioloop=0x563d01f73ed0) at ioloop.c:762 #23 0x00007f4e593a6a23 in master_service_run (service=0x563d01f73d30, callback=<optimized out>) at master-service.c:863 #24 0x0000563d014e6922 in main (argc=<optimized out>, argv=<optimized out>) at main.c:124
--
Dr. Andreas Piper, Hochschulrechenzentrum der Philipps-Univ. Marburg Hans-Meerwein-Straße 6, 35032 Marburg, Germany Phone: +49 6421 28-23521 Fax: -26994 E-Mail: piper@HRZ.Uni-Marburg.DE
Hi,
I have the same error in my replica server, it appears many times per day.
Log:
/Dec 13 13:01:23 Error: doveadm(xxx)<25513><vsFCF5w1t2GpYwAAbKtC3g>: read(xxx.xxx.xxx) failed: EOF (last sent=mailbox, last recv=mailbox)// //Dec 13 13:34:43 Warning: doveadm(xxx)<26277><9e2rJOM9t2GlZgAAbKtC3g>: /mnt/mail-storage-lv0003/xxx/Maildir/dovecot-uidlist: Duplicate file entry at line 2162: 1639396906.M258993P16725.xxx,S=7304,W=7467 (uid 71966 -> 71970)// //Dec 13 14:26:20 Error: doveadm(27294): write(xxx) failed: Timed out after 60 seconds// //Dec 13 14:26:20 Panic: doveadm(27294): file ioloop.c: line 865 (io_loop_destroy): assertion failed: (ioloop == current_ioloop)// //Dec 13 14:26:20 Error: doveadm(27294): Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x3d) [0x7f74d96f96ed] -> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7f74d96f980e] -> /usr/lib/dovecot/libdovecot.so.0(+0xff5cb) [0x7f74d97065cb] -> /usr/lib/dovecot/libdovecot.so.0(+0xff661) [0x7f74d9706661] -> /usr/lib/dovecot/libdovecot.so.0(+0x54a45) [0x7f74d965ba45] -> /usr/lib/dovecot/libdovecot.so.0(+0x57d95) [0x7f74d965ed95] -> dovecot/doveadm-server(+0x4af0f) [0x556a7a4b4f0f] -> dovecot/doveadm-server(doveadm_print_flush+0x1f) [0x556a7a4ae7af] -> dovecot/doveadm-server(doveadm_cmd_ver2_to_mail_cmd_wrapper+0x32b) [0x556a7a49dbcb] -> dovecot/doveadm-server(doveadm_cmd_run_ver2+0x4f7) [0x556a7a4ae307] -> dovecot/doveadm-server(+0x4874a) [0x556a7a4b274a] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7f74d971c679] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x131) [0x7f74d971dc61] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x4c) [0x7f74d971c71c] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x40) [0x7f74d971c8a0] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f74d96903d3] -> dovecot/doveadm-server(main+0x102) [0x556a7a48d9f2] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb) [0x7f74d92e709b] -> dovecot/doveadm-server(_start+0x2a) [0x556a7a48da7a]// //Dec 13 14:26:20 Fatal: doveadm(27294): master: service(doveadm): child 27294 killed with signal 6 (core dumps disabled - https://dovecot.org/bugreport.html#coredumps)// //Dec 13 14:29:06 Error: doveadm(27364): write(xxx) failed: Timed out after 60 seconds// //Dec 13 14:29:06 Panic: doveadm(27364): file ioloop.c: line 865 (io_loop_destroy): assertion failed: (ioloop == current_ioloop)// //Dec 13 14:29:06 Error: doveadm(27364): Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x3d) [0x7f05717756ed] -> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7f057177580e] -> /usr/lib/dovecot/libdovecot.so.0(+0xff5cb) [0x7f05717825cb] -> /usr/lib/dovecot/libdovecot.so.0(+0xff661) [0x7f0571782661] -> /usr/lib/dovecot/libdovecot.so.0(+0x54a45) [0x7f05716d7a45] -> /usr/lib/dovecot/libdovecot.so.0(+0x57d95) [0x7f05716dad95] -> dovecot/doveadm-server(+0x4af0f) [0x558edcd93f0f] -> dovecot/doveadm-server(doveadm_print_flush+0x1f) [0x558edcd8d7af] -> dovecot/doveadm-server(doveadm_cmd_ver2_to_mail_cmd_wrapper+0x32b) [0x558edcd7cbcb] -> dovecot/doveadm-server(doveadm_cmd_run_ver2+0x4f7) [0x558edcd8d307] -> dovecot/doveadm-server(+0x4874a) [0x558edcd9174a] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7f0571798679] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x131) [0x7f0571799c61] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x4c) [0x7f057179871c] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x40) [0x7f05717988a0] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f057170c3d3] -> dovecot/doveadm-server(main+0x102) [0x558edcd6c9f2] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb) [0x7f057136309b] -> dovecot/doveadm-server(_start+0x2a) [0x558edcd6ca7a]// //Dec 13 14:29:06 Fatal: doveadm(27364): master: service(doveadm): child 27364 killed with signal 6 (core dumps disabled - https://dovecot.org/bugreport.html#coredumps)/
My 'dovecot.conf':/ /
/# 2.3.17.1 (476cd46418): /etc/dovecot/dovecot.conf # Pigeonhole version 0.5.17.1 (a1a0b892) # OS: Linux 4.19.0-18-amd64 x86_64 Debian 10.11 # Hostname: xxx auth_mechanisms = plain login default_vsz_limit = 1 G doveadm_password = # hidden, use -P to show it doveadm_port = xxx mail_gid = 501 mail_location = maildir:~/Maildir:INDEX=/var/indexes/%n mail_max_userip_connections = 50 mail_plugins = " notify replication" mail_privileged_group = vmail mail_uid = 501 managesieve_notify_capability = mailto managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext vacation-seconds imapflags notify namespace inbox { inbox = yes location = mailbox Drafts { special_use = \Drafts } mailbox Junk { special_use = \Junk } mailbox "Posta inviata" { special_use = \Sent } mailbox Sent { special_use = \Sent } mailbox "Sent Messages" { special_use = \Sent } mailbox Trash { special_use = \Trash } prefix = } passdb { args = /etc/dovecot/dovecot-ldap.conf driver = ldap } plugin { mail_replica = tcp:xxx sieve = file:/var/www/html/ooo/%n;active=/var/www/html/ooo/%n/script sieve_default = /var/www/html/ooo/default.sieve sieve_duplicate_default_period = 14d sieve_duplicate_max_period = 7d sieve_extensions = +duplicate +notify +imapflags +vacation-seconds sieve_user_log = /var/www/html/ooo/%n/sieve_error.log } protocols = " imap lmtp sieve pop3" replication_max_conns = 1 service aggregator { fifo_listener replication-notify-fifo { user = vmail } unix_listener replication-notify { user = vmail } } service auth { unix_listener /var/spool/postfix/private/auth { group = postfix mode = 0660 user = postfix } unix_listener auth-master { group = vmail mode = 0660 user = vmail } } service doveadm { inet_listener { port = 12345 } } service imap-login { process_limit = 200 } service lmtp { process_min_avail = 5 unix_listener /var/spool/postfix/private/dovecot-lmtp { group = postfix mode = 0600 user = postfix } user = vmail } service managesieve-login { inet_listener sieve { port = 4190 } process_min_avail = 10 service_count = 1 } service replicator { process_min_avail = 1 unix_listener replicator-doveadm { mode = 0666 user = vmail } } service stats { unix_listener stats-reader { group = vmail mode = 0666 user = vmail } unix_listener stats-writer { group = vmail mode = 0666 user = vmail } } ssl_cert =
My 'dovecot-ldap.conf':
/hosts = xxx base = dc=xxx,dc=lan ldap_version=3 auth_bind = yes dn = cn=xxx,cn=xxx,dc=xxx,dc=lan dnpass = #hidden scope = subtree #user_attrs = sAMAccountName=home=/mnt/mail-storage-lv0003/%$,=uid=501,=gid=501 user_attrs = sAMAccountName=user,=home=/mnt/mail-storage-lv0003/%n,=uid=501,=gid=501 pass_attrs = sAMAccountName=user user_filter = (&(objectclass=person)(samaccountname=%n)) pass_filter= (&(objectclass=person)(samaccountname=%n)) iterate_attrs = sAMAccountName=user # With following filter we exclude all objects without an email address, all computers and all inactive accounts iterate_filter = (&(objectCategory=person)(mail=*)(objectClass=user)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))/
Regards
On 19/11/21 10:27, Andreas Piper wrote:
Hello,
as these 'Panic'-Events continue to appear every few minutes in my replicator-log, I would like to ask, if you have any hints how to repair the correlated mailbox?
Andreas
Am 11.11.21 um 15:04 schrieb Piper Andreas:
Hello,
Can you install dovecot debug symbols and redo the gdb backtrace? Package name is likely
dovecot-dbg.done, thanks for the hint.
Andreas
(gdb) bt #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x00007f4e58be0921 in __GI_abort () at abort.c:79 #2 0x00007f4e59427a11 in default_fatal_finish (status=0, type=LOG_TYPE_PANIC) at failures.c:459 #3 fatal_handler_real (ctx=<optimized out>, format=<optimized out>, args=<optimized out>) at failures.c:471 #4 0x00007f4e59427b01 in i_internal_fatal_handler (ctx=<optimized out>, format=<optimized out>, args=<optimized out>) at failures.c:872 #5 0x00007f4e593744a7 in i_panic (format=format@entry=0x7f4e5946ab88 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:524 #6 0x00007f4e5944121c in io_loop_destroy (_ioloop=_ioloop@entry=0x7ffd0e203e68) at ioloop.c:865 #7 0x0000563d0150ffb6 in doveadm_print_server_flush () at doveadm-print-server.c:106 #8 0x0000563d01508fb7 in doveadm_print ( value=0x563d0269ced8 "AQAAAFoYRziwaStNyDEAAEKX94joZK5JAwAAAAQ", 'A'
, "DAAAAMu73BjjBalfFPQAANPIMJnB2rkkBAAAAAg", 'A' , "EAAAD3Gkc4sGkrTcgxAABCl/eI02OuSQIAAAAD", 'A' , "gAAAD4SNQYx691evBkAANXM"...) at doveadm-print.c:87 #9 0x0000563d014f4427 in cmd_dsync_run (_ctx=0x563d01fc7cc8, user=<optimized out>) at doveadm-dsync.c:713 #10 0x0000563d014f6a44 in doveadm_mail_next_user (ctx=ctx@entry=0x563d01fc7cc8, error_r=error_r@entry=0x7ffd0e204108) at doveadm-mail.c:462 #11 0x0000563d014f76be in doveadm_mail_single_user (ctx=ctx@entry=0x563d01fc7cc8, error_r=error_r@entry=0x7ffd0e204108) at doveadm-mail.c:493 #12 0x0000563d014f7c6f in doveadm_mail_cmd_exec (wildcard_user=0x0, ctx=0x563d01fc7cc8) at doveadm-mail.c:654 #13 doveadm_cmd_ver2_to_mail_cmd_wrapper (cctx=<optimized out>) at doveadm-mail.c:983 #14 0x0000563d01508c1b in doveadm_cmd_run_ver2 (argc=argc@entry=9, argv=argv@entry=0x563d01f6b3d0, cctx=cctx@entry=0x7ffd0e204260) at doveadm-cmd.c:465 #15 0x0000563d0150d612 in doveadm_cmd_server_run_ver2 (cctx=0x7ffd0e204260, argv=0x563d01f6b3d0, argc=9, conn=0x563d01fc1988) at client-connection-tcp.c:187 #16 doveadm_cmd_handle (cctx=0x7ffd0e204260, argv=<optimized out>, argc=<optimized out>, cmd_name=<optimized out>, conn=0x563d01fc1988) at client-connection-tcp.c:214 #17 client_handle_command (args=<optimized out>, conn=0x563d01fc1988) at client-connection-tcp.c:284 #18 client_connection_tcp_input (conn=0x563d01fc1988) at client-connection-tcp.c:436 #19 0x00007f4e594406d9 in io_loop_call_io (io=0x563d01fc5950) at ioloop.c:737 #20 0x00007f4e59442034 in io_loop_handler_run_internal (ioloop=ioloop@entry=0x563d01f73ed0) at ioloop-epoll.c:222 #21 0x00007f4e594407e0 in io_loop_handler_run (ioloop=<optimized out>) at ioloop.c:789 #22 0x00007f4e59440a18 in io_loop_run (ioloop=0x563d01f73ed0) at ioloop.c:762 #23 0x00007f4e593a6a23 in master_service_run (service=0x563d01f73d30, callback=<optimized out>) at master-service.c:863 #24 0x0000563d014e6922 in main (argc=<optimized out>, argv=<optimized out>) at main.c:124
Hi,
I can add a detail, every time I get the "panic error" on the replica node, I had one minute before this log written on the first node:
/dic 14 11:04:50 xxx dovecot[16715]: doveadm(xxx)<16454><uKbQLCJsuGFGQAAAIQnY6g>: Error: Couldn't lock /mnt/mail-storage-lv0007/xxx/.dovecot-sync.lock: fcntl(/mnt/mail-storage-lv0007/xxx/.dovecot-sync.lock, write-lock, F_SETLKW) locking failed: Timed out after 30 seconds (WRITE lock held by pid 16215)/
Regards
On 13/12/21 14:57, Claudio Corvino wrote:
Hi,
I have the same error in my replica server, it appears many times per day.
Log:
/Dec 13 13:01:23 Error: doveadm(xxx)<25513><vsFCF5w1t2GpYwAAbKtC3g>: read(xxx.xxx.xxx) failed: EOF (last sent=mailbox, last recv=mailbox)// //Dec 13 13:34:43 Warning: doveadm(xxx)<26277><9e2rJOM9t2GlZgAAbKtC3g>: /mnt/mail-storage-lv0003/xxx/Maildir/dovecot-uidlist: Duplicate file entry at line 2162: 1639396906.M258993P16725.xxx,S=7304,W=7467 (uid 71966 -> 71970)// //Dec 13 14:26:20 Error: doveadm(27294): write(xxx) failed: Timed out after 60 seconds// //Dec 13 14:26:20 Panic: doveadm(27294): file ioloop.c: line 865 (io_loop_destroy): assertion failed: (ioloop == current_ioloop)// //Dec 13 14:26:20 Error: doveadm(27294): Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x3d) [0x7f74d96f96ed] -> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7f74d96f980e] -> /usr/lib/dovecot/libdovecot.so.0(+0xff5cb) [0x7f74d97065cb] -> /usr/lib/dovecot/libdovecot.so.0(+0xff661) [0x7f74d9706661] -> /usr/lib/dovecot/libdovecot.so.0(+0x54a45) [0x7f74d965ba45] -> /usr/lib/dovecot/libdovecot.so.0(+0x57d95) [0x7f74d965ed95] -> dovecot/doveadm-server(+0x4af0f) [0x556a7a4b4f0f] -> dovecot/doveadm-server(doveadm_print_flush+0x1f) [0x556a7a4ae7af] -> dovecot/doveadm-server(doveadm_cmd_ver2_to_mail_cmd_wrapper+0x32b) [0x556a7a49dbcb] -> dovecot/doveadm-server(doveadm_cmd_run_ver2+0x4f7) [0x556a7a4ae307] -> dovecot/doveadm-server(+0x4874a) [0x556a7a4b274a] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7f74d971c679] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x131) [0x7f74d971dc61] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x4c) [0x7f74d971c71c] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x40) [0x7f74d971c8a0] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f74d96903d3] -> dovecot/doveadm-server(main+0x102) [0x556a7a48d9f2] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb) [0x7f74d92e709b] -> dovecot/doveadm-server(_start+0x2a) [0x556a7a48da7a]// //Dec 13 14:26:20 Fatal: doveadm(27294): master: service(doveadm): child 27294 killed with signal 6 (core dumps disabled - https://dovecot.org/bugreport.html#coredumps)// //Dec 13 14:29:06 Error: doveadm(27364): write(xxx) failed: Timed out after 60 seconds// //Dec 13 14:29:06 Panic: doveadm(27364): file ioloop.c: line 865 (io_loop_destroy): assertion failed: (ioloop == current_ioloop)// //Dec 13 14:29:06 Error: doveadm(27364): Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x3d) [0x7f05717756ed] -> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7f057177580e] -> /usr/lib/dovecot/libdovecot.so.0(+0xff5cb) [0x7f05717825cb] -> /usr/lib/dovecot/libdovecot.so.0(+0xff661) [0x7f0571782661] -> /usr/lib/dovecot/libdovecot.so.0(+0x54a45) [0x7f05716d7a45] -> /usr/lib/dovecot/libdovecot.so.0(+0x57d95) [0x7f05716dad95] -> dovecot/doveadm-server(+0x4af0f) [0x558edcd93f0f] -> dovecot/doveadm-server(doveadm_print_flush+0x1f) [0x558edcd8d7af] -> dovecot/doveadm-server(doveadm_cmd_ver2_to_mail_cmd_wrapper+0x32b) [0x558edcd7cbcb] -> dovecot/doveadm-server(doveadm_cmd_run_ver2+0x4f7) [0x558edcd8d307] -> dovecot/doveadm-server(+0x4874a) [0x558edcd9174a] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7f0571798679] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x131) [0x7f0571799c61] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x4c) [0x7f057179871c] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x40) [0x7f05717988a0] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f057170c3d3] -> dovecot/doveadm-server(main+0x102) [0x558edcd6c9f2] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb) [0x7f057136309b] -> dovecot/doveadm-server(_start+0x2a) [0x558edcd6ca7a]// //Dec 13 14:29:06 Fatal: doveadm(27364): master: service(doveadm): child 27364 killed with signal 6 (core dumps disabled - https://dovecot.org/bugreport.html#coredumps)/
My 'dovecot.conf':/ /
/# 2.3.17.1 (476cd46418): /etc/dovecot/dovecot.conf # Pigeonhole version 0.5.17.1 (a1a0b892) # OS: Linux 4.19.0-18-amd64 x86_64 Debian 10.11 # Hostname: xxx auth_mechanisms = plain login default_vsz_limit = 1 G doveadm_password = # hidden, use -P to show it doveadm_port = xxx mail_gid = 501 mail_location = maildir:~/Maildir:INDEX=/var/indexes/%n mail_max_userip_connections = 50 mail_plugins = " notify replication" mail_privileged_group = vmail mail_uid = 501 managesieve_notify_capability = mailto managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext vacation-seconds imapflags notify namespace inbox { inbox = yes location = mailbox Drafts { special_use = \Drafts } mailbox Junk { special_use = \Junk } mailbox "Posta inviata" { special_use = \Sent } mailbox Sent { special_use = \Sent } mailbox "Sent Messages" { special_use = \Sent } mailbox Trash { special_use = \Trash } prefix = } passdb { args = /etc/dovecot/dovecot-ldap.conf driver = ldap } plugin { mail_replica = tcp:xxx sieve = file:/var/www/html/ooo/%n;active=/var/www/html/ooo/%n/script sieve_default = /var/www/html/ooo/default.sieve sieve_duplicate_default_period = 14d sieve_duplicate_max_period = 7d sieve_extensions = +duplicate +notify +imapflags +vacation-seconds sieve_user_log = /var/www/html/ooo/%n/sieve_error.log } protocols = " imap lmtp sieve pop3" replication_max_conns = 1 service aggregator { fifo_listener replication-notify-fifo { user = vmail } unix_listener replication-notify { user = vmail } } service auth { unix_listener /var/spool/postfix/private/auth { group = postfix mode = 0660 user = postfix } unix_listener auth-master { group = vmail mode = 0660 user = vmail } } service doveadm { inet_listener { port = 12345 } } service imap-login { process_limit = 200 } service lmtp { process_min_avail = 5 unix_listener /var/spool/postfix/private/dovecot-lmtp { group = postfix mode = 0600 user = postfix } user = vmail } service managesieve-login { inet_listener sieve { port = 4190 } process_min_avail = 10 service_count = 1 } service replicator { process_min_avail = 1 unix_listener replicator-doveadm { mode = 0666 user = vmail } } service stats { unix_listener stats-reader { group = vmail mode = 0666 user = vmail } unix_listener stats-writer { group = vmail mode = 0666 user = vmail } } ssl_cert =
My 'dovecot-ldap.conf':
/hosts = xxx base = dc=xxx,dc=lan ldap_version=3 auth_bind = yes dn = cn=xxx,cn=xxx,dc=xxx,dc=lan dnpass = #hidden scope = subtree #user_attrs = sAMAccountName=home=/mnt/mail-storage-lv0003/%$,=uid=501,=gid=501 user_attrs = sAMAccountName=user,=home=/mnt/mail-storage-lv0003/%n,=uid=501,=gid=501 pass_attrs = sAMAccountName=user user_filter = (&(objectclass=person)(samaccountname=%n)) pass_filter= (&(objectclass=person)(samaccountname=%n)) iterate_attrs = sAMAccountName=user # With following filter we exclude all objects without an email address, all computers and all inactive accounts iterate_filter = (&(objectCategory=person)(mail=*)(objectClass=user)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))/
Regards
On 19/11/21 10:27, Andreas Piper wrote:
Hello,
as these 'Panic'-Events continue to appear every few minutes in my replicator-log, I would like to ask, if you have any hints how to repair the correlated mailbox?
Andreas
Am 11.11.21 um 15:04 schrieb Piper Andreas:
Hello,
Can you install dovecot debug symbols and redo the gdb backtrace? Package name is likely
dovecot-dbg.done, thanks for the hint.
Andreas
(gdb) bt #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x00007f4e58be0921 in __GI_abort () at abort.c:79 #2 0x00007f4e59427a11 in default_fatal_finish (status=0, type=LOG_TYPE_PANIC) at failures.c:459 #3 fatal_handler_real (ctx=<optimized out>, format=<optimized out>, args=<optimized out>) at failures.c:471 #4 0x00007f4e59427b01 in i_internal_fatal_handler (ctx=<optimized out>, format=<optimized out>, args=<optimized out>) at failures.c:872 #5 0x00007f4e593744a7 in i_panic (format=format@entry=0x7f4e5946ab88 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:524 #6 0x00007f4e5944121c in io_loop_destroy (_ioloop=_ioloop@entry=0x7ffd0e203e68) at ioloop.c:865 #7 0x0000563d0150ffb6 in doveadm_print_server_flush () at doveadm-print-server.c:106 #8 0x0000563d01508fb7 in doveadm_print ( value=0x563d0269ced8 "AQAAAFoYRziwaStNyDEAAEKX94joZK5JAwAAAAQ", 'A'
, "DAAAAMu73BjjBalfFPQAANPIMJnB2rkkBAAAAAg", 'A' , "EAAAD3Gkc4sGkrTcgxAABCl/eI02OuSQIAAAAD", 'A' , "gAAAD4SNQYx691evBkAANXM"...) at doveadm-print.c:87 #9 0x0000563d014f4427 in cmd_dsync_run (_ctx=0x563d01fc7cc8, user=<optimized out>) at doveadm-dsync.c:713 #10 0x0000563d014f6a44 in doveadm_mail_next_user (ctx=ctx@entry=0x563d01fc7cc8, error_r=error_r@entry=0x7ffd0e204108) at doveadm-mail.c:462 #11 0x0000563d014f76be in doveadm_mail_single_user (ctx=ctx@entry=0x563d01fc7cc8, error_r=error_r@entry=0x7ffd0e204108) at doveadm-mail.c:493 #12 0x0000563d014f7c6f in doveadm_mail_cmd_exec (wildcard_user=0x0, ctx=0x563d01fc7cc8) at doveadm-mail.c:654 #13 doveadm_cmd_ver2_to_mail_cmd_wrapper (cctx=<optimized out>) at doveadm-mail.c:983 #14 0x0000563d01508c1b in doveadm_cmd_run_ver2 (argc=argc@entry=9, argv=argv@entry=0x563d01f6b3d0, cctx=cctx@entry=0x7ffd0e204260) at doveadm-cmd.c:465 #15 0x0000563d0150d612 in doveadm_cmd_server_run_ver2 (cctx=0x7ffd0e204260, argv=0x563d01f6b3d0, argc=9, conn=0x563d01fc1988) at client-connection-tcp.c:187 #16 doveadm_cmd_handle (cctx=0x7ffd0e204260, argv=<optimized out>, argc=<optimized out>, cmd_name=<optimized out>, conn=0x563d01fc1988) at client-connection-tcp.c:214 #17 client_handle_command (args=<optimized out>, conn=0x563d01fc1988) at client-connection-tcp.c:284 #18 client_connection_tcp_input (conn=0x563d01fc1988) at client-connection-tcp.c:436 #19 0x00007f4e594406d9 in io_loop_call_io (io=0x563d01fc5950) at ioloop.c:737 #20 0x00007f4e59442034 in io_loop_handler_run_internal (ioloop=ioloop@entry=0x563d01f73ed0) at ioloop-epoll.c:222 #21 0x00007f4e594407e0 in io_loop_handler_run (ioloop=<optimized out>) at ioloop.c:789 #22 0x00007f4e59440a18 in io_loop_run (ioloop=0x563d01f73ed0) at ioloop.c:762 #23 0x00007f4e593a6a23 in master_service_run (service=0x563d01f73d30, callback=<optimized out>) at master-service.c:863 #24 0x0000563d014e6922 in main (argc=<optimized out>, argv=<optimized out>) at main.c:124
participants (4)
-
Aki Tuomi
-
Andreas Piper
-
Claudio Corvino
-
Piper Andreas