Dovecot 2.3.0 assertion failure on LMTP delivery
J. Nick Koston
nick at cpanel.net
Mon Jan 29 06:48:44 EET 2018
Hi Aki,
Please see below:
Thank you
-Nick
# gdb /usr/libexec/dovecot/lmtp ./3445
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `dovecot/lmtp'.
Program terminated with signal 6, Aborted.
#0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install dovecot-2.3.0-8.cp1162.x86_64
(gdb) back
#0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6
#1 0x00007f297814e8e8 in abort () from /lib64/libc.so.6
#2 0x00007f29785a3eab in default_fatal_finish (type=<optimized out>, status=status at entry=0) at failures.c:228
#3 0x00007f29785a3f9e in i_internal_fatal_handler (ctx=0x7ffe12c97a00, format=<optimized out>, args=<optimized out>) at failures.c:718
#4 0x00007f2978517190 in i_panic (format=format at entry=0x7f29785df6a8 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:306
#5 0x00007f29785bf6bb in event_unref (_event=_event at entry=0x55ac2dab3ea8) at lib-event.c:148
#6 0x00007f29788a1b64 in mail_storage_service_user_unref (_user=_user at entry=0x55ac2dab35a8) at mail-storage-service.c:1649
#7 0x000055ac2cbd03fa in lmtp_local_rcpt_deinit (rcpt=0x55ac2dab3570) at lmtp-local.c:125
#8 0x000055ac2cbd08dc in lmtp_local_deinit (_local=_local at entry=0x55ac2daa00f0) at lmtp-local.c:83
#9 0x000055ac2cbcfa5a in client_state_reset (client=0x55ac2daa0028) at client.c:177
#10 0x00007f297852e96c in smtp_server_connection_reset_state (conn=0x55ac2daaee40) at smtp-server-connection.c:1249
#11 0x00007f297852c31d in smtp_server_command_completed (cmd=0x55ac2dab3998) at smtp-server-command.c:380
#12 0x00007f297852f848 in smtp_server_connection_next_reply (conn=0x55ac2daaee40) at smtp-server-connection.c:623
#13 smtp_server_connection_send_replies (conn=0x55ac2daaee40) at smtp-server-connection.c:682
#14 smtp_server_connection_output (conn=0x55ac2daaee40) at smtp-server-connection.c:728
#15 0x00007f29785cb990 in stream_send_io (fstream=0x55ac2da75dd0) at ostream-file.c:475
#16 0x00007f29785bb4d5 in io_loop_call_io (io=0x55ac2dab75b0) at ioloop.c:614
#17 0x00007f29785bcdaf in io_loop_handler_run_internal (ioloop=ioloop at entry=0x55ac2da6ac50) at ioloop-epoll.c:222
#18 0x00007f29785bb5d2 in io_loop_handler_run (ioloop=ioloop at entry=0x55ac2da6ac50) at ioloop.c:666
#19 0x00007f29785bb7f8 in io_loop_run (ioloop=0x55ac2da6ac50) at ioloop.c:639
#20 0x00007f2978539bb3 in master_service_run (service=0x55ac2da6aae0, callback=callback at entry=0x55ac2cbcf3d0 <client_connected>) at master-service.c:767
#21 0x000055ac2cbcf1a6 in main (argc=1, argv=0x55ac2da6a8a0) at main.c:159
(gdb) bt full
#0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f297814e8e8 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x00007f29785a3eab in default_fatal_finish (type=<optimized out>, status=status at entry=0) at failures.c:228
backtrace = 0x55ac2da62088 "/usr/lib64/dovecot/libdovecot.so.0(+0xc8ee4) [0x7f29785a3ee4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f9e) [0x7f29785a3f9e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f2978517190] -> /usr"...
recursed = 0
#3 0x00007f29785a3f9e in i_internal_fatal_handler (ctx=0x7ffe12c97a00, format=<optimized out>, args=<optimized out>) at failures.c:718
status = 0
#4 0x00007f2978517190 in i_panic (format=format at entry=0x7f29785df6a8 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:306
ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0, log_prefix = 0x0}
args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area = 0x7ffe12c97af0, reg_save_area = 0x7ffe12c97a30}}
#5 0x00007f29785bf6bb in event_unref (_event=_event at entry=0x55ac2dab3ea8) at lib-event.c:148
event = 0x55ac2dab8180
__func__ = "event_unref"
#6 0x00007f29788a1b64 in mail_storage_service_user_unref (_user=_user at entry=0x55ac2dab35a8) at mail-storage-service.c:1649
user = 0x55ac2dab3e08
__func__ = "mail_storage_service_user_unref"
#7 0x000055ac2cbd03fa in lmtp_local_rcpt_deinit (rcpt=0x55ac2dab3570) at lmtp-local.c:125
No locals.
#8 0x000055ac2cbd08dc in lmtp_local_deinit (_local=_local at entry=0x55ac2daa00f0) at lmtp-local.c:83
rcptp_end = 0x55ac2da9fbb8
local = 0x55ac2dab8000
rcptp = 0x55ac2da9fbb0
#9 0x000055ac2cbcfa5a in client_state_reset (client=0x55ac2daa0028) at client.c:177
No locals.
#10 0x00007f297852e96c in smtp_server_connection_reset_state (conn=0x55ac2daaee40) at smtp-server-connection.c:1249
No locals.
#11 0x00007f297852c31d in smtp_server_command_completed (cmd=0x55ac2dab3998) at smtp-server-command.c:380
hook_completed = <optimized out>
#12 0x00007f297852f848 in smtp_server_connection_next_reply (conn=0x55ac2daaee40) at smtp-server-connection.c:623
cmd = 0x55ac2dab3998
error = 0x0
i = <optimized out>
#13 smtp_server_connection_send_replies (conn=0x55ac2daaee40) at smtp-server-connection.c:682
No locals.
#14 smtp_server_connection_output (conn=0x55ac2daaee40) at smtp-server-connection.c:728
ret = 1
#15 0x00007f29785cb990 in stream_send_io (fstream=0x55ac2da75dd0) at ostream-file.c:475
ostream = 0x55ac2da75e60
use_cork = true
ret = <optimized out>
#16 0x00007f29785bb4d5 in io_loop_call_io (io=0x55ac2dab75b0) at ioloop.c:614
ioloop = 0x55ac2da6ac50
t_id = 2
__func__ = "io_loop_call_io"
---Type <return> to continue, or q <return> to quit---
#17 0x00007f29785bcdaf in io_loop_handler_run_internal (ioloop=ioloop at entry=0x55ac2da6ac50) at ioloop-epoll.c:222
ctx = 0x55ac2da761d0
events = <optimized out>
list = 0x55ac2da9c650
io = <optimized out>
tv = {tv_sec = 2147483, tv_usec = 0}
events_count = <optimized out>
msecs = <optimized out>
ret = 1
i = 0
call = <optimized out>
__func__ = "io_loop_handler_run_internal"
#18 0x00007f29785bb5d2 in io_loop_handler_run (ioloop=ioloop at entry=0x55ac2da6ac50) at ioloop.c:666
__func__ = "io_loop_handler_run"
#19 0x00007f29785bb7f8 in io_loop_run (ioloop=0x55ac2da6ac50) at ioloop.c:639
__func__ = "io_loop_run"
#20 0x00007f2978539bb3 in master_service_run (service=0x55ac2da6aae0, callback=callback at entry=0x55ac2cbcf3d0 <client_connected>) at master-service.c:767
No locals.
#21 0x000055ac2cbcf1a6 in main (argc=1, argv=0x55ac2da6a8a0) at main.c:159
set_roots = {0x7f2978843fc0 <smtp_submit_setting_parser_info>, 0x7f2978da4820 <lda_setting_parser_info>, 0x55ac2cdd4560 <lmtp_setting_parser_info>, 0x0}
service_flags = <optimized out>
storage_service_flags = <optimized out>
tmp_base_dir = 0x55ac2da62040 "\003"
c = <optimized out>
error = 0x0
(gdb) p current_global_event
$1 = (struct event *) 0x55ac2dab8180
(gdb) p event
No symbol "event" in current context.
(gdb) p *current_global_event
$2 = {event_passthrough = {append_log_prefix = 0x7f29785bf830 <event_passthrough_set_append_log_prefix>, replace_log_prefix = 0x7f29785bf860 <event_passthrough_replace_log_prefix>,
set_name = 0x7f29785bf8d0 <event_passthrough_set_name>, set_source = 0x7f29785bf930 <event_passthrough_set_source>, set_always_log_source = 0x7f29785bf970 <event_passthrough_set_always_log_source>,
add_categories = 0x7f29785bfb00 <event_passthrough_add_categories>, add_category = 0x7f29785bfb70 <event_passthrough_add_category>, add_fields = 0x7f29785bfe80 <event_passthrough_add_fields>,
add_str = 0x7f29785bfcc0 <event_passthrough_add_str>, add_int = 0x7f29785bfd40 <event_passthrough_add_int>, add_timeval = 0x7f29785bfdd0 <event_passthrough_add_timeval>,
event = 0x7f29785bedd0 <event_passthrough_event>}, prev = 0x55ac2dac51a0, next = 0x55ac2da75be0, refcount = 0, pool = 0x55ac2dab8070, parent = 0x0, id = 2, log_prefix = 0x0,
log_prefix_from_system_pool = false, log_prefix_replace = false, passthrough = false, forced_debug = false, always_log_source = false, sending_debug_log = false, id_sent_to_stats = false,
call_free = false, tv_created = {tv_sec = 1517200513, tv_usec = 468845}, tv_last_sent = {tv_sec = 0, tv_usec = 0}, source_filename = 0x7f297894f95a "mail-storage-service.c", source_linenum = 1333,
sending_name = 0x0, categories = {arr = {buffer = 0x0, element_size = 0}, v = 0x0, v_modifiable = 0x0}, fields = {arr = {buffer = 0x55ac2dab8278, element_size = 48}, v = 0x55ac2dab8278,
v_modifiable = 0x55ac2dab8278}}
> On Jan 27, 2018, at 4:40 PM, Aki Tuomi <aki.tuomi at dovecot.fi> wrote:
>
> Hi!
>
> This is a bug in the new event code. Can you get a full gdb backtrace?
>
> Aki
>
>> On January 27, 2018 at 6:58 AM "J. Nick Koston" <nick at cpanel.net> wrote:
>>
>>
>> Hi,
>>
>> We are seeing a frequent assertion failure on LMTP delivery with 2.3.0. This only appears to happen on CentOS/RHEL 7.
>>
>> Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Error: Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0xc8ec4) [0x7f226d036ec4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f7e) [0x7f226d036f7e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f226cfaa190] -> /usr/lib64/dovecot/libdovecot.so.0(+0xe469b) [0x7f226d05269b] -> /usr/lib64/dovecot/libdovecot-storage.so.0(mail_storage_service_user_unref+0xb4) [0x7f226d334b64] -> dovecot/lmtp(+0x63fa) [0x56438f3673fa] -> dovecot/lmtp(lmtp_local_deinit+0x5c) [0x56438f3678dc] -> dovecot/lmtp(client_state_reset+0x1a) [0x56438f366a5a] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_connection_reset_state+0x3c) [0x7f226cfc196c] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_command_completed+0x4d) [0x7f226cfbf31d] -> /usr/lib64/dovecot/libdovecot.so.0(+0x54848) [0x7f226cfc2848] -> /usr/lib64/dovecot/libdovecot.so.0(+0xf0970) [0x7f226d05e970] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x65) [0x7f226d04e4b5] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f) [0x7f226d04fd8f] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x52) [0x7f226d04e5b2] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f226d04e7d8] -> /usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f226cfccbb3] -> dovecot/lmtp(main+0x226) [0x56438f3661a6] -> /lib64/libc.so.6(__libc_start_main+0xf5) [0x7f226cbccc05] -> dovecot/lmtp(+0x52d5) [0x56438f3662d5]
>> Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Fatal: master: service(lmtp): child 29540 killed with signal 6 (core dumps disabled)
>>
>>
>> From the exim log:
>> 'LOG: MAIN cwd=/usr/local/cpanel 3 args: /usr/sbin/sendmail -v bob at l56ob1nf.cptestLOG: MAIN <= root at smoker-devautomerge-c7-2.dev.cpanel.net U=root P=local S=3146285 T="Ignore this test from Cpanel-Email-Message-Retrieve.t"LOG: MAIN cwd=/var/spool/exim 4 args: /usr/sbin/exim -v -Mc 1eemjH-0006uB-C2delivering 1eemjH-0006uB-C2 LMTP<< 220 smoker-devautomerge-c7-2.dev.cpanel.net Dovecot ready. LMTP>> LHLO smoker-devautomerge-c7-2.dev.cpanel.net
>> LMTP<< 250-smoker-devautomerge-c7-2.dev.cpanel.net LMTP<< 250-8BITMIME LMTP<< 250-CHUNKING LMTP<< 250-ENHANCEDSTATUSCODES LMTP<< 250-PIPELINING LMTP<< 250-STARTTLS LMTP<< 250-VRFY LMTP<< 250 XCLIENT ADDR PORT PROTO HELO LOGIN TTL TIMEOUT LMTP>> MAIL FROM:<root at smoker-devautomerge-c7-2.dev.cpanel.net>
>> LMTP<< 250 2.1.0 OK LMTP>> RCPT TO:<bob at l56ob1nf.cptest>
>> LMTP<< 250 2.1.5 OK LMTP>> DATA
>> LMTP<< 354 OK LMTP>> writing message and terminating "." LMTP>> QUIT
>> LOG: MAIN == bob at l56ob1nf.cptest R=virtual_user T=dovecot_virtual_delivery defer (-1): LMTP connection closed after end of data’,
>>
>>
>>
>> Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30217): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30233): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 10:05:15 smoker-devautomerge-c7-1 dovecot: lmtp(32618): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(375): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(403): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(424): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(430): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 12:32:16 smoker-devautomerge-c7-1 dovecot: lmtp(4435): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15232): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15238): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>> Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15244): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
>>
>> doveconf -n:
>>
>> # 2.3.0 (c8b89eb): /etc/dovecot/dovecot.conf
>> # OS: Linux 3.10.0-693.2.2.el7.x86_64 x86_64 CentOS Linux release 7.4.1708 (Core)
>> auth_cache_size = 1 M
>> auth_mechanisms = plain login
>> auth_username_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$-=?^_{}~./@+%"
>> disable_plaintext_auth = no
>> first_valid_uid = 201
>> lda_mailbox_autocreate = yes
>> lmtp_rcpt_check_quota = yes
>> lmtp_save_to_detail_mailbox = yes
>> lmtp_user_concurrency_limit = 4
>> mail_access_groups = dovecot
>> mail_plugins = quota quota_clone zlib
>> mail_prefetch_count = 20
>> maildir_very_dirty_syncs = yes
>> namespace inbox {
>> inbox = yes
>> location =
>> mailbox Archive {
>> auto = create
>> special_use = \Archive
>> }
>> mailbox Archives {
>> auto = no
>> special_use = \Archive
>> }
>> mailbox Drafts {
>> auto = subscribe
>> special_use = \Drafts
>> }
>> mailbox Junk {
>> auto = create
>> special_use = \Junk
>> }
>> mailbox Sent {
>> auto = subscribe
>> special_use = \Sent
>> }
>> mailbox "Sent Messages" {
>> auto = no
>> special_use = \Sent
>> }
>> mailbox Trash {
>> auto = subscribe
>> special_use = \Trash
>> }
>> mailbox spam {
>> auto = create
>> special_use = \Junk
>> }
>> prefix = INBOX.
>> separator = .
>> type = private
>> }
>> passdb {
>> args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf
>> driver = dict
>> result_failure = return-fail
>> }
>> passdb {
>> args = /usr/local/cpanel/bin/dovecot-wrap
>> driver = checkpassword
>> skip = authenticated
>> }
>> plugin {
>> acl = vfile:cache_secs=86400
>> quota_exceeded_message = Mailbox is full / Blocks limit exceeded / Inode limit exceeded
>> }
>> protocols = lmtp imap pop3
>> service auth {
>> unix_listener auth-client {
>> mode = 0666
>> }
>> }
>> service config {
>> vsz_limit = 2 G
>> }
>> service dict {
>> unix_listener dict {
>> group = dovecot
>> mode = 0660
>> }
>> }
>> service imap-login {
>> client_limit = 500
>> inet_listener imap {
>> address = *,::
>> }
>> inet_listener imaps {
>> address = *,::
>> }
>> process_limit = 50
>> process_min_avail = 2
>> service_count = 0
>> vsz_limit = 128 M
>> }
>> service imap {
>> process_limit = 512
>> vsz_limit = 512 M
>> }
>> service lmtp {
>> client_limit = 1
>> process_limit = 500
>> unix_listener lmtp {
>> group = mail
>> mode = 0660
>> user = mailnull
>> }
>> vsz_limit = 512 M
>> }
>> service managesieve-login {
>> client_limit = 500
>> process_limit = 50
>> process_min_avail = 2
>> service_count = 0
>> vsz_limit = 128 M
>> }
>> service managesieve {
>> process_limit = 512
>> vsz_limit = 512 M
>> }
>> service pop3-login {
>> client_limit = 500
>> inet_listener pop3 {
>> address = *,::
>> }
>> inet_listener pop3s {
>> address = *,::
>> }
>> process_limit = 50
>> process_min_avail = 2
>> service_count = 0
>> vsz_limit = 128 M
>> }
>> service pop3 {
>> process_limit = 512
>> vsz_limit = 512 M
>> }
>> service quota-status {
>> executable = quota-status -p postfix
>> unix_listener quota-status {
>> mode = 0666
>> }
>> }
>> ssl_cert = </etc/dovecot/ssl/dovecot.crt
>> ssl_cipher_list = ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
>> ssl_dh = # hidden, use -P to show it
>> ssl_key = # hidden, use -P to show it
>> userdb {
>> driver = prefetch
>> }
>> userdb {
>> args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf
>> driver = dict
>> }
>> userdb {
>> args = /usr/local/cpanel/bin/dovecot-wrap
>> driver = checkpassword
>> }
>> protocol imap {
>> imap_capability = +NAMESPACE
>> imap_idle_notify_interval = 24 mins
>> imap_logout_format = in=%i, out=%o, bytes=%i/%o
>> mail_max_userip_connections = 20
>> mail_plugins = acl quota imap_quota zlib imap_zlib quota_clone virtual
>> namespace sent {
>> hidden = yes
>> list = no
>> location = virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent
>> prefix = sent
>> separator = .
>> }
>> namespace spam {
>> hidden = yes
>> list = no
>> location = virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam
>> prefix = spam
>> separator = .
>> }
>> }
>> protocol pop3 {
>> mail_max_userip_connections = 3
>> mail_plugins = quota quota quota_clone virtual zlib
>> namespace sent {
>> hidden = yes
>> list = no
>> location = virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent
>> prefix = sent
>> separator = .
>> }
>> namespace spam {
>> hidden = yes
>> list = no
>> location = virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam
>> prefix = spam
>> separator = .
>> }
>> pop3_logout_format = top=%t/%p, retr=%r/%b, del=%d/%m, size=%s, bytes=%i/%o
>> pop3_uidl_format = UID%u-%v
>> }
>> protocol lmtp {
>> mail_plugins = quota quota_clone zlib
>> postmaster_address = root
>> quota_full_tempfail = no
>> }
>> protocol lda {
>> mail_plugins = quota quota_clone zlib
>> postmaster_address = root
>> quota_full_tempfail = no
>> }
>> local_name smoker-devautomerge-c7-1.dev.cpanel.net {
>> ssl_cert = </etc/dovecot/ssl/dovecot.crt
>> ssl_key = # hidden, use -P to show it
>> }
>>
>>
>
More information about the dovecot
mailing list