Re: Dovecot 2.3.0 assertion failure on LMTP delivery

29 Jan 2018


      Hi Aki,
Please see below:
Thank you
-Nick
# gdb /usr/libexec/dovecot/lmtp ./3445
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `dovecot/lmtp'.
Program terminated with signal 6, Aborted.
#0  0x00007f297814d1f7 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install dovecot-2.3.0-8.cp1162.x86_64
(gdb) back
#0  0x00007f297814d1f7 in raise () from /lib64/libc.so.6
#1  0x00007f297814e8e8 in abort () from /lib64/libc.so.6
#2  0x00007f29785a3eab in default_fatal_finish (type=<optimized out>, status=status@entry=0) at failures.c:228
#3  0x00007f29785a3f9e in i_internal_fatal_handler (ctx=0x7ffe12c97a00, format=<optimized out>, args=<optimized out>) at failures.c:718
#4  0x00007f2978517190 in i_panic (format=format@entry=0x7f29785df6a8 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:306
#5  0x00007f29785bf6bb in event_unref (_event=_event@entry=0x55ac2dab3ea8) at lib-event.c:148
#6  0x00007f29788a1b64 in mail_storage_service_user_unref (_user=_user@entry=0x55ac2dab35a8) at mail-storage-service.c:1649
#7  0x000055ac2cbd03fa in lmtp_local_rcpt_deinit (rcpt=0x55ac2dab3570) at lmtp-local.c:125
#8  0x000055ac2cbd08dc in lmtp_local_deinit (_local=_local@entry=0x55ac2daa00f0) at lmtp-local.c:83
#9  0x000055ac2cbcfa5a in client_state_reset (client=0x55ac2daa0028) at client.c:177
#10 0x00007f297852e96c in smtp_server_connection_reset_state (conn=0x55ac2daaee40) at smtp-server-connection.c:1249
#11 0x00007f297852c31d in smtp_server_command_completed (cmd=0x55ac2dab3998) at smtp-server-command.c:380
#12 0x00007f297852f848 in smtp_server_connection_next_reply (conn=0x55ac2daaee40) at smtp-server-connection.c:623
#13 smtp_server_connection_send_replies (conn=0x55ac2daaee40) at smtp-server-connection.c:682
#14 smtp_server_connection_output (conn=0x55ac2daaee40) at smtp-server-connection.c:728
#15 0x00007f29785cb990 in stream_send_io (fstream=0x55ac2da75dd0) at ostream-file.c:475
#16 0x00007f29785bb4d5 in io_loop_call_io (io=0x55ac2dab75b0) at ioloop.c:614
#17 0x00007f29785bcdaf in io_loop_handler_run_internal (ioloop=ioloop@entry=0x55ac2da6ac50) at ioloop-epoll.c:222
#18 0x00007f29785bb5d2 in io_loop_handler_run (ioloop=ioloop@entry=0x55ac2da6ac50) at ioloop.c:666
#19 0x00007f29785bb7f8 in io_loop_run (ioloop=0x55ac2da6ac50) at ioloop.c:639
#20 0x00007f2978539bb3 in master_service_run (service=0x55ac2da6aae0, callback=callback@entry=0x55ac2cbcf3d0 ) at master-service.c:767
#21 0x000055ac2cbcf1a6 in main (argc=1, argv=0x55ac2da6a8a0) at main.c:159
(gdb) bt full
#0  0x00007f297814d1f7 in raise () from /lib64/libc.so.6
No symbol table info available.
#1  0x00007f297814e8e8 in abort () from /lib64/libc.so.6
No symbol table info available.
#2  0x00007f29785a3eab in default_fatal_finish (type=<optimized out>, status=status@entry=0) at failures.c:228
backtrace = 0x55ac2da62088 "/usr/lib64/dovecot/libdovecot.so.0(+0xc8ee4) [0x7f29785a3ee4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f9e) [0x7f29785a3f9e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f2978517190] -> /usr"...
recursed = 0
#3  0x00007f29785a3f9e in i_internal_fatal_handler (ctx=0x7ffe12c97a00, format=<optimized out>, args=<optimized out>) at failures.c:718
status = 0
#4  0x00007f2978517190 in i_panic (format=format@entry=0x7f29785df6a8 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:306
ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0, log_prefix = 0x0}
args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area = 0x7ffe12c97af0, reg_save_area = 0x7ffe12c97a30}}
#5  0x00007f29785bf6bb in event_unref (_event=_event@entry=0x55ac2dab3ea8) at lib-event.c:148
event = 0x55ac2dab8180
__func__ = "event_unref"
#6  0x00007f29788a1b64 in mail_storage_service_user_unref (_user=_user@entry=0x55ac2dab35a8) at mail-storage-service.c:1649
user = 0x55ac2dab3e08
__func__ = "mail_storage_service_user_unref"
#7  0x000055ac2cbd03fa in lmtp_local_rcpt_deinit (rcpt=0x55ac2dab3570) at lmtp-local.c:125
No locals.
#8  0x000055ac2cbd08dc in lmtp_local_deinit (_local=_local@entry=0x55ac2daa00f0) at lmtp-local.c:83
rcptp_end = 0x55ac2da9fbb8
local = 0x55ac2dab8000
rcptp = 0x55ac2da9fbb0
#9  0x000055ac2cbcfa5a in client_state_reset (client=0x55ac2daa0028) at client.c:177
No locals.
#10 0x00007f297852e96c in smtp_server_connection_reset_state (conn=0x55ac2daaee40) at smtp-server-connection.c:1249
No locals.
#11 0x00007f297852c31d in smtp_server_command_completed (cmd=0x55ac2dab3998) at smtp-server-command.c:380
hook_completed = <optimized out>
#12 0x00007f297852f848 in smtp_server_connection_next_reply (conn=0x55ac2daaee40) at smtp-server-connection.c:623
cmd = 0x55ac2dab3998
error = 0x0
i = <optimized out>
#13 smtp_server_connection_send_replies (conn=0x55ac2daaee40) at smtp-server-connection.c:682
No locals.
#14 smtp_server_connection_output (conn=0x55ac2daaee40) at smtp-server-connection.c:728
ret = 1
#15 0x00007f29785cb990 in stream_send_io (fstream=0x55ac2da75dd0) at ostream-file.c:475
ostream = 0x55ac2da75e60
use_cork = true
ret = <optimized out>
#16 0x00007f29785bb4d5 in io_loop_call_io (io=0x55ac2dab75b0) at ioloop.c:614
ioloop = 0x55ac2da6ac50
t_id = 2
__func__ = "io_loop_call_io"
---Type <return> to continue, or q <return> to quit---
#17 0x00007f29785bcdaf in io_loop_handler_run_internal (ioloop=ioloop@entry=0x55ac2da6ac50) at ioloop-epoll.c:222
ctx = 0x55ac2da761d0
events = <optimized out>
list = 0x55ac2da9c650
io = <optimized out>
tv = {tv_sec = 2147483, tv_usec = 0}
events_count = <optimized out>
msecs = <optimized out>
ret = 1
i = 0
call = <optimized out>
__func__ = "io_loop_handler_run_internal"
#18 0x00007f29785bb5d2 in io_loop_handler_run (ioloop=ioloop@entry=0x55ac2da6ac50) at ioloop.c:666
__func__ = "io_loop_handler_run"
#19 0x00007f29785bb7f8 in io_loop_run (ioloop=0x55ac2da6ac50) at ioloop.c:639
__func__ = "io_loop_run"
#20 0x00007f2978539bb3 in master_service_run (service=0x55ac2da6aae0, callback=callback@entry=0x55ac2cbcf3d0 ) at master-service.c:767
No locals.
#21 0x000055ac2cbcf1a6 in main (argc=1, argv=0x55ac2da6a8a0) at main.c:159
set_roots = {0x7f2978843fc0 , 0x7f2978da4820 , 0x55ac2cdd4560 , 0x0}
service_flags = <optimized out>
storage_service_flags = <optimized out>
tmp_base_dir = 0x55ac2da62040 "\003"
c = <optimized out>
error = 0x0
(gdb) p current_global_event
$1 = (struct event *) 0x55ac2dab8180
(gdb) p event
No symbol "event" in current context.
(gdb) p *current_global_event
$2 = {event_passthrough = {append_log_prefix = 0x7f29785bf830 , replace_log_prefix = 0x7f29785bf860 ,
set_name = 0x7f29785bf8d0 , set_source = 0x7f29785bf930 , set_always_log_source = 0x7f29785bf970 ,
add_categories = 0x7f29785bfb00 , add_category = 0x7f29785bfb70 , add_fields = 0x7f29785bfe80 ,
add_str = 0x7f29785bfcc0 , add_int = 0x7f29785bfd40 , add_timeval = 0x7f29785bfdd0 ,
event = 0x7f29785bedd0 }, prev = 0x55ac2dac51a0, next = 0x55ac2da75be0, refcount = 0, pool = 0x55ac2dab8070, parent = 0x0, id = 2, log_prefix = 0x0,
log_prefix_from_system_pool = false, log_prefix_replace = false, passthrough = false, forced_debug = false, always_log_source = false, sending_debug_log = false, id_sent_to_stats = false,
call_free = false, tv_created = {tv_sec = 1517200513, tv_usec = 468845}, tv_last_sent = {tv_sec = 0, tv_usec = 0}, source_filename = 0x7f297894f95a "mail-storage-service.c", source_linenum = 1333,
sending_name = 0x0, categories = {arr = {buffer = 0x0, element_size = 0}, v = 0x0, v_modifiable = 0x0}, fields = {arr = {buffer = 0x55ac2dab8278, element_size = 48}, v = 0x55ac2dab8278,
v_modifiable = 0x55ac2dab8278}}
...
On Jan 27, 2018, at 4:40 PM, Aki Tuomi aki.tuomi@dovecot.fi wrote:
Hi!
This is a bug in the new event code. Can you get a full gdb backtrace?
Aki
...
On January 27, 2018 at 6:58 AM "J. Nick Koston" nick@cpanel.net wrote:
Hi,
We are seeing a frequent assertion failure on LMTP delivery with 2.3.0.  This only appears to happen on CentOS/RHEL 7.
Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Error: Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0xc8ec4) [0x7f226d036ec4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f7e) [0x7f226d036f7e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f226cfaa190] -> /usr/lib64/dovecot/libdovecot.so.0(+0xe469b) [0x7f226d05269b] -> /usr/lib64/dovecot/libdovecot-storage.so.0(mail_storage_service_user_unref+0xb4) [0x7f226d334b64] -> dovecot/lmtp(+0x63fa) [0x56438f3673fa] -> dovecot/lmtp(lmtp_local_deinit+0x5c) [0x56438f3678dc] -> dovecot/lmtp(client_state_reset+0x1a) [0x56438f366a5a] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_connection_reset_state+0x3c) [0x7f226cfc196c] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_command_completed+0x4d) [0x7f226cfbf31d] -> /usr/lib64/dovecot/libdovecot.so.0(+0x54848) [0x7f226cfc2848] -> /usr/lib64/dovecot/libdovecot.so.0(+0xf0970) [0x7f226d05e970] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x65) [0x7f226d04e4b5] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f) [0x7f226d04fd8f] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x52) [0x7f226d04e5b2] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f226d04e7d8] -> /usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f226cfccbb3] -> dovecot/lmtp(main+0x226) [0x56438f3661a6] -> /lib64/libc.so.6(__libc_start_main+0xf5) [0x7f226cbccc05] -> dovecot/lmtp(+0x52d5) [0x56438f3662d5]
Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Fatal: master: service(lmtp): child 29540 killed with signal 6 (core dumps disabled)
From the exim log:
'LOG: MAIN  cwd=/usr/local/cpanel 3 args: /usr/sbin/sendmail -v bob@l56ob1nf.cptestLOG: MAIN  <= root@smoker-devautomerge-c7-2.dev.cpanel.net U=root P=local S=3146285 T="Ignore this test from Cpanel-Email-Message-Retrieve.t"LOG: MAIN  cwd=/var/spool/exim 4 args: /usr/sbin/exim -v -Mc 1eemjH-0006uB-C2delivering 1eemjH-0006uB-C2  LMTP<< 220 smoker-devautomerge-c7-2.dev.cpanel.net Dovecot ready.  LMTP>> LHLO smoker-devautomerge-c7-2.dev.cpanel.net
LMTP<< 250-smoker-devautomerge-c7-2.dev.cpanel.net  LMTP<< 250-8BITMIME  LMTP<< 250-CHUNKING  LMTP<< 250-ENHANCEDSTATUSCODES  LMTP<< 250-PIPELINING  LMTP<< 250-STARTTLS  LMTP<< 250-VRFY  LMTP<< 250 XCLIENT ADDR PORT PROTO HELO LOGIN TTL TIMEOUT  LMTP>> MAIL FROM:root@smoker-devautomerge-c7-2.dev.cpanel.net
LMTP<< 250 2.1.0 OK  LMTP>> RCPT TO:bob@l56ob1nf.cptest
LMTP<< 250 2.1.5 OK  LMTP>> DATA
LMTP<< 354 OK  LMTP>> writing message and terminating "."  LMTP>> QUIT
LOG: MAIN  == bob@l56ob1nf.cptest R=virtual_user T=dovecot_virtual_delivery defer (-1): LMTP connection closed after end of data’,
Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30217): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30233): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 10:05:15 smoker-devautomerge-c7-1 dovecot: lmtp(32618): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(375): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(403): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(424): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(430): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 12:32:16 smoker-devautomerge-c7-1 dovecot: lmtp(4435): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15232): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15238): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15244): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event)
doveconf -n:
# 2.3.0 (c8b89eb): /etc/dovecot/dovecot.conf
# OS: Linux 3.10.0-693.2.2.el7.x86_64 x86_64 CentOS Linux release 7.4.1708 (Core)

auth_cache_size = 1 M
auth_mechanisms = plain login
auth_username_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$-=?^_{}~./@+%"
disable_plaintext_auth = no
first_valid_uid = 201
lda_mailbox_autocreate = yes
lmtp_rcpt_check_quota = yes
lmtp_save_to_detail_mailbox = yes
lmtp_user_concurrency_limit = 4
mail_access_groups = dovecot
mail_plugins = quota quota_clone zlib
mail_prefetch_count = 20
maildir_very_dirty_syncs = yes
namespace inbox {
inbox = yes
location =
mailbox Archive {
auto = create
special_use = \Archive
}
mailbox Archives {
auto = no
special_use = \Archive
}
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
auto = create
special_use = \Junk
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox "Sent Messages" {
auto = no
special_use = \Sent
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
mailbox spam {
auto = create
special_use = \Junk
}
prefix = INBOX.
separator = .
type = private
}
passdb {
args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf
driver = dict
result_failure = return-fail
}
passdb {
args = /usr/local/cpanel/bin/dovecot-wrap
driver = checkpassword
skip = authenticated
}
plugin {
acl = vfile:cache_secs=86400
quota_exceeded_message = Mailbox is full / Blocks limit exceeded / Inode limit exceeded
}
protocols = lmtp imap pop3
service auth {
unix_listener auth-client {
mode = 0666
}
}
service config {
vsz_limit = 2 G
}
service dict {
unix_listener dict {
group = dovecot
mode = 0660
}
}
service imap-login {
client_limit = 500
inet_listener imap {
address = *,::
}
inet_listener imaps {
address = *,::
}
process_limit = 50
process_min_avail = 2
service_count = 0
vsz_limit = 128 M
}
service imap {
process_limit = 512
vsz_limit = 512 M
}
service lmtp {
client_limit = 1
process_limit = 500
unix_listener lmtp {
group = mail
mode = 0660
user = mailnull
}
vsz_limit = 512 M
}
service managesieve-login {
client_limit = 500
process_limit = 50
process_min_avail = 2
service_count = 0
vsz_limit = 128 M
}
service managesieve {
process_limit = 512
vsz_limit = 512 M
}
service pop3-login {
client_limit = 500
inet_listener pop3 {
address = *,::
}
inet_listener pop3s {
address = *,::
}
process_limit = 50
process_min_avail = 2
service_count = 0
vsz_limit = 128 M
}
service pop3 {
process_limit = 512
vsz_limit = 512 M
}
service quota-status {
executable = quota-status -p postfix
unix_listener quota-status {
mode = 0666
}
}
ssl_cert =