Core Dumps with dovecot-2.2 -git as of 2.2.devel (eaf276b33)

This appears to be breaking dsync at the moment and is caused by something committed in the last week or so. This is with the current -git of dovecot-2.2 (eaf276b33).

The crash happens all the time - the logs are full of this message.

Oct 7 20:31:24 thunderstorm.reub.net dovecot: doveadm: Warning: I/O leak: 0x7f8bb6581b4c (ostream-file.c:350, fd 14) Oct 7 20:31:24 thunderstorm.reub.net dovecot: doveadm: Panic: file ioloop.c: line 126 (io_remove_full): assertion failed: (io->callback != NULL) Oct 7 20:31:24 thunderstorm.reub.net dovecot: doveadm: Error: Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0xd48a6) [0x7f8bb654a8a6] -> /usr/lib64/dovecot/li bdovecot.so.0(+0xd5e26) [0x7f8bb654be26] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f8bb654ac39] -> /usr/lib64/dovecot/libdovecot.so.0(+0xf5cb0) [0x7f8bb6 56bcb0] -> /usr/lib64/dovecot/libdovecot.so.0(io_remove+0x1d) [0x7f8bb656be31] -> /usr/lib64/dovecot/libdovecot.so.0(+0x10b5f8) [0x7f8bb65815f8] -> /usr/lib64/dovec ot/libdovecot.so.0(o_stream_cork+0x5a) [0x7f8bb657e289] -> /usr/lib64/dovecot/libssl_iostream_openssl.so(+0x6406) [0x7f8bb12d6406] -> /usr/lib64/dovecot/libssl_iost ream_openssl.so(openssl_iostream_bio_sync+0x18) [0x7f8bb12d6af6] -> /usr/lib64/dovecot/libssl_iostream_openssl.so(+0xa52e) [0x7f8bb12da52e] -> / usr/lib64/dovecot/libssl_iostream_openssl.so(+0xa6ef) [0x7f8bb12da6ef] -> /usr/lib64/dovecot/libdovecot.so.0(o_stream_sendv+0xc7) [0x7f8bb657e765] -> /usr/lib64/dovecot/libdovecot.so.0(o_stream_send+0x61) [0x7f8bb657e688] -> /usr/lib64/dovecot/libdovecot.so.0(+0x10dc2c) [0x7f8bb6583c2c] -> /usr/lib64/dovecot/libdovecot.so.0(+0x10ddd4) [0x7f8bb6583dd4] -> /usr/lib64/dovecot/libdovecot.so.0(o_stream_sendv+0xc7) [0x7f8bb657e765] -> /usr/lib64/dovecot/libdovecot.so.0(o_stream_nsendv+0x55) [0x7f8bb657e93f] -> /usr/lib64/dovecot/libdovecot.so.0(o_stream_nsend+0x61) [0x7f8bb657e8d3] -> dovecot/doveadm-server() [0x444a35] -> /usr/lib64/dovecot/libdovecot.so.0(i_warning+0xce) [0x7f8bb654afdc] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_destroy+0x2ca) [0x7f8bb656d8cd] -> dovecot/doveadm-server() [0x4454d1] -> dovecot/doveadm-server() [0x445793] -> dovecot/doveadm-server() [0x445dd6] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0xd6) [0x7f8bb656d241] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x225) [0x7f8bb656fe58] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x3e) [0x7f8bb656d422] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0xbe) [0x7f8bb656d345] Oct 7 20:31:24 thunderstorm.reub.net dovecot: doveadm: Fatal: master: service(doveadm): child 27186 killed with signal 6 (core dumped)

thunderstorm reuben # file core core: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'dovecot/doveadm-server', real uid: 0, effective uid: 0, real gid: 0, effective gid: 0, execfn: '/usr/libexec/dovecot/doveadm-server', platform: 'x86_64'

thunderstorm reuben # gdb /usr/libexec/dovecot/doveadm-server core GNU gdb (Gentoo 8.0.1 vanilla) 8.0.1 Copyright (C) 2017 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-pc-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: https://bugs.gentoo.org/. Find the GDB manual and other documentation resources online at: http://www.gnu.org/software/gdb/documentation/. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/libexec/dovecot/doveadm-server...done. [New LWP 27186] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by dovecot/doveadm-server'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f8bb60e7f70 in raise () from /lib64/libc.so.6 (gdb) bt full #0 0x00007f8bb60e7f70 in raise () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f8bb60e9aa7 in abort () from /lib64/libc.so.6 No symbol table info available. #2 0x00007f8bb654a8f5 in default_fatal_finish (type=LOG_TYPE_PANIC, status=0) at failures.c:201 backtrace = 0x228e4d8 "/usr/lib64/dovecot/libdovecot.so.0(+0xd48a6) [0x7f8bb654a8a6] -> /usr/lib64/dovecot/libdovecot.so.0(+0xd5e26) [0x7f8bb654be26] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f8bb654ac39] -> /usr"... #3 0x00007f8bb654be26 in i_internal_fatal_handler (ctx=0x7ffef81daa50, format=0x7f8bb65ba898 "file %s: line %d (%s): assertion failed: (%s)", args=0x7ffef81daa70) at failures.c:670 status = 0 #4 0x00007f8bb654ac39 in i_panic (format=0x7f8bb65ba898 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:275 ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0} args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area = 0x7ffef81dab50, reg_save_area = 0x7ffef81daa90}} #5 0x00007f8bb656bcb0 in io_remove_full (_io=0x220a100, closed=false) at ioloop.c:126 io = 0x22694a0 __FUNCTION__ = "io_remove_full" #6 0x00007f8bb656be31 in io_remove (io=0x220a100) at ioloop.c:169 No locals. #7 0x00007f8bb65815f8 in o_stream_file_cork (stream=0x220a010, set=true) at ostream-file.c:342 fstream = 0x220a010 ret = 32766 __FUNCTION__ = "o_stream_file_cork" #8 0x00007f8bb657e289 in o_stream_cork (stream=0x220a0a0) at ostream.c:146 _stream = 0x220a010 #9 0x00007f8bb12d6406 in openssl_iostream_bio_output (ssl_io=0x220aa00) at iostream-openssl.c:325 bytes = 140238036407941 max_bytes = 18446742978492891135 sent = 35692560 buffer = "\027\003\003\000\001\000\000\000\020\240 \002\000\000\000\000\200\254\035\370\376\177\000\000\022\343W\266\213\177\000\000\214\033\002\000\000\000\000\240\240 \002\000\000\000\000\214\033\002\000\000\000\000\020\240 \002\000\000\000\000\320\314\035\370\376\177\000\000\236f-\261\213\177\000\000\025\071\260fK\247\000\000\000\252 \002\000\000\000\000\214\033", '\000' , "\315\035\370\376\177\000\000\236f-\261\213\177\000\000\225\305^\001?\236,,\000\252 \002\000\000\000\000=$h\000\272Å«\033\000\000\000\000\000\000\000\000\221\371ͦf\237\035\224\335-\365\214l\333\347|\005\310\345\035/\200r\242\222}\313I\226G\313*\223\364\312\310\313j\263\260/\245"... bytes_sent = false ret = 32766 __FUNCTION__ = "openssl_iostream_bio_output" #10 0x00007f8bb12d6af6 in openssl_iostream_bio_sync (ssl_io=0x220aa00) at iostream-openssl.c:459 ret = false #11 0x00007f8bb12da52e in o_stream_ssl_flush_buffer (sstream=0x220aca0) at ostream-openssl.c:110 pos = 6 ret = 6 #12 0x00007f8bb12da6ef in o_stream_ssl_sendv (stream=0x220aca0, iov=0x7ffef81dcd70, iov_count=1) at ostream-openssl.c:155 sstream = 0x220aca0 bytes_sent = 6 #13 0x00007f8bb657e765 in o_stream_sendv (stream=0x220ad30, iov=0x7ffef81dcd70, iov_count=1) at ostream.c:260 _stream = 0x220aca0 i = 1 total_size = 6 ret = 35771392 __FUNCTION__ = "o_stream_sendv" #14 0x00007f8bb657e688 in o_stream_send (stream=0x220ad30, data=0x2308850, size=6) at ostream.c:239 iov = {iov_base = 0x2308850, iov_len = 6} #15 0x00007f8bb6583c2c in o_stream_multiplex_sendv (mstream=0x221d270) at ostream-multiplex.c:90 channel = 0x0 ret = 0 #16 0x00007f8bb6583dd4 in o_stream_multiplex_ochannel_sendv (stream=0x221d720, iov=0x7ffef81dcef0, iov_count=1) at ostream-multiplex.c:124 channel = 0x221d720 ret = 7354428521 total = 1 #17 0x00007f8bb657e765 in o_stream_sendv (stream=0x221d7b0, iov=0x7ffef81dcef0, iov_count=1) at ostream.c:260 _stream = 0x221d720 i = 1 total_size = 1 ret = 36233296 __FUNCTION__ = "o_stream_sendv" #18 0x00007f8bb657e93f in o_stream_nsendv (stream=0x221d7b0, iov=0x7ffef81dcef0, iov_count=1) at ostream.c:295 No locals. #19 0x00007f8bb657e8d3 in o_stream_nsend (stream=0x221d7b0, data=0x7ffef81dcf42, size=1) at ostream.c:287 iov = {iov_base = 0x7ffef81dcf42, iov_len = 1} #20 0x0000000000444a35 in doveadm_server_log_handler (ctx=0x7f8bb6805bd0 , format=0x7f8bb65bac69 "Raw backtrace for leaks: %s", args=0x7ffef81dcfb0) at client-connection.c:64 c = 3 '\003' corked = false str = 0x228e050 ptr = 0x0 start = 0x228e088 "Raw backtrace for leaks: /usr/lib64/dovecot/libdovecot.so.0(io_loop_destroy+0x2ae) [0x7f8bb656d8b1] -> dovecot/doveadm-server() [0x4454d1] -> dovecot/doveadm-server() [0x445793] -> dovecot/doveadm-ser"... va = {{gp_offset = 16, fp_offset = 48, overflow_arg_area = 0x7ffef81dd090, reg_save_area = 0x7ffef81dcfd0}} _data_stack_cur_id = 4 #21 0x00007f8bb654afdc in i_warning (format=0x7f8bb65bac69 "Raw backtrace for leaks: %s") at failures.c:325 old_errno = 0 args = {{gp_offset = 8, fp_offset = 48, overflow_arg_area = 0x7ffef81dd090, reg_save_area = 0x7ffef81dcfd0}} #22 0x00007f8bb656d8cd in io_loop_destroy (_ioloop=0x7ffef81dd138) at ioloop.c:766 backtrace = 0x228dc38 "/usr/lib64/dovecot/libdovecot.so.0(io_loop_destroy+0x2ae) [0x7f8bb656d8b1] -> dovecot/doveadm-server() [0x4454d1] -> dovecot/doveadm-server() [0x445793] -> dovecot/doveadm-server() [0x445dd6] -> /usr/"... ioloop = 0x221ddc0 to_idx = 0x221dfe0 item = 0x0 leaks = true __FUNCTION__ = "io_loop_destroy" #23 0x00000000004454d1 in doveadm_cmd_handle (conn=0x22059d8, cmd_name=0x21b8c20 "dsync-server", argc=3, argv=0x21b8c50, cctx=0x7ffef81dd1b0) at client-connection.c:337 ioloop = 0x0 prev_ioloop = 0x21bfc40 cmd = 0x0 mail_cmd = 0x21c65e8 mctx = 0x221d9f8 cmd_ver2 = 0x0 #24 0x0000000000445793 in client_handle_command (conn=0x22059d8, args=0x21b8c40) at client-connection.c:399 cctx = {cmd = 0x0, argc = 0, argv = 0x0, username = 0x21b8c19 "reuben", cli = false, tcp_server = true, local_ip = {family = 10, u = {ip6 = {__in6_u = { ---Type <return> to continue, or q <return> to quit---

Reuben