Hi Aki,
Aki Tuomi <aki.tuomi@open-xchange.com> hat am 30. März 2020 16:23 geschrieben:
Can you install dovecot-dbg to get debug symbols, open the core in gdb and run
bt full
Full backtrace:
:~# gdb /usr/lib/dovecot/auth core.juu GNU gdb (Debian 7.12-6) 7.12.0.20161007-git Copyright (C) 2016 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/lib/dovecot/auth...Reading symbols from /usr/lib/debug/.build-id/cb/2618dd0e1b77c4402bec008554fe08e287dbdd.debug...done. done. [New LWP 6133] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `dovecot/auth'. Program terminated with signal SIGABRT, Aborted. #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: Datei oder Verzeichnis nicht gefunden. (gdb) bt full #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 set = {__val = {0, 94322623831496, 1064, 139808200491339, 139808199398721, 139808200312250, 139808199398721, 121, 206158430224, 140729958716272, 140729958716064, 139808200058129, 139808202947872, 139808200088406, 94322623831496, 0}} pid = <optimized out> tid = <optimized out> #1 0x00007f27a14a442a in __GI_abort () at abort.c:89 save_stage = 2 act = {__sigaction_handler = {sa_handler = 0x7ffe3f32ccb0, sa_sigaction = 0x7ffe3f32ccb0}, sa_mask = {__val = {139808123523248, 139808199398721, 94322623829368, 139808199398721, 139808200058129, 94322623829368, 1048, 94322623829424, 94322624549952, 0, 139808200311414, 94322623829368, 140729958716272, 139808199398721, 139808200311801, 139808199398721}}, sa_flags = -1575372310, sa_restorer = 0x5} sigs = {__val = {32, 0 <repeats 15 times>}} #2 0x00007f27a21a68a4 in default_fatal_finish (status=0, type=LOG_TYPE_PANIC) at failures.c:459 backtrace = 0x55c9327cd5b0 "#0 t_askpass[0x7f27a219b5f0] -> #1 backtrace_append[0x7f27a219b860] -> #2 backtrace_get[0x7f27a219b9c0] -> #3 i_syslog_error_handler[0x7f27a21a6840] -> #4 i_syslog_fatal_handler[0x7f27a21a6970] -> #5 "... recursed = 0 recursed = 0 #3 fatal_handler_real (ctx=<optimized out>, format=<optimized out>, args=<optimized out>) at failures.c:471 status = 0 #4 0x00007f27a21a6991 in i_internal_fatal_handler (ctx=<optimized out>, format=<optimized out>, args=<optimized out>) at failures.c:848 No locals. #5 0x00007f27a20fc483 in i_panic (format=format@entry=0x7f27a21e7680 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:523 ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0, log_prefix = 0x0, log_prefix_type_pos = 0} args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area = 0x7ffe3f32d4c0, reg_save_area = 0x7ffe3f32d400}} #6 0x00007f27a2170278 in dns_client_lookup_common (client=<optimized out>, cmd=cmd@entry=0x7f27a21ed0eb "IP", param=param@entry=0x55c930eca584 "", ptr_lookup=ptr_lookup@entry=false, callback=callback@entry=0x55c930e9b490 <auth_request_proxy_dns_callback>, context=context@entry=0x55c93287d440, lookup_r=0x55c93287d450) at dns-lookup.c:371 lookup = 0x55c932882320 ret = <optimized out> __func__ = "dns_client_lookup_common" pool = <optimized out> e = <optimized out> #7 0x00007f27a21702bd in dns_client_lookup (client=<optimized out>, host=host@entry=0x55c930eca584 "", callback=callback@entry=0x55c930e9b490 <auth_request_proxy_dns_callback>, context=context@entry=0x55c93287d440, lookup_r=lookup_r@entry=0x7f27a21702bd <dns_client_lookup+29>) at dns-lookup.c:421 No locals. #8 0x00007f27a2170313 in dns_lookup (host=host@entry=0x55c930eca584 "", set=set@entry=0x7ffe3f32d550, callback=callback@entry=0x55c930e9b490 <auth_request_proxy_dns_callback>, context=context@entry=0x55c93287d440, lookup_r=0x7f27a21702bd <dns_client_lookup+29>, lookup_r@entry=0x55c93287d450) at dns-lookup.c:206 client = <optimized out> #9 0x000055c930e9b423 in auth_request_proxy_host_lookup (callback=0x55c930e9cab0 <auth_request_handler_proxy_callback>, host=0x55c930eca584 "", request=0x55c932881628) at auth-request.c:2584 dns_set = {dns_client_socket_path = 0x55c930ec5eaf "dns-client", timeout_msecs = 10000, idle_timeout_msecs = 0, ioloop = 0x0, event_parent = 0x0} value = <optimized out> secs = 32551 #10 auth_request_proxy_finish (request=request@entry=0x55c932881628, callback=callback@entry=0x55c930e9cab0 <auth_request_handler_proxy_callback>) at auth-request.c:2631 host = 0x55c930eca584 "" hostip = 0x0 ip = {family = 54288, u = {ip6 = {__in6_u = {__u6_addr8 = "\311U\000\000\035\000\000\000\000\000\000\000\360\247\001", __u6_addr16 = {21961, 0, 29, 0, 0, 0, 42992, 1}, __u6_addr32 = {21961, 29, 0, 108528}}}, ip4 = { s_addr = 21961}}} proxy_host_is_self = <optimized out> #11 0x000055c930e9d024 in auth_request_handler_reply (request=0x55c932881628, result=<optimized out>, auth_reply=0x55c93287d420, reply_size=0) at auth-request-handler.c:394 handler = 0x55c932875548 str = <optimized out> ret = <optimized out> #12 0x000055c930e93a3e in auth_policy_check (request=request@entry=0x55c932881628, password=<optimized out>, cb=cb@entry=0x55c930e9bc70 <auth_request_policy_check_callback>, context=context@entry=0x55c93287d3c0) at auth-policy.c:573 pool = <optimized out> ---Type <return> to continue, or q <return> to quit--- #13 0x000055c930e9bd8e in auth_request_success (request=0x55c932881628, data=0x55c930eca584, data_size=0) at auth-request.c:212 data_size = 0 data = 0x55c930eca584 #14 0x000055c930e9a6c8 in auth_request_verify_plain_callback_finish (result=<optimized out>, request=<optimized out>, request@entry=0x55c932881628) at auth-request.c:1157 error = 0x55c932816a68 "\210\017\177\062\311U" #15 0x000055c930e9a7f1 in auth_request_verify_plain_callback (result=<optimized out>, request=0x55c932881628) at auth-request.c:1193 passdb = <optimized out> __func__ = "auth_request_verify_plain_callback" #16 0x00007f279faaa0dc in ldap_lookup_finish (res=<optimized out>, ldap_request=0x55c93287d250, auth_request=0x55c932881628) at passdb-ldap.c:112 passdb_result = PASSDB_RESULT_USER_UNKNOWN ret = <optimized out> password = 0x0 scheme = <optimized out> #17 ldap_lookup_pass_callback (conn=<optimized out>, request=0x55c93287d250, res=<optimized out>) at passdb-ldap.c:126 ldap_request = 0x55c93287d250 auth_request = 0x55c932881628 #18 0x00007f279faa8255 in db_ldap_handle_request_result (res=<optimized out>, idx=<optimized out>, request=<optimized out>, conn=<optimized out>) at db-ldap.c:841 _data_stack_cur_id = 3 srequest = <optimized out> ret = <optimized out> final_result = <optimized out> #19 db_ldap_handle_result (res=<optimized out>, conn=<optimized out>) at db-ldap.c:908 auth_request = 0x55c932881628 idx = <optimized out> msgid = <optimized out> #20 ldap_input (conn=<optimized out>) at db-ldap.c:940 timeout = {tv_sec = 0, tv_usec = 0} res = 0x55c9327f16c0 msg = 0x55c93289e0b0 prev_reply_diff = <optimized out> ret = <optimized out> #21 0x00007f27a21c0505 in io_loop_call_io (io=0x55c93283f260) at ioloop.c:718 ioloop = 0x55c9327d5df0 t_id = 2 __func__ = "io_loop_call_io" #22 0x00007f27a21c1f3b in io_loop_handler_run_internal (ioloop=ioloop@entry=0x55c9327d5df0) at ioloop-epoll.c:222 ctx = 0x55c9327d8960 io = <optimized out> tv = {tv_sec = 0, tv_usec = 296410} events_count = <optimized out> msecs = <optimized out> ret = <optimized out> i = 0 j = <optimized out> call = <optimized out> __func__ = "io_loop_handler_run_internal" #23 0x00007f27a21c0619 in io_loop_handler_run (ioloop=<optimized out>) at ioloop.c:770 No locals. #24 0x00007f27a21c0848 in io_loop_run (ioloop=0x55c9327d5df0) at ioloop.c:743 __func__ = "io_loop_run" #25 0x00007f27a212d5c3 in master_service_run (service=0x55c9327d5c80, callback=<optimized out>) at master-service.c:809 ---Type <return> to continue, or q <return> to quit--- No locals. #26 0x000055c930e8e137 in main (argc=<optimized out>, argv=<optimized out>) at main.c:395 c = <optimized out>
On 30/03/2020 17:21 tim@linux-daus.de wrote:
Hi,
currently we deploying Dovecot as imap/pop3 proxy. Every few minutes some panic/assert occurred (we connect roughly 7k - 8k user at one imap proxy with a connection rate of 200/s).
We activate core dumps. Concerning the sensitive information in the dump we would prefer to not share the dump (e.g. i found our ssl private key in the dump).
Log/Stack trace:
Mar 30 15:54:06 imap16 dovecot: auth: Panic: file dns-lookup.c: line 371 (dns_client_lookup_common): assertion failed: (param != NULL && *param != '\0') Mar 30 15:54:06 imap16 dovecot: auth: Error: Raw backtrace: #0 t_askpass[0x7f27a219b5f0] -> #1 backtrace_append[0x7f27a219b860] -> #2 backtrace_get[0x7f27a219b9c0] -> #3 i_syslog_error_handler[0x7f27a21a6840] -> #4 i_syslog_fatal_handler[0x7f27a21a6970] -> #5 i_fatal[0x7f27a20fc3b7] -> #6 dns_client_connect[0x7f27a216ffb0] -> #7 dns_client_lookup[0x7f27a21702a0] -> #8 auth_request_proxy_finish[0x55c930e9b200] -> #9 auth_request_handler_reply[0x55c930e9cee0] -> #10 auth_policy_check[0x55c930e93a10] -> #11 auth_request_success[0x55c930e9bcf0] -> #12 auth_request_verify_plain_callback_finish[0x55c930e9a650] -> #13 auth_request_verify_plain_callback[0x55c930e9a7a0] -> #14 authdb_ldap_deinit[0x7f279faa9f10] -> #15 db_ldap_result_iterate_deinit[0x7f279faa7f70] -> #16 io_loop_call_io[0x7f27a21c0490] -> #17 io_loop_handler_run_internal[0x7f27a21c1e20] -> #18 io_loop_handler_run[0x7f27a21c05c0] -> #19 io_loop_run[0x7f27a21c0810] -> #20 master_service_run[0x7f27a212d5b0] -> #21 main[0x55c930 e8 dd10] -> #22 __libc_start_main[0x7f27a14901f0] -> #23 _start[0x55c930e8e2c0] -> #24 [no start/end information] Mar 30 15:54:06 imap16 dovecot: auth: Fatal: master: service(auth): child 6133 killed with signal 6 (core dumped)
Config:
# 2.3.9.2 (844fc8246): /etc/dovecot/dovecot.conf # Pigeonhole version 0.5.9 (db4e9a2f) # OS: Linux 4.9.0-12-amd64 x86_64 Debian 9.12 # Hostname: imap16.domain.de auth_default_realm = domain.de auth_failure_delay = 0 auth_mechanisms = plain login cram-md5 auth_username_format = %{if;%d;eq;domain.de;%n@olddomain.de;%u} auth_verbose = yes base_dir = /var/run/dovecot/ default_client_limit = 4096 default_internal_user = pop default_process_limit = 400 default_vsz_limit = 1 G doveadm_password = # hidden, use -P to show it first_valid_uid = 48 import_environment = TZ last_valid_uid = 48 login_trusted_networks = 192.168.11.0/24 mail_gid = pop mail_plugins = " mail_log notify zlib quota" mail_uid = pop passdb { args = /etc/dovecot/conf.d/dovecot-ldap-domain-proxy.conf.ext driver = ldap result_failure = return-fail result_success = continue-ok } passdb { args = allow_real_nets=192.168.11.0/24 driver = static result_failure = continue-ok } passdb { args = /etc/dovecot/conf.d/dovecot-ldap-domain-protocol-deny.conf.ext driver = ldap result_failure = return-ok result_success = return-fail } passdb { args = /etc/dovecot/passdb-domain-ldap-cram.conf.ext driver = ldap mechanisms = CRAM-MD5 result_failure = continue-fail result_success = continue-ok } passdb { args = /etc/dovecot/passdb-domain-ldap.conf.ext driver = ldap mechanisms = LOGIN,PLAIN result_failure = return-fail result_success = continue-ok } plugin { mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename mail_log_fields = uid box msgid size zlib_save = gz zlib_save_level = 6 } protocols = " imap pop3" service auth { unix_listener auth-client { group = dovecot_auth mode = 0660 user = $default_internal_user } } service doveadm { group = pop inet_listener { port = 12345 } user = pop } service imap-login { process_min_avail = 24 service_count = 0 } service pop3-login { process_min_avail = 24 service_count = 0 } ssl = required ssl_cert = </etc/ssl/certs/star_domain_de.crt ssl_cipher_list = ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-CAMELLIA128-SHA:DHE-RSA-CAMELLIA256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:CAMELLIA128-SHA:CAMELLIA256-SHA ssl_dh = # hidden, use -P to show it ssl_key = # hidden, use -P to show it verbose_proctitle = yes
Best regards, Tim