On 10/05/2015 5:47 PM, Timo Sirainen wrote:
How do I go about debugging this further? How do I go about getting a backtrace on what is essentially a child process that doesn't produce a core? (Note that my installation normally does produce cores when it crashes, so there's something different about these indexer processes..)
One thing that often works is:
mkdir /var/core; chmod 0777 /var/core echo "/var/core/%p" > /proc/sys/kernel/core_pattern
If that doesn't help, you can also try: sysctl -w fs.suid_dumpable=2 (and disable afterwards)
Also if you can get indexer-worker to start without crashing (new mail delivery works?) you can attach gdb into it and wait for it to crash (gdb -p <indexer-worker pid>, cont).
Ok we have some (well, lots actually) of coredumps now:
tornado core # gdb /usr/libexec/dovecot/indexer-worker 26697 GNU gdb (Gentoo 7.9 vanilla) 7.9 Copyright (C) 2015 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-pc-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://bugs.gentoo.org/>. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/libexec/dovecot/indexer-worker...done. Attaching to program: /usr/libexec/dovecot/indexer-worker, process 26697 ptrace: No such process. [New LWP 26697]
warning: Could not load shared library symbols for linux-vdso.so.1. Do you need "set solib-search-path" or "set sysroot"? [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `dovecot/indexer-worker'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x00007f4834fcc222 in fts_user_get_data_lang (user=0x16c7900) at fts-user.c:302 302 fts-user.c: No such file or directory. (gdb) info locals hdr = 0x1701f08 key = {uid = 8553, type = FTS_BACKEND_BUILD_KEY_HDR, part = 0x1696860, hdr_name = 0x1702120 "To", body_content_type = 0x0, body_content_disposition = 0x0}
(gdb) bt full #0 0x00007f4834fcc222 in fts_user_get_data_lang (user=0x16c7900) at fts-user.c:302 fuser = 0x0 lang = 0x0 error = 0xf <error: Cannot access memory at address 0xf> __FUNCTION__ = "fts_user_get_data_lang" #1 0x00007f4834fc1bdb in fts_build_mail_header (ctx=0x7ffd5e021ca0, block=0x7ffd5e021c80) at fts-build-mail.c:164 hdr = 0x1701f08 key = {uid = 8553, type = FTS_BACKEND_BUILD_KEY_HDR, part = 0x1696860, hdr_name = 0x1702120 "To", body_content_type = 0x0, body_content_disposition = 0x0} #2 0x00007f4834fc2b68 in fts_build_mail_real (update_ctx=0x16f5130, mail=0x17010f0) at fts-build-mail.c:540 ctx = {mail = 0x17010f0, update_ctx = 0x16f5130, content_type = 0x0, content_disposition = 0x0, body_parser = 0x0, word_buf = 0x0, pending_input = 0x0, cur_user_lang = 0x0} input = 0x1701990 parser = 0x1701b20 decoder = 0x1701ef0 raw_block = {part = 0x1696860, hdr = 0x1702050, data = 0x0, size = 0} block = {part = 0x1696860, hdr = 0x1701f08, data = 0x7ffd5e021cd8 "", size = 0} prev_part = 0x1696860 parts = 0x7f4835cc94c0 <static_system_pool> skip_body = false body_part = false body_added = false binary_body = 192 ret = 1 __FUNCTION__ = "fts_build_mail_real" #3 0x00007f4834fc2dcf in fts_build_mail (update_ctx=0x16f5130, mail=0x17010f0) at fts-build-mail.c:583 _data_stack_cur_id = 6 ret = 0 #4 0x00007f4834fca343 in fts_mail_index (_mail=0x17010f0) at fts-storage.c:484 ft = 0x16fd570 flist = 0x16cc690 #5 0x00007f4834fca440 in fts_mail_precache (_mail=0x17010f0) at fts-storage.c:501 _data_stack_cur_id = 5 mail = 0x17010f0 fmail = 0x1701580 ft = 0x16fd570 #6 0x00007f4835cf633a in mail_precache (mail=0x17010f0) at mail.c:401 _data_stack_cur_id = 4 p = 0x17010f0 #7 0x00000000004024b5 in index_mailbox_precache (conn=0x16a3040, box=0x16d0580) at master-connection.c:93 storage = 0x16cba20 username = 0x16c79e8 "reuben" box_vname = 0x16d0888 "Sent" status = {messages = 453, recent = 0, unseen = 0, uidvalidity = 1267935879, uidnext = 8558, first_unseen_seq = 0, first_recent_uid = 8557, last_cached_seq = 448, highest_modseq = 0, highest_pvt_modseq = 0, keywords = 0x0, permanent_flags = 0, permanent_keywords = 0, allow_new_keywords = 0, nonpermanent_modseqs = 0, no_modseq_tracking = 0, have_guids = 1, have_save_guids = 1, have_only_guid128 = 0} trans = 0x16e4b80 search_args = 0x0 ctx = 0x16d0370 mail = 0x17010f0 metadata = {guid = '\000' <repeats 15 times>, virtual_size = 0, cache_fields = 0x0, precache_fields = (MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY | MAIL_FETCH_RECEIVED_DATE | MAIL_FETCH_SAVE_DATE | MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE | MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_GUID), backend_ns_prefix = 0x0, backend_ns_type = (unknown: 0)} seq = 449 percentage_str = "\000\000\000" counter = 0 max = 5 percentage = 0 percentage_sent = 0 ret = 0 #8 0x000000000040295d in index_mailbox (conn=0x16a3040, user=0x16c7900, mailbox=0x1694067 "Sent", max_recent_msgs=0, what=0x169406e "i") at master-connection.c:186 ns = 0x16cb8d0 box = 0x16d0580 status = {messages = 23709136, recent = 0, unseen = 23709136, uidvalidity = 0, uidnext = 4294966784, first_unseen_seq = 4294967295, first_recent_uid = 23683432, last_cached_seq = 0, highest_modseq = 140726180650944, highest_pvt_modseq = 4203059, keywords = 0x16a1bc0, permanent_flags = 0, permanent_keywords = 0, allow_new_keywords = 0, nonpermanent_modseqs = 0, no_modseq_tracking = 0, have_guids = 0, have_save_guids = 0, have_only_guid128 = 0} path = 0x1696180 "/home/reuben/Maildir/.Sent" errstr = 0x7f48359cafc1 <master_service_settings_get+31> "H\211E\370H\213E\370H\213" error = MAIL_ERROR_NONE sync_flags = MAILBOX_SYNC_FLAG_FULL_READ ret = 0 #9 0x0000000000402b1e in master_connection_input_line (conn=0x16a3040, line=0x16a3f42 "reuben\tSent\t0\ti") at master-connection.c:222 args = 0x1694070 input = {module = 0x403134 "mail", service = 0x403139 "indexer-worker", username = 0x1694060 "reuben", session_id = 0x0, local_ip = {family = 0, u = {ip6 = {__in6_u = { __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, ip4 = { s_addr = 0}}}, remote_ip = {family = 0, u = {ip6 = {__in6_u = { __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, ip4 = { s_addr = 0}}}, local_port = 0, remote_port = 0, userdb_fields = 0x0, flags_override_add = (unknown: 0), flags_override_remove = (unknown: 0), no_userdb_lookup = 0} service_user = 0x16a6000 user = 0x16c7900 str = 0x16a3f3e "1\t0" error = 0x7f4835ff3bb0 "I\211\303f\017\032\\$0f\017\032T$ f\017\032L$\020f\017\032\004$L\213L$pL\213D$hH\213|$`H\213t$XH\213T$PH\213L$HH\213D$@H\201\304\210" max_recent_msgs = 0 ret = 0 #10 0x0000000000402c94 in master_connection_input (conn=0x16a3040) at master-connection.c:259 _data_stack_cur_id = 3 line = 0x16a3f42 "reuben\tSent\t0\ti" ret = 32584 #11 0x00007f4835a53548 in io_loop_call_io (io=0x16a3080) at ioloop.c:501 ioloop = 0x169c740 t_id = 2 __FUNCTION__ = "io_loop_call_io" #12 0x00007f4835a557f6 in io_loop_handler_run_internal (ioloop=0x169c740) at ioloop-epoll.c:220 ctx = 0x16a15f0 events = 0x16a2430 event = 0x16a2430 list = 0x16a30e0 io = 0x16a3080 tv = {tv_sec = 2147483, tv_usec = 0} events_count = 4 msecs = -1 ret = 1 i = 0 j = 0 call = true __FUNCTION__ = "io_loop_handler_run_internal" #13 0x00007f4835a536e5 in io_loop_handler_run (ioloop=0x169c740) at ioloop.c:548 No locals. #14 0x00007f4835a5363d in io_loop_run (ioloop=0x169c740) at ioloop.c:525 __FUNCTION__ = "io_loop_run" #15 0x00007f48359c8215 in master_service_run (service=0x169c5d0, callback=0x401fb6 <client_connected>) at master-service.c:569 No locals. #16 0x0000000000402182 in main (argc=1, argv=0x169c390) at indexer-worker.c:78 service_flags = MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN storage_service_flags = (MAIL_STORAGE_SERVICE_FLAG_DISALLOW_ROOT | MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP | MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP | MAIL_STORAGE_SERVICE_FLAG_NO_IDLE_TIMEOUT) c = -1 (gdb)
Thanks, Reuben