Hi,
after the upgrade to dovecot 2.3.15 the Panic: indexer-worker
error is still present. I don't know if is related to searching in
a Virtual folder (look for my previous email on list with subject
"Panic on indexer-worker and Dovecot stop to search on Solr" for
more details).
Here the relevant configuration of FTS plugin:
plugin {
fts = solr
fts_autoindex = no
fts_autoindex_exclude = \Junk
fts_autoindex_exclude2 = \Drafts
fts_decoder = decode2text
fts_enforced = body
fts_index_timeout = 5s
fts_solr = url=http://10.0.0.32:8983/solr/dovecot/
[...]
I'm running Dovecot with FTS Solr 7.7, here the log error and the
bt full:
Jun 29 13:06:26 Panic: indexer-worker(user1@company.net)
session=<SLTLmOXFFIy5Ydlu:6BC4HLH+2mBHQwAAxbYlJw>: file
http-clien
t-request.c: line 1240 (http_client_request_send_more): assertion
failed: (req->payload_input != NULL)
Jun 29 13:06:26 Error: indexer-worker(user1@company.net)
session=<SLTLmOXFFIy5Ydlu:6BC4HLH+2mBHQwAAxbYlJw>: Raw
backtrace:
/usr/lib64/dovecot/libdovecot.so.0(backtrace_append+0x42)
[0x7f7212b36862] ->
/usr/lib64/dovecot/libdovecot.so.0(backtrace_get+0x1e)
[0x7f7212b3696e] ->
/usr/lib64/dovecot/libdovecot.so.0(+0xf50fe) [0x7f7212b440fe]
-> /usr/lib64/dovecot/libdovecot.so.0(+0xf51a1) [
0x7f7212b441a1] ->
/usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f7212a9460c]
-> /usr/lib64/dovecot/libdovecot.so.0(http_client_
request_send_more+0x3dd) [0x7f7212add54d] ->
/usr/lib64/dovecot/libdovecot.so.0(http_client_connection_output+0xf1)
[0x7f7212ae1ca1]
-> /usr/lib64/dovecot/libdovecot.so.0(+0x11e870)
[0x7f7212b6d870] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x65) [0x7f
7212b5c425] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x12b)
[0x7f7212b5ddab] -> /usr/lib64/dovecot/libdove
cot.so.0(io_loop_handler_run+0x59) [0x7f7212b5c529] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38)
[0x7f7212b5c768] -> /usr
/lib64/dovecot/libdovecot.so.0(http_client_wait+0xcd)
[0x7f7212aeadbd] ->
/usr/lib64/dovecot/lib21_fts_solr_plugin.so(solr_connectio
n_select+0xe5) [0x7f7210a79265] ->
/usr/lib64/dovecot/lib21_fts_solr_plugin.so(+0x4684)
[0x7f7210a75684] -> /usr/lib64/dovecot/lib20
_fts_plugin.so(fts_backend_get_last_uid+0x6e) [0x7f7211e1944e]
-> /usr/lib64/dovecot/lib20_fts_plugin.so(+0xf9d2)
[0x7f7211e1f9d2] -
> /usr/lib64/dovecot/lib20_fts_plugin.so(+0x10f56)
[0x7f7211e20f56] ->
/usr/lib64/dovecot/lib20_virtual_plugin.so(+0x94da) [0x7f7211
a014da] -> /usr/lib64/dovecot/lib20_fts_plugin.so(+0x10ba6)
[0x7f7211e20ba6] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mail_prec
ache+0x2e) [0x7f7212e48fae] -> dovecot/indexer-worker
[user1@company.net Virtual/All](+0x2bb1) [0x55559e522bb1] ->
/usr/lib
64/dovecot/libdovecot.so.0(io_loop_call_io+0x65) [0x7f7212b5c425]
->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal
+0x12b) [0x7f7212b5ddab] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x59)
[0x7f7212b5c529] -> /usr/lib64/dovecot/libd
ovecot.so.0(io_loop_run+0x38) [0x7f7212b5c768] ->
/usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13)
[0x7f7212ac63c3] -> do
vecot/indexer-worker [user1@company.net Virtual/All](main+0xd7)
[0x55559e522437] -> /lib64/libc.so.6(__libc_start_main+0xf5
) [0x7f72126a3555] -> dovecot/indexer-worker [user1@company.net
Virtual/All](+0x24fa) [0x55559e5224fa]
Jun 29 13:06:26 Fatal: indexer-worker(user1@company.net)
session=<SLTLmOXFFIy5Ydlu:6BC4HLH+2mBHQwAAxbYlJw>: master:
service(indexer-worker): child 17223 killed with signal 6 (core
dumped)
# gdb /usr/libexec/dovecot/indexer-worker
/var/core/core.indexer-worker.17223
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later
<http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show
copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/libexec/dovecot/indexer-worker...Reading
symbols from
/usr/lib/debug/usr/libexec/dovecot/indexer-worker.debug...done.
done.
[New LWP 17223]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `dovecot/indexer-worker'.
Program terminated with signal 6, Aborted.
#0 0x00007f72126b7387 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install
bzip2-libs-1.0.6-13.el7.x86_64 expat-2.1.0-12.el7.x86_64
glibc-2.17-324.el7_9.x86_64 libgcc-4.8.5-44.el7.x86_64
libzstd-1.5.0-1.el7.x86_64 xz-libs-5.2.2-1.el7.x86_64
zlib-1.2.7-19.el7_9.x86_64
(gdb) bt full
#0 0x00007f72126b7387 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f72126b8a78 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x00007f7212b440b7 in default_fatal_finish (status=0,
type=<optimized out>) at failures.c:459
backtrace = 0x55559eb75970
"/usr/lib64/dovecot/libdovecot.so.0(backtrace_append+0x42)
[0x7f7212b36862] ->
/usr/lib64/dovecot/libdovecot.so.0(backtrace_get+0x1e)
[0x7f7212b3696e] ->
/usr/lib64/dovecot/libdovecot.so.0(+0xf50fe) [0"...
recursed = 0
#3 fatal_handler_real (ctx=<optimized out>,
format=<optimized out>, args=<optimized out>) at
failures.c:471
status = 0
#4 0x00007f7212b441a1 in i_internal_fatal_handler
(ctx=<optimized out>, format=<optimized out>,
args=<optimized out>)
at failures.c:866
No locals.
#5 0x00007f7212a9460c in i_panic
(format=format@entry=0x7f7212b85cf8 "file %s: line %d (%s):
assertion failed: (%s)")
at failures.c:523
ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp =
0x0, timestamp_usecs = 0, log_prefix = 0x0,
log_prefix_type_pos = 0}
args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area
= 0x7ffe7b6e7150, reg_save_area = 0x7ffe7b6e7090}}
#6 0x00007f7212add54d in http_client_request_send_more
(req=req@entry=0x55559eca9db8, pipelined=pipelined@entry=false)
at http-client-request.c:1240
conn = 0x55559ec8b750
cctx = 0x55559ebb63a8
output = 0x55559ecafa70
res = <optimized out>
error = <optimized out>
offset = <optimized out>
__func__ = "http_client_request_send_more"
#7 0x00007f7212ae1ca1 in http_client_connection_continue_request
(conn=0x55559ec8b750) at http-client-connection.c:1342
tmp_conn = 0x55559ec8b750
count = 1
ret = <optimized out>
reqs = <optimized out>
req = 0x55559eca9db8
pipelined = false
#8 http_client_connection_output (conn=0x55559ec8b750) at
http-client-connection.c:1374
output = <optimized out>
ret = <optimized out>
#9 0x00007f7212b6d870 in stream_send_io (fstream=0x55559ec8cb20)
at ostream-file.c:526
ostream = 0x55559ec8cbc0
am = 0x55559ec8cb20
use_cork = true
ret = <optimized out>
---Type <return> to continue, or q <return> to quit---
#10 0x00007f7212b5c425 in io_loop_call_io (io=0x55559ec925f0) at
ioloop.c:715
ioloop = 0x55559ecaf4d0
t_id = 6
__func__ = "io_loop_call_io"
#11 0x00007f7212b5ddab in io_loop_handler_run_internal
(ioloop=ioloop@entry=0x55559ecaf4d0) at ioloop-epoll.c:222
ctx = 0x55559ecab870
events = <optimized out>
list = 0x55559ec8c8e0
io = <optimized out>
tv = {tv_sec = 0, tv_usec = 0}
events_count = <optimized out>
msecs = <optimized out>
ret = 1
i = 0
call = <optimized out>
__func__ = "io_loop_handler_run_internal"
#12 0x00007f7212b5c529 in io_loop_handler_run
(ioloop=ioloop@entry=0x55559ecaf4d0) at ioloop.c:767
__func__ = "io_loop_handler_run"
#13 0x00007f7212b5c768 in io_loop_run (ioloop=0x55559ecaf4d0) at
ioloop.c:740
__func__ = "io_loop_run"
#14 0x00007f7212aeadbd in http_client_wait () at http-client.c:339
prev_ioloop = 0x55559eb7b050
client_ioloop = 0x55559ecaf4d0
prev_client_ioloop = 0x55559eb7b050
__func__ = "http_client_wait"
#15 0x00007f7210a79265 in solr_connection_select
(conn=0x55559eb8b150, query=<optimized out>,
pool=<optimized out>,
box_results_r=0x7ffe7b6e7400) at solr-connection.c:211
lctx = {result_pool = 0x55559ec03910, payload = 0x0, io =
0x0, request_status = 0, parser = 0x0, results = 0x0}
http_req = 0x55559ecb3b48
url = <optimized out>
#16 0x00007f7210a75684 in fts_backend_solr_get_last_uid
(_backend=0x55559eb8b090, box=0x55559ebe59a8,
last_uid_r=0x7ffe7b6e74a4)
at fts-backend-solr.c:238
hdr = {last_indexed_uid = 0, settings_checksum = 0, unused
= 0}
#17 0x00007f7211e1944e in fts_backend_get_last_uid
(backend=<optimized out>, box=box@entry=0x55559ebe59a8,
last_uid_r=last_uid_r@entry=0x7ffe7b6e74a4) at fts-api.c:106
hdr = {last_indexed_uid = 2, settings_checksum = 0, unused
= 317606002}
#18 0x00007f7211e1f9d2 in fts_mailbox_get_last_cached_seq
(box=box@entry=0x55559ebe59a8, seq_r=seq_r@entry=0x7ffe7b6e74f4)
at fts-storage.c:85
flist = <optimized out>
seq1 = 21845
seq2 = 2663132384
last_uid = 21845
---Type <return> to continue, or q <return> to quit---
#19 0x00007f7211e20f56 in fts_mail_precache_init
(_mail=<optimized out>, _mail=<optimized out>) at
fts-storage.c:478
ft = 0x55559ec8c820
flist = 0x55559ebca0a0
last_seq = 21845
#20 fts_mail_index (_mail=0x55559ecaed08) at fts-storage.c:501
ft = 0x55559ec8c820
flist = 0x55559ebca0a0
pmail = 0x55559ecaed08
#21 fts_mail_precache (_mail=0x55559ecaed08) at fts-storage.c:562
_data_stack_cur_id = 5
mail = 0x55559ecaed08
fmail = <optimized out>
ft = 0x55559ec8c820
ret = 0
__func__ = "fts_mail_precache"
#22 0x00007f7211a014da in virtual_mail_precache
(mail=<optimized out>) at virtual-mail.c:243
vmail = <optimized out>
backend_mail = 0x55559ecaed08
p = <optimized out>
#23 0x00007f7211e20ba6 in fts_mail_precache (_mail=0x55559ec027f8)
at fts-storage.c:555
mail = 0x55559ec027f8
fmail = 0x55559ec02f70
ft = 0x55559ebff760
ret = 0
__func__ = "fts_mail_precache"
#24 0x00007f7212e48fae in mail_precache (mail=0x55559ec027f8) at
mail.c:455
_data_stack_cur_id = 4
p = 0x55559ec027f8
ret = <optimized out>
#25 0x000055559e522bb1 in index_mailbox_precache
(conn=<optimized out>, box=0x55559eb9c498) at
master-connection.c:119
counter = 1
max = 93
percentage_sent = 0
storage = <optimized out>
status = {messages = 93, recent = 0, unseen = 0,
uidvalidity = 1624964785, uidnext = 94, first_unseen_seq = 0,
first_recent_uid = 94, last_cached_seq = 0,
highest_modseq = 0, highest_pvt_modseq = 0, keywords = 0x0,
permanent_flags = 0, flags = 0, permanent_keywords =
false, allow_new_keywords = false, nonpermanent_modseqs = false,
no_modseq_tracking = false, have_guids = true,
have_save_guids = true, have_only_guid128 = false}
uids = <optimized out>
username = 0x55559eba0e18 "user1@company.net"
first_uid = 1
percentage_str = "\261\376\332`"
---Type <return> to continue, or q <return> to quit---
percentage = <optimized out>
error = MAIL_ERROR_NONE
trans = 0x55559ebf1b00
ctx = 0x55559ebf3640
last_uid = 2
ret = 0
index_event = 0x55559ebf3250
box_vname = 0x55559eb9c830 "Virtual/All"
errstr = <optimized out>
search_args = 0x0
mail = 0x55559ec027f8
metadata = {guid = '\000' <repeats 15 times>,
virtual_size = 0, physical_size = 0, first_save_date = 0,
cache_fields = 0x0,
precache_fields = (MAIL_FETCH_STREAM_HEADER |
MAIL_FETCH_STREAM_BODY | MAIL_FETCH_RECEIVED_DATE |
MAIL_FETCH_SAVE_DATE | MAIL_FETCH_PHYSICAL_SIZE |
MAIL_FETCH_VIRTUAL_SIZE | MAIL_FETCH_UIDL_BACKEND |
MAIL_FETCH_GUID | MAIL_FETCH_POP3_ORDER),
backend_ns_prefix = 0x0, backend_ns_type = (unknown: 0)}
seq = <optimized out>
#26 index_mailbox (user=<optimized out>, user=<optimized
out>, what=<optimized out>, max_recent_msgs=<optimized
out>,
mailbox=<optimized out>, conn=0x55559ebd8a80) at
master-connection.c:238
box = 0x55559eb9c498
status = {messages = 2662839088, recent = 21845, unseen =
1, uidvalidity = 0, uidnext = 2662837936,
first_unseen_seq = 21845, first_recent_uid = 313289502,
last_cached_seq = 32626, highest_modseq = 140730969257744,
highest_pvt_modseq = 93826223419056, keywords =
0x55559eb7aeb0, permanent_flags = 313289927, flags = 32626,
permanent_keywords = false, allow_new_keywords = false,
nonpermanent_modseqs = false, no_modseq_tracking = false,
have_guids = true, have_save_guids = false,
have_only_guid128 = false}
errstr = <optimized out>
ret = 0
ns = <optimized out>
path = 0x55559eb74a50
"/home/vmail/domains/2/company.net/0/user1/Maildir/virtual/All"
error = 93
sync_flags = <optimized out>
#27 master_connection_input_line (line=<optimized out>,
conn=0x55559ebd8a80) at master-connection.c:280
str = <optimized out>
max_recent_msgs = 0
ret = <optimized out>
args = <optimized out>
input = {event_parent = 0x0, module = 0x55559e52343f
"mail", service = 0x55559e523430 "indexer-worker",
username = 0x55559eb72038 "user1@company.net",
session_id = 0x0,
session_id_prefix = 0x55559eb7205f "SLTLmOXFFIy5Ydlu",
session_create_time = 0, local_ip = {family = 0, u = {ip6 = {
__in6_u = {__u6_addr8 = '\000' <repeats 15
times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 =
{0, 0, 0,
0}}}, ip4 = {s_addr = 0}}}, remote_ip =
{family = 0, u = {ip6 = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>,
__u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0,
0}}},
---Type <return> to continue, or q <return> to quit---
ip4 = {s_addr = 0}}}, local_port = 0, remote_port =
0, userdb_fields = 0x0, forward_fields = 0x0,
flags_override_add = (unknown: 0), flags_override_remove
= (unknown: 0), no_userdb_lookup = false, debug = false,
conn_secured = false, conn_ssl_secured = false}
service_user = 0x55559ebcb318
user = 0x55559eba0cf8
error = 0x0
#28 master_connection_input (conn=0x55559ebd8a80) at
master-connection.c:320
_data_stack_cur_id = 3
line = <optimized out>
#29 0x00007f7212b5c425 in io_loop_call_io (io=0x55559ec54760) at
ioloop.c:715
ioloop = 0x55559eb7b050
t_id = 2
__func__ = "io_loop_call_io"
#30 0x00007f7212b5ddab in io_loop_handler_run_internal
(ioloop=ioloop@entry=0x55559eb7b050) at ioloop-epoll.c:222
ctx = 0x55559eb7bc40
events = <optimized out>
list = 0x55559eb86770
io = <optimized out>
tv = {tv_sec = 2147483, tv_usec = 0}
events_count = <optimized out>
msecs = <optimized out>
ret = 1
i = 0
call = <optimized out>
__func__ = "io_loop_handler_run_internal"
#31 0x00007f7212b5c529 in io_loop_handler_run
(ioloop=ioloop@entry=0x55559eb7b050) at ioloop.c:767
__func__ = "io_loop_handler_run"
#32 0x00007f7212b5c768 in io_loop_run (ioloop=0x55559eb7b050) at
ioloop.c:740
__func__ = "io_loop_run"
#33 0x00007f7212ac63c3 in master_service_run
(service=0x55559eb7aeb0, callback=callback@entry=0x55559e5225f0
<client_connected>)
at master-service.c:862
No locals.
#34 0x000055559e522437 in main (argc=1, argv=0x55559eb7ab90) at
indexer-worker.c:76
storage_service_flags =
(MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP |
MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP |
MAIL_STORAGE_SERVICE_FLAG_NO_IDLE_TIMEOUT)
c = <optimized out>
(gdb) quit
Hope can be fixed.
Thanks
-- Alessio Cecchi Postmaster @ http://www.qboxmail.it https://www.linkedin.com/in/alessice