Re: fts lucene crashes in 2.2.24

On 29.05.2016 10:56, Wolfgang Rosenauer wrote:

Hi,

I've just enabled FTS via Lucene on my Dovecot 2.2.24 installation but I see the indexer crashing ?always?.

This simple testcase with a very tiny testing mailbox exposes the issue immediately:

doveadm -v index -u anmesse INBOX

Program received signal SIGSEGV, Segmentation fault. rescan_clear_unseen_mailbox (rescan_ctx=rescan_ctx@entry=0x0, vname=0x555555839820 "INBOX.Testfolder 2", hdr=hdr@entry=0x7fffffffdaf0) at lucene-wrapper.cc:831 831 (enum mailbox_flags)0); (gdb) bt full #0 rescan_clear_unseen_mailbox (rescan_ctx=rescan_ctx@entry=0x0, vname=0x555555839820 "INBOX.Testfolder 2", hdr=hdr@entry=0x7fffffffdaf0) at lucene-wrapper.cc:831 box = 0x7ffff76fd0ad metadata = {guid = "\230\332\377\377\377\177\000\000\200߈UUU\000", virtual_size = 140737340823160, physical_size = 140737488345756, first_save_date = 140737488345840, cache_fields = 0x555500000000, precache_fields = (MAIL_FETCH_SAVE_DATE | MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_NUL_STATE | MAIL_FETCH_STREAM_BINARY | MAIL_FETCH_IMAP_BODY | MAIL_FETCH_IMAP_BODYSTRUCTURE | MAIL_FETCH_IMAP_ENVELOPE | MAIL_FETCH_FROM_ENVELOPE | MAIL_FETCH_REFCOUNT | MAIL_FETCH_BODY_SNIPPET | unknown: 1409286144), backend_ns_prefix = 0x0, backend_ns_type = (unknown: 4294957808)} #1 0x00007ffff489aade in rescan_clear_unseen_mailboxes (index=index@entry=0x555555822e20, rescan_ctx=rescan_ctx@entry=0x0) at lucene-wrapper.cc:863 iter = 0x55555588d530 info = <optimized out> vname = <optimized out> hdr = {last_indexed_uid = 0, settings_checksum = 3784394109, unused = 0} ns = 0x55555580f6c0 #2 0x00007ffff489b700 in lucene_settings_check (index=0x555555822e20) at lucene-wrapper.cc:429 set_checksum = <optimized out> ret = <optimized out> #3 lucene_index_build_init (index=0x555555822e20) at lucene-wrapper.cc:448 lock_path = 0x5555557dd320 "/srv/dovecot/anmesse/maildir/lucene-indexes/write.lock" st = {st_dev = 93823560581121, st_ino = 0, st_nlink = 140737344007992, st_mode = 4150696184, st_uid = 32767, st_gid = 5, __pad0 = 0, st_rdev = 93824995156880, st_size = 93824995175648, st_blksize = 140737351975397, st_blocks = 93824995096256, st_atim = {tv_sec = 140737488346056, tv_nsec = 5}, st_mtim = {tv_sec = 5, tv_nsec = 93824995156880}, st_ctim = { tv_sec = 93824995175648, tv_nsec = 140737488346352}, __glibc_reserved = {140737488346352, 93824995267672, 140737344364485}} exists = <optimized out> #4 0x00007ffff4899dbe in fts_backend_lucene_update_set_build_key (_ctx=0x55555583a550, key=0x7fffffffdcf0) at fts-backend-lucene.c:366 ctx = 0x55555583a550 backend = 0x555555822ce0 __FUNCTION__ = "fts_backend_lucene_update_set_build_key" #5 0x00007ffff674c984 in fts_backend_update_set_build_key (ctx=0x55555583a550, key=key@entry=0x7fffffffdcf0) at fts-api.c:175 __FUNCTION__ = "fts_backend_update_set_build_key" #6 0x00007ffff674dbb8 in fts_build_mail_header (block=0x7fffffffdcd0, block=0x7fffffffdcd0, ctx=0x7fffffffdd20) at fts-build-mail.c:174 hdr = <optimized out> key = {uid = 96, type = FTS_BACKEND_BUILD_KEY_HDR, part = 0x5555557dd2c0, hdr_name = 0x5555558396b0 "Return-Path", body_content_type = 0x0, body_content_disposition = 0x0} ret = <optimized out> #7 fts_build_mail_real (mail=0x555555838190, update_ctx=0x55555583a550) at fts-build-mail.c:548 block = {part = 0x5555557dd2c0, hdr = 0x555555839458, data = 0x55550000007c , size = 0} ret = <optimized out> input = 0x555555838eb0 raw_block = {part = 0x5555557dd2c0, hdr = 0x5555558395e0, data = 0x0, size = 0} skip_body = false ctx = {mail = 0x555555838190, update_ctx = 0x55555583a550, content_type = 0x0, content_disposition = 0x0, body_parser = 0x0, word_buf = 0x0, pending_input = 0x0, cur_user_lang = 0x0} prev_part = 0x5555557dd2c0 parser = 0x555555839070 decoder = 0x555555839440 parts = 0x31 body_part = false body_added = false binary_body = <optimized out> error = 0x60 #8 fts_build_mail (update_ctx=0x55555583a550, mail=mail@entry=0x555555838190) at fts-build-mail.c:594 _data_stack_cur_id = 5 #9 0x00007ffff675393e in fts_mail_index (_mail=0x555555838190) at fts-storage.c:501 ft = 0x5555558366a0 flist = 0x55555581ea68 #10 fts_mail_precache (_mail=0x555555838190) at fts-storage.c:520 _data_stack_cur_id = 4 mail = 0x555555838190 fmail = <optimized out> ft = <optimized out> #11 0x00007ffff76987b9 in mail_precache (mail=0x555555838190) at mail.c:401 _data_stack_cur_id = 3 p = 0x555555838190 #12 0x0000555555582fa7 in cmd_index_box_precache (box=0x55555582fd80) at doveadm-mail-index.c:75 ctx = 0x555555839be0 metadata = {guid = '\000' , virtual_size = 0, physical_size = 0, first_save_date = 0, cache_fields = 0x0, precache_fields = (MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY | MAIL_FETCH_RECEIVED_DATE | MAIL_FETCH_SAVE_DATE | MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE | MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_GUID | MAIL_FETCH_POP3_ORDER), backend_ns_prefix = 0x0, backend_ns_type = (unknown: 0)} counter = 0 max = 15 status = {messages = 15, recent = 0, unseen = 0, uidvalidity = 1048765368, uidnext = 151, first_unseen_seq = 0, first_recent_uid = 151, last_cached_seq = 0, highest_modseq = 0, highest_pvt_modseq = 0, keywords = 0x0, permanent_flags = 0, permanent_keywords = 0, allow_new_keywords = 0, nonpermanent_modseqs = 0, no_modseq_tracking = 0, have_guids = 1, have_save_guids = 1, have_only_guid128 = 0} mail = 0x555555838190 ret = 0 trans = 0x555555834310 search_args = 0x0 seq = <optimized out> #13 cmd_index_box (info=<optimized out>, info=<optimized out>, ctx=0x555555807410) at doveadm-mail-index.c:130 box = 0x55555582fd80 status = {messages = 130, recent = 0, unseen = 0, uidvalidity = 0, uidnext = 1434552496, first_unseen_seq = 21845, first_recent_uid = 4294958896, last_cached_seq = 32767, highest_modseq = 93824995069728, highest_pvt_modseq = 93824995074385, keywords = 0x555555819868, permanent_flags = 4150947016, permanent_keywords = 1, allow_new_keywords = 1, nonpermanent_modseqs = 1, no_modseq_tracking = 1, have_guids = 1, have_save_guids = 1, have_only_guid128 = 1} ret = 0 #14 cmd_index_run (_ctx=0x555555807410, user=0x5555558188b0) at doveadm-mail-index.c:201 _data_stack_cur_id = 2 iter = 0x55555582b9f0 info = <optimized out> ret = 0 user = 0x5555558188b0 _ctx = 0x555555807410 ctx = 0x555555807410 i = <optimized out> #15 0x000055555557dedf in doveadm_mail_next_user (ctx=ctx@entry=0x555555807410, cctx=cctx@entry=0x7fffffffe2a0, error_r=error_r@entry=0x7fffffffe0b8) at doveadm-mail.c:400 input = {module = 0x0, service = 0x5555555b15a9 "doveadm", username = 0x5555557da298 "anmesse", session_id = 0x0, session_id_prefix = 0x0, local_ip = {family = 0, u = {ip6 = {__in6_u = { ---Type <return> to continue, or q <return> to quit--- __u6_addr8 = '\000' , __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, ip4 = {s_addr = 0}}}, remote_ip = {family = 0, u = {ip6 = { __in6_u = {__u6_addr8 = '\000' , __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, ip4 = {s_addr = 0}}}, local_port = 0, remote_port = 0, userdb_fields = 0x0, flags_override_add = (unknown: 0), flags_override_remove = (unknown: 0), no_userdb_lookup = 0, debug = 0} error = 0x20 ip = <optimized out> ret = <optimized out> __FUNCTION__ = "doveadm_mail_next_user" #16 0x000055555557e843 in doveadm_mail_single_user (ctx=ctx@entry=0x555555807410, cctx=cctx@entry=0x7fffffffe2a0, error_r=error_r@entry=0x7fffffffe0b8) at doveadm-mail.c:431 __FUNCTION__ = "doveadm_mail_single_user" #17 0x000055555557e8fd in doveadm_mail_cmd_exec (ctx=ctx@entry=0x555555807410, cctx=cctx@entry=0x7fffffffe2a0, wildcard_user=wildcard_user@entry=0x0) at doveadm-mail.c:589 ret = <optimized out> error = 0x7ffff7fe89e8 "" #18 0x000055555557f6ef in doveadm_cmd_ver2_to_mail_cmd_wrapper (cctx=0x7fffffffe2a0) at doveadm-mail.c:1030 mctx = 0x555555807410 wildcard_user = <optimized out> username_args = {0x5555555b1479 "-u", 0x5555557da298 "anmesse", 0x0} fieldstr = 0x7ffff73d7423 "H\203\304\bH\211\330[]\303\017\037" username_args_count = <optimized out> i = <optimized out> mail_cmd = {alloc = 0x555555582ad0 , name = 0x5555555b3e7b "index", usage_args = 0x5555555b2ee8 "[-u <user>|-A] [-S ] [-q] [-n <max recent>] <mailbox mask>"} #19 0x000055555558c93c in doveadm_cmd_run_ver2 (argc=4, argv=0x5555557e13a0, cctx=cctx@entry=0x7fffffffe2a0) at doveadm-cmd.c:524 param = <optimized out> pargv = {arr = {buffer = 0x5555557d9a40, element_size = 104}, v = 0x5555557d9a40, v_modifiable = 0x5555557d9a40} opts = {arr = {buffer = <optimized out>, element_size = 32}, v = <optimized out>, v_modifiable = <optimized out>} pargc = 7 c = -1 li = 21845 pool = 0x5555557d9778 optbuf = 0x5555557d9790 __FUNCTION__ = "doveadm_cmd_run_ver2" #20 0x000055555558c9d7 in doveadm_cmd_try_run_ver2 (cmd_name=<optimized out>, argc=4, argv=0x5555557e13a0, cctx=0x7fffffffe2a0) at doveadm-cmd.c:447 cmd = <optimized out> #21 0x000055555556fb1f in main (argc=4, argv=0x5555557e13a0) at doveadm.c:376 cctx = {cmd = 0x5555557e8208, argc = 7, argv = 0x5555557d9a78, username = 0x5555557da298 "anmesse", cli = true, local_ip = {family = 0, u = {ip6 = {__in6_u = { __u6_addr8 = '\000' , __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, ip4 = {s_addr = 0}}}, remote_ip = {family = 0, u = {ip6 = { __in6_u = {__u6_addr8 = '\000' , __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, ip4 = {s_addr = 0}}}, local_port = 0, remote_port = 0} cmd_name = 0x5555557e13dc "index" quick_init = false c = <optimized out>

doveconf -n:

# 2.2.24 (a82c823): /etc/dovecot/dovecot.conf # Pigeonhole version 0.4.14 (099a97c) # OS: Linux 4.1.21-14-default x86_64 openSUSE 42.1 (x86_64) auth_mechanisms = plain login imap_id_log = * mail_gid = vmail mail_home = /srv/dovecot/%u mail_location = maildir:~/maildir mail_plugins = acl fts fts_lucene virtual zlib quota mail_uid = vmail mailbox_list_index = yes managesieve_notify_capability = mailto managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext namespace { list = children location = maildir:/srv/dovecot/%%u/maildir:INDEX=~/maildir/shared/%%u:CONTROL=~/maildir/shared/%%u prefix = shared.%%n. separator = . subscriptions = yes type = shared } namespace inbox { inbox = yes location = mailbox Drafts { special_use = \Drafts } mailbox Entwürfe { special_use = \Drafts } mailbox "Gesendete Objekte" { special_use = \Sent } mailbox Sent { special_use = \Sent } mailbox "Sent Items" { special_use = \Sent } mailbox Spam { auto = create special_use = \Junk } mailbox Trash { auto = subscribe special_use = \Trash } prefix = INBOX. separator = . type = private } passdb { args = /etc/dovecot/dovecot-ldap.conf.ext driver = ldap } plugin { acl = vfile acl_shared_dict = file:/srv/dovecot/shared-mailboxes.db fts = lucene fts_autoindex = yes fts_lucene = whitespace_chars="@" quota = dict:User quota::file:%h/dovecot-quota quota_grace = 10%% quota_rule = *:storage=100M quota_rule2 = INBOX.Trash:storage=+10M quota_status_nosuser = DUNNO quota_status_overquota = 552 5.2.2 Mailbox is full / Mailbox ist voll quota_status_success = DUNNO sieve = file:~/sieve;active=~/.dovecot.sieve zlib_save = gz zlib_save_level = 6 } protocols = imap pop3 lmtp sieve service auth { unix_listener /var/spool/postfix/private/auth { group = postfix mode = 0660 user = postfix } unix_listener auth-userdb { mode = 0777 } } service lmtp { unix_listener /var/spool/postfix/private/dovecot-lmtp { group = postfix mode = 0660 user = postfix } } service managesieve-login { inet_listener sieve { port = 4190 } inet_listener sieve_deprecated { port = 2000 } } service quota-status { client_limit = 1 executable = quota-status -p postfix inet_listener { address = 127.0.0.1,::1 port = 12340 } } ssl_cert = < ssl_dh_parameters_length = 2048 ssl_key = < ssl_options = no_compression ssl_prefer_server_ciphers = yes userdb { driver = prefetch } userdb { args = /etc/dovecot/dovecot-ldap.conf.ext driver = ldap } userdb { args = username_format=%n /etc/dovecot/shared-mailboxes default_fields = uid=600 gid=600 home=/srv/dovecot/%n driver = passwd-file } protocol lmtp { mail_plugins = acl fts fts_lucene virtual zlib quota sieve } protocol imap { mail_max_userip_connections = 30 mail_plugins = acl fts fts_lucene virtual zlib quota imap_acl imap_zlib imap_quota namespace inbox { location = mailbox Spam { autoexpunge = 90 days } mailbox Trash { autoexpunge = 370 days } prefix = } }

Thank you for your report, we are looking into it.

Aki