lmtp panic

[Timo Sirainen]

Do you have a huge >4GB dovecot.index.cache file?

> On 17 Nov 2015, at 13:04, Sergey Schwartz <sergey.schwartz at bgoperator.com> wrote:
> 
> Gentlemen,
> 
> imap hits the same problem
> 
> Nov 17 14:02:06 mx10 dovecot: imap(oleg.vasilyev at bgoperator.com): Panic: file mail-index-util.c: line 37 (mail_index_uint32_to_offset): assertion failed: (offset < 0x40000000)
> Nov 17 14:02:06 mx10 dovecot: imap(oleg.vasilyev at bgoperator.com): Error: Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(+0x8215e) [0x7f8c8ce0915e] -> /usr/lib/dovecot/libdovecot.so.0(+0x8224c) [0x7f8c8ce0924c] -> /usr/lib/dovecot/libdovecot.so.0(i_fatal+0) [0x7f8c8cdb090e] -> /usr/lib/dovecot/libdovecot-storage.so.0(mail_index_uint32_to_offset+0x78) [0x7f8c8d161c58] -> /usr/lib/dovecot/libdovecot-storage.so.0(mail_cache_add+0x6eb) [0x7f8c8d14babb] -> /usr/lib/dovecot/libdovecot-storage.so.0(index_mail_parse_header+0x280) [0x7f8c8d131f90] -> /usr/lib/dovecot/libdovecot.so.0(+0x6bace) [0x7f8c8cdf2ace] -> /usr/lib/dovecot/libdovecot.so.0(i_stream_read+0x53) [0x7f8c8ce13483] -> /usr/lib/dovecot/libdovecot.so.0(i_stream_read_data+0x3d) [0x7f8c8ce13e7d] -> /usr/lib/dovecot/libdovecot.so.0(message_parse_header_next+0x82) [0x7f8c8cdf7cb2] -> /usr/lib/dovecot/libdovecot.so.0(+0x6b429) [0x7f8c8cdf2429] -> /usr/lib/dovecot/libdovecot.so.0(i_stream_read+0x53) [0x7f8c8ce13483] -> /usr/lib/dovecot/libdovecot.so.0(i_stream_read_data+0x3d) [0x7f8c8ce13e7d] -> /usr/lib/dovecot/libdovecot.so.0(message_get_header_size+0x74) [0x7f8c8cdfb564] -> /usr/lib/dovecot/libdovecot-storage.so.0(imap_msgpart_open+0x317) [0x7f8c8d16f827] -> dovecot/imap(+0x1effe) [0x7f8c8d5ffffe] -> dovecot/imap(+0x1d3fc) [0x7f8c8d5fe3fc] -> dovecot/imap(imap_fetch_more+0x3a) [0x7f8c8d5ff4fa] -> dovecot/imap(cmd_fetch+0x354) [0x7f8c8d5f1934] -> dovecot/imap(command_exec+0x85) [0x7f8c8d5fcbe5] -> dovecot/imap(+0x1a152) [0x7f8c8d5fb152] -> dovecot/imap(+0x1a1e0) [0x7f8c8d5fb1e0] -> dovecot/imap(client_handle_input+0x13d) [0x7f8c8d5fb54d] -> dovecot/imap(client_input+0x85) [0x7f8c8d5fb915] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x4c) [0x7f8c8ce1cc4c] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x101) [0x7f8c8ce1e0b1] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x25) [0x7f8c8ce1ccd5] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f8c8ce1ce78]
> Nov 17 14:02:06 mx10 dovecot: imap(oleg.vasilyev at bgoperator.com): Fatal: master: service(imap): child 7445 killed with signal 6 (core dumped)
> 
> any ideas how to fix this ?
> 
> Best regards,
> Sergey Schwartz
> 
> Senior System Administrator
> Biblio Globus Tour Operator
> www.bgoperator.ru
> 
> T: +7 495 5042500 ext 1532
> E: sergey.schwartz at bgoperator.com
> 
> 13.11.2015 12:40, Sergey Schwartz пишет:
>> root at mx10:~# doveadm force-resync -u oleg.vasilyev at bgoperator.com noreply-05-08-2015
>> doveadm(oleg.vasilyev at bgoperator.com): Warning: fscking index file /srv/vmail/current/bgoperator.com/oleg.vasilyev/storage/dovecot.map.index
>> doveadm(oleg.vasilyev at bgoperator.com): Warning: mdbox /srv/vmail/current/bgoperator.com/oleg.vasilyev/storage: rebuilding indexes
>> doveadm(oleg.vasilyev at bgoperator.com): Panic: file mail-index-util.c: line 37 (mail_index_uint32_to_offset): assertion failed: (offset < 0x40000000)
>> doveadm(oleg.vasilyev at bgoperator.com): Error: Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(+0x820de) [0x7f0f6f04e0de] -> /usr/lib/dovecot/libdovecot.so.0(+0x82158) [0x7f0f6f04e158] -> /usr/lib/dovecot/libdovecot.so.0(i_fatal+0) [0x7f0f6eff58de] -> /usr/lib/dovecot/libdovecot-storage.so.0(mail_index_uint32_to_offset+0x78) [0x7f0f6f3a6c58] -> /usr/lib/dovecot/libdovecot-storage.so.0(mail_cache_compress+0xaad) [0x7f0f6f38c83d] -> /usr/lib/dovecot/libdovecot-storage.so.0(mail_index_sync_commit+0x2c7) [0x7f0f6f39f4b7] -> /usr/lib/dovecot/libdovecot-storage.so.0(mdbox_storage_rebuild_in_context+0x1109) [0x7f0f6f329189] -> /usr/lib/dovecot/libdovecot-storage.so.0(mdbox_sync_begin+0x825) [0x7f0f6f32aaf5] -> /usr/lib/dovecot/libdovecot-storage.so.0(mdbox_sync+0x4c) [0x7f0f6f32ac7c] -> /usr/lib/dovecot/libdovecot-storage.so.0(mdbox_storage_sync_init+0x42) [0x7f0f6f32ad32] -> /usr/lib/dovecot/libdovecot-storage.so.0(mailbox_sync_init+0x3b) [0x7f0f6f3106eb] -> /usr/lib/dovecot/libdovecot-storage.so.0(mailbox_sync+0x37) [0x7f0f6f310837] -> doveadm(+0x258e4) [0x7f0f6fc9d8e4] -> doveadm(+0x2547e) [0x7f0f6fc9d47e] -> doveadm(doveadm_mail_try_run+0x2ac) [0x7f0f6fc9e4dc] -> doveadm(main+0x3a5) [0x7f0f6fc8f245] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5) [0x7f0f6ec28ec5] -> doveadm(+0x1734e) [0x7f0f6fc8f34e]
>> 
>> 
>> root at mx10:~# doveconf -n
>> # 2.2.19 (0b1c73b01a5a): /etc/dovecot/dovecot.conf
>> # Pigeonhole version 0.4.9
>> # OS: Linux 3.13.0-34-generic x86_64 Ubuntu 14.04.3 LTS xfs
>> auth_cache_negative_ttl = 0
>> auth_cache_size = 512 M
>> auth_default_realm = bgoperator.com
>> auth_failure_delay = 5 secs
>> auth_worker_max_count = 4096
>> default_client_limit = 40960
>> default_process_limit = 8192
>> default_vsz_limit = 2 G
>> disable_plaintext_auth = no
>> first_valid_gid = 500
>> mail_location = mdbox:/srv/vmail/current/%d/%n
>> mail_plugins = acl zlib notify replication
>> mail_shared_explicit_inbox = yes
>> mailbox_list_index = yes
>> managesieve_notify_capability = mailto
>> managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate
>> mdbox_rotate_size = 32 M
>> namespace {
>>  list = yes
>>  location = mdbox:/srv/vmail/current/%%d/%%n
>>  prefix = shared/%%u/
>>  separator = /
>>  subscriptions = no
>>  type = shared
>> }
>> namespace inbox {
>>  inbox = yes
>>  location =
>>  mailbox Drafts {
>>    auto = subscribe
>>    special_use = \Drafts
>>  }
>>  mailbox Junk {
>>    auto = subscribe
>>    special_use = \Junk
>>  }
>>  mailbox Sent {
>>    auto = subscribe
>>    special_use = \Sent
>>  }
>>  mailbox Trash {
>>    auto = subscribe
>>    special_use = \Trash
>>  }
>>  prefix =
>>  separator = /
>> }
>> passdb {
>>  args = /etc/dovecot/virtual/base.conf.ext
>>  driver = sql
>> }
>> passdb {
>>  args = /etc/dovecot/virtual/master.conf.ext
>>  driver = sql
>> }
>> plugin {
>>  acl = vfile
>>  acl_shared_dict = file:/srv/vmail/current/shared.db/000-shared-mailboxes.db
>>  sieve = file:/srv/vmail/current/%d/%n/sieve;active=/srv/vmail/current/%d/%n/active.sieve
>>  sieve_max_redirects = 0
>>  zlib_save = xz
>>  zlib_save_level = 6
>> }
>> protocols = " imap lmtp sieve"
>> service auth-worker {
>>  process_limit = 4096
>> }
>> service auth {
>>  unix_listener /var/spool/postfix/private/auth {
>>    group = postfix
>>    mode = 0666
>>    user = postfix
>>  }
>> }
>> service imap-login {
>>  process_limit = 4096
>>  process_min_avail = 24
>> }
>> service imap {
>>  process_limit = 4096
>> }
>> service lmtp {
>>  unix_listener /var/spool/postfix/private/dovecot-lmtp {
>>    group = postfix
>>    mode = 0600
>>    user = postfix
>>  }
>> }
>> ssl_cert = </etc/ssl/mxbgcom/2015/bgoperator.com.crt.chained
>> ssl_key = </etc/ssl/mxbgcom/2015/bgoperator.com.key
>> syslog_facility = local0
>> userdb {
>>  args = /etc/dovecot/virtual/base.conf.ext
>>  driver = sql
>> }
>> protocol lmtp {
>>  mail_plugins = acl zlib notify replication sieve
>> }
>> protocol imap {
>>  mail_max_userip_connections = 50
>>  mail_plugins = acl zlib notify replication imap_acl
>> }
>> 
>> 
>> 
>> Best regards,
>> Sergey Schwartz
>> 
>> Senior System Administrator
>> Biblio Globus Tour Operator
>> www.bgoperator.ru
>> 
>> T: +7 495 5042500 ext 1532
>> E: sergey.schwartz at bgoperator.com
>> 
>> 11.11.2015 17:06, Sergey Schwartz пишет:
>>> After 1 or 2 hours of recieveing this kind of errors mail server runs out of file descriptors
>>> 
>>> Nov 11 16:22:20 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:24 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:25 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:27 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:27 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:27 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:28 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:28 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:30 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:31 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:31 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:32 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:32 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:55 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:22:55 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> Nov 11 16:23:00 mx10 dovecot: config: Error: net_accept() failed: Too many open files
>>> 
>>> 
>>> Best regards,
>>> Sergey Schwartz
>>> 
>>> Senior System Administrator
>>> Biblio Globus Tour Operator
>>> www.bgoperator.ru
>>> 
>>> T: +7 495 5042500 ext 1532
>>> E: sergey.schwartz at bgoperator.com
>>> 
>>> 10.11.2015 15:46, Sergey Schwartz пишет:
>>>> Gents,
>>>> 
>>>> 
>>>> 
>>>> I've just upgraded to the latest build of dovecot , now lmtp delivery process panics for just one user
>>>> 
>>>> 
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Panic: file str.c: line 22 (str_new_const): assertion failed: (str[len] == '\0')
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Panic: file str.c: line 22 (str_new_const): assertion failed: (str[len] == '\0')
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Error: Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(+0x820de) [0x7f50e596b0de] -> /usr/lib/dovecot/libdovecot.so.0(+0x821cc) [0x7f50e596b1cc] -> /usr/lib/dovecot/libdovecot.so.0(i_fatal+0) [0x7f50e59128de] -> /usr/lib/dovecot/libdovecot.so.0(+0xa8bf8) [0x7f50e5991bf8] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x5ddbd) [0x7f50e380ddbd] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_match+0xf1) [0x7f50e37f43b1] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x5f555) [0x7f50e380f555] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_interpreter_continue+0xe7) [0x7f50e37eb2e7] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_interpreter_run+0x2b) [0x7f50e37eb46b] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x4e6ea) [0x7f50e37fe6ea] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_execute+0x47) [0x7f50e37ff277] -> /usr/lib/dovecot/modules/lib90_sieve_plugin.so(+0x3b81) [0x7f50e3a62b81] -> /usr/lib/dovecot/libdovecot-lda.so.0(mail_deliver+0x49) [0x7f50e5f26899] -> dovecot/lmtp(+0x6a04) [0x7f50e6357a04] -> dovecot/lmtp(+0x72d7) [0x7f50e63582d7] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x4c) [0x7f50e597ebbc] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x101) [0x7f50e597ffb1] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x25) [0x7f50e597ec45] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f50e597ede8] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f50e59182e3] -> dovecot/lmtp(main+0x165) [0x7f50e6356135] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5) [0x7f50e5545ec5] -> dovecot/lmtp(+0x5223) [0x7f50e6356223]
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Error: Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(+0x820de) [0x7f887f40e0de] -> /usr/lib/dovecot/libdovecot.so.0(+0x821cc) [0x7f887f40e1cc] -> /usr/lib/dovecot/libdovecot.so.0(i_fatal+0) [0x7f887f3b58de] -> /usr/lib/dovecot/libdovecot.so.0(+0xa8bf8) [0x7f887f434bf8] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x5ddbd) [0x7f887d2b0dbd] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_match+0xf1) [0x7f887d2973b1] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x5f555) [0x7f887d2b2555] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_interpreter_continue+0xe7) [0x7f887d28e2e7] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_interpreter_run+0x2b) [0x7f887d28e46b] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x4e6ea) [0x7f887d2a16ea] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_execute+0x47) [0x7f887d2a2277] -> /usr/lib/dovecot/modules/lib90_sieve_plugin.so(+0x3b81) [0x7f887d505b81] -> /usr/lib/dovecot/libdovecot-lda.so.0(mail_deliver+0x49) [0x7f887f9c9899] -> dovecot/lmtp(+0x6a04) [0x7f887fdfaa04] -> dovecot/lmtp(+0x72d7) [0x7f887fdfb2d7] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x4c) [0x7f887f421bbc] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x101) [0x7f887f422fb1] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x25) [0x7f887f421c45] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f887f421de8] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f887f3bb2e3] -> dovecot/lmtp(main+0x165) [0x7f887fdf9135] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5) [0x7f887efe8ec5] -> dovecot/lmtp(+0x5223) [0x7f887fdf9223]
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(7502): Connect from local
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(7509): Connect from local
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Panic: file str.c: line 22 (str_new_const): assertion failed: (str[len] == '\0')
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Error: Raw backtrace: /usr/lib/dovecot/libdovecot.so.0(+0x820de) [0x7fd26d0530de] -> /usr/lib/dovecot/libdovecot.so.0(+0x821cc) [0x7fd26d0531cc] -> /usr/lib/dovecot/libdovecot.so.0(i_fatal+0) [0x7fd26cffa8de] -> /usr/lib/dovecot/libdovecot.so.0(+0xa8bf8) [0x7fd26d079bf8] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x5ddbd) [0x7fd26aef5dbd] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_match+0xf1) [0x7fd26aedc3b1] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x5f555) [0x7fd26aef7555] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_interpreter_continue+0xe7) [0x7fd26aed32e7] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_interpreter_run+0x2b) [0x7fd26aed346b] -> /usr/lib/dovecot/libdovecot-sieve.so.0(+0x4e6ea) [0x7fd26aee66ea] -> /usr/lib/dovecot/libdovecot-sieve.so.0(sieve_execute+0x47) [0x7fd26aee7277] -> /usr/lib/dovecot/modules/lib90_sieve_plugin.so(+0x3b81) [0x7fd26b14ab81] -> /usr/lib/dovecot/libdovecot-lda.so.0(mail_deliver+0x49) [0x7fd26d60e899] -> dovecot/lmtp(+0x6a04) [0x7fd26da3fa04] -> dovecot/lmtp(+0x72d7) [0x7fd26da402d7] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x4c) [0x7fd26d066bbc] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x101) [0x7fd26d067fb1] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x25) [0x7fd26d066c45] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7fd26d066de8] -> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7fd26d0002e3] -> dovecot/lmtp(main+0x165) [0x7fd26da3e135] -> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5) [0x7fd26cc2dec5] -> dovecot/lmtp(+0x5223) [0x7fd26da3e223]
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Fatal: master: service(lmtp): child 7494 killed with signal 6 (core dumped)
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Fatal: master: service(lmtp): child 7497 killed with signal 6 (core dumped)
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Fatal: master: service(lmtp): child 7499 killed with signal 6 (core dumped)
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Fatal: master: service(lmtp): child 6719 killed with signal 6 (core dumped)
>>>> Nov 10 15:36:49 mx10 dovecot: lmtp(oleg.vasilyev at bgoperator.com): Fatal: master: service(lmtp): child 7502 killed with signal 6 (core dumped)
>>>> Nov 10 15:36:51 mx10 dovecot: lmtp(7509): Disconnect from local: Connection closed (in DATA finished)
>>>> 
>>>> 
>>>> Oleg has lots and lots of small emails :)
>>>> 
>>>> I'm running ubuntu server 14.04 x86_64
>>>> dovecot 2.2.19 (0b1c73b01a5a) build from http://xi.rename-it.nl/debian/ jessie-auto/dovecot-2.2 main
>>>>