Re: dotlock causing crashes

On 16.08.2017 21:17, Ian Bobbitt wrote:

OS: CentOS 7 x86_64 Dovecot version: 2.2.31 (65cde28) (GhettoForge RPM) Filesystem: GlusterFS, but working on changing that. Only one server is receiving activity.

Was getting messages about corrupt dovecot.map.index files. Changed to dotlock from fcntl to try to fix that.

Reading symbols from /usr/libexec/dovecot/imap...(no debugging symbols found)...done. [New LWP 74012] Core was generated by `dovecot/imap'. Program terminated with signal 6, Aborted. #0 0x00007fa262c741d7 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56 56 return INLINE_SYSCALL (tgkill, 3, pid, selftid, sig); (gdb) bt full #0 0x00007fa262c741d7 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56 resultvar = 0 pid = 74012 selftid = 74012 #1 0x00007fa262c758c8 in __GI_abort () at abort.c:90 save_stage = 2 act = {__sigaction_handler = {sa_handler = 0x7ffd7009f401, sa_sigaction = 0x7ffd7009f401}, sa_mask = {__val = {0, 0, 140335431377968, 140335423109592, 140335422613219, 4246482, 140335418575669, 12278048, 4192326493288016896, 12278592, 140335423192931, 0, 0, 140335425698848, 12280232, 140726483153732}}, sa_flags = 1657305400, sa_restorer = 0x79a} sigs = {__val = {32, 0 }} #2 0x00007fa26309eac6 in default_fatal_finish (type=<optimized out>, status=status@entry=0) at failures.c:201 backtrace = 0xbb5958 "/usr/lib64/dovecot/libdovecot.so.0(+0x9eace) [0x7fa26309eace] -> /usr/lib64/dovecot/libdovecot.so.0(+0x9ebae) [0x7fa26309ebae] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7fa26303012c] -> /usr"... #3 0x00007fa26309ebae in i_internal_fatal_handler (ctx=0x7ffd7009f4d0, format=<optimized out>, args=<optimized out>) at failures.c:670 status = 0 #4 0x00007fa26303012c in i_panic (format=format@entry=0x7fa2630d11de "file %s: line %d: unreached") at failures.c:275 ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0} args = {{gp_offset = 24, fp_offset = 48, overflow_arg_area = 0x7ffd7009f5d0, reg_save_area = 0x7ffd7009f510}} #5 0x00007fa2630a344f in file_lock_do (fd=fd@entry=20, path=path@entry=0xbb5868 "/gnoc/mail/home/bgeels/mail/mailboxes/Junk/dbox-Mails/.vsize.lock23f657caa43d8796", lock_type=lock_type@entry=1, lock_method=lock_method@entry=FILE_LOCK_METHOD_DOTLOCK, timeout_secs=0, error_r=error_r@entry=0x7ffd7009f768) at file-lock.c:285 lock_type_str = 0x7fa2630e6948 "write-lock" started = 1502905468 ret = <optimized out> __FUNCTION__ = "file_lock_do" #6 0x00007fa2630a3796 in file_wait_lock_error (fd=20, path=0xbb5868 "/gnoc/mail/home/bgeels/mail/mailboxes/Junk/dbox-Mails/.vsize.lock23f657caa43d8796", lock_type=1, lock_method=FILE_LOCK_METHOD_DOTLOCK, timeout_secs=<optimized out>, lock_r=0xc4ec10, error_r=0x7ffd7009f768) at file-lock.c:314 ret = <optimized out> #7 0x00007fa2630a3813 in file_try_lock_error (fd=<optimized out>, path=<optimized out>, lock_type=lock_type@entry=1, lock_method=lock_method@entry=FILE_LOCK_METHOD_DOTLOCK, lock_r=lock_r@entry=0xc4ec10, error_r=error_r@entry=0x7ffd7009f768) at file-lock.c:66 No locals. #8 0x00007fa2630a0955 in try_create_new (error_r=0x7ffd7009f768, lock_r=0xc4ec10, fd_r=0x7ffd7009f700, set=0x7ffd7009f770, path=0xc2f930 "/gnoc/mail/home/bgeels/mail/mailboxes/Junk/dbox-Mails/.vsize.lock") at file-create-locked.c:65 fd = 20 orig_errno = <optimized out> ret = -1 temp_path = 0xbb5830 mode = 0 uid = <optimized out> gid = 4294967295 #9 file_create_locked (path=0xc2f930 "/gnoc/mail/home/bgeels/mail/mailboxes/Junk/dbox-Mails/.vsize.lock", set=set@entry=0x7ffd7009f770, lock_r=lock_r@entry=0xc4ec10, created_r=created_r@entry=0x7ffd7009f767, error_r=error_r@entry=0x7ffd7009f768) at file-create-locked.c:118 i = 0 fd = <optimized out> ret = <optimized out> __FUNCTION__ = "file_create_locked" #10 0x00007fa2633e8f80 in vsize_update_lock_full (update=0xc4ebd0, lock_secs=lock_secs@entry=0) at index-mailbox-size.c:150 box = 0xc2e268 perm = 0xc2e440 set = {lock_timeout_secs = 0, lock_method = FILE_LOCK_METHOD_DOTLOCK, mode = 384, uid = 0, gid = 4294967295, gid_origin = 0xc2ea58 "/gnoc/mail/home/bgeels/mail/mailboxes/Junk"} error = 0x7fa2633f2062 "1\300[]A\\\303\017\037\200" created = false #11 0x00007fa2633e9057 in index_mailbox_vsize_update_try_lock (update=<optimized out>) at index-mailbox-size.c:167 No locals. #12 0x00007fa2633e9755 in index_mailbox_vsize_update_appends (box=<optimized out>) at index-mailbox-size.c:479 update = 0xc4ebd0 status = {messages = 1323, recent = 0, unseen = 0, uidvalidity = 1413091786, uidnext = 6750, first_unseen_seq = 0, first_recent_uid = 5886, last_cached_seq = 0, highest_modseq = 0, highest_pvt_modseq = 0, keywords = 0x0, permanent_flags = 0, flags = 0, permanent_keywords = 0, allow_new_keywords = 0, nonpermanent_modseqs = 0, no_modseq_tracking = 0, have_guids = 1, have_save_guids = 1, have_only_guid128 = 0} #13 0x00007fa2633f633c in index_mailbox_sync_deinit (_ctx=0xc4ece0, status_r=0x7ffd7009f900) at index-sync.c:346 ctx = 0xc4ece0 sync_rec = {seq1 = 0, seq2 = 0, type = (unknown: 0)} delayed_expunges = false ret = <optimized out> #14 0x00007fa2633784b2 in mailbox_sync_deinit (_ctx=_ctx@entry=0x7ffd7009f910, status_r=status_r@entry=0x7ffd7009f900) at mail-storage.c:1868 ctx = <optimized out> box = 0xc2e268 errormsg = <optimized out> error = MAIL_ERROR_NONE ret = <optimized out> #15 0x00007fa263378589 in mailbox_sync (box=<optimized out>, flags=<optimized out>, flags@entry=(unknown: 0)) at mail-storage.c:1894 ctx = 0x0 status = {sync_delayed_expunges = 0} #16 0x00007fa26221ae4f in imap_sieve_mailbox_transaction_run (changes=0x7ffd7009fb50, dest_box=0xc00a08, ismt=<optimized out>) at imap-sieve-storage.c:727 sbox = 0xc2e268 isrun = 0xc27580 cause = 0x7fa26221c447 "COPY" ret = <optimized out> isuser = <optimized out> mevent = <optimized out> headers_ctx = 0xbb53a0 st = 0x1 isrun_src = 0x0 siter = {array = 0xbb57f8, prev_n = 8, prev_idx = 0} script_name = 0x0 can_discard = true src_mail = 0x0 wanted_headers = {0x7fa26221c567 "From", 0x7fa26221c56c "To", 0x7fa26221c136 "Message-ID", 0x7fa26221c56f "Subject", 0x7fa26221c577 "Return-Path", 0x0} src_box = 0xbe2f78 user = <optimized out> mail = 0xbd7228 #17 imap_sieve_mailbox_transaction_commit (t=<optimized out>, changes_r=0x7ffd7009fb50) at imap-sieve-storage.c:808 box = 0xc00a08 user = <optimized out> ismt = <optimized out> lbox = <optimized out> isuser = 0xbd6d30 ret = 0 #18 0x00007fa2628337fa in acl_transaction_commit (ctx=0xc28060, changes_r=0x7ffd7009fb50) at acl-mailbox.c:460 abox = 0xc01018 at = <optimized out> ret = <optimized out> #19 0x00007fa2633789b1 in mailbox_transaction_commit_get_changes (_t=<optimized out>, changes_r=0x7ffd7009fb50) at mail-storage.c:2042 _data_stack_cur_id = 4 t = 0xc28060 box = 0xc00a08 save_count = 1 ret = <optimized out> __FUNCTION__ = "mailbox_transaction_commit_get_changes" #20 0x000000000040e94c in cmd_copy_full () No symbol table info available. #21 0x000000000041accc in command_exec () No symbol table info available. #22 0x000000000041932f in client_command_input () No symbol table info available. #23 0x00000000004193c1 in client_command_input () No symbol table info available. #24 0x0000000000419765 in client_handle_input () No symbol table info available. #25 0x0000000000419c42 in client_input () No symbol table info available. #26 0x00007fa2630b3782 in io_loop_call_io (io=0xbd1dc0) at ioloop.c:599 ioloop = 0xbbd9a0 t_id = 2 __FUNCTION__ = "io_loop_call_io" #27 0x00007fa2630b4e0f in io_loop_handler_run_internal (ioloop=ioloop@entry=0xbbd9a0) at ioloop-epoll.c:223 ctx = 0xbbf4c0 events = <optimized out> list = 0xbc0f20 io = <optimized out> tv = {tv_sec = 1799, tv_usec = 999317} events_count = <optimized out> msecs = <optimized out> ret = 1 i = 0 call = <optimized out> __FUNCTION__ = "io_loop_handler_run_internal" #28 0x00007fa2630b381c in io_loop_handler_run (ioloop=ioloop@entry=0xbbd9a0) at ioloop.c:648 No locals. #29 0x00007fa2630b39d8 in io_loop_run (ioloop=0xbbd9a0) at ioloop.c:623 __FUNCTION__ = "io_loop_run" #30 0x00007fa26303a6e3 in master_service_run (service=0xbbd840, callback=<optimized out>) at master-service.c:666 No locals. #31 0x000000000040cbd2 in main () No symbol table info available.

# 2.2.31 (65cde28): /etc/dovecot/dovecot.conf # Pigeonhole version 0.4.19 (e5c7051) doveconf: Warning: service auth { client_limit=20480 } is lower than required under max. load (24576) # OS: Linux 3.10.0-514.21.2.el7.x86_64 x86_64 CentOS Linux release 7.3.1611 (Core) auth_cache_negative_ttl = 0 auth_cache_size = 10 M auth_master_user_separator = * auth_mechanisms = plain login auth_username_format = %Ln auth_verbose = yes auth_worker_max_count = 100 base_dir = /var/run/dovecot/ default_client_limit = 2048 default_process_limit = 4096 default_vsz_limit = 1 G dict { acl = mysql:/etc/dovecot/dovecot-sql-dict.conf.ext } first_valid_uid = 1000 lda_mailbox_autocreate = yes lda_mailbox_autosubscribe = yes lock_method = dotlock mail_attachment_dir = /gnoc/mail/attachments/ mail_attachment_fs = sis-queue /gnoc/mail/attachments/queue/:posix mail_attachment_hash = %{sha256} mail_fsync = always mail_gid = vmail mail_location = mdbox:~/mail mail_nfs_index = yes mail_nfs_storage = yes mail_plugins = " listescape acl" mail_uid = vmail managesieve_notify_capability = mailto managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext spamtest spamtestplus editheader vnd.dovecot.debug imapsieve vnd.dovecot.imapsieve mdbox_rotate_size = 20 M mmap_disable = yes namespace { list = children location = mdbox:%%h/mail:INDEXPVT=~/mail/shared/%%u prefix = shared/%%u/ separator = / subscriptions = no type = shared } namespace inbox { inbox = yes location = mailbox Archive { special_use = \Archive } mailbox Archives { special_use = \Archive } mailbox Drafts { auto = subscribe special_use = \Drafts } mailbox Junk { special_use = \Junk } mailbox Sent { special_use = \Sent } mailbox "Sent Messages" { special_use = \Sent } mailbox Spam { special_use = \Junk } mailbox Trash { auto = subscribe special_use = \Trash } mailbox sent-mail { special_use = \Sent } prefix = separator = / } passdb { args = /etc/dovecot/dovecot-sql-pass-sieve.conf.ext driver = sql } passdb { args = /etc/dovecot/dovecot-sql-pass-nologin.conf.ext driver = sql } passdb { args = /etc/dovecot/dovecot-ldap-proxy.conf.ext driver = ldap override_fields = proxy=Y host=mail1-new.grnoc.iu.edu starttls=any-cert noauthenticate=yes } passdb { args = /etc/dovecot/passwd.masterusers driver = passwd-file master = yes pass = yes } passdb { args = /etc/dovecot/dovecot-sql.conf.ext driver = sql } plugin { acl = vfile:/gnoc/mail/dict/acls:cache_secs=300 acl_shared_dict = proxy::acl imapsieve_mailbox1_before = file:/etc/dovecot/sieve/report-spam.sieve imapsieve_mailbox1_causes = COPY imapsieve_mailbox1_name = Spam imapsieve_mailbox2_before = file:/etc/dovecot/sieve/report-spam.sieve imapsieve_mailbox2_causes = COPY imapsieve_mailbox2_name = Junk imapsieve_mailbox3_before = file:/etc/dovecot/sieve/report-ham.sieve imapsieve_mailbox3_causes = COPY imapsieve_mailbox3_from = Spam imapsieve_mailbox3_name = * imapsieve_mailbox4_before = file:/etc/dovecot/sieve/report-ham.sieve imapsieve_mailbox4_causes = COPY imapsieve_mailbox4_from = Junk imapsieve_mailbox4_name = * sieve = ~/.dovecot.sieve sieve_before = /etc/dovecot/sieve/before sieve_before2 = /gnoc/mail/home/%Ln/sieve-before sieve_dir = ~/sieve sieve_extensions = +spamtest +spamtestplus +vnd.dovecot.debug +editheader sieve_global_dir = /etc/dovecot/sieve/global sieve_global_extensions = +vnd.dovecot.pipe sieve_pipe_bin_dir = /etc/dovecot/sieve/bin sieve_plugins = sieve_imapsieve sieve_extprograms sieve_spamtest_max_header = X-Spam-Status: \w+, score=-?[[:digit:]]+\.[[:digit:]]+ required=([[:digit:]]+\.[[:digit:]]+) sieve_spamtest_status_header = X-Spam-Status: \w+, score=(-?[[:digit:]]+\.[[:digit:]]) sieve_spamtest_status_type = score } protocols = imap pop3 lmtp sieve service anvil { client_limit = 16387 } service auth { client_limit = 20480 unix_listener /var/spool/postfix/private/auth { group = postfix mode = 0666 user = postfix } } service dict { unix_listener dict { group = vmail mode = 0660 user = vmail } } service imap { process_limit = 4096 vsz_limit = 1 G } service lmtp { unix_listener /var/spool/postfix/private/dovecot-lmtp { group = postfix mode = 0600 user = postfix } vsz_limit = 1 G } service managesieve-login { inet_listener sieve { port = 4190 } } ssl = required ssl_ca =

Hi!

This problem is caused by mailbox vsize code not supporting dotlocks. I have opened an issue about this.

Aki