Dovecot 2.2.15, Panic: array.h: line 197 (array_idx_i): assertion failed: (idx * array->element_size < array->buffer->used)

Hi all

Version: 2.2.15 OS: SLES11 SP3 x86_64

Problem with replication for some users:

Log Server 1: Feb 24 09:29:42 v063ik dovecot: dsync-local(user@domain.com): Error: read(v063il.pnet.ch) failed: <no error>

Log Server 2: Feb 24 09:29:42 v063il dovecot: dsync-server(user@domain.com): Panic: file ../../../src/lib/array.h: line 197 (array_idx_i): assertion failed: (idx * array->element_size < array->buffer->used) Feb 24 09:29:42 v063il dovecot: dsync-server(user@domain.com): Error: Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0x75f63) [0x7fac77eeaf63] -> /usr/lib64/dovecot/libdovecot.so.0(+0x75ff6) [0x7fac77eeaff6] -> /usr/lib64/dovecot/libdovecot.so.0(+0x27595) [0x7fac77e9c595] -> /usr/lib64/dovecot/modules/lib01_acl_plugin.so(+0x71ba) [0x7fac772c41ba] -> /usr/lib64/dovecot/modules/lib01_acl_plugin.so(acl_attribute_iter_next+0x64) [0x7fac772c5bb4] -> /usr/lib64/dovecot/libdovecot-storage.so.0(+0xb11ca) [0x7fac782071ca] -> /usr/lib64/dovecot/libdovecot-storage.so.0(index_storage_mailbox_delete_pre+0xd8) [0x7fac782074a8] -> /usr/lib64/dovecot/libdovecot-storage.so.0(index_storage_mailbox_delete+0x9) [0x7fac782075a9] -> /usr/lib64/dovecot/modules/lib15_notify_plugin.so(+0x2080) [0x7fac76eaa080] -> /usr/lib64/dovecot/modules/lib01_acl_plugin.so(+0xea66) [0x7fac772cba66] -> /usr/lib64/dovecot/libdovecot-storage.so.0(mailbox_delete+0x2f) [0x7fac781db70f] -> dovecot/doveadm-server 10.1.100.49 user@domain.com recv_mailbox_tree_deletes [0x7fac786b6249] -> dovecot/doveadm-server 10.1.100.49 user@domain.com recv_mailbox_tree_deletes [0x7fac786b54d2] -> dovecot/doveadm-server 10.1.100.49 user@domain.com recv_mailbox_tree_deletes [0x7fac786b28db] -> dovecot/doveadm-server 10.1.100.49 user@domain.com recv_mailbox_tree_deletes [0x7fac786b2f7d] -> dovecot/doveadm-server 10.1.100.49 user@domain.com recv_mailbox_tree_deletes [0x7fac786c603f] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x49) [0x7fac77efcc29] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0xb4) [0x7fac77efde24] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x9) [0x7fac77efccc9] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7fac77efcd58] -> dovecot/doveadm-server 10.1.100.49 user@domain.com recv_mailbox_tree_deletes [0x7fac786a44c0] -> dovecot/doveadm-server 10.1.100.49 user@domain.com recv_mailbox_tree_deletes [0x7fac786a75c3] -> dovecot/doveadm-server 10.1.100.49 user@domain.com recv_mailbox_tree_deletes [0x7fac786b13d2] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x49) [0x7fac77efcc29] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0xb4) [0x7fac77efde24] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x9) [0x7fac77efccc9] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7fac77efcd58] -> /usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7fac77ea14d3] Feb 24 09:29:42 v063il dovecot: dsync-server(user@domain.com): Fatal: master: service(doveadm): child 12663 killed with signal 6 (core dumped)

Backtrace of core dump of server 2: v063il:/usr/lib64/dovecot # gdb /usr/lib/dovecot/doveadm-server /data/vmail/epost.ch/1003539/core GNU gdb (GDB) SUSE (7.5.1-0.7.29) Copyright (C) 2012 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-suse-linux". For bug reporting instructions, please see: http://www.gnu.org/software/gdb/bugs/... Reading symbols from /usr/lib/dovecot/doveadm-server...done. [New LWP 12509] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Missing separate debuginfo for /lib64/libbz2.so.1 Try: zypper install -C "debuginfo(build-id)=9d16aacb0b213c6ff10b2f162549f2cefee24697" Missing separate debuginfo for /lib64/libgcc_s.so.1 Try: zypper install -C "debuginfo(build-id)=3b149eccd897f1f37dce50ad22614043eba757a2" Core was generated by `dovecot/doveadm-server'. Program terminated with signal 6, Aborted. #0 0x00007fe4cc640885 in *__GI_raise (sig=<optimized out>) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64 64 ../nptl/sysdeps/unix/sysv/linux/raise.c: No such file or directory. (gdb) bt full #0 0x00007fe4cc640885 in *__GI_raise (sig=<optimized out>) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64 pid = <optimized out> selftid = <optimized out> #1 0x00007fe4cc641e61 in *__GI_abort () at abort.c:92 act = {__sigaction_handler = {sa_handler = 0x1, sa_sigaction = 0x1}, sa_mask = {__val = {140620662524321, 32, 4294967295, 33, 140620662296361, 140620658199328, 140620670309600, 206158430224, 4294967295, 140736446740304, 157882997, 2994704, 140620661813248, 4294967295, 33, 0}}, sa_flags = -856089480, sa_restorer = 0x7fe400000001} sigs = {__val = {32, 0 }} #2 0x00007fe4cc9fcf82 in default_fatal_finish (type=<optimized out>, status=0) at failures.c:202 backtrace = 0x7fe4cee02c90 "/usr/lib64/dovecot/libdovecot.so.0(+0x75f63) [0x7fe4cc9fcf63] -> /usr/lib64/dovecot/libdovecot.so.0(+0x75ff6) [0x7fe4cc9fcff6] -> /usr/lib64/dovecot/libdovecot.so.0(+0x27595) [0x7fe4cc9ae595] -> /usr/"... #3 0x00007fe4cc9fcff6 in i_internal_fatal_handler (ctx=0x7fffc1ea3890, format=<optimized out>, args=<optimized out>) at failures.c:666 status = 0 #4 0x00007fe4cc9ae595 in i_panic (format=<optimized out>) at failures.c:276 ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0} args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area = 0x7fffc1ea3970, reg_save_area = 0x7fffc1ea38b0}} #5 0x00007fe4cbdd61ba in array_idx_i (idx=<optimized out>, array=<optimized out>) at ../../../src/lib/array.h:197 __FUNCTION__ = "array_idx_i" #6 acl_default_object_list_next (iter=<optimized out>, rights_r=<optimized out>) at acl-api.c:216 No locals. #7 0x00007fe4cbdd7bb4 in acl_attribute_iter_next_acl (aiter=<optimized out>) at acl-attributes.c:193 No locals. #8 acl_attribute_iter_next (iter=0x7fe4cede87f0) at acl-attributes.c:219 key = 0x0 #9 0x00007fe4ccd191ca in mailbox_delete_all_attributes (t=0x7fe4ceddbf80, type=MAIL_ATTRIBUTE_TYPE_SHARED) at index-storage.c:632 iter = 0x7fe4cede87f0 key = <optimized out> ret = 0 #10 0x00007fe4ccd194a8 in mailbox_expunge_all_data (box=<optimized out>) at index-storage.c:670 No locals. #11 index_storage_mailbox_delete_pre (box=0x7fe4cede5cb0) at index-storage.c:707 status = {messages = 3435634784, recent = 32740, unseen = 3470679216, uidvalidity = 32740, uidnext = 3470679216, first_unseen_seq = 32740, first_recent_uid = 1, last_cached_seq = 0, highest_modseq = 140620699884664, highest_pvt_modseq = 140620668173509, keywords = 0x0, permanent_flags = MAIL_FLAGGED, permanent_keywords = 0, allow_new_keywords = 0, nonpermanent_modseqs = 0, no_modseq_tracking = 0, have_guids = 0, have_save_guids = 0, have_only_guid128 = 0} #12 0x00007fe4ccd195a9 in index_storage_mailbox_delete (box=0x30dd) at index-storage.c:764 ret = <optimized out> #13 0x00007fe4cb9bc080 in notify_mailbox_delete (box=0x7fe4cede5cb0) at notify-storage.c:231 lbox = 0x7fe4cede65f0 #14 0x00007fe4cbddda66 in acl_mailbox_delete (box=0x7fe4cede5cb0) at acl-mailbox.c:190 ret = 0 #15 0x00007fe4ccced70f in mailbox_delete (box=0x7fe4cede5cb0) at mail-storage.c:1357 ret = <optimized out> #16 0x00007fe4cd1c8249 in dsync_brain_mailbox_tree_sync_change (brain=0x7fe4cedbc7d0, change=0x7fe4cedd5c60) at dsync-brain-mailbox-tree-sync.c:182 box = 0x7fe4cede5cb0 destbox = <optimized out> errstr = <optimized out> func_name = 0x7fe4cd1dedf5 "mailbox_delete" storage_name = <optimized out> error = <optimized out> ret = 1 __FUNCTION__ = "dsync_brain_mailbox_tree_sync_change" #17 0x00007fe4cd1c74d2 in dsync_brain_mailbox_trees_sync (brain=<optimized out>) at dsync-brain-mailbox-tree.c:295 No locals. #18 dsync_brain_recv_mailbox_tree_deletes (brain=0x7fe4cedbc7d0) at dsync-brain-mailbox-tree.c:471 node = 0x0 status = 0x7fe4cd1de985 "not found" deletes = 0x7fe4cedc32b0 i = 30 count = 30 sep = <optimized out> __FUNCTION__ = "dsync_brain_recv_mailbox_tree_deletes" #19 0x00007fe4cd1c48db in dsync_brain_run_real (changed_r=<optimized out>, brain=<optimized out>) at dsync-brain.c:568 No locals. #20 dsync_brain_run (brain=0x7fe4cedbc7d0, changed_r=0x7fffc1ea3d5f) at dsync-brain.c:608 _data_stack_cur_id = 5 ret = true #21 0x00007fe4cd1c4f7d in dsync_brain_run_io (context=<optimized out>) at dsync-brain.c:89 brain = 0x7fe4cedbc7d0 changed = false try_pending = true #22 0x00007fe4cd1d803f in dsync_ibc_stream_input (ibc=0x7fe4cedba7b0) at dsync-ibc-stream.c:199 No locals. #23 0x00007fe4cca0ec29 in io_loop_call_io (io=0x7fe4cedba660) at ioloop.c:498 ioloop = 0x7fe4ced58110 t_id = 4 __FUNCTION__ = "io_loop_call_io" #24 0x00007fe4cca0fe24 in io_loop_handler_run_internal (ioloop=<optimized out>) at ioloop-epoll.c:220 ctx = 0x7fe4ceda5920 event = 0x7fe4ceda5a60 list = 0x7fe4cedba640 io = 0x30dd tv = {tv_sec = 59, tv_usec = 997329} msecs = <optimized out> ret = <optimized out> call = false __FUNCTION__ = "io_loop_handler_run_internal" #25 0x00007fe4cca0ecc9 in io_loop_handler_run (ioloop=0x30dd) at ioloop.c:545 No locals. #26 0x00007fe4cca0ed58 in io_loop_run (ioloop=0x7fe4ced58110) at ioloop.c:522 __FUNCTION__ = "io_loop_run" #27 0x00007fe4cd1b64c0 in cmd_dsync_server_run (_ctx=0x7fe4ceda4310, user=0x7fe4cedb0d60) at doveadm-dsync.c:1043 ibc = 0x7fe4cedba7b0 brain = 0x7fe4cedbc7d0 temp_prefix = <optimized out> state_str = 0x0 sync_type = <optimized out> name = 0x7fe4ced7c730 "SSL 10.1.100.49" process_title_prefix = 0x7fe4ced46da8 "10.1.100.49 " #28 0x00007fe4cd1b95c3 in doveadm_mail_next_user (ctx=0x7fe4ceda4310, input=<optimized out>, error_r=0x7fffc1ea3f00) at doveadm-mail.c:314 error = <optimized out> ip = <optimized out> ret = 0 __FUNCTION__ = "doveadm_mail_next_user" #29 0x00007fe4cd1c33d2 in doveadm_mail_cmd_server_run (input=<optimized out>, ctx=<optimized out>, conn=<optimized out>) at client-connection.c:123 No locals. #30 client_handle_command (args=<optimized out>, conn=<optimized out>) at client-connection.c:232 ctx = 0x7fe4ceda4310 flags = <optimized out> cmd_name = <optimized out> input = {module = 0x0, service = 0x7fe4cd1db44b "doveadm", username = 0x7fe4ced44679 "user@domain.com", session_id = 0x0, local_ip = {family = 2, u = { ip6 = {__in6_u = {__u6_addr8 = "\n\001d2", '\000' , __u6_addr16 = {266, 12900, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {845414666, 0, 0, 0}}}, ip4 = {s_addr = 845414666}}}, remote_ip = {family = 2, u = {ip6 = {__in6_u = {__u6_addr8 = "\n\001d1", '\000' , __u6_addr16 = {266, 12644, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {828637450, 0, 0, 0}}}, ip4 = {s_addr = 828637450}}}, local_port = 24042, remote_port = 43404, userdb_fields = 0x0, flags_override_add = (unknown: 0), flags_override_remove = (unknown: 0), no_userdb_lookup = 0} argc = <optimized out> #31 client_connection_input (conn=0x7fe4ced77dd0) at client-connection.c:345 args = <optimized out> _data_stack_cur_id = 3 line = <optimized out> ok = <optimized out> #32 0x00007fe4cca0ec29 in io_loop_call_io (io=0x7fe4ced76d80) at ioloop.c:498 ioloop = 0x7fe4ced4b730 t_id = 2 __FUNCTION__ = "io_loop_call_io" #33 0x00007fe4cca0fe24 in io_loop_handler_run_internal (ioloop=<optimized out>) at ioloop-epoll.c:220 ctx = 0x7fe4ced57f80 event = 0x7fe4ced50f10 list = 0x7fe4ced76de0 io = 0x30dd tv = {tv_sec = 2147483, tv_usec = 0} msecs = <optimized out> ret = <optimized out> call = false __FUNCTION__ = "io_loop_handler_run_internal" #34 0x00007fe4cca0ecc9 in io_loop_handler_run (ioloop=0x30dd) at ioloop.c:545 No locals. #35 0x00007fe4cca0ed58 in io_loop_run (ioloop=0x7fe4ced4b730) at ioloop.c:522 __FUNCTION__ = "io_loop_run" #36 0x00007fe4cc9b34d3 in master_service_run (service=0x7fe4ced4b5c0, callback=0x30dd) at master-service.c:566 No locals. #37 0x00007fe4cd1c3bbb in main (argc=1, argv=0x7fe4ced4b390) at main.c:92 set_roots = {0x7fe4cd3e8cc0 , 0x0} error = 0x7fe4cd1b60b0 <_start> "1\355I\211\321^H\211\342H\203\344\360PTL\213\005*.#" (gdb)

Thanks for help.

Thom