Assertion during dsync receive
Ian Bobbitt
ibobbitt at globalnoc.iu.edu
Fri Feb 23 17:52:53 EET 2018
Hi,
I'm getting an assertion failed on the receiving side, causing syncs to fail for one user. The servers are setup so that
only one is receiving any traffic other than replication at any time. The one that's only receiving replications is the
one that's failing.
I've tried deleting the user's home on the receiving server, but it still crashes during the sync. Oddly, the user's
home is 7.4G on the sending server, but ends up at 42G on the receiving side, even after deleting and trying a fresh sync.
The mailbox implicated in the backtrace ("Spam") does have a very large number of messages in it.
On sender:
Spam messages=1217764 recent=0 uidnext=1218103 uidvalidity=1379509105 unseen=16 highestmodseq=744588 vsize=34468460093
guid=090ed93a7a055559abf10200fdf6807a firstsaved=1498744186
On receiver:
Spam messages=1217766 recent=352 uidnext=1218105 uidvalidity=1379509105 unseen=16 highestmodseq=744589 vsize=34468496809
guid=090ed93a7a055559abf10200fdf6807a firstsaved=1519396172
Feb 23 14:57:20 dovecot: dsync-local(bgeels): Warning: fscking index file
/gnoc/mail/home/bgeels/mail/storage/dovecot.map.index
Feb 23 14:57:20 dovecot: dsync-local(bgeels): Warning: mdbox /gnoc/mail/home/bgeels/mail/storage: rebuilding indexes
Feb 23 14:57:33 dovecot: dsync-local(bgeels): Panic: file mail-index-util.c: line 10 (mail_index_uint32_to_offset):
assertion failed: (offset < 0x40000000)
Feb 23 14:57:33 dovecot: dsync-local(bgeels): Error: Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0x9f3de)
[0x7feb584143de] -> /usr/lib64/dovecot/libdovecot.so.0(+0x9f4be) [0x7feb584144be] ->
/usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7feb583a577c] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mail_index_uint32_to_offset+0xa0) [0x7feb587906d0] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mail_cache_compress+0x854) [0x7feb58774f34] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mail_index_sync_commit+0x25f) [0x7feb587884ff] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mdbox_storage_rebuild_in_context+0x10de) [0x7feb5870b3ae] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mdbox_sync_begin+0x858) [0x7feb5870ccd8] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mdbox_sync+0x4c) [0x7feb5870ce7c] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mdbox_storage_sync_init+0x4b) [0x7feb5870cf3b] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mailbox_sync_init+0x44) [0x7feb586f2834] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mailbox_sync+0x37) [0x7feb586f28d7] ->
dovecot/doveadm-server(dsync_mailbox_import_deinit+0x475) [0x445495] -> dovecot/doveadm-server() [0x43edc0] ->
dovecot/doveadm-server(dsync_brain_sync_mails+0x743) [0x43f653] -> dovecot/doveadm-server(dsync_brain_run+0x541)
[0x43acf1] -> dovecot/doveadm-server() [0x43b070] -> dovecot/doveadm-server() [0x44fe5f] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x52) [0x7feb58429cd2] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f) [0x7feb5842b3bf] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x3c) [0x7feb58429d6c] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7feb58429f28] -> dovecot/doveadm-server() [0x4209c5] ->
dovecot/doveadm-server() [0x422df6] -> dovecot/doveadm-server() [0x4377f4] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x52) [0x7feb58429cd2] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f) [0x7feb5842b3bf] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x3c) [0x7feb58429d6c]
Feb 23 14:57:33 dsync-local(bgeels): Fatal: master: service(doveadm): child 82098 killed with signal 6 (core dumped)
I've attached the output of `doveconf -n` and the full backtrace from a core dump.
Dovecot 2.2.33.2 (GhettoForge package)
CentOS 7 x86_64
XFS, no NFS.
-- Ian
-------------- next part --------------
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-100.el7_4.1
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/bin/doveadm...(no debugging symbols found)...done.
[New LWP 146391]
Core was generated by `/usr/bin/doveadm dsync-server -ubgeels -U'.
Program terminated with signal 6, Aborted.
#0 0x00007fc20bc531f7 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install dovecot22-2.2.33.2-2.gf.el7.x86_64
(gdb) #0 0x00007fc20bc531f7 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007fc20bc548e8 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x00007fc20c0803d6 in default_fatal_finish (type=<optimized out>, status=status at entry=0) at failures.c:201
backtrace = 0x1bcdfd0 "/usr/lib64/dovecot/libdovecot.so.0(+0x9f3de) [0x7fc20c0803de] -> /usr/lib64/dovecot/libdovecot.so.0(default_fatal_handler+0x2a) [0x7fc20c08044a] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7fc"...
#3 0x00007fc20c08044a in default_fatal_handler (ctx=0x7ffff51be4b0, format=<optimized out>, args=<optimized out>) at failures.c:215
status = 0
#4 0x00007fc20c01177c in i_panic (format=format at entry=0x7fc20c40ba20 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:275
ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0}
args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area = 0x7ffff51be5b0, reg_save_area = 0x7ffff51be4f0}}
#5 0x00007fc20c3fc6d0 in mail_index_uint32_to_offset (offset=<optimized out>) at mail-index-util.c:10
__FUNCTION__ = "mail_index_uint32_to_offset"
#6 0x00007fc20c3e0f34 in mail_cache_copy (ext_offsets=0x7ffff51be690, max_uid_r=0x7ffff51be64c, file_size_r=<synthetic pointer>, file_seq_r=<synthetic pointer>, fd=28, trans=0x1c48dd0, cache=<optimized out>) at mail-cache-compress.c:293
iter = {view = 0x1c19230, remap_counter = 2438427, seq = 1217766, rec = 0x7fc1f8deca18, pos = 832, rec_size = 832, offset = 0, trans_next_idx = 0, stop = 0, failed = 0, memory_appends_checked = 0, disk_appends_checked = 0}
first_new_seq = 1
i = <optimized out>
field = {field_idx = 30, size = 237, data = 0x7fc1f8decc68, offset = 1627311208}
view = 0x1c1a620
hdr = {major_version = 1 '\001', compat_sizeof_uoff_t = 8 '\b', minor_version = 1 '\001', unused = 0 '\000', indexid = 1519395846, file_seq = 1519396173, continued_record_count = 0, record_count = 1217766, backwards_compat_used_file_size = 0, deleted_record_count = 0, field_header_offset = 0}
output = 0x1c462e0
ext_offset = 1554079344
record_count = <optimized out>
max_drop_time = <optimized out>
ctx = {cache = 0x1c14790, buffer = 0x1c0eab0, field_seen = 0x1c0ea70, bitmask_pos = {arr = {buffer = 0x1bcdec0, element_size = 4}, v = 0x1bcdec0, v_modifiable = 0x1bcdec0}, field_file_map = 0x1bcde20, field_seen_value = 141 '\215', new_msg = true}
cache_view = 0x1c19230
cache_rec = {prev_offset = 0, size = 876}
message_count = 1217766
seq = <optimized out>
used_fields_count = <optimized out>
orig_fields_count = 33
idx_hdr = <optimized out>
#7 mail_cache_compress_write (unlock=<synthetic pointer>, temp_path=0x1bcddd0 "/gnoc/mail/home/bgeels/mail/mailboxes/Spam/dbox-Mails/dovecot.index.cache.tmp", fd=28, trans=0x1c48dd0, cache=<optimized out>) at mail-cache-compress.c:339
file_seq = <optimized out>
max_uid = 1218104
i = <optimized out>
count = <optimized out>
st = {st_dev = 25, st_ino = 1180560, st_nlink = 25, st_mode = 106699888, st_uid = 0, st_gid = 204982256, __pad0 = 32706, st_rdev = 140471401555808, st_size = 0, st_blksize = 1180560, st_blocks = 16, st_atim = {tv_sec = 29420496, tv_nsec = 140737305634904}, st_mtim = {tv_sec = 29779808, tv_nsec = 140737305634904}, st_ctim = {tv_sec = 29737888,
tv_nsec = 140737305634904}, __unused = {29659536, 140737305634904, 29737888}}
old_offset = 29443984
ext_offsets = {arr = {buffer = 0x1c1bc40, element_size = 4}, v = 0x1c1bc40, v_modifiable = 0x1c1bc40}
offsets = <optimized out>
file_size = <optimized out>
#8 mail_cache_compress_locked (dotlock_r=0x7ffff51be668, unlock=<synthetic pointer>, trans=0x1c48dd0, cache=<optimized out>) at mail-cache-compress.c:478
data = 0x7fc197e00000
fd = 28
temp_path = 0x1bcddd0 "/gnoc/mail/home/bgeels/mail/mailboxes/Spam/dbox-Mails/dovecot.index.cache.tmp"
#9 mail_cache_compress (cache=<optimized out>, trans=0x1c48dd0, lock_r=lock_r at entry=0x7ffff51be868) at mail-cache-compress.c:544
dotlock = 0x1c0ea00
unlock = true
ret = <optimized out>
__FUNCTION__ = "mail_cache_compress"
#10 0x00007fc20c3f44ff in mail_index_sync_commit (_ctx=_ctx at entry=0x7ffff51be988) at mail-index-sync.c:860
ctx = 0x1c65540
index = 0x1c0aa30
cache_lock = 0x0
next_uid = 6
want_rotate = <optimized out>
index_undeleted = false
delete_index = <optimized out>
ret = 0
ret2 = <optimized out>
#11 0x00007fc20c3773ae in rebuild_mailbox (ns=<optimized out>, vname=<optimized out>, ctx=0x1c08250) at mdbox-storage-rebuild.c:588
box = 0x1c15df8
mbox = 0x1c15df8
rebuild_ctx = 0x0
ret = <optimized out>
sync_ctx = 0x1c65540
view = 0x1c1a620
trans = 0x1c48dd0
error = <optimized out>
#12 rebuild_namespace_mailboxes (ns=<optimized out>, ctx=0x1c08250) at mdbox-storage-rebuild.c:616
_data_stack_cur_id = 5
iter = 0x1c3a648
info = <optimized out>
ret = 0
#13 rebuild_mailboxes (ctx=0x1c08250) at mdbox-storage-rebuild.c:636
storage = <optimized out>
ns = <optimized out>
#14 mdbox_storage_rebuild_scan (ctx=0x1c08250) at mdbox-storage-rebuild.c:948
data = 0x1c3bb50
data_size = 8
#15 mdbox_storage_rebuild_in_context (storage=0x1bf2238, atomic=atomic at entry=0x1c6c680) at mdbox-storage-rebuild.c:971
No locals.
#16 0x00007fc20c378cd8 in mdbox_sync_begin (mbox=mbox at entry=0x1c1de98, flags=flags at entry=(unknown: 0), atomic=0x1c6c680, ctx_r=ctx_r at entry=0x7ffff51beba8) at mdbox-sync.c:245
storage = 0x1bf2238
hdr = <optimized out>
ctx = <optimized out>
reason = <optimized out>
sync_flags = <optimized out>
rebuild = true
storage_rebuilt = false
#17 0x00007fc20c378e7c in mdbox_sync (mbox=mbox at entry=0x1c1de98, flags=flags at entry=(unknown: 0)) at mdbox-sync.c:345
sync_ctx = 0x0
atomic = 0x1c6c680
ret = <optimized out>
#18 0x00007fc20c378f3b in mdbox_storage_sync_init (box=0x1c1de98, flags=MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) at mdbox-sync.c:368
mbox = 0x1c1de98
mdbox_sync_flags = (unknown: 0)
ret = 0
#19 0x00007fc20c35e834 in mailbox_sync_init (box=0x1c1de98, flags=MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) at mail-storage.c:1891
_data_stack_cur_id = 4
ctx = <optimized out>
#20 0x00007fc20c35e8d7 in mailbox_sync (box=<optimized out>, flags=<optimized out>) at mail-storage.c:1947
ctx = 0x1c67a38
status = {sync_delayed_expunges = 0}
#21 0x0000000000443415 in dsync_mailbox_import_deinit ()
No symbol table info available.
#22 0x000000000043cd40 in dsync_brain_sync_half_finished.part.0 ()
No symbol table info available.
#23 0x000000000043d5d3 in dsync_brain_sync_mails ()
No symbol table info available.
#24 0x0000000000438c71 in dsync_brain_run ()
No symbol table info available.
#25 0x0000000000438fb1 in dsync_brain_run_io ()
No symbol table info available.
#26 0x000000000044dddf in dsync_ibc_stream_input ()
No symbol table info available.
#27 0x00007fc20c095cd2 in io_loop_call_io (io=0x1bf8690) at ioloop.c:600
ioloop = 0x1bd3890
t_id = 2
__FUNCTION__ = "io_loop_call_io"
#28 0x00007fc20c0973bf in io_loop_handler_run_internal (ioloop=ioloop at entry=0x1bd3890) at ioloop-epoll.c:223
ctx = 0x1bdf0d0
events = <optimized out>
list = 0x1bf86f0
io = <optimized out>
tv = {tv_sec = 4, tv_usec = 524014}
events_count = <optimized out>
msecs = <optimized out>
ret = 1
i = 0
call = <optimized out>
__FUNCTION__ = "io_loop_handler_run_internal"
#29 0x00007fc20c095d6c in io_loop_handler_run (ioloop=ioloop at entry=0x1bd3890) at ioloop.c:649
No locals.
#30 0x00007fc20c095f28 in io_loop_run (ioloop=0x1bd3890) at ioloop.c:624
__FUNCTION__ = "io_loop_run"
#31 0x0000000000421d4e in cmd_dsync_server_run ()
No symbol table info available.
#32 0x0000000000423706 in doveadm_mail_next_user ()
No symbol table info available.
#33 0x0000000000424286 in doveadm_mail_cmd_exec ()
No symbol table info available.
#34 0x0000000000424b3c in doveadm_mail_try_run ()
No symbol table info available.
#35 0x0000000000414c65 in main ()
No symbol table info available.
(gdb) quit
-------------- next part --------------
# 2.2.33.2 (d6601f4ec): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.4.21 (92477967)
# OS: Linux 3.10.0-514.21.2.el7.x86_64 x86_64 CentOS Linux release 7.4.1708 (Core)
auth_cache_negative_ttl = 0
auth_cache_size = 10 M
auth_master_user_separator = *
auth_mechanisms = plain login
auth_username_format = %Ln
auth_verbose = yes
auth_worker_max_count = 100
base_dir = /var/run/dovecot/
default_client_limit = 2048
default_process_limit = 4096
default_vsz_limit = 1 G
dict {
acl = mysql:/etc/dovecot/dovecot-sql-dict.conf.ext
}
dsync_remote_cmd = ssh -l vmail -i /home/vmail/.ssh/id_rsa repl.peer doveadm dsync-server -u%u -U
first_valid_uid = 1000
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
mail_attachment_dir = /gnoc/mail/attachments/
mail_attachment_fs = sis-queue /gnoc/mail/attachments/queue/:posix
mail_attachment_hash = %{sha256}
mail_gid = vmail
mail_location = mdbox:~/mail
mail_plugins = " listescape acl notify replication"
mail_uid = vmail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext spamtest spamtestplus editheader vnd.dovecot.debug imapsieve vnd.dovecot.imapsieve
mbox_write_locks = fcntl
mdbox_rotate_size = 20 M
namespace inbox {
inbox = yes
location =
mailbox Archive {
special_use = \Archive
}
mailbox Archives {
special_use = \Archive
}
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Spam {
special_use = \Junk
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
mailbox sent-mail {
special_use = \Sent
}
prefix =
separator = /
}
namespace sharedmail {
list = children
location = mdbox:%%h/mail:INDEXPVT=~/mail/shared/%%u
prefix = shared/%%u/
separator = /
subscriptions = no
type = shared
}
passdb {
args = /etc/dovecot/dovecot-sql-master.conf.ext
default_fields = userdb_master_user=%{login_user}
driver = sql
master = yes
pass = yes
}
passdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
plugin {
acl = vfile:/gnoc/mail/dict/acls:cache_secs=300
acl_shared_dict = proxy::acl
imapsieve_mailbox1_before = file:/etc/dovecot/sieve/report-spam.sieve
imapsieve_mailbox1_causes = COPY
imapsieve_mailbox1_name = Spam
imapsieve_mailbox2_before = file:/etc/dovecot/sieve/report-spam.sieve
imapsieve_mailbox2_causes = COPY
imapsieve_mailbox2_name = Junk
imapsieve_mailbox3_before = file:/etc/dovecot/sieve/report-ham.sieve
imapsieve_mailbox3_causes = COPY
imapsieve_mailbox3_from = Spam
imapsieve_mailbox3_name = *
imapsieve_mailbox4_before = file:/etc/dovecot/sieve/report-ham.sieve
imapsieve_mailbox4_causes = COPY
imapsieve_mailbox4_from = Junk
imapsieve_mailbox4_name = *
mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
mail_log_fields = uid box msgid size
mail_replica = remote:vmail at repl.peer
sieve = ~/.dovecot.sieve
sieve_before = /etc/dovecot/sieve/before
sieve_before2 = /gnoc/mail/home/%Ln/sieve-before
sieve_dir = ~/sieve
sieve_extensions = +spamtest +spamtestplus +vnd.dovecot.debug +editheader
sieve_global_dir = /etc/dovecot/sieve/global
sieve_global_extensions = +vnd.dovecot.pipe
sieve_pipe_bin_dir = /etc/dovecot/sieve/bin
sieve_plugins = sieve_imapsieve sieve_extprograms
sieve_spamtest_max_header = X-Spam-Status: \w+, score=-?[[:digit:]]+\.[[:digit:]]+ required=([[:digit:]]+\.[[:digit:]]+)
sieve_spamtest_status_header = X-Spam-Status: \w+, score=(-?[[:digit:]]+\.[[:digit:]])
sieve_spamtest_status_type = score
}
protocols = imap pop3 lmtp sieve
service aggregator {
fifo_listener replication-notify-fifo {
mode = 0666
user = vmail
}
unix_listener replication-notify {
mode = 0666
user = vmail
}
}
service anvil {
client_limit = 16387
}
service auth {
client_limit = 327680
unix_listener /var/spool/postfix/private/auth {
group = postfix
mode = 0666
user = postfix
}
}
service dict {
unix_listener dict {
group = vmail
mode = 0660
user = vmail
}
}
service doveadm {
vsz_limit = 2 G
}
service imap {
executable = imap postlogin
process_limit = 4096
vsz_limit = 1 G
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
vsz_limit = 1 G
}
service managesieve-login {
inet_listener sieve {
port = 4190
}
}
service postlogin {
executable = script-login -d rawlog -I -t
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
mode = 0666
}
}
ssl = required
ssl_ca = </etc/pki/tls/cert.pem
ssl_cert = </var/lib/acme/live/mail.ctc.grnoc.iu.edu/fullchain
ssl_cipher_list = ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
ssl_dh_parameters_length = 2048
ssl_key = # hidden, use -P to show it
ssl_prefer_server_ciphers = yes
ssl_protocols = !SSLv2 !SSLv3
ssl_require_crl = no
userdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
protocol lmtp {
mail_plugins = " listescape acl notify replication sieve"
postmaster_address = postmaster at globalnoc.iu.edu
}
protocol lda {
mail_plugins = " listescape acl notify replication sieve"
}
protocol imap {
mail_max_userip_connections = 256
mail_plugins = " listescape acl notify replication imap_acl imap_sieve"
}
-------------- next part --------------
A non-text attachment was scrubbed...
Name: smime.p7s
Type: application/pkcs7-signature
Size: 4090 bytes
Desc: S/MIME Cryptographic Signature
URL: <https://dovecot.org/pipermail/dovecot/attachments/20180223/42744840/attachment-0001.p7s>
More information about the dovecot
mailing list