MDBOX DSYNC error: Broken physical size in mailbox

[paul-dovecotlist at trainor.nz]

Hi All,

I currently have a small Dovecot server (Server A) and I've installed a 
second dovecot mailbox server (server B) and am trying to sync mailboxes 
from Server A to Server B.

Both servers sit behind two dovecot directors on separate hosts and have 
identical config files (doveconf -n output below). Mailboxes on Server A 
are on an NFS mount.

Server A has been working fine with no issues for a couple of years.

I'm running this command on Server A:
doveadm sync -u #user# tcp:#newservername#:24245

Small mailboxes with few items are syncing OK.. but most mailboxes fail 
(sycn stops) with the following type of error: (Mailbox name, storage 
file, UID and sizes vary).

dsync-local(#username#)<b2vTDKerOV/fAQAAZMK/0g>: Error: Mailbox INBOX: 
Cache 
/#usershomedirectory#/mdbox/mailboxes/INBOX/dbox-Mails/dovecot.index.cache: 
Deleting corrupted cache record uid=9: UID 9: Broken physical size in 
mailbox INBOX: read(/#usershomedirectory#/mdbox/storage/m.1) failed: 
Cached message size larger than expected (60616 > 8688, box=INBOX, UID=9)

If I run it again, I'll get the same error, but with a new UID.

If I do a local migration instead of server to server, I still get the 
errors.
e.g. doveadm sync -u #user# maildir:~/Maildir-test/

I tried running a force resync (doveadm force-resync -u #user# -f "*) 
but it didn't help.

There's no issues from an imap client accessing any emails. Using 
imapsync to sync affected mailboxes works fine for example. But... if I 
then run a dsync on Server B (doveadm sync -u #user# 
maildir:~/Maildir-test/) on an account that was successfully migrated 
with imapsync, I get the issue again.

How worried should I be about these corrupted cache records, and any 
ideas how to resolve?

Thanks in advance for your help!

Paul.

--- output of doveconf -n ---

# 2.3.11.3 (502c39af9): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.11 (6c69c917)
# OS: Linux 5.3.0-1032-aws x86_64 Ubuntu 18.04.5 LTS
hostname = ###
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
login_trusted_networks = ###
mail_fsync = always
mail_location = mdbox:~/mdbox
mail_plugins = " acl notify replication"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope 
encoded-character vacation subaddress comparator-i;ascii-numeric 
relational regex imap4flags copy include variables body enotify 
environment mailbox date index ihave duplicate mime foreverypart extracttext
mmap_disable = yes
namespace inbox {
   inbox = yes
   location =
   mailbox Drafts {
     special_use = \Drafts
   }
   mailbox Junk {
     special_use = \Junk
   }
   mailbox Sent {
     special_use = \Sent
   }
   mailbox "Sent Messages" {
     special_use = \Sent
   }
   mailbox Trash {
     special_use = \Trash
   }
   prefix =
   separator = /
   type = private
}
namespace shared {
   list = children
   location = 
mdbox:/mailhomes/%%d/%%n/mdbox:INDEXPVT=/mailhomes/%d/%n/shared
   prefix = shared/%%u/
   separator = /
   subscriptions = no
   type = shared
}
passdb {
   args = /etc/dovecot/dovecot-ldap.conf.ext
   driver = ldap
}
plugin {
   acl = vfile:/tmp/global-acls:cache_secs=5
   mail_plugins = " acl notify replication"
   sieve = file:~/sieve;active=~/.dovecot.sieve
   sieve_before = /mailhomes/sieve/global/SpamToJunk.sieve
   sieve_max_actions = 32
   sieve_max_redirects = 4
   sieve_max_script_size = 1M
   sieve_quota_max_scripts = 20
   sieve_quota_max_storage = 10M
   sieve_redirect_envelope_from = sender
   zlib_save = bz2
   zlib_save_level = 6
}
protocols = " imap lmtp sieve submission sieve"
recipient_delimiter = -
service doveadm {
   inet_listener {
     port = 24245
   }
}
service imap-login {
   process_min_avail = 2
}
service imap {
   process_limit = 4096
}
service lmtp {
   inet_listener lmtp {
     port = 24
   }
}
service managesieve-login {
   inet_listener sieve {
     port = 4190
   }
   process_min_avail = 2
   service_count = 1
}
service managesieve {
   process_limit = 1024
}
service submission-login {
   inet_listener submission {
     port = 587
   }
}
ssl = required
ssl_cert = </etc/dovecot/dovecot-backend.pem
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
submission_client_workarounds = whitespace-before-path
submission_host = ###
submission_relay_host = ###
submission_relay_ssl = starttls
submission_relay_trusted = yes
userdb {
   driver = prefetch
}
userdb {
   args = /etc/dovecot/dovecot-ldap-userdb.conf.ext
   driver = ldap
}
protocol lda {
   mail_plugins = " acl notify replication zlib"
}
protocol imap {
   mail_max_userip_connections = 100
   mail_plugins = " acl notify replication zlib imap_acl"
}
protocol lmtp {
   mail_plugins = " acl notify replication zlib sieve"
}
protocol sieve {
   mail_max_userip_connections = 10
   managesieve_implementation_string = Dovecot Pigeonhole
   managesieve_logout_format = bytes=%i/%o
   managesieve_max_compile_errors = 5
   managesieve_max_line_length = 64 k
}
protocol submission {
   mail_plugins = " acl notify replication"
}
local ### {
   doveadm_password = # hidden, use -P to show it
}
local ### {
   doveadm_password = # hidden, use -P to show it
}