Dovecot sync broke after upgrade to OpenBSD 6.9

[William Orr]

Hey,

I have two mailservers running OpenBSD 6.9, and I use bidirectional
syncing of my maildirs through doveadm. After the
upgrade, I noticed that the sync process was failing.

OpenBSD 6.8 shipped with Dovecot 2.3.13, 6.9 ships with 2.3.14. Here's
the output of the dovecot --build-options:

kefka|~|03:01:50|89$ dovecot --build-options
Build options: ioloop=kqueue notify=kqueue openssl io_block_size=8192
SQL driver plugins: mysql postgresql sqlite
Passdb: bsdauth checkpassword ldap passwd passwd-file sql
Userdb: checkpassword ldap(plugin) passwd prefetch passwd-file sql

I did not compile this manually, this is from packages. More info about
configure options passed are available here, in the Makefile from ports
https://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/mail/dovecot/Makefile?rev=1.297&content-type=text/x-cvsweb-markup

Here's a sample run of the sync:

kefka|~|02:57:05|0$ doas doveadm sync -u worr at worrbase.com remote:sabin.worrbase.com
doveadm(VERSION	dsync	3	5): Error: User doesn't exist
dsync-local(worr)<FNQIMvI9omB4VwEADTvxNg>: Error: read(sabin.worrbase.com) failed: EOF (version not received)
dsync-local(worr)<FNQIMvI9omB4VwEADTvxNg>: Error: Remote command returned error 67: /usr/bin/ssh -i /root/.ssh/id_ed25519.dsync sabin.worrbase.com /usr/local/bin/dsync-in-wrapper.sh

kefka|~|02:57:08|75$ cat /usr/local/bin/dsync-in-wrapper.sh
#!/bin/ksh
read username
/usr/local/bin/doveadm dsync-server -u "$username"

I ktraced the process, and noticed that in the communication with the
remote mail server, that a bunch of doveadm plugins fail to load. It's
worth noting that these are plugins that are dlopen(3)ed in response to
certain commands sent over the wire, so they don't show up in ldd(1) output.

kefka|~|03:00:08|0$ doas kdump | grep symbol                                        
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_user_module'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_mailbox_get_aclobj'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_mailbox_list_get_backend'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_object_list_init'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_object_list_next'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_backend_rights_match_me'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_rights_get_id'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_object_list_deinit'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_object_get_my_rights'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_rights_update_import'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_mailbox_update_acl'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_lookup_dict_rebuild'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_backend_nonowner_lookups_iter_init'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_backend_nonowner_lookups_iter_next'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_backend_nonowner_lookups_iter_deinit'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_backend_nonowner_lookups_rebuild'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_lookup_dict_is_enabled'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_lookup_dict_iterate_visible_init'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_lookup_dict_iterate_visible_next'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so: undefined symbol 'acl_lookup_dict_iterate_visible_deinit'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_quota_plugin.so: undefined symbol 'quota_user_module'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_quota_plugin.so: undefined symbol 'quota_root_get_resources'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib10_doveadm_quota_plugin.so: undefined symbol 'quota_get_resource'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_list_backend'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_backend_lookup'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_backend_lookup_done'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_search_args_expand'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_language_find'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_user_get_language_list'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_language_detect'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_language_list_get_first'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_user_language_find'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_tokenizer_reset'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_tokenizer_final'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_tokenizer_next'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_filter_filter'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_backend_optimize'
       "doveadm:/usr/local/lib/dovecot/doveadm/lib20_doveadm_fts_plugin.so: undefined symbol 'fts_backend_rescan'
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_user_get_public_key'
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_user_generate_keypair'
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_box_get_public_key'
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_box_generate_keypair'
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_box_get_pvt_digests'
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_acl_secure_sharing_enabled'"
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_box_share_private_keys'
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_box_set_shared_key'
       "doveadm:/usr/local/lib/dovecot/doveadm/libdoveadm_mail_crypt_plugin.so: undefined symbol 'mail_crypt_get_private_key'

I checked on both machines, and these symbols are all undefined in the
associated libraries. I do notice though, that these symbols seem to be
present in similar dovecot, non-doveadm plugins:

kefka|~|03:04:56|130$ nm -A /usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so  | grep acl_user_module
/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so:         U acl_user_module
kefka|~|03:05:14|0$ nm -A /usr/local/lib/dovecot/lib01_acl_plugin.so   | grep acl_user_module               
/usr/local/lib/dovecot/lib01_acl_plugin.so:0001b008 D acl_user_module

However, the doveadm plugins don't link against them?

kefka|~|03:04:24|1$ ldd /usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so 
/usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so:
	Start            End              Type  Open Ref GrpRef Name
	000001d89e043000 000001d89e04d000 dlib  1    0   0      /usr/local/lib/dovecot/doveadm/lib10_doveadm_acl_plugin.so

Is anyone else seeing issues like this? Is there perhaps a
misconfiguration on my end?

Here's the full output of dovecot -n

# 2.3.14 (cee3cbc0d): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.14 (1b5c82b2)
# OS: OpenBSD 6.9 amd64  
# Hostname: kefka.worrbase.com
auth_username_format = %n
default_vsz_limit = 512 M
doveadm_password = # hidden, use -P to show it
dsync_remote_cmd = /usr/bin/ssh -i /root/.ssh/id_ed25519.dsync %{host} /usr/local/bin/dsync-in-wrapper.sh
first_valid_uid = 1000
imap_client_workarounds = delay-newmail tb-extra-mailbox-sep tb-lsub-flags
mail_location = maildir:~/Maildir
mail_plugins = " notify replication"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext
mbox_write_locks = fcntl
mmap_disable = yes
namespace inbox {
  inbox = yes
  location = 
  mailbox Drafts {
    special_use = \Drafts
  }
  mailbox Sent {
    special_use = \Sent
  }
  mailbox Spam {
    special_use = \Junk
  }
  mailbox Trash {
    special_use = \Trash
  }
  prefix = 
  separator = /
}
passdb {
  driver = bsdauth
}
plugin {
  mail_replica = remoteprefix:root at sabin.worrbase.com
  sieve = ~/.dovecot.sieve
  sieve_dir = ~/.sieve
  sieve_user_log = ~/.dovecot.sieve.log
}
pop3_client_workarounds = outlook-no-nuls oe-ns-eoh
postmaster_address = postmaster at worrbase.com
protocols = imap lmtp
service aggregator {
  fifo_listener replication-notify-fifo {
    mode = 0666
  }
  unix_listener replication-notify {
    mode = 0666
  }
}
service imap-login {
  inet_listener imap {
    port = 143
  }
  inet_listener imaps {
    port = 993
    ssl = yes
  }
}
service lmtp {
  inet_listener lmtp {
    address = localhost
    port = 2525
  }
}
service managesieve-login {
  inet_listener sieve {
    port = 4190
  }
  inet_listener sieve_deprecated {
    port = 2000
  }
}
service pop3-login {
  inet_listener pop3 {
    port = 0
  }
  inet_listener pop3s {
    port = 0
  }
}
service replicator {
  process_min_avail = 1
  unix_listener replicator-doveadm {
    mode = 0666
  }
}
ssl = required
ssl_cert = </etc/mail/certs/mail.worrbase.com.crt
ssl_cipher_list = ALL:HIGH:!TLSv1:!SSLv3:!SSLv2:!MEDIUM:!LOW:!EXP:!RC4:!MD5:!aNULL:@STRENGTH
ssl_client_ca_file = /etc/ssl/cert.pem
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
ssl_min_protocol = TLSv1.1
userdb {
  driver = passwd
}
protocol lmtp {
  mail_plugins = " notify replication sieve"
}
protocol lda {
  mail_plugins = " notify replication sieve"
}
protocol imap {
  mail_max_userip_connections = 20
  mail_plugins = " notify replication"
}

Let me know if I need to provide more info.

Thanks so much for the help!