Hi, I’m preparing the migration from an old Rocky 8 with Dovecot 2.3 to Rocky 9 with Dovecot 2.4.5. I’m using Apache Solr as the FTS driver for the messages and Maildir as storage. Everything works great on 2.3, but since I started experimenting on 2.4 I noticed a big performance reduction on FTS queries in IMAP, specifically on the virtual all mailbox.

I’ve noticed that in Dovecot 2.4 the SORT command is taking 100x more than v2.3 to complete. Running the same command (SORT (REVERSE DATE) US-ASCII BODY "Hi") on a medium sized mailbox (about 8GB) on Dovecot 2.3 it completes in seconds and on Dovecot 2.4 it takes about 3-5 minutes. On both cases the query returns about 80k messages.

To exclude Solr as a culprit, using a network sniff tool, I checked that when Dovecot queries Solr for the FTS, it responds within seconds (as the SEARCH command does) so most of the execution time is spent on actually sorting the messages. Is this a regression? Am I missing some cache optimization?

For context, here is my Dovecot 2.3 config:

2.3.21.1 (d492236fa0): /etc/dovecot/dovecot.conf

Pigeonhole version 0.5.21.1 (49005e73)

OS: Linux 4.18.0-553.137.1.el8_10.x86_64 x86_64 Rocky Linux release

8.10 (Green Obsidian)

mail_attribute_dict = file:%h/Maildir/dovecot-attributes mail_fsync = always mail_location = maildir:~/Maildir:VOLATILEDIR=/dev/shm/dovecot/%2.256Nu/%u

mail_plugins = quota quota_clone acl zlib fts fts_solr virtual notify mail_log maildir_very_dirty_syncs = yes

mmap_disable = yes

namespace Virtual { hidden = yes list = no location = virtual:/etc/dovecot/virtual:INDEX=~/Maildir/virtual prefix = Virtual/ separator = / subscriptions = no }

namespace inbox { inbox = yes location = mailbox Archive { auto = subscribe special_use = \Archive } mailbox Drafts { auto = subscribe special_use = \Drafts } mailbox Sent { auto = subscribe special_use = \Sent } mailbox "Sent Messages" { special_use = \Sent } mailbox Spam { auto = subscribe special_use = \Junk } mailbox Trash { auto = subscribe special_use = \Trash } mailbox virtual/All { comment = All my messages special_use = \All } prefix = separator = / } plugin { acl = vfile acl_shared_dict = proxy::acl fts = solr fts_autoindex = no fts_autoindex_exclude = Drafts fts_autoindex_exclude2 = Spam fts_autoindex_exclude3 = Trash fts_autoindex_exclude4 = Sent fts_autoindex_max_recent_msgs = 500 fts_decoder = decode2text fts_header_excludes = Received Authentication-Results DKIM-* ARC-* X-* Feedback-* List-* fts_header_includes = From To Cc Bcc Subject Message-ID X-Identifier fts_index_timeout = 8s fts_solr = url=.../solr/dovecot/ zlib_save = gz zlib_save_level = 6 }

service decode2text { executable = script /usr/libexec/dovecot/decode2text.sh unix_listener decode2text { mode = 0666 } user = dovecot }

service imap { executable = imap imap-postlogin process_limit = 8000 vsz_limit = 2 G }

Dovecot 2.4 config:

2.4.5-2 (0cbb641e3e): /etc/dovecot/dovecot.conf

Pigeonhole version 2.4.5-2 (742bb892)

OS: Linux 5.14.0-687.36.1.el9_8.x86_64 x86_64 Rocky Linux release 9.8

(Blue Onyx)

4 default setting changes since version 2.4.4

dovecot_config_version = 2.4.4 acl_dict_index = yes acl_driver = vfile

dovecot_storage_version = 2.4.4

fts_header_excludes { Received = yes Authentication-Results = yes DKIM-* = yes ARC-* = yes X-* = yes Feedback-* = yes List-* = yes } fts_header_includes { From = yes To = yes Cc = yes Bcc = yes Subject = yes Message-ID = yes X-Identifier = yes } fts_search_timeout = 8s

mail_compress_write_method = gz mail_driver = maildir mail_fsync = always mail_gid = 89

mail_path = ~/Maildir mail_plugins = acl mail_compress virtual fts fts_solr notify quota quota_clone mail_uid = 89 mail_volatile_path = /dev/shm/dovecot/%{user | md5 | substr(0, 8) % 256 | hex(2)}/%{user} mmap_disable = yes

verbose_proctitle = yes

mail_attribute { dict file { path = %{home}/Maildir/dovecot-attributes } } maildir { very_dirty_syncs = yes } namespace inbox { inbox = yes separator = / mailbox Drafts { auto = subscribe special_use = "\\Drafts" } mailbox Spam { auto = subscribe autoexpunge = 31 days special_use = "\\Junk" } mailbox Junk { auto = no } mailbox Trash { auto = subscribe autoexpunge = 31 days special_use = "\\Trash" quota_storage_extra = 100M } mailbox Sent { auto = subscribe special_use = "\\Sent" } mailbox Archive { auto = subscribe special_use = "\\Archive" } mailbox "Sent Messages" { special_use = "\\Sent" } }

namespace Virtual { mail_cache_path = ~/Maildir/virtual mail_driver = virtual mail_index_path = ~/Maildir/virtual mail_path = /var/lib/dovecot/virtual hidden = yes list = no prefix = Virtual/ separator = / subscriptions = no mailbox All { comment = All my messages special_use = "\\All" } }

protocol imap { mail_plugins { imap_acl = yes last_login = yes imap_quota = yes virtual = yes } }

fts solr { soft_commit = yes } mailbox Drafts { fts_autoindex = no } mailbox Spam { fts_autoindex = no } mailbox Trash { fts_autoindex = no } mailbox Sent { fts_autoindex = no } language en { default = yes } language it { }

Thanks

--

Gioele Pannetto