Bug: Dovecot index loosing sync with FTS despite "fts_autoindex = yes"

[kadafax at gmail.com]

Le 21/05/2018 à 12:38, Aki Tuomi a écrit :
> can you try turning on pluign { fts_enforced = yes } and repeat your test?

Same (wrong) result:
1. Send an email with "too6Ouka" in the body

2. Search against "too6Ouka":
# doveadm search -u username mailbox INBOX body too6Ouka
--> No result

3. Force re-index:
# doveadm fts rescan -u username

4. Search again against "too6Ouka":
# doveadm search -u username mailbox INBOX body too6Ouka
--> e09cce0283e8695ab760000002deed92 29055

Don't know if relevant, but on a side note, if I send a second message
with "too6Ouka" in the body, followed by:
# doveadm -v index -u username Inbox
--> doveadm(username): Info: INBOX: Cache is already up to date
And a search against the pattern immediately return only one result
instead of two.

# doveconf -n
# 2.2.34 (874deae): /usr/local/etc/dovecot/dovecot.conf
# Pigeonhole version 0.4.22 (22940fb7)
# OS: Linux 3.10.0-862.2.3.el7.x86_64 x86_64 CentOS Linux release
7.5.1804 (Core)  ext4
# Hostname: mailserver.domain
auth_master_user_separator = *
auth_socket_path = /var/run/dovecot/auth-userdb
auth_username_chars =
abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@*
auth_verbose = yes
base_dir = /var/run/dovecot/
debug_log_path = /var/log/dovecot-debug.log
default_client_limit = 5120
default_process_limit = 1024
default_vsz_limit = 2 G
hostname = domain
info_log_path = /var/log/dovecot.log
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
listen = *
log_path = /var/log/dovecot-error.log
login_greeting = Welcome to dovecot on mydomain
login_log_format_elements = user=<%u> method=%m rip=%r lip=%l mpid=%e %c
mail_gid = vmail
mail_location = maildir:/var/vmail/%Ln
mail_plugins = " quota stats fts fts_solr"
mail_uid = vmail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope
encoded-character vacation subaddress comparator-i;ascii-numeric
relational regex imap4flags copy include variables body enotify
environment mailbox date index ihave duplicate mime foreverypart extracttext
namespace {
  inbox = yes
  location = maildir:/var/vmail/%Ln
  mailbox Drafts {
    auto = subscribe
    special_use = \Drafts
  }
  mailbox Sent {
    auto = subscribe
    special_use = \Sent
  }
  mailbox "Sent Messages" {
    special_use = \Sent
  }
  mailbox Trash {
    auto = subscribe
    special_use = \Trash
  }
  prefix =
  type = private
}
passdb {
  args = /usr/local/etc/dovecot/deny-users
  deny = yes
  driver = passwd-file
}
passdb {
  args = /usr/local/etc/dovecot/master-users
  driver = passwd-file
  master = yes
}
passdb {
  args = /usr/local/etc/dovecot/dovecot-ldap.conf.ext
  driver = ldap
}
plugin {
  fts = solr
  fts_autoindex = yes
  fts_enforced = yes
  fts_solr = url=http://localhost:8983/solr/dovecot/ break-imap-search
  quota = maildir:User quota
  quota_rule = *:storage=20GB
  quota_rule2 = Trash:storage=+200M
  quota_warning = storage=95%% quota-warning 95 %u
  quota_warning2 = storage=80%% quota-warning 80 %u
  sieve = /var/sieve-scripts/%Ln.sieve
  sieve_dir = /var/vmail/%Ln/home/sieve
  stats_command_min_time = 1 mins
  stats_domain_min_time = 12 hours
  stats_ip_min_time = 12 hours
  stats_memory_limit = 16 M
  stats_refresh = 30 secs
  stats_session_min_time = 15 mins
  stats_track_cmds = yes
  stats_user_min_time = 1 hours
}
postmaster_address = postmaster at domain
protocols = imap pop3 sieve sieve
service auth-worker {
  user = root
}
service auth {
  unix_listener /var/spool/postfix/private/auth {
    mode = 0777
    user = vmail
  }
}
service imap-login {
  inet_listener imaps {
    port = 993
    ssl = yes
  }
  process_min_avail = 10
}
service imap {
  process_limit = 2128
}
service managesieve-login {
  inet_listener sieve {
    port = 4190
  }
}
service pop3-login {
  inet_listener pop3s {
    port = 995
    ssl = yes
  }
}
service quota-warning {
  executable = script /usr/local/bin/quota-warning.sh
  user = nobody
}
service stats {
  fifo_listener stats-mail {
    mode = 0600
    user = vmail
  }
}
ssl_cert = </etc/letsencrypt/live/mailserver.domain/fullchain.pem
ssl_key =  # hidden, use -P to show it
userdb {
  args = /usr/local/etc/dovecot/dovecot-ldap-userdb.conf.ext
  driver = ldap
  override_fields = uid=vmail gid=vmail home=/var/vmail/%Ln/home
}
verbose_proctitle = yes
protocol lda {
  mail_plugins = " quota stats fts fts_solr sieve virtual"
}
protocol imap {
  mail_max_userip_connections = 20
  mail_plugins = " quota stats fts fts_solr imap_quota virtual imap_stats"
}
protocol sieve {
  mail_max_userip_connections = 20
}



>
> Aki
>
>
> On 21.05.2018 13:31, kadafax at gmail.com wrote:
>> Le 21/05/2018 à 11:46, Aki Tuomi a écrit :
>>> That message is coming from the temporary raw user, not from the actual
>>> user.
>>>
>>> How did you verify that the FTS indexes are not created or updated
>>> automatically?
>> I send an email containing a specific strings ('thahB8ea' in that case)
>> and search agains it:
>> # doveadm search -u username mailbox INBOX body thahB8ea
>> --> No result
>> (same with integrated search on roundcube)
>>
>> Then I force a rescan, followed by the same search:
>> # doveadm  fts rescan -u username
>> # doveadm search -u username mailbox INBOX body thahB8ea
>> (Long waiting time)
>> --> e09cce0283e8695ab760000002deed92 29048
>>
>> So I conclude that autoindexing is not working.
>>
>> Thanks,
>> kfx
>>