On 06/03/2023 15:45 EET Tim Evers <te-ml-ext@artfiles.de> wrote:
Am 06.03.23 um 14:00 schrieb Aki Tuomi:
>> On 06/03/2023 13:06 EET Tim Evers <te-ml-ext@artfiles.de> wrote:
>>
>>
>> Am 06.03.23 um 11:59 schrieb Aki Tuomi:
>>
>>>> On 06/03/2023 12:44 EET Tim Evers <te-ml-ext@artfiles.de> wrote:
>>>>
>>>>
>>>> Am 06.03.23 um 11:42 schrieb Aki Tuomi:
>>>>
>>>>>> On 06/03/2023 12:32 EET Tim Evers <te-ml-ext@artfiles.de> wrote:
>>>>>>
>>>>>>
>>>>>> Hi,
>>>>>>
>>>>>> since I did not get any feedback on my bug report post regarding
>>>>>> corrupted sizes while using zlib
>>>>>> (https://dovecot.org/pipermail/dovecot/2023-February/126105.html) I
>>>>>> would like to confirm that this bug report reached someone in charge.
>>>>>>
>>>>>> Or if not - I would kindly ask for directions on how to post it in a way
>>>>>> that spawns some action.
>>>>>>
>>>>>> Thx
>>>>>>
>>>>>> Tim
>>>>> Can you try erasing dovecot.index.cache and see if the problem fixes itself? It's possible your cache contains invalid values from old errors.
>>>>>
>>>>> Aki
>>>> Did that already several times (other dovecot.index.* files too). The
>>>> only thing that helps is actually deleting the dovecot-uidlist file but
>>>> the problem returns.
>>>>
>>>> Tim
>>> I think it's because the value on the filename is wrong. The filename contains the virtual and physical size.
>>>
>>> Can you try if https://github.com/dovecot/tools/blob/main/maildir-size-fix.pl fixes your issue?
>>>
>>> Aki
>> Quoting
>>
>> https://doc.dovecot.org/configuration_manual/zlib_plugin/ :
>>
>> "All mails must have ,|S=<size>|in their filename where <size> contains
>> the original uncompressed mail size, otherwise there will be problems
>> with quota calculation as well as other potential random failures."
>>
>> And the script you send:
>>
>> "# Check if the files are compressed. Use the uncompressed size for S=size."
>>
>> Which it does in my example. S= contains the uncompressed size, not the
>> size on disk. Is this not correct? All Files are written exclusively by
>> dovecot, no other software is involved.
>>
>> Tim
> Can you send output from doveconf -n? I am having suspicion that zlib plugin isn't always involved in your config, which causes the compressed file to be cached wrongly. Especially it feels like your IMAP protocol does not have compression on while LMTP does.
>
> Aki
Actually it is the other way round (I removed zlib from lmtp to reduce
the impact of the problem):
# 2.3.20 (80a5ac675d): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.20 (149edcf2)
# OS: Linux 4.15.0-204-generic x86_64 Ubuntu 18.04.6 LTS
# Hostname: node2.mst27.artfiles.de
auth_verbose = yes
auth_worker_max_count = 1000
default_client_limit = 5000
dict {
sqlquota = mysql:/etc/dovecot/dovecot-dict-sql.conf.ext
}
disable_plaintext_auth = no
import_environment = TZ CLUSTERNAME
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
mail_location = maildir:~/Maildir
mail_plugins = " mail_log notify old_stats"
mailbox_idle_check_interval = 10 mins
maildir_broken_filename_sizes = yes
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope
encoded-character vacation subaddress comparator-i;ascii-numeric
relational regex imap4flags copy include variables body enotify
environment mailbox date index ihave duplicate mime foreverypart
extracttext editheader imapsieve vnd.dovecot.imapsieve
namespace inbox {
inbox = yes
location =
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
auto = subscribe
special_use = \Junk
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
prefix = INBOX.
separator = .
}
passdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
plugin {
imapsieve_mailbox1_before = file:~/af_report-spam.sieve
imapsieve_mailbox1_causes = COPY
imapsieve_mailbox1_name = INBOX.Junk
imapsieve_mailbox2_before = file:~/af_report-ham.sieve
imapsieve_mailbox2_causes = COPY
imapsieve_mailbox2_from = INBOX.Junk
imapsieve_mailbox2_name = *
quota = dict:User quota::proxy::sqlquota
quota_rule = *:storage=0:messages=0
sieve = file:~/sieve;active=~/.dovecot.sieve
sieve_before = file:~/af_master.sieve
sieve_before2 = file:~/af_konto.sieve
sieve_before3 = file:~/af_domain.sieve
sieve_before4 = file:~/af_spamfilter.sieve
sieve_extensions = +editheader
sieve_filter_bin_dir = /usr/local/bin/sieve
sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.filter
+vnd.dovecot.environment
sieve_max_redirects = 100
sieve_pipe_bin_dir = /usr/local/bin/sieve
sieve_plugins = sieve_imapsieve sieve_extprograms
stats_refresh = 30 secs
zlib_save = gz
zlib_save_level = 6
}
pop3_no_flag_updates = yes
protocols = " imap lmtp sieve pop3"
service anvil {
client_limit = 6203
}
service auth-worker {
process_limit = 1000
}
service auth {
client_limit = 6300
}
service dict {
process_limit = 500
unix_listener dict {
group = vmail
mode = 0600
user = vmail
}
}
service imap-login {
process_limit = 4000
process_min_avail = 0
service_count = 0
vsz_limit = 512 M
}
service imap {
client_limit = 1
process_limit = 4000
vsz_limit = 1 G
}
service lmtp {
inet_listener lmtp {
address = 127.0.0.1
port = 10024
}
process_limit = 80
process_min_avail = 20
vsz_limit = 2 G
}
service pop3-login {
process_limit = 2000
service_count = 0
vsz_limit = 512 M
}
service pop3 {
client_limit = 1
process_limit = 2000
vsz_limit = 1 G
}
service stats {
fifo_listener stats-mail {
mode = 0600
user = vmail
}
}
ssl_cert = </etc/dovecot/private/dovecot.pem
ssl_client_ca_dir = /etc/ssl/certs
ssl_key = # hidden, use -P to show it
submission_host = smtp4.artfiles.de
syslog_facility = local5
userdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
protocol lmtp {
mail_plugins = " mail_log notify old_stats sieve quota"
}
protocol imap {
mail_max_userip_connections = 50
mail_plugins = " mail_log notify old_stats quota imap_quota
imap_sieve zlib imap_old_stats"
rawlog_dir = /var/log/rawlog/%u
}
protocol pop3 {
mail_max_userip_connections = 50
mail_plugins = " mail_log notify old_stats zlib"
}
It will probably compound on the issue though, especially when rebuilding caches and indexes, as it will record the sizes wrong. I'm not sure if this is *really* the issue, but it's most likely culprit. The 8192 is very odd number to have there, which either indicates that the zlib plugin is not consistently used, or you have some issues with the gz library itself.
Aki
Not sure if I understand this - shouldn't dovecot always note the
decompressed size in cache and filename regardless if it's compressed or
not?