[Dovecot] Cached message size larger than expected
Steven Haigh
netwiz at crc.id.au
Fri Feb 14 04:49:45 UTC 2014
On 14/02/14 15:39, Timo Sirainen wrote:
> On 14.2.2014, at 13.32, Steven Haigh <netwiz at crc.id.au> wrote:
>
>> Looking further into this, it seems like the file has been compressed twice:
>>
>> This doesn't seem right for mail that has been sent, or moved from the
>> inbox to the trash folder...
>>
>> Is it possible that when the mail is moved between folders, it is
>> somehow compressed a second time?
>
> Can you reproduce this by copying/moving a mail? I can't. What's your doveconf -n output?
I'm digging a bit futher... I can't get the trash folder to update at
all now via thunderbird - I'm trying to get a common method - but can't
quite figure it out.
Config attached.
I'm wondering... After I enabled compression on store, I ran a script
that went through and compressed files with gzip and compressed existing
content - also attached.
It seemed to make sense while reading through the script, so I'm not
quite sure if its something I missed either...
--
Steven Haigh
Email: netwiz at crc.id.au
Web: https://www.crc.id.au
Phone: (03) 9001 6090 - 0412 935 897
Fax: (03) 8338 0299
-------------- next part --------------
# 2.2.12: /etc/dovecot/dovecot.conf
# OS: Linux 3.12.10-1.el6xen.x86_64 x86_64 Scientific Linux release 6.5 (Carbon) ext4
auth_cache_negative_ttl = 2 hours
auth_cache_size = 10 M
auth_cache_ttl = 2 hours
auth_mechanisms = plain login
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
login_log_format_elements = user=<%u> method=%m rip=%r lip=%l mpid=%e %c
mail_gid = 5000
mail_location = maildir:/home/%d/%n/
mail_plugins = zlib quota
mail_uid = 5000
mailbox_idle_check_interval = 5 secs
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date ihave editheader
mbox_write_locks = fcntl
namespace inbox {
inbox = yes
location =
mailbox Drafts {
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Trash {
special_use = \Trash
}
prefix =
separator = .
type = private
}
passdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
plugin {
quota = maildir:Filesystem Quota
quota_rule = *:bytes=2G
quota_warning = storage=95%% quota-warning 95 %u
quota_warning2 = storage=80%% quota-warning 80 %u
sieve = /home/%d/%n/dovecot.sieve
sieve_dir = /home/%d/%n/sieve
sieve_editheader_max_header_size = 1k
sieve_editheader_protected = X-Verified
sieve_extensions = +editheader
sieve_max_actions = 64
sieve_max_redirects = 25
zlib_save = gz
zlib_save_level = 9
}
protocols = imap lmtp sieve
service auth {
unix_listener /var/spool/postfix/private/auth {
group = vmail
mode = 0660
user = postfix
}
unix_listener auth-userdb {
group = vmail
mode = 0600
user = vmail
}
}
service dict {
unix_listener dict {
group = vmail
mode = 0660
user = dovecot
}
}
service managesieve-login {
executable = managesieve-login -D
inet_listener sieve {
port = 4190
}
}
ssl_ca = </root/sub.class1.server.ca.pem
ssl_cert = </root/startssl-mail.crc.id.au.crt
ssl_key = </root/startssl-mail.crc.id.au.key
userdb {
driver = prefetch
}
userdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
protocol lmtp {
mail_plugins = zlib quota sieve
}
protocol lda {
mail_plugins = zlib quota sieve
}
protocol imap {
imap_logout_format =
mail_max_userip_connections = 20
mail_plugins = zlib quota imap_zlib imap_quota
}
-------------- next part --------------
#!/bin/sh
# Find the mails you want to compress in a single maildir.
#
# Skip files that don't have ,S=<size> in the filename.
#
# Compress the mails to tmp/
#
# Update the compressed files' mtimes to be the same as they were in the original files (e.g. touch command)
#
# Run maildirlock <path> <timeout>. It writes PID to stdout, save it.
#
# <path> is path to the directory containing Maildir's dovecot-uidlist (the control directory, if it's separate)
#
# <timeout> specifies how long to wait for the lock before failing.
#
# If maildirlock grabbed the lock successfully (exit code 0) you can continue.
# For each mail you compressed:
#
# Verify that it still exists where you last saw it.
# If it doesn't exist, delete the compressed file. Its flags may have been changed or it may have been expunged. This happens rarely, so just let the next run handle it.
#
# If the file does exist, rename() (mv) the compressed file over the original file.
#
# Dovecot can now read the file, but to avoid compressing it again on the next run, you'll probably want to rename it again to include e.g. a "Z" flag in the file name to mark that it was compressed (e.g. 1223212411.M907959P17184.host,S=3271:2,SZ). Remember that the Maildir specifications require that the flags are sorted by their ASCII value, although Dovecot itself doesn't care about that.
#
# Unlock the maildir by sending a TERM signal to the maildirlock process (killing the PID it wrote to stdout).
##
## <http://ivaldi.nl/blog/2011/12/06/compressed-mail-in-dovecot/>
##
store=/home/
compress=gzip
#compress=bzip2
find "$store" -type d -name "cur" | while read maildir;
do
tmpdir=$(cd "$maildir/../tmp" &>/dev/null && pwd) || exit 1
find=$(find "$maildir" -type f -name "*,S=*" ! -name "*,*:2,*,*Z*" -printf "%f\n")
if [ -z "$find" ];
then
continue
fi
echo "$find" | while read filename;
do
srcfile="$maildir/$filename"
tmpfile="$tmpdir/$filename"
$compress --best --stdout "$srcfile" > "$tmpfile" &&
# Copy over some things
chown --reference="$srcfile" "$tmpfile" &&
chmod --reference="$srcfile" "$tmpfile" &&
touch --reference="$srcfile" "$tmpfile"
done
# Should really check dovecot-uidlist is in $maildir/..
if lock=$(/usr/libexec/dovecot/maildirlock "$maildir/.." 10);
then
# The directory is locked now
echo "Processing $maildir/..."
echo "$find" | while read filename;
do
srcfile="$maildir/$filename"
tmpfile="$tmpdir/$filename"
$compress --best --stdout "$srcfile" > "$tmpfile" &&
# Copy over some things
chown --reference="$srcfile" "$tmpfile" &&
chmod --reference="$srcfile" "$tmpfile" &&
touch --reference="$srcfile" "$tmpfile"
done
# Should really check dovecot-uidlist is in $maildir/..
if lock=$(/usr/libexec/dovecot/maildirlock "$maildir/.." 10);
then
# The directory is locked now
echo "Processing $maildir/..."
echo "$find" | while read filename;
do
flags=$(echo $filename | awk -F:2, '{print $2}')
if echo $flags | grep ',';
then
newname=$filename"Z"
else
newname=$filename",Z"
fi
srcfile=$maildir/$filename
tmpfile=$tmpdir/$filename
dstfile=$maildir/$newname
if [ -f "$srcfile" ] && [ -f "$tmpfile" ];
then
#echo "$srcfile -> $dstfile"
mv "$tmpfile" "$srcfile" &&
mv "$srcfile" "$dstfile"
else
rm -f "$tmpfile"
fi
done
kill $lock
else
echo "Failed to lock: $maildir" >&2
echo "$find" | while read filename;
do
rm -f "$tmpdir/$filename"
done
fi
done
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 819 bytes
Desc: OpenPGP digital signature
URL: <http://dovecot.org/pipermail/dovecot/attachments/20140214/97ffc0a2/attachment.sig>
More information about the dovecot
mailing list