Mail Archiving/Lucene Indexes/mbox

Fri Aug 1 14:22:42 UTC 2014

Yes, I do.

thebighonker.lerctr.org /home/ler $ doveconf -n
# 2.2.13: /usr/local/etc/dovecot/dovecot.conf
# OS: FreeBSD 10.0-STABLE amd64
auth_default_realm = lerctr.org
auth_mechanisms = plain login
auth_realms = lerctr.org thebighonker.lerctr.org tbh.lerctr.org
auth_username_format = %Ln
lda_mailbox_autocreate = yes
lmtp_save_to_detail_mailbox = yes
login_access_sockets = tcpwrap
mail_debug = yes
mail_location = mbox:~/mail:INBOX=~/mail/INBOX
mail_plugins = fts fts_lucene stats
mail_privileged_group = mail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags
copy include variables body enotify environment mailbox date ihave
duplicate vacation-seconds editheader
namespace archive {
  hidden = no
  inbox = no
  list = no
  location = mbox:~/MAILARCHIVE
  prefix = "#ARCHIVE/"
  separator = /
}
namespace inbox {
  inbox = yes
  location =
  mailbox Drafts {
    special_use = \Drafts
  }
  mailbox INBOX {
    auto = create
  }
  mailbox Junk {
    special_use = \Junk
  }
  mailbox Sent {
    special_use = \Sent
  }
  mailbox "Sent Messages" {
    special_use = \Sent
  }
  mailbox Trash {
    special_use = \Trash
  }
  prefix =
}
passdb {
  args = failure_show_msg=yes session=yes max_requests=20
  driver = pam
}
plugin {
  fts = lucene
  fts_autoindex = yes
  fts_lucene = whitespace_chars=@. normalize no_snowball
  recipient_delimiter = +
  sieve = ~/.dovecot.sieve
  sieve_dir = ~/sieve
  sieve_extensions = +editheader +vacation-seconds
  stats_command_min_time = 1 mins
  stats_domain_min_time = 12 hours
  stats_ip_min_time = 12 hours
  stats_memory_limit = 16 M
  stats_refresh = 5s
  stats_session_min_time = 15 mins
  stats_track_cmds = yes
  stats_user_min_time = 1 hours
}
protocols = imap pop3 lmtp sieve
service auth {
  unix_listener auth-client {
    mode = 0666
  }
  unix_listener auth-master {
    mode = 0666
  }
}
service stats {
  chroot = empty
  client_limit = 0
  drop_priv_before_exec = no
  executable = stats
  extra_groups =
  fifo_listener stats-mail {
    group =
    mode = 0666
    user =
  }
  group =
  idle_kill = 4294967295 secs
  privileged_group =
  process_limit = 1
  process_min_avail = 0
  protocol =
  service_count = 0
  type =
  unix_listener stats {
    group =
    mode = 0666
    user =
  }
  user = $default_internal_user
  vsz_limit = 18446744073709551615 B
}
service tcpwrap {
  unix_listener login/tcpwrap {
    group = $default_login_user
    mode = 0600
    user = $default_login_user
  }
}
ssl_cert = </etc/ssl/certs/CHAIN.crt
ssl_key = </etc/ssl/certs/thebighonker.lerctr.org.key.open
userdb {
  driver = passwd
}
verbose_proctitle = yes
protocol lmtp {
  mail_plugins = fts fts_lucene sieve
}
protocol lda {
  mail_plugins = fts fts_lucene sieve
}
protocol imap {
  imap_client_workarounds = tb-extra-mailbox-sep tb-lsub-flags
  mail_max_userip_connections = 50
  mail_plugins = fts fts_lucene stats imap_stats
}
thebighonker.lerctr.org /home/ler $

The following was put out while the script was running:
Aug  1 00:05:00 thebighonker dovecot: indexer-worker(ler): Error: lucene:
Failed to sync mailbox INBOX: Mailbox isn't selectable
Aug  1 00:05:31 thebighonker last message repeated 24 times
Aug  1 00:05:45 thebighonker last message repeated 95 times
Aug  1 00:05:45 thebighonker exim[69950]: 1XD51Y-000ICE-2D <= ler at lerctr.org
U=ler P=local S=11222
Aug  1 00:05:45 thebighonker dovecot: lda(ler): sieve: msgid=<
E1XD51Y-000ICE-2D at thebighonker.lerctr.org>: stored mail into mailbox 'INBOX'
Aug  1 00:05:45 thebighonker exim[70321]: 1XD51Y-000ICE-2D => ler <
ler at lerctr.org> R=localuser T=local_delivery_dovecot S=11327 QT=45s DT=0s
Aug  1 00:05:45 thebighonker exim[70321]: 1XD51Y-000ICE-2D Completed QT=45s
Aug  1 00:05:46 thebighonker dovecot: indexer-worker(ler): Indexed 1
messages in INBOX

the script:
thebighonker.lerctr.org /home/ler $ cat bin/archive-mail
#!/bin/sh
PATH=$PATH:/usr/local/bin
#Expects to be run after midnight on the first of the month
#  to archive all the previous months mail
#Date Run:
TODAY=`date "+%Y-%m-%d"`
#last month in YYYY/MM
YEAR_LAST_MONTH=`date -v-1d "+%Y/%m"`
#1st of last month as 01-Mon-YYYY
FIRST_LAST_MONTH=`date -v-1d "+01-%b-%Y"`
echo 'TODAY=' ${TODAY}
echo 'YEAR_LAST_MONTH=' ${YEAR_LAST_MONTH}
echo 'FIRST_LAST_MONTH=' ${FIRST_LAST_MONTH}
# get a list of all the mailboxes with at least one real message
doveadm -f table mailbox status vsize \* 2>/dev/null | \
        awk '{if ($2 > 0) print $1}' >/tmp/ler-mailbox-list.$$ 2>/dev/null
# archive all the mails
for i in `cat /tmp/ler-mailbox-list.$$`
do
   echo `date` start ${i}
   doveadm mailbox create \#ARCHIVE/${YEAR_LAST_MONTH}/${i}
   doveadm -f tab mailbox status messages ${i}
   doveadm move \#ARCHIVE/${YEAR_LAST_MONTH}/${i} mailbox \
            ${i} BEFORE ${TODAY} SINCE ${FIRST_LAST_MONTH}
   doveadm -f tab mailbox status messages ${i}
   echo `date` done  ${i}
done
rm /tmp/ler-mailbox-list.$$
thebighonker.lerctr.org /home/ler $

I am running the hg patch that fixes the filenames to be in the .imap
space, but that is the only patch I have applied on top of 2.2.13.

On Fri, Aug 1, 2014 at 9:09 AM, Timo Sirainen <tss at iki.fi> wrote:

> On 01 Aug 2014, at 08:40, Larry Rosenman <larryrtx at gmail.com> wrote:
>
> > I have a script (available on demand) that moves all my mail for a given
> > month to a separate NAMESPACE (#ARCHIVE).
> >
> > When I do that, the lucene-indexes folder in the PRIMARY namespace seems
> to
> > get cleaned out, and NOTHING gets put in the lucene-indexes folder
> > (although it gets created) in the #ARCHIVE namespace.
> >
> > Is this to spec?
> >
> > the archiving is done via doveadm move commands.
>
> Do you have fts_autoindex=yes? If not, the Lucene index update is delayed
> until the first search is issued.
>
>

-- 
Larry Rosenman                     http://www.lerctr.org/~ler
Phone: +1 214-642-9640 (c)     E-Mail: larryrtx at gmail.com
US Mail: 108 Turvey Cove, Hutto, TX 78634-5688