High I/O wait with Dovecot POP3

[Frido Otten]

Hello,

In the past we've had 4 loadbalanced servers configured with courier
pop3 and dovecot handling respectively POP3 and IMAP connections. These
4 servers have a shared NFS storage on a Netapp FAS3220 controller with
SAS disks. This setup is handling 50 POP3 sessions per second without
any problems.

Now we're installing 6 new loadbalanced servers to do the same job on
the same storage, but using Dovecot to handle the POP3 sessions instead
of Courier. The IMAP sessions are already handled by those 6 new
servers, but we're encountering high iowait issues when using POP3 on
those servers.

I've applied the hints from http://wiki2.dovecot.org/NFS but this had no
improvement.

The 6 new servers are quad core Intel(R) Xeon(R) CPU X3323  @ 2.50GHz,
with 4 GB RAM connected with gigabit ethernet connection to the storage.
The 4 old servers are single core (w/o HT) P4 2,8 GHz with 1 GB RAM.

# dovecot --version
2.1.7

# /usr/sbin/dovecot -n
# 2.1.7: /etc/dovecot/dovecot.conf
# OS: Linux 2.6.32-5-686-bigmem i686 Debian 6.0.10 ext4
auth_cache_negative_ttl = 0
auth_cache_size = 8 k
auth_mechanisms = plain digest-md5 cram-md5 login
base_dir = /var/run/imap/
dict {
  acl = mysql:/etc/dovecot/dovecot-dict-sql.conf.ext
}
disable_plaintext_auth = no
login_greeting = Dovecot MX.
login_log_format_elements = user=<%u> method=%m rip=%r lip=%l %c
mail_fsync = always
mail_location = maildir:/var/spool/mail/%1u/%1.1u/%u
mail_nfs_index = yes
mail_nfs_storage = yes
maildir_stat_dirs = yes
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope
encoded-character vacation subaddress comparator-i;ascii-numeric
relational regex imap4flags copy include variables body enotify
environment mailbox date ihave
mmap_disable = yes
namespace {
  inbox = yes
  location =
  prefix =
  separator = .
  type = private
}
namespace {
  list = children
  location =
maildir:/var/spool/mail/%%1u/%%1.1u/%%u/:INDEX=/var/spool/mail/%1u/%1.1u/%u/shared/%%u
  prefix = shared.%%u.
  separator = .
  subscriptions = no
  type = shared
}
passdb {
  args = /etc/dovecot/dovecot-sql-auth.conf
  driver = sql
}
plugin {
  acl = vfile
  acl_shared_dict = proxy::acl
  quota = maildir
  quota_rule2 = Trash:storage=+25M
}
protocols = imap
service anvil {
  client_limit = 4614
}
service auth {
  client_limit = 2560
  executable = /usr/lib/dovecot/auth
  unix_listener auth-master {
    mode = 0600
    user = exim
  }
  user = dovecot
}
service dict {
  unix_listener dict {
    group = exim
    mode = 0660
    user = exim
  }
}
service imap-login {
  chroot = login
  client_limit = 2048
  executable = /usr/lib/dovecot/imap-login
  inet_listener imap {
    address = *, ::
    port = 143
  }
  inet_listener imaps {
    address = *, ::
    port = 993
  }
  process_limit = 2048
  process_min_avail = 3
  service_count = 0
  user = dovecot
  vsz_limit = 128 M
}
service imap {
  executable = /usr/lib/dovecot/imap
  process_limit = 2048
}
service managesieve-login {
  chroot = login
  client_limit = 2048
  process_limit = 2048
  process_min_avail = 3
  service_count = 0
  user = dovecot
  vsz_limit = 128 M
}
service managesieve {
  process_limit = 2048
}
service pop3-login {
  chroot = login
  client_limit = 512
  inet_listener pop3 {
    address = 1.2.3.4
    port = 110
  }
  inet_listener pop3s {
    address = 1.2.3.4
    port = 0
  }
  process_limit = 512
  process_min_avail = 3
  service_count = 0
  user = dovecot
  vsz_limit = 128 M
}
service pop3 {
  process_limit = 2560
}
ssl_cert = </etc/ssl/certs/wildcardcertificate.crt
ssl_key = </etc/ssl/private/wildcardcertificate.key
syslog_facility = local7
userdb {
  driver = prefetch
}
userdb {
  args = /etc/dovecot/dovecot-sql-auth.conf
  driver = sql
}
protocol imap {
  mail_plugins = quota imap_quota acl imap_acl
}
protocol pop3 {
  pop3_uidl_format = %v.%u
}
protocol lda {
  auth_socket_path = /var/run/imap/auth-master
  info_log_path = /var/log/dovecot-deliver.log
  log_path = /var/log/dovecot-deliver.log
  mail_plugins = sieve quota
  postmaster_address = Mailer-Daemon at mx.domain.nl
  rejection_reason = Your message with subject "%s" was automatically
rejected:%n%r
  sendmail_path = /usr/sbin/exim
}




Best regards,

Frido