Hitting wall at 2048 IMAP connections

[Ron Cleven]

We are gradually rolling out Dovecot (IMAP only, no POP3) to our 
customer base.  We are replicating between a pair of CentOS 7 boxes.  
All has been working wonderfully.  However, to be sure our rollout 
continues to go smoothly, we put together a simple benchmark client 
program to fire up X persistent IMAP connections (using hundreds of 
mailboxes) that login, list the folders, select the INBOX, periodically 
FETCH an email from the INBOX, and otherwise sit in an IDLE loop.  As 
long as we keep the number of concurrent IMAP connections under 2k, 
everything works fine.  2,000 processes generate a minimal load on the 
server.  However, 2048 is a "hard limit" for number of IMAP processes on 
a box the way things are configured right now.  The number (2048) is a 
bit too magical to be anything but a misconfiguration of either some 
kernel limit or a dovecot configuration limit.  I have been unable to 
figure out where else to look.  I have pasted in my dovecot config below 
along with the kernel  setting for max_user_instances.

Hoping someone can tell me what stupid mistake I have made or what else 
to check.

Just an aside, we front-end this with a separate set of proxy servers 
that provide an SSL front-end, hence, no SSL in the dovecot config.

/proc/sys/fs/inotify/max_user_instances = 4096

# 2.2.10: /etc/dovecot/dovecot.conf
# OS: Linux 3.10.0-123.13.1.el7.x86_64 x86_64 CentOS Linux release 
7.0.1406 (Cor
e)
auth_master_user_separator = *
auth_mechanisms = plain login
default_internal_user = sysadm
default_login_user = sysadm
default_vsz_limit = 512 M
disable_plaintext_auth = no
doveadm_password = **************
doveadm_port = 12345
imap_max_line_length = 128 k
listen = *
mail_gid = 2001
mail_location = maildir:~/Maildir
mail_plugins = notify replication
mail_uid = 2001
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope 
encoded-character vacation subaddress comparator-i;ascii-numeric 
relational regex imap4flags copy include variables body enotify 
environment mailbox date ihave
mbox_write_locks = fcntl
namespace inbox {
   inbox = yes
   location =
   mailbox Drafts {
     special_use = \Drafts
   }
   mailbox Junk {
     special_use = \Junk
   }
   mailbox Sent {
     special_use = \Sent
   }
   mailbox "Sent Messages" {
     special_use = \Sent
   }
   mailbox Trash {
     special_use = \Trash
   }
   prefix =
}

passdb {
   args = scheme=PLAIN /z/imap/usr/master-users
   driver = passwd-file
   master = yes
   pass = yes
}
passdb {
   args = scheme=PLAIN username_format=%u /z/imap/usr/users
   driver = passwd-file
}
plugin {
   mail_replica = tcp:imap0s
   sieve = ~/.dovecot.sieve
   sieve_dir = ~/sieve
   sieve_max_actions = 32
   sieve_max_redirects = 4
   sieve_max_script_size = 1M
}
protocols = imap pop3 sieve
service aggregator {
   fifo_listener replication-notify-fifo {
     mode = 0666
     user = sysadm
   }
   unix_listener replication-notify {
     mode = 0666
     user = sysadm
   }
}
service auth {
   unix_listener auth-userdb {
     mode = 0777
   }
}
service doveadm {
   inet_listener {
     port = 12345
   }
   user = sysadm
}
service imap {
   process_limit = 4096
}
service managesieve-login {
   inet_listener sieve {
     port = 4190
   }
}
service managesieve {
   process_limit = 4096
}
service replicator {
   process_min_avail = 1
}
ssl = no
userdb {
   args = username_format=%u /z/imap/usr/users
   driver = passwd-file
}
protocol lda {
   mail_plugins = sieve notify replication
}
protocol imap {
   mail_max_userip_connections = 40
}
protocol sieve {
   managesieve_max_line_length = 65536
}