August 2017 - dovecot

proxy-dict with tcp connection
by Ralf Becker 04 Aug '17

04 Aug '17

I try to create a patch to allow (proxy-)dict to use tcp connections instead of a unix domain socket. I'm replacing connection_init_client_unix with connection_init_client_ip: --- ./src/lib-dict/dict-client.c.orig +++ ./src/lib-dict/dict-client.c @@ -721,6 +721,10 @@ client_dict_init(struct dict *driver, const char *uri, struct ioloop *old_ioloop = current_ioloop; struct client_dict *dict; const char *p, *dest_uri, *path; + const char *const *args; + unsigned int argc; + struct ip_addr ip; + in_port_t port=0; unsigned int idle_msecs = DICT_CLIENT_DEFAULT_TIMEOUT_MSECS; unsigned int warn_slow_msecs = DICT_CLIENT_DEFAULT_WARN_SLOW_MSECS; @@ -772,7 +776,21 @@ client_dict_init(struct dict *driver, const char *uri, dict->warn_slow_msecs = warn_slow_msecs; i_array_init(&dict->cmds, 32); - if (uri[0] == ':') { + args = t_strsplit(uri, ":"); + for(argc=0; args[argc] != NULL; argc++); + + if (argc == 3) { /* host:ip:somewhere --> argc == 3 */ + if (net_addr2ip(args[0], &ip) < 0) { + *error_r = t_strdup_printf("Invalid IP: %s in URI: %s", args[0], uri); + return -1; + } + if (net_str2port(args[1], &port) < 0) { + *error_r = t_strdup_printf("Invalid port: %s in URI: %s", args[1], uri); + return -1; + } + dest_uri = strrchr(uri, ':'); + } else if (uri[0] == ':') { /* default path */ path = t_strconcat(set->base_dir, "/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL); @@ -784,7 +802,13 @@ client_dict_init(struct dict *driver, const char *uri, path = t_strconcat(set->base_dir, "/", t_strdup_until(uri, dest_uri), NULL); } - connection_init_client_unix(dict_connections, &dict->conn.conn, path); + if (port > 0) { + connection_init_client_ip(dict_connections, &dict->conn.conn, &ip, port); + } else { + connection_init_client_unix(dict_connections, &dict->conn.conn, path); + } dict->uri = i_strdup(dest_uri + 1); dict->ioloop = io_loop_create(); But unfortunately this crashes: Jul 28 13:20:04 auth: Error: auth worker: Aborted PASSL request for info(a)outdoor-training.de: Worker process died unexpectedly Jul 28 13:20:04 auth-worker(705): Fatal: master: service(auth-worker): child 705 killed with signal 11 (core dumped) Jul 28 13:20:04 doveadm(10.44.88.1,info(a)outdoor-training.de): Error: user info(a)outdoor-training.de: Auth PASS lookup failed It looks like the tcp connection gets opened non-blocking and the first write / dict lookup happens to early: 4303041 13:44:25.120398220 0 auth (29884) < connect res=-115(EINPROGRESS) tuple=172.18.0.2:47552->10.44.99.180:2001 Looking at dict-memcached-ascii.c I probably need to do something like: i_array_init(&dict->input_states, 4); i_array_init(&dict->replies, 4); dict->ioloop = io_loop_create(); io_loop_set_current(old_ioloop); *dict_r = &dict->dict; to wait until the socket is ready ... Any idea / tips? Ralf -- Ralf Becker EGroupware GmbH [www.egroupware.org] Handelsregister HRB Kaiserslautern 3587 Geschäftsführer Birgit und Ralf Becker Leibnizstr. 17, 67663 Kaiserslautern, Germany Telefon +49 631 31657-0

2 2

rawlog
by Matt Bryant 04 Aug '17

04 Aug '17

Hi, Trying to get rawlog working on dovecot 2.2.31 configured as per https://wiki2.dovecot.org/Debugging/Rawlog but a) it doesnt appear to be loggin anything b) rawlog_dir which is supposed to be v2.2.26+ seems no where in site .. in fact complains about unknown variable So does rawlog still do anything ???? Or am I missing something ... config is below .... # 2.2.19: /etc/dovecot/dovecot.conf # Pigeonhole version 0.4.9 (357ac0a0e68b+) doveconf: Warning: service auth { client_limit=30000 } is lower than required under max. load (150032) doveconf: Warning: service anvil { client_limit=22000 } is lower than required under max. load (50027) # OS: Linux 3.10.0-327.4.4.el7.x86_64 x86_64 CentOS Linux release 7.2.1511 (Core) auth_cache_negative_ttl = 2 mins auth_cache_size = 10 M auth_cache_ttl = 10 mins auth_master_user_separator = * auth_mechanisms = plain login auth_worker_max_count = 10000 default_client_limit = 50000 default_process_limit = 50000 disable_plaintext_auth = no doveadm_password = # hidden, use -P to show it imap_client_workarounds = delay-newmail tb-extra-mailbox-sep tb-lsub-flags imap_hibernate_timeout = 1 mins imap_idle_notify_interval = 1 mins login_greeting = IMAP/POP3 ready - dev-dh-ro-ms-001-b mail_attachment_dir = /var/lib/dovecot/attachments/%Ld mail_cache_min_mail_count = 5 mail_plugins = " notify replication quota virtual" mailbox_list_index = yes managesieve_notify_capability = mailto managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate vacation-seconds spamtest spamtestplus editheader mbox_write_locks = fcntl mmap_disable = yes namespace { inbox = yes list = yes location = mailbox Archive { auto = create special_use = \Archive } mailbox Drafts { auto = create special_use = \Drafts } mailbox Sent { auto = create special_use = \Sent } mailbox Spam { auto = create special_use = \Junk } mailbox Trash { auto = create special_use = \Trash } prefix = INBOX/ separator = / type = private } namespace { list = no location = virtual:/var/lib/dovecot/virtual:INDEXPVT=~/virtual prefix = virtual/ separator = / type = private } passdb { args = /etc/dovecot/sql_users.conf driver = sql } plugin { mail_log_events = delete expunge mail_log_fields = uid box msgid size mail_replica = tcp:dev-ms-001-a:4000 quota = dict:UserQuota::file:%h/dovecot-quota quota_rule2 = INBOX/Trash:storage=+10%% sieve = file:~/sieve/user;active=~/.dovecot.sieve sieve_default = file:/var/lib/dovecot/sieve/default.sieve sieve_default_name = default sieve_editheader_max_header_size = 1k sieve_extensions = +spamtest +spamtestplus +editheader +vacation-seconds sieve_global = file:/var/lib/dovecot/sieve sieve_quota_max_scripts = 5 sieve_spamtest_max_value = 200 sieve_spamtest_status_header = X-Spam-score-int: -?([[:digit:]]+) sieve_spamtest_status_type = score sieve_vacation_default_period = 10d sieve_vacation_max_period = 30d sieve_vacation_min_period = 1s } pop3_client_workarounds = outlook-no-nuls oe-ns-eoh pop3_fast_size_lookups = yes protocols = imap pop3 lmtp sieve replication_dsync_parameters = -d -n INBOX -l 30 -U service aggregator { fifo_listener replication-notify-fifo { user = vmail } unix_listener replication-notify { user = vmail } } service anvil { client_limit = 22000 } service auth-worker { user = $default_internal_user } service auth { client_limit = 30000 unix_listener auth_client { mode = 0660 user = exim } } service doveadm { client_limit = 1 inet_listener { address = * port = 4000 } process_limit = 80 process_min_avail = 8 service_count = 10 } service imap-hibernate { client_limit = 8000 process_limit = 8 process_min_avail = 8 service_count = 0 unix_listener imap-hibernate { group = vmail mode = 0660 } } service imap-login { client_limit = 8000 inet_listener imap { port = 143 } process_limit = 8 process_min_avail = 8 service_count = 0 } service imap { client_limit = 1 process_limit = 50000 service_count = 100 } service managesieve-login { client_limit = 1000 inet_listener sieve { port = 4190 } process_limit = 8 process_min_avail = 8 service_count = 0 } service managesieve { process_limit = 1024 } service pop3-login { client_limit = 1000 inet_listener pop3 { port = 110 } process_limit = 8 process_min_avail = 8 service_count = 0 } service pop3 { process_limit = 10000 } service postlogin { executable = script-login -d rawlog unix_listener postlogin { group = atmail mode = 0660 } } service replicator { process_min_avail = 1 unix_listener replicator-doveadm { group = atmail mode = 0660 } } shutdown_clients = no ssl_cert = </etc/pki/dovecot/certs/dovecot.pem ssl_key = </etc/pki/dovecot/private/dovecot.pem userdb { driver = prefetch } userdb { args = /etc/dovecot/sql_users.conf driver = sql } protocol imap { mail_max_userip_connections = 30000 mail_plugins = " notify replication quota virtual imap_quota" } protocol pop3 { mail_max_userip_connections = 30000 mail_plugins = " notify replication quota virtual" } protocol lmtp { auth_username_format = %Lu mail_plugins = " notify replication quota virtual sieve quota" postmaster_address = mailer-daemon }

3 3

Re: rawlog
by Joseph Tam 04 Aug '17

04 Aug '17

Matt Bryant wrote: > a) it doesnt appear to be loggin anything > b) rawlog_dir which is supposed to be v2.2.26+ seems no where in site .. > in fact complains about unknown variable > ... > service postlogin { > executable = script-login -d rawlog > unix_listener postlogin { > group = atmail > mode = 0660 > } > } In my test setup, I use the first option protocol imap { ... rawlog_dir = /data/rawlogs/%u } You may also want it into your protocol pop3 section. Joseph Tam <jtam.home(a)gmail.com>

1 0

pam auth problem
by Randy Bush 03 Aug '17

03 Aug '17

# dovecot --version 2.2.31 (65cde28) on freebsd 64 10.3 system converted to dovecot 2 against my will and consuming a lot of time sorting it out. i am glad google does not charge. have spent two hours on this one alone; and undoubtedly it is my st00pidity. so excuse my desperate posting to lazynet. cram-md5 works, pam not so much Aug 3 06:06:35 psg auth: in openpam_parse_chain(): /etc/pam.d/dovecot(1): missing or invalid facility Aug 3 06:06:35 psg dovecot: auth-worker(48815): Error: pam(elb,2604:6000:130d:c31b:d250:99ff:fe90:14dd,<Lyh6LtNVMq8mBGAAEw3DG9JQmf/+kBTd>): pam_start() failed: system error Aug 3 06:06:35 psg auth: in openpam_parse_chain(): /etc/pam.d/dovecot(1): missing or invalid facility Aug 3 06:06:35 psg dovecot: auth-worker(48815): Error: pam(elb,2604:6000:130d:c31b:d250:99ff:fe90:14dd,<Lyh6LtNVMq8mBGAAEw3DG9JQmf/+kBTd>): pam_start() failed: system error # 2.2.31 (65cde28): /usr/local/etc/dovecot/dovecot.conf # OS: FreeBSD 10.3-RELEASE-p20 amd64 auth_mechanisms = plain login cram-md5 first_valid_gid = 0 mail_location = mbox:~/mail/:INBOX=/var/mail/%u mail_privileged_group = mail namespace inbox { inbox = yes location = prefix = } passdb { args = scheme=cram-md5 /usr/local/etc/dovecot.cram-md5 driver = passwd-file name = passwd-file } passdb { driver = pam } passdb { driver = pam name = pam } pop3_client_workarounds = outlook-no-nuls oe-ns-eoh protocols = imap pop3 service auth { unix_listener auth-userdb { group = mail } } service imap-login { inet_listener imaps { port = 993 ssl = yes } } service pop3-login { inet_listener pop3 { port = 110 } inet_listener pop3s { port = 995 ssl = yes } } ssl_cert = </etc/ssl/certs/dovecot.pem ssl_key = # hidden, use -P to show it userdb { driver = passwd } # cat /etc/pam.d/dovecot passdb { driver = pam args = failure_show_msg=yes args = %s }

5 10

Replication and public folders with private (seen) flags
by Ralf Becker 03 Aug '17

03 Aug '17

We started using Dovecot replication between two nodes and noticed that our configured private flags (INDEXPVT) in public/shared mailboxes are not replicated. We are only replicating INBOX namespace, as we dont want to replicate content of shared mailboxes for every user again. Is there a way to replicate the INDEXPVT or is that not (yet) implemented? Dovecot versions: # 2.2.31 (65cde28): /etc/dovecot/dovecot.conf # Pigeonhole version 0.4.19 (e5c7051) # OS: Linux 4.4.0-87-generic x86_64 Using following namespaces: namespace inboxes { inbox = yes location = mailbox Drafts { auto = subscribe special_use = \Drafts } mailbox Junk { auto = subscribe special_use = \Junk } mailbox Sent { auto = subscribe special_use = \Sent } mailbox Templates { auto = subscribe } mailbox Trash { auto = subscribe special_use = \Trash } prefix = INBOX/ separator = / subscriptions = no } namespace subs { hidden = yes list = no location = prefix = separator = / } namespace users { location = mdbox:%%h/mdbox:INDEXPVT=~/shared/%%u prefix = user/%%n/ separator = / subscriptions = no type = shared } And the following replication config: root@ka-nfs-mail:~# cat /etc/dovecot/replication.conf service aggregator { fifo_listener replication-notify-fifo { user = dovecot } unix_listener replication-notify { user = dovecot } } service replicator { process_min_avail = 1 unix_listener replicator-doveadm { group = dovecot user = dovecot mode = 0660 } } service doveadm { inet_listener { port = 12345 #ssl = yes } } doveadm_port = 12345 doveadm_password = ******* plugin { #mail_replica = tcp:10.44.99.1 # use doveadm_port mail_replica = tcp:10.44.88.1 # use doveadm_port } replication_dsync_parameters = -d -n INBOX -l 30 -U -- Ralf Becker EGroupware GmbH [www.egroupware.org] Handelsregister HRB Kaiserslautern 3587 Geschäftsführer Birgit und Ralf Becker Leibnizstr. 17, 67663 Kaiserslautern, Germany Telefon +49 631 31657-0

1 0

Doveadm-sync SSH practicalities
by Terry Jones 02 Aug '17

02 Aug '17

Hi, The documentation is somewhat silent on this subject. What permissions does the SSH user need ? How associated does it need to be with things like dovecot directory ownership etc ? Obviously my dovecot daemon processes are running as restricted users with "nologin" shells etc., and I don't really want to go opening them up if I don't have to. So could someone kindly please expand a little on how exactly doveadm-sync is supposed to work with SSH ? Thanks Terry

1 0

result of uid sort by subject
by TACHIBANA Masashi 02 Aug '17

02 Aug '17

Result of uid sort by subject is not expected. for example: Japanese => English => Japanese => English I'm now using Dovecot 2.2.31. command example: HNKK6 UID SORT (REVERSE SUBJECT) utf-7 ALL * SORT 3 1 7 2 4 10 11 8 12 HNKK6 OK Sort completed (0.002 + 0.041 + 0.001 secs). Any hint? -- Masashi Astro TACHIBANA QUALITIA CO., LTD. mailto:tachibana@qualitia.co.jp

2 2

Corrupt index files
by Bruce Guenter 02 Aug '17

02 Aug '17

I am running Dovecot IMAP on Linux, on a LizardFS storage cluster with Maildir storage. This has worked well for most of the accounts for several months. However in the last couple of weeks we are seeing increasing errors regarding corrupted index files. Some of the accounts affected are unable to retrieve messages due to timeouts. It appeared the problems were due to the accounts being accessed from multiple servers simultaneously, so I forced them all to access one server, but the errors remained. It looks like it has something to do with file locking, but LizardFS supports advisory file locking and I do have it enabled. Deleting the corrupted indexes fixes the problem for a while, but it eventually returns, particularly for some accounts. Here are some errors I'm seeing (just a random grab). Actual home directories are munged for confidentiality. imap[25157]: (clientes.standby) Error: Failed to fix view for HOME/clientes:standby/dovecot.index: Missing middle file seq=1 (between 1..1, we have seqs 8): File is already open imap[5565]: (stadiumchair) Error: Transaction log file HOME/stadiumchair/.Drafts/dovecot.index.log: marked corrupted imap[5005]: (stadiumchair) Error: Corrupted transaction log file HOME/stadiumchair/.Drafts/dovecot.index.log seq 2: indexid changed 1418941056 -> 1500658549 (sync_offset=0) imap[20243]: (martha) Error: Transaction log HOME/martha/dovecot.index.log: duplicate transaction log sequence (539) imap[4665]: (emsspam) Error: Index file HOME/emsspam/dovecot.index: indexid changed: 1500658479 -> 1297175382 imap[4665]: (emsspam) Error: Corrupted transaction log file HOME/emsspam/dovecot.index.log seq 3: indexid changed: 1500658479 -> 1297175382 (sync_offset=316) imap[22985]: (emsspam) Error: Corrupted transaction log file HOME/emsspam/dovecot.index.log seq 10742: Invalid transaction log size (9296 vs 9296): HOME/emsspam/dovecot.index.log (sync_offset=9296) imap[3267]: (emsspam) Error: Failed to map view for HOME/emsspam/dovecot.index: Failed to map file seq=10742 offset=9052..18446744073709551615 (ret=0): corrupted, indexid=0 imap[3267]: (emsspam) Error: HOME/emsspam/dovecot.index view is inconsistent: uid=3062271 inserted in the middle of mailbox The output of dovecot -n is pasted in below. Note that some of the boxes are running 4.9, some running 4.4, all have the same problems. Also note that I am using a custom authentication front end for our virtual mailboxes, but it just sets up the minimal environment variables and runs imap. Is there anything I can change to eliminate these problems? Are there any other diagnostics I can provide to shed light on this? # 2.2.31 (65cde28): /etc/dovecot/dovecot.conf # OS: Linux 4.4.66 x86_64 Gentoo Base System release 2.3 log_path = /dev/stderr mail_debug = yes mail_fsync = always mail_location = maildir:~/.maildir mail_log_prefix = "%s[%p]: (%u) " mmap_disable = yes namespace inbox { inbox = yes location = mailbox Drafts { special_use = \Drafts } mailbox Junk { special_use = \Junk } mailbox Sent { special_use = \Sent } mailbox "Sent Messages" { special_use = \Sent } mailbox Trash { special_use = \Trash } prefix = INBOX separator = type = private } passdb { args = * driver = pam } passdb { args = /etc/dovecot/dovecot-sql.conf.ext driver = sql } plugin { mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename } ssl_cert = </etc/ssl/dovecot/server.pem ssl_key = # hidden, use -P to show it userdb { driver = passwd } userdb { args = /etc/dovecot/dovecot-sql.conf.ext driver = sql } -- Bruce Guenter <bruce(a)untroubled.org> http://untroubled.org/

5 11