Hi all,

I'm trying to replace a shared NFS (rock solid, btw) with a replicated two-node setup. However I've been
struggling with an issue connected in some way with the replication between the two servers (master-master).

The issue is once a folder (say "aaa") is created one one account, it cannot be renamed or deleted. If deleted, it will instantly reappear (when replication is running) or when renamed to "bbb", the operation will fail with an error like the one below:

Jan 15 18:21:22 ms1 dovecot: dsync-local(test2@mydomain.ext)<JtJJARLPAWBPJAAAxdOQ8Q>: Error: rename(/var/vmail-local/mydomain.ext/test2/idx/.bbb, /var/vmail-local/mydomain.ext/test2/idx/.aaa) failed: Directory not empty
Jan 15 18:21:22 ms1 dovecot: dsync-local(test2@mydomain.ext)<JtJJARLPAWBPJAAAxdOQ8Q>: Error: rename(/var/vmail-local/mydomain.ext/test2/idx/.bbb, /var/vmail-local/mydomain.ext/test2/idx/.aaa) failed: Directory not empty

I can see that the index directory contains both the .aaa and .bbb index directories with dovecot.log files inside.

If I stop the one of the nodes all works nicely and I can create, delete and rename folders at will.

My setup is as below.

dovecot --version -> 2.3.4.1 (f79e8e7e4)
the indexes are in a separate directory on /var/vmail-local/ (performance reasons).
the mail spool is mounted on /var/vmail/
One node has local disks for everything. It runs with mmap_disable=no, mail_fsync=optimized, mail_nfs_storage=no, mail_nfs_index=no
The other node has an NFS based /var/vmail/ and local disks for indexes. On this node I'm using the following settings:
mmap_disable=yes, mail_fsync=always, mail_nfs_storage=yes, mail_nfs_index=no
The nfs server is v4 

The two servers are "orchestrated" by a two-node director cluster.

I'm sure I'm missing or misconfigured something.

The doveconf -n config is, part from the NFS quick, the same on both nodes as follows: 
--------
# 2.3.4.1 (f79e8e7e4): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.4 ()
doveconf: Warning: service auth { client_limit=1000 } is lower than required under max. load (2012)
doveconf: Warning: service anvil { client_limit=1000 } is lower than required under max. load (1515)
# OS: Linux 5.4.78-2-pve x86_64 Debian 10.7
# Hostname: dovecot1.mydomain.ext
auth_master_user_separator = +
auth_mechanisms = plain login
default_process_limit = 500
disable_plaintext_auth = no
doveadm_password = # hidden, use -P to show it
doveadm_port = 12727
imap_capability = +SPECIAL-USE XLIST
listen = *,[::]
lmtp_rcpt_check_quota = yes
log_timestamp = "%Y-%m-%d %H:%M:%S "
login_trusted_networks = 172.16.20.0/24 172.16.30.0/24
mail_fsync = always
mail_max_userip_connections = 100
mail_nfs_storage = yes
mail_plugins = quota notify replication lazy_expunge acl
mail_privileged_group = vmail
mmap_disable = yes
namespace inbox {
  inbox = yes
  location =
  mailbox Drafts {
    special_use = \Drafts
  }
  mailbox EXPUNGED {
    autoexpunge = 1 days
    autoexpunge_max_mails = 20
  }
  mailbox Junk {
    special_use = \Junk
  }
  mailbox Sent {
    special_use = \Sent
  }
  mailbox "Sent Messages" {
    special_use = \Sent
  }
  mailbox Trash {
    special_use = \Trash
  }
  prefix =
  separator = .
}
passdb {
  args = /etc/dovecot/dovecot-sql.conf
  driver = sql
}
passdb {
  args = /etc/dovecot/master-users
  driver = passwd-file
  master = yes
  result_success = continue
}
passdb {
  driver = pam
}
plugin {
  acl = vfile:/etc/dovecot/dovecot.acl
  lazy_expunge = EXPUNGED
  mail_replica = tcp:172.16.30.100:12727
  quota = dict:user::file:/var/vmail/%d/%n/.quotausage
  quota_rule = EXPUNGED:ignore
  quota_status_nouser = DUNNO
  quota_status_overquota = 552 5.2.2 Mailbox is full
  quota_status_success = DUNNO
  sieve = /var/vmail/%d/%n/.sieve
  sieve_after = /var/vmail/%d/%n/.ispconfig.sieve
  sieve_before = /var/vmail/%d/%n/.ispconfig-before.sieve
  sieve_max_actions = 100
  sieve_max_redirects = 25
  sieve_max_script_size = 2M
}
protocols = imap pop3 lmtp
service aggregator {
  fifo_listener replication-notify-fifo {
    group = root
    mode = 0660
    user = vmail
  }
  unix_listener replication-notify {
    group = root
    mode = 0660
    user = vmail
  }
}
service auth {
  unix_listener /var/spool/postfix/private/auth {
    group = postfix
    mode = 0660
    user = postfix
  }
  unix_listener auth-userdb {
    group = vmail
    mode = 0600
    user = vmail
  }
  user = root
}
service doveadm {
  inet_listener {
    port = 12727
  }
}
service imap-login {
  client_limit = 1000
  process_limit = 512
}
service lmtp {
  inet_listener lmtp {
    port = 24
  }
  unix_listener /var/spool/postfix/private/dovecot-lmtp {
    group = postfix
    mode = 0600
    user = postfix
  }
}
service quota-status {
  client_limit = 1
  executable = quota-status -p postfix
  unix_listener /var/spool/postfix/private/quota-status {
    group = postfix
    mode = 0660
    user = postfix
  }
}
service replicator {
  process_min_avail = 1
  unix_listener replicator-doveadm {
    mode = 0600
    user = vmail
  }
}
service stats {
  unix_listener stats-reader {
    group = vmail
    mode = 0660
    user = vmail
  }
  unix_listener stats-writer {
    group = vmail
    mode = 0660
    user = vmail
  }
}
ssl = required
ssl_cert = </etc/letsencrypt/live/mail.mydomain.ext/fullchain.pem
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
userdb {
  driver = prefetch
}
userdb {
  args = /etc/dovecot/dovecot-sql.conf
  driver = sql
}
userdb {
  driver = passwd
}
protocol imap {
  auth_verbose = yes
  mail_plugins = quota imap_quota notify replication lazy_expunge acl
}
protocol pop3 {
  auth_verbose = yes
  mail_plugins = quota notify replication lazy_expunge acl
  pop3_uidl_format = %08Xu%08Xv
}
protocol lda {
  mail_plugins = sieve quota notify replication lazy_expunge acl
  postmaster_address = postmaster@dovecot1.mydomain.ext
}
protocol lmtp {
  mail_plugins = quota sieve notify replication lazy_expunge acl
  postmaster_address = postmaster@dovecot1.mydomain.ext
}
--------------

My sql config is this:
---
password_query = SELECT email as user, password, maildir as userdb_home, CONCAT( maildir_format, ':', maildir, '/', IF(maildir_format='maildir','Maildir',maildir_format)) as userdb_mail, uid as userdb_uid, gid as userdb_gid, CONCAT('*:storage=', quota, 'B') AS userdb_quota_rule, CONCAT(maildir, '/.sieve') as userdb_sieve FROM mail_user WHERE (login = '%u' OR email = '%u') AND `disable%Ls` = 'n' AND server_id = '20' AND EXISTS (SELECT domain_id FROM mail_domain WHERE domain = '%d' AND active = 'y' AND server_id = 20)

user_query = SELECT email as user, maildir as home, CONCAT( maildir_format, ':', maildir, '/', IF(maildir_format='maildir','Maildir',maildir_format),':INDEX=/var/vmail-local/%d/%n/idx') as mail, uid, gid, CONCAT('*:storage=', quota, 'B') AS quota_rule, CONCAT(maildir, '/.sieve') as sieve FROM mail_user WHERE (login = '%u' OR email = '%u') AND `disable%Ls` = 'n' AND server_id = '20'

---

Best regards,
Dave