Reproducible SIGSEGV when Dovecot 2.3 compiled against glibc-2.28
Reuben Farrelly
reuben-dovecot at reub.net
Wed Aug 8 06:54:03 EEST 2018
Hi,
Dovecot 2.3 (release and current -git) versions compile, but fail to run
when compiled against glibc-2.28.
This is what is logged on startup:
Aug 8 08:24:39 thunderstorm.reub.net dovecot[569]: master: Dovecot
v2.3.2.1 (0719df592) starting up for imap, lmtp, sieve, submission, sieve
Aug 8 08:24:39 thunderstorm.reub.net dovecot[569]: master: Error:
service(auth): command startup failed, throttling for 2 secs
Aug 8 08:24:39 thunderstorm.reub.net dovecot[574]: auth: Fatal: master:
service(auth): child 582 killed with signal 11 (core dumped)
Aug 8 08:24:39 thunderstorm.reub.net dovecot[574]: replicator: Error:
userdb lookup: Disconnected unexpectedly
Aug 8 08:24:52 thunderstorm.reub.net dovecot[569]: master: Warning:
Killed with signal 15 (by pid=670 uid=0 code=kill)
The issue is specifically with the 'auth' binary. Other components all
appear to be unaffected. The 'auth' binary dies with a Segmentation
Fault when run as a standalone executable too.
As the auth binary is critical to many different parts of Dovecot, a
failure of this is catastrophic.
This is a 100% reproducible problem. The platform is Gentoo x86_64.
thunderstorm /usr/libexec/dovecot # ./auth-old
Segmentation fault
thunderstorm /usr/libexec/dovecot #
[I've renamed the original binary to auth-old, and put in it's place a
working 'auth' binary built against glibc-2.27 in order to have a
functioning system]
Problem matrix looks like this:
Build on a glibc-2.27 system, run on a glibc-2.27 - OK
Build on a glibc-2.27 system, run on a glibc-2.28 - OK
Build on a glibc-2.28 system, run on a glibc-2.27 - SEGFAULT
Build on a glibc-2.28 system, run on a glibc-2.28 - SEGFAULT
(All other components including gcc otherwise identical)
./configure --prefix=/usr --build=x86_64-pc-linux-gnu
--host=x86_64-pc-linux-gnu --mandir=/usr/share/man
--infodir=/usr/share/info --datadir=/usr/share --sysconfdir=/etc
--localstatedir=/var/lib --disable-dependency-tracking
--disable-silent-rules --docdir=/usr/share/doc/dovecot-9999_p20180807
--htmldir=/usr/share/doc/dovecot-9999_p20180807/html --libdir=/usr/lib64
--with-rundir=/run/dovecot --with-statedir=/var/lib/dovecot
--with-moduledir=/usr/lib64/dovecot --without-stemmer --disable-rpath
--without-libbsd --with-icu --with-ssl
--with-systemdsystemunitdir=/lib/systemd/system --with-sodium
--with-bzlib --without-libcap --without-gssapi --without-lua
--without-ldap --with-lucene --with-lz4 --with-lzma --without-mysql
--with-pam --without-pgsql --without-sqlite --without-solr
--with-libwrap --without-textcat --without-vpopmail --with-zlib
--disable-static
Strace:
thunderstorm /usr/libexec/dovecot # strace ./auth-old
execve("./auth-old", ["./auth-old"], 0x7ffd17c804c0 /* 27 vars */) = 0
brk(NULL) = 0x557e9dc28000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
directory)
openat(AT_FDCWD,
"/usr/lib64/dovecot/old-stats/tls/x86_64/x86_64/libstats_auth.so",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/old-stats/tls/x86_64/x86_64", 0x7ffcc7973020) =
-1 ENOENT (No such file or directory)
openat(AT_FDCWD,
"/usr/lib64/dovecot/old-stats/tls/x86_64/libstats_auth.so",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/old-stats/tls/x86_64", 0x7ffcc7973020) = -1
ENOENT (No such file or directory)
openat(AT_FDCWD,
"/usr/lib64/dovecot/old-stats/tls/x86_64/libstats_auth.so",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/old-stats/tls/x86_64", 0x7ffcc7973020) = -1
ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/tls/libstats_auth.so",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/old-stats/tls", 0x7ffcc7973020) = -1 ENOENT (No
such file or directory)
openat(AT_FDCWD,
"/usr/lib64/dovecot/old-stats/x86_64/x86_64/libstats_auth.so",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/old-stats/x86_64/x86_64", 0x7ffcc7973020) = -1
ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/x86_64/libstats_auth.so",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/old-stats/x86_64", 0x7ffcc7973020) = -1 ENOENT
(No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/x86_64/libstats_auth.so",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/old-stats/x86_64", 0x7ffcc7973020) = -1 ENOENT
(No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/libstats_auth.so",
O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\t\0\0\0\0\0\0"..., 832)
= 832
fstat(3, {st_mode=S_IFREG|0755, st_size=18848, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f3eef676000
mmap(NULL, 2105632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f3eef24f000
mprotect(0x7f3eef251000, 2093056, PROT_NONE) = 0
mmap(0x7f3eef450000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f3eef450000
close(3) = 0
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/tls/x86_64/x86_64/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/tls/x86_64/x86_64", 0x7ffcc7973000) = -1 ENOENT
(No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/tls/x86_64/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/tls/x86_64", 0x7ffcc7973000) = -1 ENOENT (No
such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/tls/x86_64/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/tls/x86_64", 0x7ffcc7973000) = -1 ENOENT (No
such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/tls/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/tls", 0x7ffcc7973000) = -1 ENOENT (No such file
or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/x86_64/x86_64/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/x86_64/x86_64", 0x7ffcc7973000) = -1 ENOENT (No
such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/x86_64/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/x86_64", 0x7ffcc7973000) = -1 ENOENT (No such
file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/x86_64/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/dovecot/x86_64", 0x7ffcc7973000) = -1 ENOENT (No such
file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/libdovecot.so.0",
O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\277\3\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=4783816, ...}) = 0
mmap(NULL, 4186392, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f3eeee50000
mprotect(0x7f3eef043000, 2093056, PROT_NONE) = 0
mmap(0x7f3eef242000, 40960, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f2000) = 0x7f3eef242000
mmap(0x7f3eef24c000, 8472, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3eef24c000
close(3) = 0
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/libcrypt.so.1",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/libcrypt.so.1", O_RDONLY|O_CLOEXEC)
= -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=54433, ...}) = 0
mmap(NULL, 54433, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3eef668000
close(3) = 0
openat(AT_FDCWD, "/lib64/libcrypt.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0
\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=39064, ...}) = 0
mmap(NULL, 2322976, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f3eeec18000
mprotect(0x7f3eeec20000, 2097152, PROT_NONE) = 0
mmap(0x7f3eeee20000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x7f3eeee20000
mmap(0x7f3eeee22000, 184864, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3eeee22000
close(3) = 0
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/libpam.so.0",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/libpam.so.0", O_RDONLY|O_CLOEXEC) =
-1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libpam.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200&\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=55840, ...}) = 0
mmap(NULL, 2151000, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f3eeea0a000
mprotect(0x7f3eeea17000, 2093056, PROT_NONE) = 0
mmap(0x7f3eeec16000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc000) = 0x7f3eeec16000
close(3) = 0
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/libsodium.so.23",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/libsodium.so.23",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/libsodium.so.23", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\305\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=318056, ...}) = 0
mmap(NULL, 2413576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f3eee7bc000
mprotect(0x7f3eee809000, 2093056, PROT_NONE) = 0
mmap(0x7f3eeea08000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4c000) = 0x7f3eeea08000
close(3) = 0
openat(AT_FDCWD, "/usr/lib64/dovecot/old-stats/libc.so.6",
O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/dovecot/libc.so.6", O_RDONLY|O_CLOEXEC) =
-1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\3407\2\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1869376, ...}) = 0
mmap(NULL, 3975016, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f3eee3f1000
mprotect(0x7f3eee5b3000, 2093056, PROT_NONE) = 0
mmap(0x7f3eee7b2000, 24576, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c1000) = 0x7f3eee7b2000
mmap(0x7f3eee7b8000, 14184, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3eee7b8000
close(3) = 0
openat(AT_FDCWD, "/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\16\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=14424, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f3eef666000
mmap(NULL, 2109712, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f3eee1ed000
mprotect(0x7f3eee1f0000, 2093056, PROT_NONE) = 0
mmap(0x7f3eee3ef000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f3eee3ef000
close(3) = 0
openat(AT_FDCWD, "/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0j\0\0\0\0\0\0"..., 832)
= 832
fstat(3, {st_mode=S_IFREG|0755, st_size=118024, ...}) = 0
mmap(NULL, 2229408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f3eedfcc000
mprotect(0x7f3eedfe8000, 2093056, PROT_NONE) = 0
mmap(0x7f3eee1e7000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b000) = 0x7f3eee1e7000
mmap(0x7f3eee1e9000, 13472, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3eee1e9000
close(3) = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f3eef663000
arch_prctl(ARCH_SET_FS, 0x7f3eef663b80) = 0
mprotect(0x7f3eee7b2000, 16384, PROT_READ) = 0
mprotect(0x7f3eee1e7000, 4096, PROT_READ) = 0
mprotect(0x7f3eee3ef000, 4096, PROT_READ) = 0
mprotect(0x7f3eeea08000, 4096, PROT_READ) = 0
mprotect(0x7f3eeec16000, 4096, PROT_READ) = 0
mprotect(0x7f3eeee20000, 4096, PROT_READ) = 0
mprotect(0x7f3eef242000, 28672, PROT_READ) = 0
mprotect(0x7f3eef450000, 4096, PROT_READ) = 0
mprotect(0x557e9c5e7000, 12288, PROT_READ) = 0
mprotect(0x7f3eef678000, 4096, PROT_READ) = 0
munmap(0x7f3eef668000, 54433) = 0
set_tid_address(0x7f3eef663e50) = 19762
set_robust_list(0x7f3eef663e60, 24) = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7f3eedfd2380, sa_mask=[],
sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f3eedfe0400}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7f3eedfd2430, sa_mask=[],
sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f3eedfe0400},
NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024,
rlim_max=RLIM64_INFINITY}) = 0
getrandom("\xfb\x47\x75\x83", 4, 0) = 4
brk(NULL) = 0x557e9dc28000
brk(0x557e9dc49000) = 0x557e9dc49000
uname({sysname="Linux", nodename="thunderstorm", ...}) = 0
getpid() = 19762
openat(AT_FDCWD, "/dev/null", O_WRONLY) = 3
fcntl(3, F_GETFD) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
rt_sigaction(SIGPIPE, {sa_handler=SIG_IGN, sa_mask=[],
sa_flags=SA_RESTORER|SA_RESTART, sa_restorer=0x7f3eee4297c0}, NULL, 8) = 0
rt_sigaction(SIGALRM, {sa_handler=0x7f3eeef9899b, sa_mask=[],
sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f3eee4297c0}, NULL, 8) = 0
openat(AT_FDCWD, "/proc/sys/crypto/fips_enabled", O_RDONLY) = -1 ENOENT
(No such file or directory)
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR,
si_addr=0xffffffffeee4f200} ---
+++ killed by SIGSEGV +++
Segmentation fault
thunderstorm /usr/libexec/dovecot #
gdb:
thunderstorm /var/core # gdb auth core.auth.18428
GNU gdb (Gentoo 8.1.1 p1) 8.1.1
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later
<http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-pc-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<https://bugs.gentoo.org/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from auth...done.
warning: exec file is newer than core file.
[New LWP 18428]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `dovecot/auth'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 __strcmp_sse2_unaligned ()
at ../sysdeps/x86_64/multiarch/strcmp-sse2-unaligned.S:31
31 ../sysdeps/x86_64/multiarch/strcmp-sse2-unaligned.S: No such
file or directory.
(gdb) bt full
#0 __strcmp_sse2_unaligned ()
at ../sysdeps/x86_64/multiarch/strcmp-sse2-unaligned.S:31
No locals.
#1 0x000055bbe3362dcf in password_scheme_register_crypt ()
at password-scheme-crypt.c:191
i = 0
crypted = 0xfffffffffdbcf200 <error: Cannot access memory at
address 0xfffffffffdbcf200>
__func__ = <optimized out>
#2 0x000055bbe33627cb in password_schemes_init () at password-scheme.c:874
i = 27
#3 0x000055bbe332a82a in main_preinit () at main.c:185
mod_set = {
abi_version = 0xfe2096c0 <error: Cannot access memory at
address 0xfe2096c0>,
binary_name = 0x599ce8cff6a85000 <error: Cannot access memory
at address 0x599ce8cff6a85000>,
setting_name = 0x7f1efdc1b011 <__x86_return_thunk+5>
"\363\220\017\256\350\353\371H\215d$\b\303\350\a",
filter_callback = 0x7f1efdc51029 <master_getopt+149>,
filter_context = 0x7f1efdc1b011 <__x86_return_thunk+5>,
require_init_funcs = false, debug = false,
ignore_dlopen_errors = false, ignore_missing = false}
services = 0x55bbe3819fa0
#4 0x000055bbe332aef5 in main (argc=1, argv=0x55bbe3819ae0) at main.c:392
c = -1
(gdb)
Release notes for glibc are here:
https://www.sourceware.org/ml/libc-alpha/2018-08/msg00003.htm
There are some notes about changes to crypt functions which could be
relevant given the gdb has references to crypt password schemes.
I have libgcrypt-1.8.3 installed but I _haven't_ specifically disabled
crypt in glibc (see release notes).
Thanks,
Reuben
More information about the dovecot
mailing list