Userspace code coverage - ligurio/openbsd-tests GitHub Wiki

Preparation

Install required packages:

  • Regression suite requirements: pkg_add net-snmp scapy py-libdnet p5-Net-Flow p5-BSD-Socket-Splice p5-IO-Socket-INET6 p5-Socket6 p5-File-Slurp p5-BSD-Resource p5-IO-Socket-SSL p5-ldap openldap-client-- p5-AnyEvent p5-Hash-Merge p5-YAML-1.19 p5-NetPacket p5-Net-Pcap p5-Crypt-Random

  • cmake (to build compiler-rt)

  • llvm (to build compiler-rt)

  • python-2.7

  • git

  • py-pip (gcovr)

  • pip install gcovr

  • the_silver_searcher (for convenience)

  • vim--no_x11 (for convenience)

$ doas pkg_add cmake llvm python-2.7 git py-pip the_silver_searcher vim--no_x11 mc
$ cd /usr/src
$ ftp https://mirror.yandex.ru/pub/OpenBSD/6.2/src.tar.gz
$ ftp https://mirror.yandex.ru/pub/OpenBSD/6.2/sys.tar.gz
$ tar xvzf src.tar.gz
$ tar xvzf sys.tar.gz
or
CVS (https://www.openbsd.org/anoncvs.html)
or
$ git clone https://github.com/openbsd/src
$ ftp -o - https://ftp.eu.openbsd.org/pub/OpenBSD/snapshots/timestamp
$ cd src; git reset --hard $(git rev-list -1 $(git rev-parse --until=TIMESTAMP) master)

/etc/mk.conf

COMPILER_VERSION?=clang

SKIPDIR+=       lib/csu
SKIPDIR+=       libexec/ld.so
SKIPDIR+=       games                   # not interested
SKIPDIR+=       gnu                     # not interested
SKIPDIR+=       sys
SKIPDIR+=       regress/sys             # save time on running tests
SKIPDIR+=       regress/gnu             # save time on running tests
SKIPDIR+=       usr.sbin/bind           # configure: error: C compiler cannot create executables
SKIPDIR+=       usr.sbin/nsd            # configure: error: C compiler cannot create executables
SKIPDIR+=       usr.sbin/unbound        # configure: error: C compiler cannot create executables

.if defined(COVERAGE)

CFLAGS+=        --coverage
CLEANFILES+=    *.gcov *.gcda *.gcno coverage.info

.if ${COMPILER_VERSION} == "clang"
LDADD+=         -lclang_rt.profile
LDFLAGS+=       -lclang_rt.profile
LIBS+=          -lclang_rt.profile
.elif ${COMPILER_VERSION} == "gcc4" || ${COMPILER_VERSION} == "gcc3"
LDADD+=         -lgcov
LDFLAGS+=       -lgcov
LIBS+=          -lgcov
.endif

.endif # COVERAGE

Disable pledge(2)

Since version 5.9 OpenBSD includes pledge() system call. It is a useful for mitigation security attacks but unfortunately it prevents to collecting of code coverage data. You will get SIGABRT in a target testing binary on attempt to run tests with it:

Testing suffix "/usr/bin/gcc" "xx"
===> bc
rm -f *.log t19
t1
Abort trap (core dumped)
*** Error 134 in target 'regress' (ignored)
Abort trap (core dumped)
*** Error 134 in bc (<bsd.regress.mk>:106 'regress': @echo usr.bin/bc/t1 | tee -a /dev/null /dev/null 2>&1 > /dev/null)
*** Error 1 in /usr/src/regress/usr.bin (<bsd.subdir.mk>:48 'all')
# dmesg | tail -2
tee(5031): syscall 5 "rpath"
tee(79812): syscall 5 "rpath"
Abort trap (core dumped)
#

To avoid such issues we need to add additional promises to each program where it absent using script addpromise.py. From my experience only four promises are required: wpath, rpath, cpath and flock.

$ find /usr/src \( -path /usr/src/gnu -o -path /usr/src/sys -o -path /usr/src/games -o -path /usr/src/regress \) -prune -o -name '*.[c,y]' -type f -exec addpromise.py {} \;

For example cat(1) source file will look after this like:

diff --git a/bin/cat/cat.c b/bin/cat/cat.c
index ed28a7f44bf..9a1d17c4a2f 100644
--- a/bin/cat/cat.c
+++ b/bin/cat/cat.c
@@ -63,7 +63,7 @@ main(int argc, char *argv[])
 {
        int ch;

-       if (pledge("stdio rpath", NULL) == -1)
+       if (pledge("stdio rpath cpath wpath", NULL) == -1)
                err(1, "pledge");

        while ((ch = getopt(argc, argv, "benstuv")) != -1)
  • Now we are ready to build a system from sources. All required steps are described in the release(8). But there is one note: don't do 'make obj' because we should place obj files to the same directory where sources are place.

Disable chroot(2)

There is only one way to disable chroot - comment out appropriate lines in a source code.

Disable privilege separation in daemons

There two ways to disable privilege separation:

  • comment out appropriate lines in a source code
setgroups(1, &pw->pw_gid) == -1)
        err(1, "setgroups() failed");
if (setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) == -1)
        err(1, "setresgid() failed");
if (setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid) == -1)
        err(1, "setresuid() failed");

I did it for each component in userspace except: unbound, nsd, inetd, lpd, chroot and authpf.

  • give more rights to user account using as separation account

Compiler-specific tweaks in a system

Clang
  • unfortunately libcompiler_rt is in OpenBSD source tree, but without required functions (libprofile)
  • git clone https://github.com/llvm-mirror/compiler-rt
  • echo "set(COMPILER_RT_HAS_PROFILE TRUE)" >> cmake/config-ix.cmake
  • mkdir build; cd build; cmake ../; make install
  • ln -s /usr/local/lib/openbsd/libclang_rt.profile-x86_64.a /usr/lib/libclang_rt.profile.a
GCC
  • create symlink for gcov library: ln -s /usr/lib/gcc-lib/amd64-unknown-openbsd6.2/4.2.1/libgcov.a /usr/lib/libgcov.a
  • use COMPILER_VERSION=gcc4 instead of clang

Code coverage measurement

# cd /usr/src
# COVERAGE=1 COMPILER_VERSION="clang" make clean
# COVERAGE=1 COMPILER_VERSION="clang" make 2>&1 | tee build.log
# COVERAGE=1 COMPILER_VERSION="clang" make install
# ??? COVERAGE=1 COMPILER_VERSION="clang" make REGRESS_LOG=/var/log/regress-tests regress
# COVERAGE=1 COMPILER_VERSION="clang" make regression-tests 2>&1 | tee regress.log  # don't forget to install regress dependencies (https://github.com/ligurio/openbsd-tests/wiki/How-to-run-tests#how-to-run-regress-tests)
# ln lib/libcurses/tinfo/doalloc.c lib/libcurses/doalloc.c
# gcovr --html --html-details --output=coverage.html --verbose --exclude-directories="regress" --print-summary --root `pwd` .
# gcovr --xml --xml-pretty --verbose --exclude-directories="regress" --print-summary --root `pwd` .

$ cd /usr/src
$ COVERAGE=1 COMPILER_VERSION="clang" make 2>&1 | tee build.log
$ COVERAGE=1 COMPILER_VERSION="clang" make install
$ COVERAGE=1 COMPILER_VERSION="clang" make -C regress/sbin/iked/parser/ 2>&1 | tee regress.log
$ COVERAGE=1 COMPILER_VERSION="clang" make -C regress/usr.bin/mandoc/db/dbm_dump 2>&1 | tee regress.log
$ make -C regress 2>&1 | tee regress.log

$ COVERAGE=1 COMPILER_VERSION="clang" make -C libexec 2>&1 | tee -a build.log
$ COVERAGE=1 COMPILER_VERSION="clang" make -C libexec install
$ make -C regress/libexec 2>&1 | tee -a regress.log

$ COVERAGE=1 COMPILER_VERSION="clang" make -C lib 2>&1 | tee -a build.log
$ COVERAGE=1 COMPILER_VERSION="clang" make -C lib install
$ COVERAGE=1 COMPILER_VERSION="clang" make -C regress/lib 2>&1 | tee -a regress.log

$ ln lib/libcurses/tinfo/doalloc.c lib/libcurses/doalloc.c
$ gcovr --html --html-details --output=coverage.html --exclude-directories="regress" --print-summary --root `pwd` .
$ gcovr --xml --xml-pretty --exclude-directories="regress" --print-summary --root `pwd` .