- Fix CVE-2025-11411 (possible domain hijacking attack), reported by Yuxiao Wu,

Yunyi Zhang, Baojun Liu and Haixin Duan from Tsinghua University.

.github/workflows/analysis_ports.yml

@@ -32,12 +32,12 @@ jobs:
           - name: OS X
             os: macos-latest
             install_expat: "yes"
-            config: "--enable-debug --disable-flto --with-ssl=/usr/local/opt/openssl --with-libexpat=/usr/local/opt/expat"
+            config: "--enable-debug --disable-flto --with-ssl=/opt/homebrew/opt/openssl --with-libexpat=/opt/homebrew/opt/expat"
             make_test: "yes"
           - name: Clang on OS X
             os: macos-latest
             install_expat: "yes"
-            config: "CC=clang --enable-debug --disable-flto --with-ssl=/usr/local/opt/openssl --with-libexpat=/usr/local/opt/expat --disable-static"
+            config: "CC=clang --enable-debug --disable-flto --with-ssl=/opt/homebrew/opt/openssl --with-libexpat=/opt/homebrew/opt/expat --disable-static"
             make_test: "yes"
             clang_analysis: "yes"
           - name: ubsan (gcc undefined behaviour sanitizer)
@@ -161,13 +161,44 @@ jobs:
             test_windows: "yes"
             config: "no"
             make: "no"
+          - name: FreeBSD
+            os: ubuntu-latest
+            config: "no"
+            make: "no"
+            with_cross_platform_action: "yes"
+            cross_platform_os: "freebsd"
+            cross_platform_arch: "x86-64"
+            cross_platform_version: "14.1"
+            cross_platform_config: "--enable-debug --disable-flto --with-libevent --disable-static"
+          - name: OpenBSD
+            os: ubuntu-latest
+            config: "no"
+            make: "no"
+            with_cross_platform_action: "yes"
+            cross_platform_os: "openbsd"
+            cross_platform_arch: "x86-64"
+            cross_platform_version: "7.5"
+            cross_platform_config: "--enable-debug --disable-flto --with-libevent --disable-static"
+          - name: NetBSD
+            os: ubuntu-latest
+            config: "no"
+            make: "no"
+            with_cross_platform_action: "yes"
+            cross_platform_os: "netbsd"
+            cross_platform_arch: "x86-64"
+            cross_platform_version: "10.0"
+            cross_platform_config: "--enable-debug --disable-flto --with-libevent --disable-static"
  
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: false
+          persist-credentials: false
       - name: test_windows
         if: ${{ matrix.test_windows == 'yes' }}
+        env:
+          LIBEXPAT_FNAME: expat-2.7.0
+          LIBEXPAT_VERSION_DIR: R_2_7_0
         shell: bash
         run: |
           export unboundpath=`pwd`
@@ -181,21 +212,20 @@ jobs:
           export LEX="win_flex"
           echo 'YACC="win_bison -y"'
           export YACC="win_bison -y"
-          #echo "curl cpanm"
-          #curl -L -k -s -S -o cpanm https://cpanmin.us/
-          #echo "perl cpanm Pod::Usage"
-          #perl cpanm Pod::Usage
           mkdir openssl
           echo "curl openssl"
-          curl -L -k -s -S -o openssl-1.1.1j.tar.gz https://www.openssl.org/source/openssl-1.1.1j.tar.gz
-          tar xzf openssl-1.1.1j.tar.gz
-          cd openssl-1.1.1j
+          curl -L -k -s -S -o openssl-3.2.0.tar.gz https://www.openssl.org/source/openssl-3.2.0.tar.gz
+          tar xzf openssl-3.2.0.tar.gz
+          cd openssl-3.2.0
           # remove pod::Usage because we do not need -help or -man output
-          # from the Configure script
-          echo "Fixup ./Configure by removing use Pod::Usage require"
-          sed -e 's/use Pod::Usage//' < Configure > Configure.fix
-          echo "./Configure.fix no-shared no-asm -DOPENSSL_NO_CAPIENG mingw64 --prefix=\""$prepath/openssl\"""
-          ./Configure.fix no-shared no-asm -DOPENSSL_NO_CAPIENG mingw64 --prefix="$prepath/openssl"
+          # from the Configure script, this fixes it for openssl 1.1.1
+          #echo "Fixup ./Configure by removing use Pod::Usage require"
+          #sed -e 's/use Pod::Usage//' < Configure > Configure.fix
+          # ./Configure.fix no-shared no-asm -DOPENSSL_NO_CAPIENG mingw64 --prefix="$prepath/openssl"
+          # pacman is used to install for msys2, with
+          # C:/msys64/usr/bin/pacman -S perl
+          echo "C:/msys64/usr/bin/perl ./Configure no-shared no-asm -DOPENSSL_NO_CAPIENG mingw64 --prefix=\""$prepath/openssl"\" PERL=\"C:/msys64/usr/bin/perl\""
+          C:/msys64/usr/bin/perl ./Configure no-shared no-asm -DOPENSSL_NO_CAPIENG mingw64 --prefix="$prepath/openssl" PERL="C:/msys64/usr/bin/perl"
           # make the libs only, build faster
           echo "make build_libs"
           #make
@@ -211,9 +241,9 @@ jobs:
           cd ..
           mkdir expat
           echo "curl expat"
-          curl -L -k -s -S -o expat-2.2.10.tar.gz https://github.com/libexpat/libexpat/releases/download/R_2_2_10/expat-2.2.10.tar.gz
-          tar xzf expat-2.2.10.tar.gz
-          cd expat-2.2.10
+          curl -L -k -s -S -o $LIBEXPAT_FNAME.tar.gz https://github.com/libexpat/libexpat/releases/download/$LIBEXPAT_VERSION_DIR/$LIBEXPAT_FNAME.tar.gz
+          tar xzf $LIBEXPAT_FNAME.tar.gz
+          cd $LIBEXPAT_FNAME
           echo "./configure SHELL=/usr/bin/bash CONFIG_SHELL=/usr/bin/bash --prefix=\"$prepath/expat\" --exec-prefix=\"$prepath/expat\" --bindir=\"$prepath/expat/bin\" --includedir=\"$prepath/expat/include\" --mandir=\"$prepath/expat/man\" --libdir=\"$prepath/expat/lib\""
           ./configure SHELL=/usr/bin/bash CONFIG_SHELL=/usr/bin/bash --prefix="$prepath/expat" --exec-prefix="$prepath/expat" --bindir="$prepath/expat/bin" --includedir="$prepath/expat/include" --mandir="$prepath/expat/man" --libdir="$prepath/expat/lib"
           # fixup SHELL is treated specially, but SHELZZ is not by make.
@@ -332,6 +362,33 @@ jobs:
           echo "::group::make install"
           make install
           echo "::endgroup::"
+      - name: cross-platform-action on ${{ matrix.cross_platform_os }} ${{ matrix.cross_platform_version }}
+        if: ${{ matrix.with_cross_platform_action == 'yes' }}
+        uses: cross-platform-actions/action@v0.25.0
+        env:
+          CROSS_PLATFORM_OS: ${{ matrix.cross_platform_os }}
+        with:
+          environment_variables: CROSS_PLATFORM_OS
+          operating_system: ${{ matrix.cross_platform_os }}
+          architecture: ${{ matrix.cross_platform_arch }}
+          version: ${{ matrix.cross_platform_version }}
+          shell: bash
+          memory: 4G
+          cpu_count: 2
+          run: |
+            set -e -x
+            if test "$CROSS_PLATFORM_OS" = "freebsd"; then sudo pkg install -y openssl libevent expat; fi
+            if test "$CROSS_PLATFORM_OS" = "openbsd"; then sudo pkg_add libevent; fi
+            if test "$CROSS_PLATFORM_OS" = "netbsd"; then sudo pkgin -y install libevent; fi
+            echo "::group::configure"
+            ./configure ${{ matrix.cross_platform_config }}
+            echo "::endgroup::"
+            echo "::group::make"
+            make
+            echo "::endgroup::"
+            echo "::group::make test"
+            make test
+            echo "::endgroup::"
       - name: install libevent
         if: ${{ matrix.install_libevent == 'yes' }}
         run: sudo apt-get install libevent-dev

.github/workflows/ci.yml

@@ -12,7 +12,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
     - name: configure
       run: ./configure --enable-debug
     - name: make

.gitignore

@@ -10,6 +10,9 @@
 /config.status
 /dnstap/dnstap_config.h
 /dnscrypt/dnscrypt_config.h
+/util/configlexer.c
+/util/configparser.c
+/util/configparser.h
 /clubsyms.def
 /doc/example.conf
 /doc/libunbound.3
@@ -33,6 +36,7 @@
 /asynclook
 /delayer
 /dohclient
+/doqclient
 /lock-verify
 /memstats
 /perf
@@ -55,6 +59,7 @@
 /pythonmod/unboundmodule.py
 /testdata/result.*
 /testdata/.done-*
+/testdata/.skip-*
 /testdata/.perfstats.txt
 /doc/html
 /doc/xml

acx_nlnetlabs.m4

@@ -2,7 +2,10 @@
 # Copyright 2009, Wouter Wijngaards, NLnet Labs.   
 # BSD licensed.
 #
-# Version 46
+# Version 48
+# 2024-01-16 fix to add -l:libssp.a to -lcrypto link check.
+#	     and check for getaddrinfo with only header.
+# 2024-01-15 fix to add crypt32 to -lcrypto link check when checking for gdi32.
 # 2023-05-04 fix to remove unused whitespace.
 # 2023-01-26 fix -Wstrict-prototypes.
 # 2022-09-01 fix checking if nonblocking sockets work on OpenBSD.
@@ -707,7 +710,7 @@ AC_DEFUN([ACX_SSL_CHECKS], [
 		    LIBSSL_LDFLAGS="$LIBSSL_LDFLAGS -L$ssldir_lib"
 	    	    ACX_RUNTIME_PATH_ADD([$ssldir_lib])
 	    fi
-        
+
             AC_MSG_CHECKING([for EVP_sha256 in -lcrypto])
             LIBS="$LIBS -lcrypto"
             LIBSSL_LIBS="$LIBSSL_LIBS -lcrypto"
@@ -732,40 +735,73 @@ AC_DEFUN([ACX_SSL_CHECKS], [
                   ]])],[
                     AC_DEFINE([HAVE_EVP_SHA256], 1,
                         [If you have EVP_sha256])
-                    AC_MSG_RESULT(yes) 
+                    AC_MSG_RESULT(yes)
                   ],[
                     AC_MSG_RESULT(no)
                     LIBS="$BAKLIBS"
                     LIBSSL_LIBS="$BAKSSLLIBS"
-                    LIBS="$LIBS -ldl"
-                    LIBSSL_LIBS="$LIBSSL_LIBS -ldl"
-                    AC_MSG_CHECKING([if -lcrypto needs -ldl])
-                    AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[
-                        int EVP_sha256(void);
-                        (void)EVP_sha256();
-                      ]])],[
-                        AC_DEFINE([HAVE_EVP_SHA256], 1,
-                            [If you have EVP_sha256])
-                        AC_MSG_RESULT(yes) 
-                      ],[
-                        AC_MSG_RESULT(no)
-                        LIBS="$BAKLIBS"
-                        LIBSSL_LIBS="$BAKSSLLIBS"
-                        LIBS="$LIBS -ldl -pthread"
-                        LIBSSL_LIBS="$LIBSSL_LIBS -ldl -pthread"
-                        AC_MSG_CHECKING([if -lcrypto needs -ldl -pthread])
-                        AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[
-                            int EVP_sha256(void);
-                            (void)EVP_sha256();
-                          ]])],[
-                            AC_DEFINE([HAVE_EVP_SHA256], 1,
-                                [If you have EVP_sha256])
-                            AC_MSG_RESULT(yes) 
-                          ],[
-                            AC_MSG_RESULT(no)
-                            AC_MSG_ERROR([OpenSSL found in $ssldir, but version 0.9.7 or higher is required])
+
+		    LIBS="$LIBS -lgdi32 -lws2_32 -lcrypt32"
+		    LIBSSL_LIBS="$LIBSSL_LIBS -lgdi32 -lws2_32 -lcrypt32"
+                    AC_MSG_CHECKING([if -lcrypto needs -lgdi32 -lws2_32 -lcrypt32])
+		    AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[
+			int EVP_sha256(void);
+			(void)EVP_sha256();
+		      ]])],[
+			AC_DEFINE([HAVE_EVP_SHA256], 1,
+			    [If you have EVP_sha256])
+			AC_MSG_RESULT(yes)
+		      ],[
+			AC_MSG_RESULT(no)
+			LIBS="$BAKLIBS"
+			LIBSSL_LIBS="$BAKSSLLIBS"
+
+			LIBS="$LIBS -lgdi32 -lws2_32 -lcrypt32 -l:libssp.a"
+			LIBSSL_LIBS="$LIBSSL_LIBS -lgdi32 -lws2_32 -lcrypt32 -l:libssp.a"
+			AC_MSG_CHECKING([if -lcrypto needs -lgdi32 -lws2_32 -lcrypt32 -l:libssp.a])
+			AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[
+			    int EVP_sha256(void);
+			    (void)EVP_sha256();
+			  ]])],[
+			    AC_DEFINE([HAVE_EVP_SHA256], 1,
+				[If you have EVP_sha256])
+			    AC_MSG_RESULT(yes)
+			  ],[
+			    AC_MSG_RESULT(no)
+			    LIBS="$BAKLIBS"
+			    LIBSSL_LIBS="$BAKSSLLIBS"
+
+			    LIBS="$LIBS -ldl"
+			    LIBSSL_LIBS="$LIBSSL_LIBS -ldl"
+			    AC_MSG_CHECKING([if -lcrypto needs -ldl])
+			    AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[
+				int EVP_sha256(void);
+				(void)EVP_sha256();
+			      ]])],[
+				AC_DEFINE([HAVE_EVP_SHA256], 1,
+				    [If you have EVP_sha256])
+				AC_MSG_RESULT(yes)
+			      ],[
+				AC_MSG_RESULT(no)
+				LIBS="$BAKLIBS"
+				LIBSSL_LIBS="$BAKSSLLIBS"
+				LIBS="$LIBS -ldl -pthread"
+				LIBSSL_LIBS="$LIBSSL_LIBS -ldl -pthread"
+				AC_MSG_CHECKING([if -lcrypto needs -ldl -pthread])
+				AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[
+				    int EVP_sha256(void);
+				    (void)EVP_sha256();
+				  ]])],[
+				    AC_DEFINE([HAVE_EVP_SHA256], 1,
+					[If you have EVP_sha256])
+				    AC_MSG_RESULT(yes)
+				  ],[
+				    AC_MSG_RESULT(no)
+				    AC_MSG_ERROR([OpenSSL found in $ssldir, but version 0.9.7 or higher is required])
+				])
+			    ])
 			])
-                    ])
+		    ])
                 ])
             ])
         fi
@@ -779,7 +815,7 @@ AC_CHECK_HEADERS([openssl/rand.h],,, [AC_INCLUDES_DEFAULT])
 
 dnl Check for SSL, where SSL is mandatory
 dnl Adds --with-ssl option, searches for openssl and defines HAVE_SSL if found
-dnl Setup of CPPFLAGS, CFLAGS.  Adds -lcrypto to LIBS. 
+dnl Setup of CPPFLAGS, CFLAGS.  Adds -lcrypto to LIBS.
 dnl Checks main header files of SSL.
 dnl
 AC_DEFUN([ACX_WITH_SSL],
@@ -872,7 +908,7 @@ dnl see if on windows
 if test "$ac_cv_header_windows_h" = "yes"; then
 	AC_DEFINE(USE_WINSOCK, 1, [Whether the windows socket API is used])
 	USE_WINSOCK="1"
-	if echo $LIBS | grep 'lws2_32' >/dev/null; then
+	if echo "$LIBS" | grep 'lws2_32' >/dev/null; then
 		:
 	else
 		LIBS="$LIBS -lws2_32"
@@ -880,6 +916,24 @@ if test "$ac_cv_header_windows_h" = "yes"; then
 fi
 ],
 dnl no quick getaddrinfo, try mingw32 and winsock2 library.
+dnl perhaps getaddrinfo needs only the include
+AC_LINK_IFELSE(
+[AC_LANG_PROGRAM(
+[
+#ifdef HAVE_WS2TCPIP_H
+#include <ws2tcpip.h>
+#endif
+],
+[
+        (void)getaddrinfo(NULL, NULL, NULL, NULL);
+]
+)],
+[
+ac_cv_func_getaddrinfo="yes"
+AC_DEFINE(USE_WINSOCK, 1, [Whether the windows socket API is used])
+USE_WINSOCK="1"
+],
+
 ORIGLIBS="$LIBS"
 LIBS="$LIBS -lws2_32"
 AC_LINK_IFELSE(
@@ -904,6 +958,7 @@ ac_cv_func_getaddrinfo="no"
 LIBS="$ORIGLIBS"
 ])
 )
+)
 
 AC_MSG_RESULT($ac_cv_func_getaddrinfo)
 if test $ac_cv_func_getaddrinfo = yes; then

acx_python.m4

@@ -9,7 +9,7 @@ AC_DEFUN([AC_PYTHON_DEVEL],[
 
         AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]])
         if test -z "$PYTHON"; then
-           AC_MSG_ERROR([Cannot find python$PYTHON_VERSION in your system path])
+           AC_MSG_ERROR([Cannot find 'python$PYTHON_VERSION' in your system path. You can use the environment variable 'PYTHON_VERSION=version_number' for an explicit version.])
            PYTHON_VERSION=""
         fi
 

ax_build_date_epoch.m4

@@ -0,0 +1,70 @@
+# ===========================================================================
+#   https://www.gnu.org/software/autoconf-archive/ax_build_date_epoch.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_BUILD_DATE_EPOCH(VARIABLE[, FORMAT[, ACTION-IF-FAIL]])
+#
+# DESCRIPTION
+#
+#   Sets VARIABLE to a string representing the current time.  It is
+#   formatted according to FORMAT if specified, otherwise it is formatted as
+#   the number of seconds (excluding leap seconds) since the UNIX epoch (01
+#   Jan 1970 00:00:00 UTC).
+#
+#   If the SOURCE_DATE_EPOCH environment variable is set, it uses the value
+#   of that variable instead of the current time.  See
+#   https://reproducible-builds.org/specs/source-date-epoch).  If
+#   SOURCE_DATE_EPOCH is set but cannot be properly interpreted as a UNIX
+#   timestamp, then execute ACTION-IF-FAIL if specified, otherwise error.
+#
+#   VARIABLE is AC_SUBST-ed.
+#
+# LICENSE
+#
+#   Copyright (c) 2016 Eric Bavier <bavier@member.fsf.org>
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+#   Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+#   As a special exception, the respective Autoconf Macro's copyright owner
+#   gives unlimited permission to copy, distribute and modify the configure
+#   scripts that are the output of Autoconf when processing the Macro. You
+#   need not follow the terms of the GNU General Public License when using
+#   or distributing such scripts, even though portions of the text of the
+#   Macro appear in them. The GNU General Public License (GPL) does govern
+#   all other use of the material that constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the Autoconf
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
+
+#serial 2
+
+AC_DEFUN([AX_BUILD_DATE_EPOCH],
+[dnl
+AC_MSG_CHECKING([for build time])
+ax_date_fmt="m4_default($2,%s)"
+AS_IF([test x"$SOURCE_DATE_EPOCH" = x],
+ [$1=`date "+$ax_date_fmt"`],
+ [ax_build_date=`date -u -d "@$SOURCE_DATE_EPOCH" "+$ax_date_fmt" 2>/dev/null \
+                 || date -u -r "$SOURCE_DATE_EPOCH" "+$ax_date_fmt" 2>/dev/null`
+  AS_IF([test x"$ax_build_date" = x],
+   [m4_ifval([$3],
+      [$3],
+      [AC_MSG_ERROR([malformed SOURCE_DATE_EPOCH])])],
+   [$1=$ax_build_date])])
+AC_MSG_RESULT([$$1])
+])dnl AX_BUILD_DATE_EPOCH

ax_pkg_swig.m4

@@ -1,39 +1,43 @@
 # ===========================================================================
-#              http://autoconf-archive.cryp.to/ac_pkg_swig.html
+#       https://www.gnu.org/software/autoconf-archive/ax_pkg_swig.html
 # ===========================================================================
 #
 # SYNOPSIS
 #
-#   AC_PROG_SWIG([major.minor.micro])
+#   AX_PKG_SWIG([major.minor.micro], [action-if-found], [action-if-not-found])
 #
 # DESCRIPTION
 #
-#   This macro searches for a SWIG installation on your system. If found you
-#   should call SWIG via $(SWIG). You can use the optional first argument to
-#   check if the version of the available SWIG is greater than or equal to
-#   the value of the argument. It should have the format: N[.N[.N]] (N is a
-#   number between 0 and 999. Only the first N is mandatory.)
+#   This macro searches for a SWIG installation on your system. If found,
+#   then SWIG is AC_SUBST'd; if not found, then $SWIG is empty.  If SWIG is
+#   found, then SWIG_LIB is set to the SWIG library path, and AC_SUBST'd.
 #
-#   If the version argument is given (e.g. 1.3.17), AC_PROG_SWIG checks that
-#   the swig package is this version number or higher.
+#   You can use the optional first argument to check if the version of the
+#   available SWIG is greater than or equal to the value of the argument. It
+#   should have the format: N[.N[.N]] (N is a number between 0 and 999. Only
+#   the first N is mandatory.) If the version argument is given (e.g.
+#   1.3.17), AX_PKG_SWIG checks that the swig package is this version number
+#   or higher.
+#
+#   As usual, action-if-found is executed if SWIG is found, otherwise
+#   action-if-not-found is executed.
 #
 #   In configure.in, use as:
 #
-#     AC_PROG_SWIG(1.3.17)
-#     SWIG_ENABLE_CXX
-#     SWIG_MULTI_MODULE_SUPPORT
-#     SWIG_PYTHON
+#     AX_PKG_SWIG(1.3.17, [], [ AC_MSG_ERROR([SWIG is required to build..]) ])
+#     AX_SWIG_ENABLE_CXX
+#     AX_SWIG_MULTI_MODULE_SUPPORT
+#     AX_SWIG_PYTHON
 #
-# LAST MODIFICATION
-#
-#   2008-04-12
-#
-# COPYLEFT
+# LICENSE
 #
 #   Copyright (c) 2008 Sebastian Huber <sebastian-huber@web.de>
-#   Copyright (c) 2008 Alan W. Irwin <irwin@beluga.phys.uvic.ca>
+#   Copyright (c) 2008 Alan W. Irwin
 #   Copyright (c) 2008 Rafael Laboissiere <rafael@laboissiere.net>
-#   Copyright (c) 2008 Andrew Collier <colliera@ukzn.ac.za>
+#   Copyright (c) 2008 Andrew Collier
+#   Copyright (c) 2011 Murray Cumming <murrayc@openismus.com>
+#   Copyright (c) 2018 Reini Urban <rurban@cpan.org>
+#   Copyright (c) 2021 Vincent Danjean <Vincent.Danjean@ens-lyon.org>
 #
 #   This program is free software; you can redistribute it and/or modify it
 #   under the terms of the GNU General Public License as published by the
@@ -46,7 +50,7 @@
 #   Public License for more details.
 #
 #   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see <http://www.gnu.org/licenses/>.
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 #   As a special exception, the respective Autoconf Macro's copyright owner
 #   gives unlimited permission to copy, distribute and modify the configure
@@ -57,17 +61,21 @@
 #   all other use of the material that constitutes the Autoconf Macro.
 #
 #   This special exception to the GPL applies to versions of the Autoconf
-#   Macro released by the Autoconf Macro Archive. When you make and
-#   distribute a modified version of the Autoconf Macro, you may extend this
-#   special exception to the GPL to apply to your modified version as well.
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
 
-AC_DEFUN([AC_PROG_SWIG],[
-        AC_PATH_PROG([SWIG],[swig])
+#serial 15
+
+AC_DEFUN([AX_PKG_SWIG],[
+        # Find path to the "swig" executable.
+        AC_PATH_PROGS([SWIG],[swig swig3.0 swig2.0])
         if test -z "$SWIG" ; then
-                AC_MSG_WARN([cannot find 'swig' program. You should look at http://www.swig.org])
-                SWIG='echo "Error: SWIG is not installed. You should look at http://www.swig.org" ; false'
-        elif test -n "$1" ; then
-                AC_MSG_CHECKING([for SWIG version])
+                m4_ifval([$3],[$3],[:])
+        elif test -z "$1" ; then
+                m4_ifval([$2],[$2],[:])
+	else
+                AC_MSG_CHECKING([SWIG version])
                 [swig_version=`$SWIG -version 2>&1 | grep 'SWIG Version' | sed 's/.*\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/g'`]
                 AC_MSG_RESULT([$swig_version])
                 if test -n "$swig_version" ; then
@@ -77,12 +85,12 @@ AC_DEFUN([AC_PROG_SWIG],[
                         if test -z "$required_major" ; then
                                 [required_major=0]
                         fi
-                        [required=`echo $required | sed 's/[0-9]*[^0-9]//'`]
+                        [required=`echo $required. | sed 's/[0-9]*[^0-9]//'`]
                         [required_minor=`echo $required | sed 's/[^0-9].*//'`]
                         if test -z "$required_minor" ; then
                                 [required_minor=0]
                         fi
-                        [required=`echo $required | sed 's/[0-9]*[^0-9]//'`]
+                        [required=`echo $required. | sed 's/[0-9]*[^0-9]//'`]
                         [required_patch=`echo $required | sed 's/[^0-9].*//'`]
                         if test -z "$required_patch" ; then
                                 [required_patch=0]
@@ -103,30 +111,28 @@ AC_DEFUN([AC_PROG_SWIG],[
                         if test -z "$available_patch" ; then
                                 [available_patch=0]
                         fi
-			[badversion=0]
-			if test $available_major -lt $required_major ; then
-				[badversion=1]
-			fi
-                        if test $available_major -eq $required_major \
-                                -a $available_minor -lt $required_minor ; then
-				[badversion=1]
-			fi
-                        if test $available_major -eq $required_major \
-                                -a $available_minor -eq $required_minor \
-                                -a $available_patch -lt $required_patch ; then
-				[badversion=1]
-			fi
-			if test $badversion -eq 1 ; then
-                                AC_MSG_WARN([SWIG version >= $1 is required.  You have $swig_version.  You should look at http://www.swig.org])
-                                SWIG='echo "Error: SWIG version >= $1 is required.  You have '"$swig_version"'.  You should look at http://www.swig.org" ; false'
+                        # Convert the version tuple into a single number for easier comparison.
+                        # Using base 100 should be safe since SWIG internally uses BCD values
+                        # to encode its version number.
+                        required_swig_vernum=`expr $required_major \* 10000 \
+                            \+ $required_minor \* 100 \+ $required_patch`
+                        available_swig_vernum=`expr $available_major \* 10000 \
+                            \+ $available_minor \* 100 \+ $available_patch`
+
+                        if test $available_swig_vernum -lt $required_swig_vernum; then
+                                AC_MSG_WARN([SWIG version >= $1 is required.  You have $swig_version.])
+                                SWIG=''
+                                m4_ifval([$3],[$3],[])
                         else
-                                AC_MSG_NOTICE([SWIG executable is '$SWIG'])
-                                SWIG_LIB=`$SWIG -swiglib`
-                                AC_MSG_NOTICE([SWIG library directory is '$SWIG_LIB'])
+                                AC_MSG_CHECKING([for SWIG library])
+                                SWIG_LIB=`$SWIG -swiglib | tr '\r\n' '  '`
+                                AC_MSG_RESULT([$SWIG_LIB])
+                                m4_ifval([$2],[$2],[])
                         fi
                 else
                         AC_MSG_WARN([cannot determine SWIG version])
-                        SWIG='echo "Error: Cannot determine SWIG version.  You should look at http://www.swig.org" ; false'
+                        SWIG=''
+                        m4_ifval([$3],[$3],[])
                 fi
         fi
         AC_SUBST([SWIG_LIB])

ax_pthread.m4

@@ -1,5 +1,5 @@
 # ===========================================================================
-#        http://www.gnu.org/software/autoconf-archive/ax_pthread.html
+#        https://www.gnu.org/software/autoconf-archive/ax_pthread.html
 # ===========================================================================
 #
 # SYNOPSIS
@@ -14,24 +14,28 @@
 #   flags that are needed. (The user can also force certain compiler
 #   flags/libs to be tested by setting these environment variables.)
 #
-#   Also sets PTHREAD_CC to any special C compiler that is needed for
-#   multi-threaded programs (defaults to the value of CC otherwise). (This
-#   is necessary on AIX to use the special cc_r compiler alias.)
+#   Also sets PTHREAD_CC and PTHREAD_CXX to any special C compiler that is
+#   needed for multi-threaded programs (defaults to the value of CC
+#   respectively CXX otherwise). (This is necessary on e.g. AIX to use the
+#   special cc_r/CC_r compiler alias.)
 #
 #   NOTE: You are assumed to not only compile your program with these flags,
-#   but also link it with them as well. e.g. you should link with
+#   but also to link with them as well. For example, you might link with
 #   $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS
+#   $PTHREAD_CXX $CXXFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS
 #
-#   If you are only building threads programs, you may wish to use these
+#   If you are only building threaded programs, you may wish to use these
 #   variables in your default LIBS, CFLAGS, and CC:
 #
 #     LIBS="$PTHREAD_LIBS $LIBS"
 #     CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+#     CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS"
 #     CC="$PTHREAD_CC"
+#     CXX="$PTHREAD_CXX"
 #
 #   In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant
-#   has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to that name
-#   (e.g. PTHREAD_CREATE_UNDETACHED on AIX).
+#   has a nonstandard name, this macro defines PTHREAD_CREATE_JOINABLE to
+#   that name (e.g. PTHREAD_CREATE_UNDETACHED on AIX).
 #
 #   Also HAVE_PTHREAD_PRIO_INHERIT is defined if pthread is found and the
 #   PTHREAD_PRIO_INHERIT symbol is defined when compiling with
@@ -55,6 +59,7 @@
 #
 #   Copyright (c) 2008 Steven G. Johnson <stevenj@alum.mit.edu>
 #   Copyright (c) 2011 Daniel Richard G. <skunk@iSKUNK.ORG>
+#   Copyright (c) 2019 Marc Stevens <marc.stevens@cwi.nl>
 #
 #   This program is free software: you can redistribute it and/or modify it
 #   under the terms of the GNU General Public License as published by the
@@ -67,7 +72,7 @@
 #   Public License for more details.
 #
 #   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see <http://www.gnu.org/licenses/>.
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 #   As a special exception, the respective Autoconf Macro's copyright owner
 #   gives unlimited permission to copy, distribute and modify the configure
@@ -82,35 +87,41 @@
 #   modified version of the Autoconf Macro, you may extend this special
 #   exception to the GPL to apply to your modified version as well.
 
-#serial 21
+#serial 31
 
 AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD])
 AC_DEFUN([AX_PTHREAD], [
 AC_REQUIRE([AC_CANONICAL_HOST])
+AC_REQUIRE([AC_PROG_CC])
+AC_REQUIRE([AC_PROG_SED])
 AC_LANG_PUSH([C])
 ax_pthread_ok=no
 
 # We used to check for pthread.h first, but this fails if pthread.h
-# requires special compiler flags (e.g. on True64 or Sequent).
+# requires special compiler flags (e.g. on Tru64 or Sequent).
 # It gets checked for in the link test anyway.
 
 # First of all, check if the user has set any of the PTHREAD_LIBS,
 # etcetera environment variables, and if threads linking works using
 # them:
-if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then
-        save_CFLAGS="$CFLAGS"
+if test "x$PTHREAD_CFLAGS$PTHREAD_LIBS" != "x"; then
+        ax_pthread_save_CC="$CC"
+        ax_pthread_save_CFLAGS="$CFLAGS"
+        ax_pthread_save_LIBS="$LIBS"
+        AS_IF([test "x$PTHREAD_CC" != "x"], [CC="$PTHREAD_CC"])
+        AS_IF([test "x$PTHREAD_CXX" != "x"], [CXX="$PTHREAD_CXX"])
         CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
-        save_LIBS="$LIBS"
         LIBS="$PTHREAD_LIBS $LIBS"
-        AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS])
-        AC_TRY_LINK_FUNC([pthread_join], [ax_pthread_ok=yes])
+        AC_MSG_CHECKING([for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS])
+        AC_LINK_IFELSE([AC_LANG_CALL([], [pthread_join])], [ax_pthread_ok=yes])
         AC_MSG_RESULT([$ax_pthread_ok])
-        if test x"$ax_pthread_ok" = xno; then
+        if test "x$ax_pthread_ok" = "xno"; then
                 PTHREAD_LIBS=""
                 PTHREAD_CFLAGS=""
         fi
-        LIBS="$save_LIBS"
-        CFLAGS="$save_CFLAGS"
+        CC="$ax_pthread_save_CC"
+        CFLAGS="$ax_pthread_save_CFLAGS"
+        LIBS="$ax_pthread_save_LIBS"
 fi
 
 # We must check for the threads library under a number of different
@@ -118,12 +129,14 @@ fi
 # (e.g. DEC) have both -lpthread and -lpthreads, where one of the
 # libraries is broken (non-POSIX).
 
-# Create a list of thread flags to try.  Items starting with a "-" are
-# C compiler flags, and other items are library names, except for "none"
-# which indicates that we try without any flags at all, and "pthread-config"
-# which is a program returning the flags for the Pth emulation library.
+# Create a list of thread flags to try. Items with a "," contain both
+# C compiler flags (before ",") and linker flags (after ","). Other items
+# starting with a "-" are C compiler flags, and remaining items are
+# library names, except for "none" which indicates that we try without
+# any flags at all, and "pthread-config" which is a program returning
+# the flags for the Pth emulation library.
 
-ax_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
+ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
 
 # The ordering *is* (sometimes) important.  Some notes on the
 # individual items follow:
@@ -132,82 +145,163 @@ ax_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mt
 # none: in case threads are in libc; should be tried before -Kthread and
 #       other compiler flags to prevent continual compiler warnings
 # -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h)
-# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able)
-# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread)
-# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads)
-# -pthreads: Solaris/gcc
-# -mthreads: Mingw32/gcc, Lynx/gcc
+# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads), Tru64
+#           (Note: HP C rejects this with "bad form for `-t' option")
+# -pthreads: Solaris/gcc (Note: HP C also rejects)
 # -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it
-#      doesn't hurt to check since this sometimes defines pthreads too;
-#      also defines -D_REENTRANT)
-#      ... -mt is also the pthreads flag for HP/aCC
+#      doesn't hurt to check since this sometimes defines pthreads and
+#      -D_REENTRANT too), HP C (must be checked before -lpthread, which
+#      is present but should not be used directly; and before -mthreads,
+#      because the compiler interprets this as "-mt" + "-hreads")
+# -mthreads: Mingw32/gcc, Lynx/gcc
 # pthread: Linux, etcetera
 # --thread-safe: KAI C++
 # pthread-config: use pthread-config program (for GNU Pth library)
 
-case ${host_os} in
+case $host_os in
+
+        freebsd*)
+
+        # -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able)
+        # lthread: LinuxThreads port on FreeBSD (also preferred to -pthread)
+
+        ax_pthread_flags="-kthread lthread $ax_pthread_flags"
+        ;;
+
+        hpux*)
+
+        # From the cc(1) man page: "[-mt] Sets various -D flags to enable
+        # multi-threading and also sets -lpthread."
+
+        ax_pthread_flags="-mt -pthread pthread $ax_pthread_flags"
+        ;;
+
+        openedition*)
+
+        # IBM z/OS requires a feature-test macro to be defined in order to
+        # enable POSIX threads at all, so give the user a hint if this is
+        # not set. (We don't define these ourselves, as they can affect
+        # other portions of the system API in unpredictable ways.)
+
+        AC_EGREP_CPP([AX_PTHREAD_ZOS_MISSING],
+            [
+#            if !defined(_OPEN_THREADS) && !defined(_UNIX03_THREADS)
+             AX_PTHREAD_ZOS_MISSING
+#            endif
+            ],
+            [AC_MSG_WARN([IBM z/OS requires -D_OPEN_THREADS or -D_UNIX03_THREADS to enable pthreads support.])])
+        ;;
+
         solaris*)
 
         # On Solaris (at least, for some versions), libc contains stubbed
         # (non-functional) versions of the pthreads routines, so link-based
-        # tests will erroneously succeed.  (We need to link with -pthreads/-mt/
-        # -lpthread.)  (The stubs are missing pthread_cleanup_push, or rather
-        # a function called by this macro, so we could check for that, but
-        # who knows whether they'll stub that too in a future libc.)  So,
-        # we'll just look for -pthreads and -lpthread first:
+        # tests will erroneously succeed. (N.B.: The stubs are missing
+        # pthread_cleanup_push, or rather a function called by this macro,
+        # so we could check for that, but who knows whether they'll stub
+        # that too in a future libc.)  So we'll check first for the
+        # standard Solaris way of linking pthreads (-mt -lpthread).
 
-        ax_pthread_flags="-pthreads pthread -mt -pthread $ax_pthread_flags"
-        ;;
-
-        darwin*)
-        ax_pthread_flags="-pthread $ax_pthread_flags"
+        ax_pthread_flags="-mt,-lpthread pthread $ax_pthread_flags"
         ;;
 esac
 
-# Clang doesn't consider unrecognized options an error unless we specify
-# -Werror. We throw in some extra Clang-specific options to ensure that
-# this doesn't happen for GCC, which also accepts -Werror.
+# Are we compiling with Clang?
 
-AC_MSG_CHECKING([if compiler needs -Werror to reject unknown flags])
-save_CFLAGS="$CFLAGS"
-ax_pthread_extra_flags="-Werror"
-CFLAGS="$CFLAGS $ax_pthread_extra_flags -Wunknown-warning-option -Wsizeof-array-argument"
-AC_COMPILE_IFELSE([AC_LANG_PROGRAM([int foo(void);],[foo()])],
-                  [AC_MSG_RESULT([yes])],
-                  [ax_pthread_extra_flags=
-                   AC_MSG_RESULT([no])])
-CFLAGS="$save_CFLAGS"
+AC_CACHE_CHECK([whether $CC is Clang],
+    [ax_cv_PTHREAD_CLANG],
+    [ax_cv_PTHREAD_CLANG=no
+     # Note that Autoconf sets GCC=yes for Clang as well as GCC
+     if test "x$GCC" = "xyes"; then
+        AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG],
+            [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */
+#            if defined(__clang__) && defined(__llvm__)
+             AX_PTHREAD_CC_IS_CLANG
+#            endif
+            ],
+            [ax_cv_PTHREAD_CLANG=yes])
+     fi
+    ])
+ax_pthread_clang="$ax_cv_PTHREAD_CLANG"
 
-if test x"$ax_pthread_ok" = xno; then
-for flag in $ax_pthread_flags; do
 
-        case $flag in
+# GCC generally uses -pthread, or -pthreads on some platforms (e.g. SPARC)
+
+# Note that for GCC and Clang -pthread generally implies -lpthread,
+# except when -nostdlib is passed.
+# This is problematic using libtool to build C++ shared libraries with pthread:
+# [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=25460
+# [2] https://bugzilla.redhat.com/show_bug.cgi?id=661333
+# [3] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=468555
+# To solve this, first try -pthread together with -lpthread for GCC
+
+AS_IF([test "x$GCC" = "xyes"],
+      [ax_pthread_flags="-pthread,-lpthread -pthread -pthreads $ax_pthread_flags"])
+
+# Clang takes -pthread (never supported any other flag), but we'll try with -lpthread first
+
+AS_IF([test "x$ax_pthread_clang" = "xyes"],
+      [ax_pthread_flags="-pthread,-lpthread -pthread"])
+
+
+# The presence of a feature test macro requesting re-entrant function
+# definitions is, on some systems, a strong hint that pthreads support is
+# correctly enabled
+
+case $host_os in
+        darwin* | hpux* | linux* | osf* | solaris*)
+        ax_pthread_check_macro="_REENTRANT"
+        ;;
+
+        aix*)
+        ax_pthread_check_macro="_THREAD_SAFE"
+        ;;
+
+        *)
+        ax_pthread_check_macro="--"
+        ;;
+esac
+AS_IF([test "x$ax_pthread_check_macro" = "x--"],
+      [ax_pthread_check_cond=0],
+      [ax_pthread_check_cond="!defined($ax_pthread_check_macro)"])
+
+
+if test "x$ax_pthread_ok" = "xno"; then
+for ax_pthread_try_flag in $ax_pthread_flags; do
+
+        case $ax_pthread_try_flag in
                 none)
                 AC_MSG_CHECKING([whether pthreads work without any flags])
                 ;;
 
+                *,*)
+                PTHREAD_CFLAGS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\1/"`
+                PTHREAD_LIBS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\2/"`
+                AC_MSG_CHECKING([whether pthreads work with "$PTHREAD_CFLAGS" and "$PTHREAD_LIBS"])
+                ;;
+
                 -*)
-                AC_MSG_CHECKING([whether pthreads work with $flag])
-                PTHREAD_CFLAGS="$flag"
+                AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag])
+                PTHREAD_CFLAGS="$ax_pthread_try_flag"
                 ;;
 
                 pthread-config)
                 AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no])
-                if test x"$ax_pthread_config" = xno; then continue; fi
+                AS_IF([test "x$ax_pthread_config" = "xno"], [continue])
                 PTHREAD_CFLAGS="`pthread-config --cflags`"
                 PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
                 ;;
 
                 *)
-                AC_MSG_CHECKING([for the pthreads library -l$flag])
-                PTHREAD_LIBS="-l$flag"
+                AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag])
+                PTHREAD_LIBS="-l$ax_pthread_try_flag"
                 ;;
         esac
 
-        save_LIBS="$LIBS"
-        save_CFLAGS="$CFLAGS"
+        ax_pthread_save_CFLAGS="$CFLAGS"
+        ax_pthread_save_LIBS="$LIBS"
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
         LIBS="$PTHREAD_LIBS $LIBS"
-        CFLAGS="$CFLAGS $PTHREAD_CFLAGS $ax_pthread_extra_flags"
 
         # Check for various functions.  We must include pthread.h,
         # since some functions may be macros.  (On the Sequent, we
@@ -218,8 +312,18 @@ for flag in $ax_pthread_flags; do
         # pthread_cleanup_push because it is one of the few pthread
         # functions on Solaris that doesn't have a non-functional libc stub.
         # We try pthread_create on general principles.
+
         AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>
-                        static void routine(void *a) { *((int*)a) = 0; }
+#                       if $ax_pthread_check_cond
+#                        error "$ax_pthread_check_macro must be defined"
+#                       endif
+                        static void *some_global = NULL;
+                        static void routine(void *a)
+                          {
+                             /* To avoid any unused-parameter or
+                                unused-but-set-parameter warning.  */
+                             some_global = a;
+                          }
                         static void *start_routine(void *a) { return a; }],
                        [pthread_t th; pthread_attr_t attr;
                         pthread_create(&th, 0, start_routine, 0);
@@ -227,101 +331,187 @@ for flag in $ax_pthread_flags; do
                         pthread_attr_init(&attr);
                         pthread_cleanup_push(routine, 0);
                         pthread_cleanup_pop(0) /* ; */])],
-                [ax_pthread_ok=yes],
-                [])
+            [ax_pthread_ok=yes],
+            [])
 
-        LIBS="$save_LIBS"
-        CFLAGS="$save_CFLAGS"
+        CFLAGS="$ax_pthread_save_CFLAGS"
+        LIBS="$ax_pthread_save_LIBS"
 
         AC_MSG_RESULT([$ax_pthread_ok])
-        if test "x$ax_pthread_ok" = xyes; then
-                break;
-        fi
+        AS_IF([test "x$ax_pthread_ok" = "xyes"], [break])
 
         PTHREAD_LIBS=""
         PTHREAD_CFLAGS=""
 done
 fi
 
+
+# Clang needs special handling, because older versions handle the -pthread
+# option in a rather... idiosyncratic way
+
+if test "x$ax_pthread_clang" = "xyes"; then
+
+        # Clang takes -pthread; it has never supported any other flag
+
+        # (Note 1: This will need to be revisited if a system that Clang
+        # supports has POSIX threads in a separate library.  This tends not
+        # to be the way of modern systems, but it's conceivable.)
+
+        # (Note 2: On some systems, notably Darwin, -pthread is not needed
+        # to get POSIX threads support; the API is always present and
+        # active.  We could reasonably leave PTHREAD_CFLAGS empty.  But
+        # -pthread does define _REENTRANT, and while the Darwin headers
+        # ignore this macro, third-party headers might not.)
+
+        # However, older versions of Clang make a point of warning the user
+        # that, in an invocation where only linking and no compilation is
+        # taking place, the -pthread option has no effect ("argument unused
+        # during compilation").  They expect -pthread to be passed in only
+        # when source code is being compiled.
+        #
+        # Problem is, this is at odds with the way Automake and most other
+        # C build frameworks function, which is that the same flags used in
+        # compilation (CFLAGS) are also used in linking.  Many systems
+        # supported by AX_PTHREAD require exactly this for POSIX threads
+        # support, and in fact it is often not straightforward to specify a
+        # flag that is used only in the compilation phase and not in
+        # linking.  Such a scenario is extremely rare in practice.
+        #
+        # Even though use of the -pthread flag in linking would only print
+        # a warning, this can be a nuisance for well-run software projects
+        # that build with -Werror.  So if the active version of Clang has
+        # this misfeature, we search for an option to squash it.
+
+        AC_CACHE_CHECK([whether Clang needs flag to prevent "argument unused" warning when linking with -pthread],
+            [ax_cv_PTHREAD_CLANG_NO_WARN_FLAG],
+            [ax_cv_PTHREAD_CLANG_NO_WARN_FLAG=unknown
+             # Create an alternate version of $ac_link that compiles and
+             # links in two steps (.c -> .o, .o -> exe) instead of one
+             # (.c -> exe), because the warning occurs only in the second
+             # step
+             ax_pthread_save_ac_link="$ac_link"
+             ax_pthread_sed='s/conftest\.\$ac_ext/conftest.$ac_objext/g'
+             ax_pthread_link_step=`AS_ECHO(["$ac_link"]) | sed "$ax_pthread_sed"`
+             ax_pthread_2step_ac_link="($ac_compile) && (echo ==== >&5) && ($ax_pthread_link_step)"
+             ax_pthread_save_CFLAGS="$CFLAGS"
+             for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do
+                AS_IF([test "x$ax_pthread_try" = "xunknown"], [break])
+                CFLAGS="-Werror -Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS"
+                ac_link="$ax_pthread_save_ac_link"
+                AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void){return 0;}]])],
+                    [ac_link="$ax_pthread_2step_ac_link"
+                     AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void){return 0;}]])],
+                         [break])
+                    ])
+             done
+             ac_link="$ax_pthread_save_ac_link"
+             CFLAGS="$ax_pthread_save_CFLAGS"
+             AS_IF([test "x$ax_pthread_try" = "x"], [ax_pthread_try=no])
+             ax_cv_PTHREAD_CLANG_NO_WARN_FLAG="$ax_pthread_try"
+            ])
+
+        case "$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" in
+                no | unknown) ;;
+                *) PTHREAD_CFLAGS="$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG $PTHREAD_CFLAGS" ;;
+        esac
+
+fi # $ax_pthread_clang = yes
+
+
+
 # Various other checks:
-if test "x$ax_pthread_ok" = xyes; then
-        save_LIBS="$LIBS"
-        LIBS="$PTHREAD_LIBS $LIBS"
-        save_CFLAGS="$CFLAGS"
+if test "x$ax_pthread_ok" = "xyes"; then
+        ax_pthread_save_CFLAGS="$CFLAGS"
+        ax_pthread_save_LIBS="$LIBS"
         CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+        LIBS="$PTHREAD_LIBS $LIBS"
 
         # Detect AIX lossage: JOINABLE attribute is called UNDETACHED.
-        AC_MSG_CHECKING([for joinable pthread attribute])
-        attr_name=unknown
-        for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do
-            AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>],
-                           [int attr = $attr; return attr /* ; */])],
-                [attr_name=$attr; break],
-                [])
-        done
-        AC_MSG_RESULT([$attr_name])
-        if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then
-            AC_DEFINE_UNQUOTED([PTHREAD_CREATE_JOINABLE], [$attr_name],
-                               [Define to necessary symbol if this constant
-                                uses a non-standard name on your system.])
-        fi
+        AC_CACHE_CHECK([for joinable pthread attribute],
+            [ax_cv_PTHREAD_JOINABLE_ATTR],
+            [ax_cv_PTHREAD_JOINABLE_ATTR=unknown
+             for ax_pthread_attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do
+                 AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>],
+                                                 [int attr = $ax_pthread_attr; return attr /* ; */])],
+                                [ax_cv_PTHREAD_JOINABLE_ATTR=$ax_pthread_attr; break],
+                                [])
+             done
+            ])
+        AS_IF([test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xunknown" && \
+               test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xPTHREAD_CREATE_JOINABLE" && \
+               test "x$ax_pthread_joinable_attr_defined" != "xyes"],
+              [AC_DEFINE_UNQUOTED([PTHREAD_CREATE_JOINABLE],
+                                  [$ax_cv_PTHREAD_JOINABLE_ATTR],
+                                  [Define to necessary symbol if this constant
+                                   uses a non-standard name on your system.])
+               ax_pthread_joinable_attr_defined=yes
+              ])
 
-        AC_MSG_CHECKING([if more special flags are required for pthreads])
-        flag=no
-        case ${host_os} in
-            aix* | freebsd* | darwin*) flag="-D_THREAD_SAFE";;
-            osf* | hpux*) flag="-D_REENTRANT";;
-            solaris*)
-            if test "$GCC" = "yes"; then
-                flag="-D_REENTRANT"
-            else
-                # TODO: What about Clang on Solaris?
-                flag="-mt -D_REENTRANT"
-            fi
-            ;;
-        esac
-        AC_MSG_RESULT([$flag])
-        if test "x$flag" != xno; then
-            PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS"
-        fi
+        AC_CACHE_CHECK([whether more special flags are required for pthreads],
+            [ax_cv_PTHREAD_SPECIAL_FLAGS],
+            [ax_cv_PTHREAD_SPECIAL_FLAGS=no
+             case $host_os in
+             solaris*)
+             ax_cv_PTHREAD_SPECIAL_FLAGS="-D_POSIX_PTHREAD_SEMANTICS"
+             ;;
+             esac
+            ])
+        AS_IF([test "x$ax_cv_PTHREAD_SPECIAL_FLAGS" != "xno" && \
+               test "x$ax_pthread_special_flags_added" != "xyes"],
+              [PTHREAD_CFLAGS="$ax_cv_PTHREAD_SPECIAL_FLAGS $PTHREAD_CFLAGS"
+               ax_pthread_special_flags_added=yes])
 
         AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT],
-            [ax_cv_PTHREAD_PRIO_INHERIT], [
-                AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <pthread.h>]],
-                                                [[int i = PTHREAD_PRIO_INHERIT;]])],
-                    [ax_cv_PTHREAD_PRIO_INHERIT=yes],
-                    [ax_cv_PTHREAD_PRIO_INHERIT=no])
+            [ax_cv_PTHREAD_PRIO_INHERIT],
+            [AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <pthread.h>]],
+                                             [[int i = PTHREAD_PRIO_INHERIT;
+                                               return i;]])],
+                            [ax_cv_PTHREAD_PRIO_INHERIT=yes],
+                            [ax_cv_PTHREAD_PRIO_INHERIT=no])
             ])
-        AS_IF([test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes"],
-            [AC_DEFINE([HAVE_PTHREAD_PRIO_INHERIT], [1], [Have PTHREAD_PRIO_INHERIT.])])
+        AS_IF([test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes" && \
+               test "x$ax_pthread_prio_inherit_defined" != "xyes"],
+              [AC_DEFINE([HAVE_PTHREAD_PRIO_INHERIT], [1], [Have PTHREAD_PRIO_INHERIT.])
+               ax_pthread_prio_inherit_defined=yes
+              ])
 
-        LIBS="$save_LIBS"
-        CFLAGS="$save_CFLAGS"
+        CFLAGS="$ax_pthread_save_CFLAGS"
+        LIBS="$ax_pthread_save_LIBS"
 
         # More AIX lossage: compile with *_r variant
-        if test "x$GCC" != xyes; then
+        if test "x$GCC" != "xyes"; then
             case $host_os in
                 aix*)
                 AS_CASE(["x/$CC"],
-                  [x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6],
-                  [#handle absolute path differently from PATH based program lookup
-                   AS_CASE(["x$CC"],
-                     [x/*],
-                     [AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])],
-                     [AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])])])
+                    [x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6],
+                    [#handle absolute path differently from PATH based program lookup
+                     AS_CASE(["x$CC"],
+                         [x/*],
+                         [
+			   AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])
+			   AS_IF([test "x${CXX}" != "x"], [AS_IF([AS_EXECUTABLE_P([${CXX}_r])],[PTHREAD_CXX="${CXX}_r"])])
+			 ],
+                         [
+			   AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])
+			   AS_IF([test "x${CXX}" != "x"], [AC_CHECK_PROGS([PTHREAD_CXX],[${CXX}_r],[$CXX])])
+			 ]
+                     )
+                    ])
                 ;;
             esac
         fi
 fi
 
 test -n "$PTHREAD_CC" || PTHREAD_CC="$CC"
+test -n "$PTHREAD_CXX" || PTHREAD_CXX="$CXX"
 
 AC_SUBST([PTHREAD_LIBS])
 AC_SUBST([PTHREAD_CFLAGS])
 AC_SUBST([PTHREAD_CC])
+AC_SUBST([PTHREAD_CXX])
 
 # Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
-if test x"$ax_pthread_ok" = xyes; then
+if test "x$ax_pthread_ok" = "xyes"; then
         ifelse([$1],,[AC_DEFINE([HAVE_PTHREAD],[1],[Define if you have POSIX threads libraries and header files.])],[$1])
         :
 else

cachedb/cachedb.c

@@ -47,9 +47,12 @@
 #include "util/regional.h"
 #include "util/net_help.h"
 #include "util/config_file.h"
+#include "util/data/dname.h"
 #include "util/data/msgreply.h"
 #include "util/data/msgencode.h"
 #include "services/cache/dns.h"
+#include "services/mesh.h"
+#include "services/modstack.h"
 #include "validator/val_neg.h"
 #include "validator/val_secalgo.h"
 #include "iterator/iter_utils.h"
@@ -265,15 +268,6 @@ cachedb_init(struct module_env* env, int id)
 		return 0;
 	}
 	cachedb_env->enabled = 1;
-	if(env->cfg->serve_expired && env->cfg->serve_expired_reply_ttl)
-		log_warn(
-			"cachedb: serve-expired-reply-ttl is set but not working for data "
-			"originating from the external cache; 0 TTL is used for those.");
-	if(env->cfg->serve_expired && env->cfg->serve_expired_client_timeout)
-		log_warn(
-			"cachedb: serve-expired-client-timeout is set but not working for "
-			"data originating from the external cache; expired data are used "
-			"in the reply without first trying to refresh the data.");
 	return 1;
 }
 
@@ -329,30 +323,31 @@ error_response(struct module_qstate* qstate, int id, int rcode)
 
 /**
  * Hash the query name, type, class and dbacess-secret into lookup buffer.
- * @param qstate: query state with query info
- * 	and env->cfg with secret.
+ * @param qinfo: query info
+ * @param env: with env->cfg with secret.
  * @param buf: returned buffer with hash to lookup
  * @param len: length of the buffer.
  */
 static void
-calc_hash(struct module_qstate* qstate, char* buf, size_t len)
+calc_hash(struct query_info* qinfo, struct module_env* env, char* buf,
+	size_t len)
 {
 	uint8_t clear[1024];
 	size_t clen = 0;
 	uint8_t hash[CACHEDB_HASHSIZE/8];
 	const char* hex = "0123456789ABCDEF";
-	const char* secret = qstate->env->cfg->cachedb_secret;
+	const char* secret = env->cfg->cachedb_secret;
 	size_t i;
 
 	/* copy the hash info into the clear buffer */
-	if(clen + qstate->qinfo.qname_len < sizeof(clear)) {
-		memmove(clear+clen, qstate->qinfo.qname,
-			qstate->qinfo.qname_len);
-		clen += qstate->qinfo.qname_len;
+	if(clen + qinfo->qname_len < sizeof(clear)) {
+		memmove(clear+clen, qinfo->qname, qinfo->qname_len);
+		query_dname_tolower(clear+clen);
+		clen += qinfo->qname_len;
 	}
 	if(clen + 4 < sizeof(clear)) {
-		uint16_t t = htons(qstate->qinfo.qtype);
-		uint16_t c = htons(qstate->qinfo.qclass);
+		uint16_t t = htons(qinfo->qtype);
+		uint16_t c = htons(qinfo->qclass);
 		memmove(clear+clen, &t, 2);
 		memmove(clear+clen+2, &c, 2);
 		clen += 4;
@@ -511,9 +506,38 @@ adjust_msg_ttl(struct dns_msg* msg, time_t adjust)
 	}
 }
 
+/* Set the TTL of the given RRset to fixed value. */
+static void
+packed_rrset_ttl_set(struct packed_rrset_data* data, time_t ttl)
+{
+	size_t i;
+	size_t total = data->count + data->rrsig_count;
+	data->ttl = ttl;
+	for(i=0; i<total; i++) {
+		data->rr_ttl[i] = ttl;
+	}
+	data->ttl_add = 0;
+}
+
+/* Set the TTL of a DNS message and its RRs by to a fixed value. */
+static void
+set_msg_ttl(struct dns_msg* msg, time_t ttl)
+{
+	size_t i;
+	msg->rep->ttl = ttl;
+	msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
+	msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
+
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		packed_rrset_ttl_set((struct packed_rrset_data*)msg->
+			rep->rrsets[i]->entry.data, ttl);
+	}
+}
+
 /** convert dns message in buffer to return_msg */
 static int
-parse_data(struct module_qstate* qstate, struct sldns_buffer* buf)
+parse_data(struct module_qstate* qstate, struct sldns_buffer* buf,
+	int* msg_expired)
 {
 	struct msg_parse* prs;
 	struct edns_data edns;
@@ -583,6 +607,7 @@ parse_data(struct module_qstate* qstate, struct sldns_buffer* buf)
 	adjust = *qstate->env->now - (time_t)timestamp;
 	if(qstate->return_msg->rep->ttl < adjust) {
 		verbose(VERB_ALGO, "cachedb msg expired");
+		*msg_expired = 1;
 		/* If serve-expired is enabled, we still use an expired message
 		 * setting the TTL to 0. */
 		if(!qstate->env->cfg->serve_expired ||
@@ -598,6 +623,9 @@ parse_data(struct module_qstate* qstate, struct sldns_buffer* buf)
 	}
 	verbose(VERB_ALGO, "cachedb msg adjusted down by %d", (int)adjust);
 	adjust_msg_ttl(qstate->return_msg, adjust);
+	if(qstate->env->cfg->aggressive_nsec) {
+		limit_nsec_ttl(qstate->return_msg);
+	}
 
 	/* Similar to the unbound worker, if serve-expired is enabled and
 	 * the msg would be considered to be expired, mark the state so a
@@ -605,6 +633,7 @@ parse_data(struct module_qstate* qstate, struct sldns_buffer* buf)
 	 * 'now' should be redundant given how these values were calculated,
 	 * but we check it just in case as does good_expiry_and_qinfo(). */
 	if(qstate->env->cfg->serve_expired &&
+		!qstate->env->cfg->serve_expired_client_timeout &&
 		(adjust == -1 || (time_t)expiry < *qstate->env->now)) {
 		qstate->need_refetch = 1;
 	}
@@ -617,10 +646,11 @@ parse_data(struct module_qstate* qstate, struct sldns_buffer* buf)
  * return true if lookup was successful.
  */
 static int
-cachedb_extcache_lookup(struct module_qstate* qstate, struct cachedb_env* ie)
+cachedb_extcache_lookup(struct module_qstate* qstate, struct cachedb_env* ie,
+	int* msg_expired)
 {
 	char key[(CACHEDB_HASHSIZE/8)*2+1];
-	calc_hash(qstate, key, sizeof(key));
+	calc_hash(&qstate->qinfo, qstate->env, key, sizeof(key));
 
 	/* call backend to fetch data for key into scratch buffer */
 	if( !(*ie->backend->lookup)(qstate->env, ie, key,
@@ -634,7 +664,7 @@ cachedb_extcache_lookup(struct module_qstate* qstate, struct cachedb_env* ie)
 	}
 
 	/* parse dns message into return_msg */
-	if( !parse_data(qstate, qstate->env->scratch_buffer) ) {
+	if( !parse_data(qstate, qstate->env->scratch_buffer, msg_expired) ) {
 		return 0;
 	}
 	return 1;
@@ -647,7 +677,7 @@ static void
 cachedb_extcache_store(struct module_qstate* qstate, struct cachedb_env* ie)
 {
 	char key[(CACHEDB_HASHSIZE/8)*2+1];
-	calc_hash(qstate, key, sizeof(key));
+	calc_hash(&qstate->qinfo, qstate->env, key, sizeof(key));
 
 	/* prepare data in scratch buffer */
 	if(!prep_data(qstate, qstate->env->scratch_buffer))
@@ -666,6 +696,7 @@ cachedb_extcache_store(struct module_qstate* qstate, struct cachedb_env* ie)
 static int
 cachedb_intcache_lookup(struct module_qstate* qstate, struct cachedb_env* cde)
 {
+	uint8_t dpname_storage[LDNS_MAX_DOMAINLEN+1];
 	uint8_t* dpname=NULL;
 	size_t dpnamelen=0;
 	struct dns_msg* msg;
@@ -674,7 +705,7 @@ cachedb_intcache_lookup(struct module_qstate* qstate, struct cachedb_env* cde)
 		return 0;
 	}
 	if(iter_stub_fwd_no_cache(qstate, &qstate->qinfo,
-		&dpname, &dpnamelen))
+		&dpname, &dpnamelen, dpname_storage, sizeof(dpname_storage)))
 		return 0; /* no cache for these queries */
 	msg = dns_cache_lookup(qstate->env, qstate->qinfo.qname,
 		qstate->qinfo.qname_len, qstate->qinfo.qtype,
@@ -705,17 +736,44 @@ cachedb_intcache_lookup(struct module_qstate* qstate, struct cachedb_env* cde)
  * Store query into the internal cache of unbound.
  */
 static void
-cachedb_intcache_store(struct module_qstate* qstate)
+cachedb_intcache_store(struct module_qstate* qstate, int msg_expired)
 {
 	uint32_t store_flags = qstate->query_flags;
+	int serve_expired = qstate->env->cfg->serve_expired;
 
 	if(qstate->env->cfg->serve_expired)
 		store_flags |= DNSCACHE_STORE_ZEROTTL;
 	if(!qstate->return_msg)
 		return;
+	if(serve_expired && msg_expired) {
+		/* Set TTLs to a value such that value + *env->now is
+		 * going to be now-3 seconds. Making it expired
+		 * in the cache. */
+		set_msg_ttl(qstate->return_msg, (time_t)-3);
+		/* The expired entry does not get checked by the validator
+		 * and we need a validation value for it. */
+		if(qstate->env->cfg->cachedb_check_when_serve_expired)
+			qstate->return_msg->rep->security = sec_status_insecure;
+	}
 	(void)dns_cache_store(qstate->env, &qstate->qinfo,
 		qstate->return_msg->rep, 0, qstate->prefetch_leeway, 0,
-		qstate->region, store_flags, qstate->qstarttime);
+		qstate->region, store_flags, qstate->qstarttime,
+		qstate->is_valrec);
+	if(serve_expired && msg_expired) {
+		if(qstate->env->cfg->serve_expired_client_timeout) {
+			/* No expired response from the query state, the
+			 * query resolution needs to continue and it can
+			 * pick up the expired result after the timer out
+			 * of cache. */
+			return;
+		}
+		/* set TTLs to zero again */
+		adjust_msg_ttl(qstate->return_msg, -1);
+		/* Send serve expired responses based on the cachedb
+		 * returned message, that was just stored in the cache.
+		 * It can then continue to work on this query. */
+		mesh_respond_serve_expired(qstate->mesh_info);
+	}
 }
 
 /**
@@ -731,6 +789,7 @@ cachedb_handle_query(struct module_qstate* qstate,
 	struct cachedb_qstate* ATTR_UNUSED(iq),
 	struct cachedb_env* ie, int id)
 {
+	int msg_expired = 0;
 	qstate->is_cachedb_answer = 0;
 	/* check if we are enabled, and skip if so */
 	if(!ie->enabled) {
@@ -765,20 +824,26 @@ cachedb_handle_query(struct module_qstate* qstate,
 	}
 
 	/* ask backend cache to see if we have data */
-	if(cachedb_extcache_lookup(qstate, ie)) {
+	if(cachedb_extcache_lookup(qstate, ie, &msg_expired)) {
 		if(verbosity >= VERB_ALGO)
 			log_dns_msg(ie->backend->name,
 				&qstate->return_msg->qinfo,
 				qstate->return_msg->rep);
 		/* store this result in internal cache */
-		cachedb_intcache_store(qstate);
+		cachedb_intcache_store(qstate, msg_expired);
 		/* In case we have expired data but there is a client timer for expired
 		 * answers, pass execution to next module in order to try updating the
 		 * data first.
-		 * TODO: this needs revisit. The expired data stored from cachedb has
-		 * 0 TTL which is picked up by iterator later when looking in the cache.
-		 * Document that ext cachedb does not work properly with
-		 * serve_stale_reply_ttl yet. */
+		 */
+		if(qstate->env->cfg->serve_expired && msg_expired) {
+			qstate->return_msg = NULL;
+			qstate->ext_state[id] = module_wait_module;
+			/* The expired reply is sent with
+			 * mesh_respond_serve_expired, and so
+			 * the need_refetch is not used. */
+			qstate->need_refetch = 0;
+			return;
+		}
 		if(qstate->need_refetch && qstate->serve_expired_data &&
 			qstate->serve_expired_data->timer) {
 				qstate->return_msg = NULL;
@@ -791,6 +856,14 @@ cachedb_handle_query(struct module_qstate* qstate,
 		return;
 	}
 
+	if(qstate->serve_expired_data &&
+		qstate->env->cfg->cachedb_check_when_serve_expired &&
+		!qstate->env->cfg->serve_expired_client_timeout) {
+		/* Reply with expired data if any to client, because cachedb
+		 * also has no useful, current data */
+		mesh_respond_serve_expired(qstate->mesh_info);
+	}
+
 	/* no cache fetches */
 	/* pass request to next module */
 	qstate->ext_state[id] = module_wait_module;
@@ -914,7 +987,7 @@ cachedb_get_mem(struct module_env* env, int id)
  */
 static struct module_func_block cachedb_block = {
 	"cachedb",
-	&cachedb_init, &cachedb_deinit, &cachedb_operate,
+	NULL, NULL, &cachedb_init, &cachedb_deinit, &cachedb_operate,
 	&cachedb_inform_super, &cachedb_clear, &cachedb_get_mem
 };
 
@@ -923,4 +996,41 @@ cachedb_get_funcblock(void)
 {
 	return &cachedb_block;
 }
+
+int
+cachedb_is_enabled(struct module_stack* mods, struct module_env* env)
+{
+	struct cachedb_env* ie;
+	int id = modstack_find(mods, "cachedb");
+	if(id == -1)
+		return 0;
+	ie = (struct cachedb_env*)env->modinfo[id];
+	if(ie && ie->enabled)
+		return 1;
+	return 0;
+}
+
+void cachedb_msg_remove(struct module_qstate* qstate)
+{
+	cachedb_msg_remove_qinfo(qstate->env, &qstate->qinfo);
+}
+
+void cachedb_msg_remove_qinfo(struct module_env* env, struct query_info* qinfo)
+{
+	char key[(CACHEDB_HASHSIZE/8)*2+1];
+	int id = modstack_find(env->modstack, "cachedb");
+	struct cachedb_env* ie = (struct cachedb_env*)env->modinfo[id];
+
+	log_query_info(VERB_ALGO, "cachedb msg remove", qinfo);
+	calc_hash(qinfo, env, key, sizeof(key));
+	sldns_buffer_clear(env->scratch_buffer);
+	sldns_buffer_write_u32(env->scratch_buffer, 0);
+	sldns_buffer_flip(env->scratch_buffer);
+
+	/* call backend */
+	(*ie->backend->store)(env, ie, key,
+		sldns_buffer_begin(env->scratch_buffer),
+		sldns_buffer_limit(env->scratch_buffer),
+		0);
+}
 #endif /* USE_CACHEDB */

cachedb/cachedb.h

@@ -41,6 +41,7 @@
  */
 #include "util/module.h"
 struct cachedb_backend;
+struct module_stack;
 
 /**
  * The global variable environment contents for the cachedb
@@ -110,3 +111,26 @@ size_t cachedb_get_mem(struct module_env* env, int id);
  */
 struct module_func_block* cachedb_get_funcblock(void);
 
+/**
+ * See if the cachedb is enabled.
+ * @param mods: module stack. It finds the cachedb module environment.
+ * @param env: module environment.
+ * @return true if exists and enabled.
+ */
+int cachedb_is_enabled(struct module_stack* mods, struct module_env* env);
+
+/**
+ * Remove a message from the global cache. Because edns subnet has a more
+ * specific entry, and if not removed when everything expires, the global
+ * entry is used, instead of a fresh lookup of the edns subnet entry.
+ * @param qstate: query state.
+ */
+void cachedb_msg_remove(struct module_qstate* qstate);
+
+/**
+ * Remove message from the cachedb cache, by query info.
+ * @param env: module environment to look up cachedb state.
+ * @param qinfo: the message to remove.
+ */
+void cachedb_msg_remove_qinfo(struct module_env* env,
+	struct query_info* qinfo);

cachedb/redis.c

@@ -46,24 +46,62 @@
 #include "cachedb/cachedb.h"
 #include "util/alloc.h"
 #include "util/config_file.h"
+#include "util/locks.h"
+#include "util/timeval_func.h"
 #include "sldns/sbuffer.h"
 
 #ifdef USE_REDIS
 #include "hiredis/hiredis.h"
 
 struct redis_moddata {
-	redisContext** ctxs;	/* thread-specific redis contexts */
-	int numctxs;		/* number of ctx entries */
-	const char* server_host; /* server's IP address or host name */
-	int server_port;	 /* server's TCP port */
-	const char* server_path; /* server's unix path, or "", NULL if unused */
-	const char* server_password; /* server's AUTH password, or "", NULL if unused */
-	struct timeval timeout;	 /* timeout for connection setup and commands */
-	int logical_db;		/* the redis logical database to use */
+	/* thread-specific redis contexts */
+	redisContext** ctxs;
+	redisContext** replica_ctxs;
+	/* number of ctx entries */
+	int numctxs;
+	/* server's IP address or host name */
+	const char* server_host;
+	const char* replica_server_host;
+	/* server's TCP port */
+	int server_port;
+	int replica_server_port;
+	/* server's unix path, or "", NULL if unused */
+	const char* server_path;
+	const char* replica_server_path;
+	/* server's AUTH password, or "", NULL if unused */
+	const char* server_password;
+	const char* replica_server_password;
+	/* timeout for commands */
+	struct timeval command_timeout;
+	struct timeval replica_command_timeout;
+	/* timeout for connection setup */
+	struct timeval connect_timeout;
+	struct timeval replica_connect_timeout;
+	/* the reconnect interval time. */
+	struct timeval reconnect_interval;
+	struct timeval replica_reconnect_interval;
+	/* reconnect attempts, 0 if connected, counts up failed reconnects. */
+	int reconnect_attempts;
+	int replica_reconnect_attempts;
+	/* Lock on reconnect_wait time. */
+	lock_basic_type wait_lock;
+	lock_basic_type replica_wait_lock;
+	/* reconnect wait time, wait until it has passed before reconnect. */
+	struct timeval reconnect_wait;
+	struct timeval replica_reconnect_wait;
+	/* the redis logical database to use */
+	int logical_db;
+	int replica_logical_db;
+	/* if the SET with EX command is supported */
+	int set_with_ex_available;
 };
 
+/** The limit on the number of redis connect attempts. After failure if
+ * the number is exceeded, the reconnects are throttled by the wait time. */
+#define REDIS_RECONNECT_ATTEMPT_LIMIT 3
+
 static redisReply* redis_command(struct module_env*, struct cachedb_env*,
-	const char*, const uint8_t*, size_t);
+	const char*, const uint8_t*, size_t, int);
 
 static void
 moddata_clean(struct redis_moddata** moddata) {
@@ -77,63 +115,139 @@ moddata_clean(struct redis_moddata** moddata) {
 		}
 		free((*moddata)->ctxs);
 	}
+	if((*moddata)->replica_ctxs) {
+		int i;
+		for(i = 0; i < (*moddata)->numctxs; i++) {
+			if((*moddata)->replica_ctxs[i])
+				redisFree((*moddata)->replica_ctxs[i]);
+		}
+		free((*moddata)->replica_ctxs);
+	}
+	lock_basic_destroy(&(*moddata)->wait_lock);
+	lock_basic_destroy(&(*moddata)->replica_wait_lock);
 	free(*moddata);
 	*moddata = NULL;
 }
 
 static redisContext*
-redis_connect(const struct redis_moddata* moddata)
+redis_connect(const char* host, int port, const char* path,
+	const char* password, int logical_db,
+	const struct timeval connect_timeout,
+	const struct timeval command_timeout,
+	const struct timeval* reconnect_interval,
+	int* reconnect_attempts,
+	struct timeval* reconnect_wait,
+	lock_basic_type* wait_lock,
+	struct timeval* now_tv,
+	const char* infostr)
 {
+	struct timeval now_val;
 	redisContext* ctx;
 
-	if(moddata->server_path && moddata->server_path[0]!=0) {
-		ctx = redisConnectUnixWithTimeout(moddata->server_path,
-			moddata->timeout);
+	/* See if the redis server is down, and reconnect has to wait. */
+	if(*reconnect_attempts > REDIS_RECONNECT_ATTEMPT_LIMIT) {
+		/* Acquire lock to look at timeval, the integer has atomic
+		 * integrity. */
+		struct timeval wait_tv;
+		if(now_tv) {
+			now_val = *now_tv;
+		} else {
+			if(gettimeofday(&now_val, NULL) < 0)
+				log_err("redis: gettimeofday: %s",
+					strerror(errno));
+		}
+		lock_basic_lock(wait_lock);
+		wait_tv = *reconnect_wait;
+		lock_basic_unlock(wait_lock);
+		if(timeval_smaller(&now_val, &wait_tv)) {
+			verbose(VERB_ALGO, "redis %sdown, reconnect wait",
+				infostr);
+			return NULL;
+		}
+	}
+
+	if(path && path[0]!=0) {
+		ctx = redisConnectUnixWithTimeout(path, connect_timeout);
 	} else {
-		ctx = redisConnectWithTimeout(moddata->server_host,
-			moddata->server_port, moddata->timeout);
+		ctx = redisConnectWithTimeout(host, port, connect_timeout);
 	}
 	if(!ctx || ctx->err) {
 		const char *errstr = "out of memory";
 		if(ctx)
 			errstr = ctx->errstr;
-		log_err("failed to connect to redis server: %s", errstr);
+		log_err("failed to connect to redis %sserver: %s", infostr, errstr);
 		goto fail;
 	}
-	if(redisSetTimeout(ctx, moddata->timeout) != REDIS_OK) {
-		log_err("failed to set redis timeout");
+	if(redisSetTimeout(ctx, command_timeout) != REDIS_OK) {
+		log_err("failed to set redis %stimeout, %s", infostr, ctx->errstr);
 		goto fail;
 	}
-	if(moddata->server_password && moddata->server_password[0]!=0) {
+	if(password && password[0]!=0) {
 		redisReply* rep;
-		rep = redisCommand(ctx, "AUTH %s", moddata->server_password);
+		rep = redisCommand(ctx, "AUTH %s", password);
 		if(!rep || rep->type == REDIS_REPLY_ERROR) {
-			log_err("failed to authenticate with password");
+			log_err("failed to authenticate %swith password", infostr);
 			freeReplyObject(rep);
 			goto fail;
 		}
 		freeReplyObject(rep);
 	}
-	if(moddata->logical_db > 0) {
+	if(logical_db > 0) {
 		redisReply* rep;
-		rep = redisCommand(ctx, "SELECT %d", moddata->logical_db);
+		rep = redisCommand(ctx, "SELECT %d", logical_db);
 		if(!rep || rep->type == REDIS_REPLY_ERROR) {
-			log_err("failed to set logical database (%d)",
-				moddata->logical_db);
+			log_err("failed %sto set logical database (%d)",
+				infostr, logical_db);
 			freeReplyObject(rep);
 			goto fail;
 		}
 		freeReplyObject(rep);
 	}
-	verbose(VERB_OPS, "Connection to Redis established");
+	*reconnect_attempts = 0;
+	if(verbosity >= VERB_OPS) {
+		char port_str[6+1];
+		port_str[0] = ' ';
+		(void)snprintf(port_str+1, sizeof(port_str)-1, "%d", port);
+		verbose(VERB_OPS, "Connection to Redis %sestablished (%s%s)",
+			infostr,
+			path&&path[0]!=0?path:host,
+			path&&path[0]!=0?"":port_str);
+	}
 	return ctx;
 
 fail:
 	if(ctx)
 		redisFree(ctx);
+	(*reconnect_attempts)++;
+	if(*reconnect_attempts > REDIS_RECONNECT_ATTEMPT_LIMIT) {
+		/* Wait for the reconnect interval before trying again. */
+		struct timeval tv;
+		if(now_tv) {
+			now_val = *now_tv;
+		} else {
+			if(gettimeofday(&now_val, NULL) < 0)
+				log_err("redis: gettimeofday: %s",
+					strerror(errno));
+		}
+		tv = now_val;
+		timeval_add(&tv, reconnect_interval);
+		lock_basic_lock(wait_lock);
+		*reconnect_wait = tv;
+		lock_basic_unlock(wait_lock);
+		verbose(VERB_ALGO, "redis %sreconnect wait until %d.%6.6d",
+			infostr, (int)tv.tv_sec, (int)tv.tv_usec);
+	}
 	return NULL;
 }
 
+static void
+set_timeout(struct timeval* timeout, int value, int explicit_value)
+{
+	int v = explicit_value != 0 ? explicit_value : value;
+	timeout->tv_sec = v / 1000;
+	timeout->tv_usec = (v % 1000) * 1000;
+}
+
 static int
 redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
 {
@@ -147,42 +261,120 @@ redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
 		log_err("out of memory");
 		goto fail;
 	}
+	lock_basic_init(&moddata->wait_lock);
+	lock_protect(&moddata->wait_lock, &moddata->reconnect_wait,
+		sizeof(moddata->reconnect_wait));
+	lock_basic_init(&moddata->replica_wait_lock);
+	lock_protect(&moddata->replica_wait_lock,
+		&moddata->replica_reconnect_wait,
+		sizeof(moddata->replica_reconnect_wait));
 	moddata->numctxs = env->cfg->num_threads;
+	/* note: server_host and similar string configuration options are
+	 * shallow references to configured strings; we don't have to free them
+	 * in this module. */
+	moddata->server_host = env->cfg->redis_server_host;
+	moddata->replica_server_host = env->cfg->redis_replica_server_host;
+
+	moddata->server_port = env->cfg->redis_server_port;
+	moddata->replica_server_port = env->cfg->redis_replica_server_port;
+
+	moddata->server_path = env->cfg->redis_server_path;
+	moddata->replica_server_path = env->cfg->redis_replica_server_path;
+
+	moddata->server_password = env->cfg->redis_server_password;
+	moddata->replica_server_password = env->cfg->redis_replica_server_password;
+
+	set_timeout(&moddata->command_timeout,
+		env->cfg->redis_timeout,
+		env->cfg->redis_command_timeout);
+	set_timeout(&moddata->replica_command_timeout,
+		env->cfg->redis_replica_timeout,
+		env->cfg->redis_replica_command_timeout);
+	set_timeout(&moddata->connect_timeout,
+		env->cfg->redis_timeout,
+		env->cfg->redis_connect_timeout);
+	set_timeout(&moddata->replica_connect_timeout,
+		env->cfg->redis_replica_timeout,
+		env->cfg->redis_replica_connect_timeout);
+	set_timeout(&moddata->reconnect_interval, 1000, 0);
+	set_timeout(&moddata->replica_reconnect_interval, 1000, 0);
+
+	moddata->logical_db = env->cfg->redis_logical_db;
+	moddata->replica_logical_db = env->cfg->redis_replica_logical_db;
+
 	moddata->ctxs = calloc(env->cfg->num_threads, sizeof(redisContext*));
 	if(!moddata->ctxs) {
 		log_err("out of memory");
 		goto fail;
 	}
-	/* note: server_host is a shallow reference to configured string.
-	 * we don't have to free it in this module. */
-	moddata->server_host = env->cfg->redis_server_host;
-	moddata->server_port = env->cfg->redis_server_port;
-	moddata->server_path = env->cfg->redis_server_path;
-	moddata->server_password = env->cfg->redis_server_password;
-	moddata->timeout.tv_sec = env->cfg->redis_timeout / 1000;
-	moddata->timeout.tv_usec = (env->cfg->redis_timeout % 1000) * 1000;
-	moddata->logical_db = env->cfg->redis_logical_db;
-	for(i = 0; i < moddata->numctxs; i++) {
-		redisContext* ctx = redis_connect(moddata);
-		if(!ctx) {
-			log_err("redis_init: failed to init redis");
+	if((moddata->replica_server_host && moddata->replica_server_host[0]!=0)
+		|| (moddata->replica_server_path && moddata->replica_server_path[0]!=0)) {
+		/* There is a replica configured, allocate ctxs */
+		moddata->replica_ctxs = calloc(env->cfg->num_threads, sizeof(redisContext*));
+		if(!moddata->replica_ctxs) {
+			log_err("out of memory");
 			goto fail;
 		}
+	}
+	for(i = 0; i < moddata->numctxs; i++) {
+		redisContext* ctx = redis_connect(
+			moddata->server_host,
+			moddata->server_port,
+			moddata->server_path,
+			moddata->server_password,
+			moddata->logical_db,
+			moddata->connect_timeout,
+			moddata->command_timeout,
+			&moddata->reconnect_interval,
+			&moddata->reconnect_attempts,
+			&moddata->reconnect_wait,
+			&moddata->wait_lock,
+			env->now_tv,
+			"");
+		if(!ctx) {
+			log_err("redis_init: failed to init redis "
+				"(for thread %d)", i);
+			/* And continue, the context can be established
+			 * later, just like after a disconnect. */
+		}
 		moddata->ctxs[i] = ctx;
 	}
+	if(moddata->replica_ctxs) {
+		for(i = 0; i < moddata->numctxs; i++) {
+			redisContext* ctx = redis_connect(
+				moddata->replica_server_host,
+				moddata->replica_server_port,
+				moddata->replica_server_path,
+				moddata->replica_server_password,
+				moddata->replica_logical_db,
+				moddata->replica_connect_timeout,
+				moddata->replica_command_timeout,
+				&moddata->replica_reconnect_interval,
+				&moddata->replica_reconnect_attempts,
+				&moddata->replica_reconnect_wait,
+				&moddata->replica_wait_lock,
+				env->now_tv,
+				"replica ");
+			if(!ctx) {
+				log_err("redis_init: failed to init redis "
+					"replica (for thread %d)", i);
+				/* And continue, the context can be established
+				* later, just like after a disconnect. */
+			}
+			moddata->replica_ctxs[i] = ctx;
+		}
+	}
 	cachedb_env->backend_data = moddata;
-	if(env->cfg->redis_expire_records) {
+	if(env->cfg->redis_expire_records &&
+		moddata->ctxs[env->alloc->thread_num] != NULL) {
 		redisReply* rep = NULL;
 		int redis_reply_type = 0;
-		/** check if setex command is supported */
+		/** check if set with ex command is supported */
 		rep = redis_command(env, cachedb_env,
-			"SETEX __UNBOUND_REDIS_CHECK__ 1 none", NULL, 0);
+			"SET __UNBOUND_REDIS_CHECK__ none EX 1", NULL, 0, 1);
 		if(!rep) {
 			/** init failed, no response from redis server*/
-			log_err("redis_init: failed to init redis, the "
-				"redis-expire-records option requires the SETEX command "
-				"(redis >= 2.0.0)");
-			goto fail;
+			goto set_with_ex_fail;
 		}
 		redis_reply_type = rep->type;
 		freeReplyObject(rep);
@@ -190,15 +382,18 @@ redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
 		case REDIS_REPLY_STATUS:
 			break;
 		default:
-			/** init failed, setex command not supported */
-			log_err("redis_init: failed to init redis, the "
-				"redis-expire-records option requires the SETEX command "
-				"(redis >= 2.0.0)");
-			goto fail;
+			/** init failed, set_with_ex command not supported */
+			goto set_with_ex_fail;
 		}
+		moddata->set_with_ex_available = 1;
 	}
 	return 1;
 
+set_with_ex_fail:
+	log_err("redis_init: failure during redis_init, the "
+		"redis-expire-records option requires the SET with EX command "
+		"(redis >= 2.6.12)");
+	return 1;
 fail:
 	moddata_clean(&moddata);
 	return 0;
@@ -229,9 +424,9 @@ redis_deinit(struct module_env* env, struct cachedb_env* cachedb_env)
  */
 static redisReply*
 redis_command(struct module_env* env, struct cachedb_env* cachedb_env,
-	const char* command, const uint8_t* data, size_t data_len)
+	const char* command, const uint8_t* data, size_t data_len, int write)
 {
-	redisContext* ctx;
+	redisContext* ctx, **ctx_selector;
 	redisReply* rep;
 	struct redis_moddata* d = (struct redis_moddata*)
 		cachedb_env->backend_data;
@@ -242,17 +437,50 @@ redis_command(struct module_env* env, struct cachedb_env* cachedb_env,
 	 * assumption throughout the unbound architecture, so we simply assert
 	 * it. */
 	log_assert(env->alloc->thread_num < d->numctxs);
-	ctx = d->ctxs[env->alloc->thread_num];
+
+	ctx_selector = !write && d->replica_ctxs
+		?d->replica_ctxs
+		:d->ctxs;
+	ctx = ctx_selector[env->alloc->thread_num];
 
 	/* If we've not established a connection to the server or we've closed
 	 * it on a failure, try to re-establish a new one.   Failures will be
 	 * logged in redis_connect(). */
 	if(!ctx) {
-		ctx = redis_connect(d);
-		d->ctxs[env->alloc->thread_num] = ctx;
+		if(!write && d->replica_ctxs) {
+			ctx = redis_connect(
+				d->replica_server_host,
+				d->replica_server_port,
+				d->replica_server_path,
+				d->replica_server_password,
+				d->replica_logical_db,
+				d->replica_connect_timeout,
+				d->replica_command_timeout,
+				&d->replica_reconnect_interval,
+				&d->replica_reconnect_attempts,
+				&d->replica_reconnect_wait,
+				&d->replica_wait_lock,
+				env->now_tv,
+				"replica ");
+		} else {
+			ctx = redis_connect(
+				d->server_host,
+				d->server_port,
+				d->server_path,
+				d->server_password,
+				d->logical_db,
+				d->connect_timeout,
+				d->command_timeout,
+				&d->reconnect_interval,
+				&d->reconnect_attempts,
+				&d->reconnect_wait,
+				&d->wait_lock,
+				env->now_tv,
+				"");
+		}
+		ctx_selector[env->alloc->thread_num] = ctx;
 	}
-	if(!ctx)
-		return NULL;
+	if(!ctx) return NULL;
 
 	/* Send the command and get a reply, synchronously. */
 	rep = (redisReply*)redisCommand(ctx, command, data, data_len);
@@ -262,7 +490,7 @@ redis_command(struct module_env* env, struct cachedb_env* cachedb_env,
 		log_err("redis_command: failed to receive a reply, "
 			"closing connection: %s", ctx->errstr);
 		redisFree(ctx);
-		d->ctxs[env->alloc->thread_num] = NULL;
+		ctx_selector[env->alloc->thread_num] = NULL;
 		return NULL;
 	}
 
@@ -280,7 +508,14 @@ redis_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
 	char* key, struct sldns_buffer* result_buffer)
 {
 	redisReply* rep;
-	char cmdbuf[4+(CACHEDB_HASHSIZE/8)*2+1]; /* "GET " + key */
+	/* Supported commands:
+	 * - "GET " + key
+	 */
+#define REDIS_LOOKUP_MAX_BUF_LEN			\
+	4				/* "GET " */	\
+	+(CACHEDB_HASHSIZE/8)*2		/* key hash */	\
+	+ 1				/* \0 */
+	char cmdbuf[REDIS_LOOKUP_MAX_BUF_LEN];
 	int n;
 	int ret = 0;
 
@@ -292,7 +527,7 @@ redis_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
 		return 0;
 	}
 
-	rep = redis_command(env, cachedb_env, cmdbuf, NULL, 0);
+	rep = redis_command(env, cachedb_env, cmdbuf, NULL, 0, 0);
 	if(!rep)
 		return 0;
 	switch(rep->type) {
@@ -329,26 +564,45 @@ redis_store(struct module_env* env, struct cachedb_env* cachedb_env,
 {
 	redisReply* rep;
 	int n;
-	int set_ttl = (env->cfg->redis_expire_records &&
+	struct redis_moddata* moddata = (struct redis_moddata*)
+		cachedb_env->backend_data;
+	int set_ttl = (moddata->set_with_ex_available &&
+		env->cfg->redis_expire_records &&
 		(!env->cfg->serve_expired || env->cfg->serve_expired_ttl > 0));
 	/* Supported commands:
 	 * - "SET " + key + " %b"
-	 * - "SETEX " + key + " " + ttl + " %b"
+	 * - "SET " + key + " %b EX " + ttl
+	 *   older redis 2.0.0 was "SETEX " + key + " " + ttl + " %b"
+	 * - "EXPIRE " + key + " 0"
 	 */
-	char cmdbuf[6+(CACHEDB_HASHSIZE/8)*2+11+3+1];
+#define REDIS_STORE_MAX_BUF_LEN				\
+	7			/* "EXPIRE " */		\
+	+(CACHEDB_HASHSIZE/8)*2	/* key hash */		\
+	+ 7			/* " %b EX " */		\
+	+ 20			/* ttl (uint64_t) */	\
+	+ 1			/* \0 */
+	char cmdbuf[REDIS_STORE_MAX_BUF_LEN];
 
 	if (!set_ttl) {
 		verbose(VERB_ALGO, "redis_store %s (%d bytes)", key, (int)data_len);
 		/* build command to set to a binary safe string */
 		n = snprintf(cmdbuf, sizeof(cmdbuf), "SET %s %%b", key);
+	} else if(ttl == 0) {
+		/* use the EXPIRE command, SET with EX 0 is an invalid time. */
+		/* Replies with REDIS_REPLY_INTEGER of 1. */
+		verbose(VERB_ALGO, "redis_store expire %s (%d bytes)",
+			key, (int)data_len);
+		n = snprintf(cmdbuf, sizeof(cmdbuf), "EXPIRE %s 0", key);
+		data = NULL;
+		data_len = 0;
 	} else {
 		/* add expired ttl time to redis ttl to avoid premature eviction of key */
 		ttl += env->cfg->serve_expired_ttl;
 		verbose(VERB_ALGO, "redis_store %s (%d bytes) with ttl %u",
-			key, (int)data_len, (uint32_t)ttl);
+			key, (int)data_len, (unsigned)(uint32_t)ttl);
 		/* build command to set to a binary safe string */
-		n = snprintf(cmdbuf, sizeof(cmdbuf), "SETEX %s %u %%b", key,
-			(uint32_t)ttl);
+		n = snprintf(cmdbuf, sizeof(cmdbuf), "SET %s %%b EX %u", key,
+			(unsigned)(uint32_t)ttl);
 	}
 
 
@@ -357,11 +611,12 @@ redis_store(struct module_env* env, struct cachedb_env* cachedb_env,
 		return;
 	}
 
-	rep = redis_command(env, cachedb_env, cmdbuf, data, data_len);
+	rep = redis_command(env, cachedb_env, cmdbuf, data, data_len, 1);
 	if(rep) {
 		verbose(VERB_ALGO, "redis_store set completed");
 		if(rep->type != REDIS_REPLY_STATUS &&
-			rep->type != REDIS_REPLY_ERROR) {
+			rep->type != REDIS_REPLY_ERROR &&
+			rep->type != REDIS_REPLY_INTEGER) {
 			log_err("redis_store: unexpected type of reply (%d)",
 				rep->type);
 		}

compat/fake-rfc2553.c

@@ -57,7 +57,7 @@ int getnameinfo(const struct sockaddr *sa, size_t ATTR_UNUSED(salen), char *host
 	}
 
 	if (host != NULL) {
-		if (flags & NI_NUMERICHOST) {
+		if ((flags & NI_NUMERICHOST)) {
 			if (strlcpy(host, inet_ntoa(sin->sin_addr),
 			    hostlen) >= hostlen)
 				return (EAI_MEMORY);
@@ -168,7 +168,7 @@ getaddrinfo(const char *hostname, const char *servname,
 			port = 0;
 	}
 
-	if (hints && hints->ai_flags & AI_PASSIVE) {
+	if (hints && (hints->ai_flags & AI_PASSIVE)) {
 		addr = htonl(0x00000000);
 		if (hostname && inet_aton(hostname, &in) != 0)
 			addr = in.s_addr;
@@ -193,7 +193,7 @@ getaddrinfo(const char *hostname, const char *servname,
 	}
 	
 	/* Don't try DNS if AI_NUMERICHOST is set */
-	if (hints && hints->ai_flags & AI_NUMERICHOST)
+	if (hints && (hints->ai_flags & AI_NUMERICHOST))
 		return (EAI_NONAME);
 	
 	hp = gethostbyname(hostname);

compat/malloc.c

@@ -5,12 +5,8 @@
 #undef malloc
 #include <sys/types.h>
 
-#ifndef USE_WINSOCK
-void *malloc ();
-#else
 /* provide a prototype */
 void *malloc (size_t n);
-#endif
 
 /* Allocate an N-byte block of memory from the heap.
    If N is zero, allocate a 1-byte block.  */

config.h.in

@@ -1,5 +1,8 @@
 /* config.h.in.  Generated from configure.ac by autoheader.  */
 
+/* apply the fallthrough attribute. */
+#undef ATTR_FALLTHROUGH
+
 /* apply the noreturn attribute to a function that exits the program */
 #undef ATTR_NORETURN
 
@@ -57,6 +60,9 @@
 /* Define to 1 if you have the <arpa/inet.h> header file. */
 #undef HAVE_ARPA_INET_H
 
+/* Whether the C compiler accepts the "fallthrough" attribute */
+#undef HAVE_ATTR_FALLTHROUGH
+
 /* Whether the C compiler accepts the "format" attribute */
 #undef HAVE_ATTR_FORMAT
 
@@ -123,6 +129,14 @@
    and to 0 if you don't. */
 #undef HAVE_DECL_NGHTTP2_SESSION_SERVER_NEW
 
+/* Define to 1 if you have the declaration of `ngtcp2_conn_server_new', and to
+   0 if you don't. */
+#undef HAVE_DECL_NGTCP2_CONN_SERVER_NEW
+
+/* Define to 1 if you have the declaration of `ngtcp2_crypto_encrypt_cb', and
+   to 0 if you don't. */
+#undef HAVE_DECL_NGTCP2_CRYPTO_ENCRYPT_CB
+
 /* Define to 1 if you have the declaration of `NID_ED25519', and to 0 if you
    don't. */
 #undef HAVE_DECL_NID_ED25519
@@ -159,6 +173,10 @@
    0 if you don't. */
 #undef HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
 
+/* Define to 1 if you have the declaration of `SSL_CTX_set_tmp_ecdh', and to 0
+   if you don't. */
+#undef HAVE_DECL_SSL_CTX_SET_TMP_ECDH
+
 /* Define to 1 if you have the declaration of `strlcat', and to 0 if you
    don't. */
 #undef HAVE_DECL_STRLCAT
@@ -364,6 +382,9 @@
 /* Define if we have LibreSSL */
 #undef HAVE_LIBRESSL
 
+/* If we have atomic_store */
+#undef HAVE_LINK_ATOMIC_STORE
+
 /* Define to 1 if you have the <linux/net_tstamp.h> header file. */
 #undef HAVE_LINUX_NET_TSTAMP_H
 
@@ -379,8 +400,8 @@
 /* Define to 1 if you have the `memmove' function. */
 #undef HAVE_MEMMOVE
 
-/* Define to 1 if you have the <memory.h> header file. */
-#undef HAVE_MEMORY_H
+/* Define to 1 if you have the <minix/config.h> header file. */
+#undef HAVE_MINIX_CONFIG_H
 
 /* Define to 1 if you have the <netdb.h> header file. */
 #undef HAVE_NETDB_H
@@ -406,12 +427,80 @@
 /* Define to 1 if you have the <net/if.h> header file. */
 #undef HAVE_NET_IF_H
 
+/* Define to 1 if you have the <net/pfvar.h> header file. */
+#undef HAVE_NET_PFVAR_H
+
 /* Define this to use nghttp2 client. */
 #undef HAVE_NGHTTP2
 
 /* Define to 1 if you have the <nghttp2/nghttp2.h> header file. */
 #undef HAVE_NGHTTP2_NGHTTP2_H
 
+/* Define this to use ngtcp2. */
+#undef HAVE_NGTCP2
+
+/* Define to 1 if you have the `ngtcp2_ccerr_default' function. */
+#undef HAVE_NGTCP2_CCERR_DEFAULT
+
+/* Define to 1 if you have the `ngtcp2_conn_encode_0rtt_transport_params'
+   function. */
+#undef HAVE_NGTCP2_CONN_ENCODE_0RTT_TRANSPORT_PARAMS
+
+/* Define to 1 if you have the `ngtcp2_conn_get_max_local_streams_uni'
+   function. */
+#undef HAVE_NGTCP2_CONN_GET_MAX_LOCAL_STREAMS_UNI
+
+/* Define to 1 if you have the `ngtcp2_conn_get_num_scid' function. */
+#undef HAVE_NGTCP2_CONN_GET_NUM_SCID
+
+/* Define to 1 if you have the `ngtcp2_conn_in_closing_period' function. */
+#undef HAVE_NGTCP2_CONN_IN_CLOSING_PERIOD
+
+/* Define to 1 if you have the `ngtcp2_conn_in_draining_period' function. */
+#undef HAVE_NGTCP2_CONN_IN_DRAINING_PERIOD
+
+/* Define if ngtcp2_conn_shutdown_stream has 4 arguments. */
+#undef HAVE_NGTCP2_CONN_SHUTDOWN_STREAM4
+
+/* Define to 1 if you have the `ngtcp2_conn_tls_early_data_rejected' function.
+   */
+#undef HAVE_NGTCP2_CONN_TLS_EARLY_DATA_REJECTED
+
+/* Define to 1 if you have the `ngtcp2_crypto_encrypt_cb' function. */
+#undef HAVE_NGTCP2_CRYPTO_ENCRYPT_CB
+
+/* Define to 1 if you have the
+   `ngtcp2_crypto_quictls_configure_client_context' function. */
+#undef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_CLIENT_CONTEXT
+
+/* Define to 1 if you have the
+   `ngtcp2_crypto_quictls_configure_server_context' function. */
+#undef HAVE_NGTCP2_CRYPTO_QUICTLS_CONFIGURE_SERVER_CONTEXT
+
+/* Define to 1 if you have the
+   `ngtcp2_crypto_quictls_from_ossl_encryption_level' function. */
+#undef HAVE_NGTCP2_CRYPTO_QUICTLS_FROM_OSSL_ENCRYPTION_LEVEL
+
+/* Define to 1 if you have the `ngtcp2_crypto_quictls_init' function. */
+#undef HAVE_NGTCP2_CRYPTO_QUICTLS_INIT
+
+/* Define to 1 if the system has the type `ngtcp2_encryption_level'. */
+#undef HAVE_NGTCP2_ENCRYPTION_LEVEL
+
+/* Define to 1 if you have the <ngtcp2/ngtcp2_crypto_openssl.h> header file.
+   */
+#undef HAVE_NGTCP2_NGTCP2_CRYPTO_OPENSSL_H
+
+/* Define to 1 if you have the <ngtcp2/ngtcp2_crypto_ossl.h> header file. */
+#undef HAVE_NGTCP2_NGTCP2_CRYPTO_OSSL_H
+
+/* Define to 1 if you have the <ngtcp2/ngtcp2_crypto_quictls.h> header file.
+   */
+#undef HAVE_NGTCP2_NGTCP2_CRYPTO_QUICTLS_H
+
+/* Define to 1 if you have the <ngtcp2/ngtcp2.h> header file. */
+#undef HAVE_NGTCP2_NGTCP2_H
+
 /* Use libnss for crypto */
 #undef HAVE_NSS
 
@@ -575,18 +664,27 @@
 /* Define to 1 if you have the `SSL_get1_peer_certificate' function. */
 #undef HAVE_SSL_GET1_PEER_CERTIFICATE
 
+/* Define to 1 if you have the `SSL_is_quic' function. */
+#undef HAVE_SSL_IS_QUIC
+
 /* Define to 1 if you have the `SSL_set1_host' function. */
 #undef HAVE_SSL_SET1_HOST
 
 /* Define to 1 if you have the <stdarg.h> header file. */
 #undef HAVE_STDARG_H
 
+/* Define to 1 if you have the <stdatomic.h> header file. */
+#undef HAVE_STDATOMIC_H
+
 /* Define to 1 if you have the <stdbool.h> header file. */
 #undef HAVE_STDBOOL_H
 
 /* Define to 1 if you have the <stdint.h> header file. */
 #undef HAVE_STDINT_H
 
+/* Define to 1 if you have the <stdio.h> header file. */
+#undef HAVE_STDIO_H
+
 /* Define to 1 if you have the <stdlib.h> header file. */
 #undef HAVE_STDLIB_H
 
@@ -614,6 +712,23 @@
 /* Define to 1 if `ipi_spec_dst' is a member of `struct in_pktinfo'. */
 #undef HAVE_STRUCT_IN_PKTINFO_IPI_SPEC_DST
 
+/* Define to 1 if `tokenlen' is a member of `struct ngtcp2_pkt_hd'. */
+#undef HAVE_STRUCT_NGTCP2_PKT_HD_TOKENLEN
+
+/* Define to 1 if `max_tx_udp_payload_size' is a member of `struct
+   ngtcp2_settings'. */
+#undef HAVE_STRUCT_NGTCP2_SETTINGS_MAX_TX_UDP_PAYLOAD_SIZE
+
+/* Define to 1 if `tokenlen' is a member of `struct ngtcp2_settings'. */
+#undef HAVE_STRUCT_NGTCP2_SETTINGS_TOKENLEN
+
+/* Define to 1 if `original_dcid_present' is a member of `struct
+   ngtcp2_transport_params'. */
+#undef HAVE_STRUCT_NGTCP2_TRANSPORT_PARAMS_ORIGINAL_DCID_PRESENT
+
+/* Define to 1 if the system has the type `struct ngtcp2_version_cid'. */
+#undef HAVE_STRUCT_NGTCP2_VERSION_CID
+
 /* Define to 1 if `sun_len' is a member of `struct sockaddr_un'. */
 #undef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN
 
@@ -689,6 +804,9 @@
 /* Define to 1 if you have the <vfork.h> header file. */
 #undef HAVE_VFORK_H
 
+/* Define to 1 if you have the <wchar.h> header file. */
+#undef HAVE_WCHAR_H
+
 /* Define to 1 if you have the <windows.h> header file. */
 #undef HAVE_WINDOWS_H
 
@@ -830,7 +948,9 @@
 /* Define to 1 if libsodium supports sodium_set_misuse_handler */
 #undef SODIUM_MISUSE_HANDLER
 
-/* Define to 1 if you have the ANSI C header files. */
+/* Define to 1 if all of the C90 standard headers exist (not just the ones
+   required in a freestanding environment). This macro is provided for
+   backward compatibility; new code need not use it. */
 #undef STDC_HEADERS
 
 /* use default strptime. */
@@ -910,6 +1030,9 @@
 /* Define this to enable client TCP Fast Open. */
 #undef USE_MSG_FASTOPEN
 
+/* Define this to use ngtcp2_crypto_ossl. */
+#undef USE_NGTCP2_CRYPTO_OSSL
+
 /* Define this to enable client TCP Fast Open. */
 #undef USE_OSX_MSG_FASTOPEN
 
@@ -926,21 +1049,87 @@
 #ifndef _ALL_SOURCE
 # undef _ALL_SOURCE
 #endif
+/* Enable general extensions on macOS.  */
+#ifndef _DARWIN_C_SOURCE
+# undef _DARWIN_C_SOURCE
+#endif
+/* Enable general extensions on Solaris.  */
+#ifndef __EXTENSIONS__
+# undef __EXTENSIONS__
+#endif
 /* Enable GNU extensions on systems that have them.  */
 #ifndef _GNU_SOURCE
 # undef _GNU_SOURCE
 #endif
-/* Enable threading extensions on Solaris.  */
+/* Enable X/Open compliant socket functions that do not require linking
+   with -lxnet on HP-UX 11.11.  */
+#ifndef _HPUX_ALT_XOPEN_SOCKET_API
+# undef _HPUX_ALT_XOPEN_SOCKET_API
+#endif
+/* Identify the host operating system as Minix.
+   This macro does not affect the system headers' behavior.
+   A future release of Autoconf may stop defining this macro.  */
+#ifndef _MINIX
+# undef _MINIX
+#endif
+/* Enable general extensions on NetBSD.
+   Enable NetBSD compatibility extensions on Minix.  */
+#ifndef _NETBSD_SOURCE
+# undef _NETBSD_SOURCE
+#endif
+/* Enable OpenBSD compatibility extensions on NetBSD.
+   Oddly enough, this does nothing on OpenBSD.  */
+#ifndef _OPENBSD_SOURCE
+# undef _OPENBSD_SOURCE
+#endif
+/* Define to 1 if needed for POSIX-compatible behavior.  */
+#ifndef _POSIX_SOURCE
+# undef _POSIX_SOURCE
+#endif
+/* Define to 2 if needed for POSIX-compatible behavior.  */
+#ifndef _POSIX_1_SOURCE
+# undef _POSIX_1_SOURCE
+#endif
+/* Enable POSIX-compatible threading on Solaris.  */
 #ifndef _POSIX_PTHREAD_SEMANTICS
 # undef _POSIX_PTHREAD_SEMANTICS
 #endif
+/* Enable extensions specified by ISO/IEC TS 18661-5:2014.  */
+#ifndef __STDC_WANT_IEC_60559_ATTRIBS_EXT__
+# undef __STDC_WANT_IEC_60559_ATTRIBS_EXT__
+#endif
+/* Enable extensions specified by ISO/IEC TS 18661-1:2014.  */
+#ifndef __STDC_WANT_IEC_60559_BFP_EXT__
+# undef __STDC_WANT_IEC_60559_BFP_EXT__
+#endif
+/* Enable extensions specified by ISO/IEC TS 18661-2:2015.  */
+#ifndef __STDC_WANT_IEC_60559_DFP_EXT__
+# undef __STDC_WANT_IEC_60559_DFP_EXT__
+#endif
+/* Enable extensions specified by ISO/IEC TS 18661-4:2015.  */
+#ifndef __STDC_WANT_IEC_60559_FUNCS_EXT__
+# undef __STDC_WANT_IEC_60559_FUNCS_EXT__
+#endif
+/* Enable extensions specified by ISO/IEC TS 18661-3:2015.  */
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+# undef __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
+/* Enable extensions specified by ISO/IEC TR 24731-2:2010.  */
+#ifndef __STDC_WANT_LIB_EXT2__
+# undef __STDC_WANT_LIB_EXT2__
+#endif
+/* Enable extensions specified by ISO/IEC 24747:2009.  */
+#ifndef __STDC_WANT_MATH_SPEC_FUNCS__
+# undef __STDC_WANT_MATH_SPEC_FUNCS__
+#endif
 /* Enable extensions on HP NonStop.  */
 #ifndef _TANDEM_SOURCE
 # undef _TANDEM_SOURCE
 #endif
-/* Enable general extensions on Solaris.  */
-#ifndef __EXTENSIONS__
-# undef __EXTENSIONS__
+/* Enable X/Open extensions.  Define to 500 only if necessary
+   to make mbstate_t available.  */
+#ifndef _XOPEN_SOURCE
+# undef _XOPEN_SOURCE
 #endif
 
 
@@ -966,11 +1155,6 @@
    `char[]'. */
 #undef YYTEXT_POINTER
 
-/* Enable large inode numbers on Mac OS X 10.5.  */
-#ifndef _DARWIN_USE_64_BIT_INODE
-# define _DARWIN_USE_64_BIT_INODE 1
-#endif
-
 /* Number of bits in a file offset, on hosts where this is settable. */
 #undef _FILE_OFFSET_BITS
 
@@ -980,19 +1164,9 @@
 /* Define for large files, on AIX-style hosts. */
 #undef _LARGE_FILES
 
-/* Define to 1 if on MINIX. */
-#undef _MINIX
-
 /* Enable for compile on Minix */
 #undef _NETBSD_SOURCE
 
-/* Define to 2 if the system does not provide POSIX.1 features except with
-   this defined. */
-#undef _POSIX_1_SOURCE
-
-/* Define to 1 if you need to in order for `stat' and other things to work. */
-#undef _POSIX_SOURCE
-
 /* defined to use gcc ansi snprintf and sscanf that understands %lld when
    compiled for windows. */
 #undef __USE_MINGW_ANSI_STDIO
@@ -1033,7 +1207,7 @@
 /* Define to `long int' if <sys/types.h> does not define. */
 #undef off_t
 
-/* Define to `int' if <sys/types.h> does not define. */
+/* Define as a signed integer type capable of holding a process identifier. */
 #undef pid_t
 
 /* Define to 'int' if not defined */
@@ -1425,6 +1599,10 @@ struct sockaddr_storage;
 #  define calloc(n,s) unbound_stat_calloc_log(n, s, __FILE__, __LINE__, __func__)
 #  define free(p) unbound_stat_free_log(p, __FILE__, __LINE__, __func__)
 #  define realloc(p,s) unbound_stat_realloc_log(p, s, __FILE__, __LINE__, __func__)
+#  define strdup(s) unbound_stat_strdup_log(s, __FILE__, __LINE__, __func__)
+#ifdef HAVE_REALLOCARRAY
+#  define reallocarray(p,n,s) unbound_stat_reallocarray_log(p, n, s, __FILE__, __LINE__, __func__)
+#endif
 void *unbound_stat_malloc(size_t size);
 void *unbound_stat_calloc(size_t nmemb, size_t size);
 void unbound_stat_free(void *ptr);
@@ -1437,6 +1615,10 @@ void unbound_stat_free_log(void *ptr, const char* file, int line,
 	const char* func);
 void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
 	int line, const char* func);
+void *unbound_stat_reallocarray_log(void *ptr, size_t nmemb, size_t size,
+	const char* file, int line, const char* func);
+char *unbound_stat_strdup_log(const char *s, const char* file, int line,
+	const char* func);
 #elif defined(UNBOUND_ALLOC_LITE)
 #  include "util/alloc.h"
 #endif /* UNBOUND_ALLOC_LITE and UNBOUND_ALLOC_STATS */
@@ -1447,6 +1629,8 @@ void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
 #define UNBOUND_DNS_OVER_TLS_PORT 853
 /** default port for DNS over HTTPS traffic. */
 #define UNBOUND_DNS_OVER_HTTPS_PORT 443
+/** default port for DNS over QUIC traffic. */
+#define UNBOUND_DNS_OVER_QUIC_PORT 853
 /** default port for unbound control traffic, registered port with IANA,
     ub-dns-control  8953/tcp    unbound dns nameserver control */
 #define UNBOUND_CONTROL_PORT 8953

configure.ac

@@ -2,23 +2,24 @@
 # Process this file with autoconf to produce a configure script.
 AC_PREREQ([2.56])
 sinclude(acx_nlnetlabs.m4)
+sinclude(ax_build_date_epoch.m4)
 sinclude(ax_pthread.m4)
 sinclude(acx_python.m4)
-sinclude(ac_pkg_swig.m4)
+sinclude(ax_pkg_swig.m4)
 sinclude(dnstap/dnstap.m4)
 sinclude(dnscrypt/dnscrypt.m4)
 
 # must be numbers. ac_defun because of later processing
 m4_define([VERSION_MAJOR],[1])
-m4_define([VERSION_MINOR],[19])
-m4_define([VERSION_MICRO],[0])
+m4_define([VERSION_MINOR],[24])
+m4_define([VERSION_MICRO],[1])
 AC_INIT([unbound],m4_defn([VERSION_MAJOR]).m4_defn([VERSION_MINOR]).m4_defn([VERSION_MICRO]),[unbound-bugs@nlnetlabs.nl or https://github.com/NLnetLabs/unbound/issues],[unbound])
 AC_SUBST(UNBOUND_VERSION_MAJOR, [VERSION_MAJOR])
 AC_SUBST(UNBOUND_VERSION_MINOR, [VERSION_MINOR])
 AC_SUBST(UNBOUND_VERSION_MICRO, [VERSION_MICRO])
 
 LIBUNBOUND_CURRENT=9
-LIBUNBOUND_REVISION=23
+LIBUNBOUND_REVISION=34
 LIBUNBOUND_AGE=1
 # 1.0.0 had 0:12:0
 # 1.0.1 had 0:13:0
@@ -109,6 +110,17 @@ LIBUNBOUND_AGE=1
 # 1.17.1 had 9:21:1
 # 1.18.0 had 9:22:1
 # 1.19.0 had 9:23:1
+# 1.19.1 had 9:24:1
+# 1.19.2 had 9:25:1
+# 1.19.3 had 9:26:1
+# 1.20.0 had 9:27:1
+# 1.21.0 had 9:28:1
+# 1.21.1 had 9:29:1
+# 1.22.0 had 9:30:1
+# 1.23.0 had 9:31:1
+# 1.23.1 had 9:32:1
+# 1.24.0 had 9:33:1
+# 1.24.1 had 9:34:1
 
 #   Current  -- the number of the binary API that we're implementing
 #   Revision -- which iteration of the implementation of the binary
@@ -270,6 +282,9 @@ AC_DEFINE(WINVER, 0x0502, [the version of the windows API enabled])
 ACX_RSRC_VERSION(wnvs)
 AC_DEFINE_UNQUOTED(RSRC_PACKAGE_VERSION, [$wnvs], [version number for resource files])
 
+# Check for 'grep -e' program, here, since ACX_CHECK_FLTO needs that.
+AC_PROG_GREP
+
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
 AC_LANG([C])
@@ -331,6 +346,8 @@ AC_MSG_RESULT($ac_cv_c_weak_attribute)
 if test $ac_cv_c_weak_attribute = yes; then
   AC_DEFINE(HAVE_ATTR_WEAK, 1, [Whether the C compiler accepts the "weak" attribute])
   AC_DEFINE(ATTR_WEAK, [__attribute__((weak))], [apply the weak attribute to a symbol])
+else
+  AC_DEFINE(ATTR_WEAK,[], [apply the weak attribute to a symbol])
 fi
 ])dnl End of CHECK_WEAK_ATTRIBUTE
 
@@ -352,11 +369,54 @@ AC_MSG_RESULT($ac_cv_c_noreturn_attribute)
 if test $ac_cv_c_noreturn_attribute = yes; then
   AC_DEFINE(HAVE_ATTR_NORETURN, 1, [Whether the C compiler accepts the "noreturn" attribute])
   AC_DEFINE(ATTR_NORETURN, [__attribute__((__noreturn__))], [apply the noreturn attribute to a function that exits the program])
+else
+  AC_DEFINE(ATTR_NORETURN,[], [apply the noreturn attribute to a function that exits the program])
 fi
 ])dnl End of CHECK_NORETURN_ATTRIBUTE
 
 CHECK_NORETURN_ATTRIBUTE
 
+AC_DEFUN([CHECK_FALLTHROUGH_ATTRIBUTE],
+[AC_REQUIRE([AC_PROG_CC])
+AC_MSG_CHECKING(whether the C compiler (${CC-cc}) accepts the "fallthrough" attribute)
+BAKCFLAGS="$CFLAGS"
+CFLAGS="$CFLAGS -Werror"
+AC_CACHE_VAL(ac_cv_c_fallthrough_attribute,
+[ac_cv_c_fallthrough_attribute=no
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <stdio.h>
+void f(int x) {
+	int y = 0;
+	switch(x) {
+	case 1:
+		y = 1;
+		__attribute__((fallthrough));
+		/* fallthrough */
+	case 2:
+		y++;
+		break;
+	case 3:
+		y = 3;
+		break;
+	}
+	printf("%d", y);
+}
+]], [[
+   f(1);
+]])],[ac_cv_c_fallthrough_attribute="yes"],[ac_cv_c_fallthrough_attribute="no"])
+])
+CFLAGS="$BAKCFLAGS"
+
+AC_MSG_RESULT($ac_cv_c_fallthrough_attribute)
+if test $ac_cv_c_fallthrough_attribute = yes; then
+  AC_DEFINE(HAVE_ATTR_FALLTHROUGH, 1, [Whether the C compiler accepts the "fallthrough" attribute])
+  AC_DEFINE(ATTR_FALLTHROUGH, [__attribute__((fallthrough));], [apply the fallthrough attribute.])
+else
+  AC_DEFINE(ATTR_FALLTHROUGH,[], [apply the fallthrough attribute.])
+fi
+])dnl End of CHECK_FALLTHROUGH_ATTRIBUTE
+
+CHECK_FALLTHROUGH_ATTRIBUTE
+
 if test "$srcdir" != "."; then
 	CPPFLAGS="$CPPFLAGS -I$srcdir"
 fi
@@ -406,7 +466,7 @@ AC_CHECK_TOOL(STRIP, strip)
 ACX_LIBTOOL_C_ONLY
 
 # pkg-config is only needed for these options, do not require it otherwise
-if test "$enable_systemd" = "yes" -o "$with_pyunbound" = "yes" -o "$with_pythonmod" = "yes"; then
+if test "$enable_systemd" = "yes" -o "$enable_dnstap" = "yes" -o "$with_pyunbound" = "yes" -o "$with_pythonmod" = "yes"; then
 PKG_PROG_PKG_CONFIG
 fi
 
@@ -467,6 +527,7 @@ AC_CHECK_HEADERS([netioapi.h],,, [AC_INCLUDES_DEFAULT
 
 # Check for Linux timestamping headers
 AC_CHECK_HEADERS([linux/net_tstamp.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([stdatomic.h],,, [AC_INCLUDES_DEFAULT])
 
 # check for types.  
 # Using own tests for int64* because autoconf builtin only give 32bit.
@@ -792,9 +853,9 @@ if test x_$ub_test_python != x_no; then
       ub_have_swig=no
       AC_ARG_ENABLE(swig-version-check, AS_HELP_STRING([--disable-swig-version-check],[Disable swig version check to build python modules with older swig even though that is unreliable]))
       if test "$enable_swig_version_check" = "yes"; then
-      	AC_PROG_SWIG(2.0.1)
+      	AX_PKG_SWIG(2.0.1)
       else
-      	AC_PROG_SWIG
+      	AX_PKG_SWIG
       fi
       AC_MSG_CHECKING(SWIG)
       if test ! -x "$SWIG"; then
@@ -852,7 +913,8 @@ if test "`uname`" = "Linux"; then
 	GCC_DOCKER_LINTFLAGS='-syntax'
 	AC_SUBST(GCC_DOCKER_LINTFLAGS)
 fi
-CONFIG_DATE=`date +%Y%m%d`
+AX_BUILD_DATE_EPOCH(CONFIG_DATE, [%Y%m%d])
+AC_ARG_VAR(SOURCE_DATE_EPOCH, [If it is set, it uses the value of that variable instead of the current time as the build timestamp. The format is a unix timestamp. This enables reproducible build output.])
 AC_SUBST(CONFIG_DATE)
 
 # Checks for libraries.
@@ -944,7 +1006,7 @@ LIBS="-lssl $LIBS"
 AC_CHECK_FUNCS([OPENSSL_init_ssl SSL_CTX_set_security_level SSL_set1_host SSL_get0_peername X509_VERIFY_PARAM_set1_host SSL_CTX_set_ciphersuites SSL_CTX_set_tlsext_ticket_key_evp_cb SSL_CTX_set_alpn_select_cb SSL_get0_alpn_selected SSL_CTX_set_alpn_protos SSL_get1_peer_certificate])
 LIBS="$BAKLIBS"
 
-AC_CHECK_DECLS([SSL_COMP_get_compression_methods,sk_SSL_COMP_pop_free,SSL_CTX_set_ecdh_auto], [], [], [
+AC_CHECK_DECLS([SSL_COMP_get_compression_methods,sk_SSL_COMP_pop_free,SSL_CTX_set_ecdh_auto,SSL_CTX_set_tmp_ecdh], [], [], [
 AC_INCLUDES_DEFAULT
 #ifdef HAVE_OPENSSL_ERR_H
 #include <openssl/err.h>
@@ -1173,8 +1235,17 @@ case "$enable_ecdsa" in
       ;;
     *)
       if test $USE_NSS = "no" -a $USE_NETTLE = "no"; then
-	      AC_CHECK_FUNC(ECDSA_sign, [], [AC_MSG_ERROR([OpenSSL does not support ECDSA: please upgrade or rerun with --disable-ecdsa])])
-	      AC_CHECK_FUNC(SHA384_Init, [], [AC_MSG_ERROR([OpenSSL does not support SHA384: please upgrade or rerun with --disable-ecdsa])])
+	      AC_CHECK_FUNC(EVP_PKEY_fromdata, [
+	          # with EVP_PKEY_fromdata, check if EC is not disabled
+	          AC_CHECK_DECL([OPENSSL_NO_EC], [AC_MSG_ERROR([OpenSSL does not support ECDSA: please upgrade or rerun with --disable-ecdsa])
+		  ], [], [AC_INCLUDES_DEFAULT
+#include <openssl/evp.h>
+		  ])
+		], [
+		  # without EVP_PKEY_fromdata, older openssl, check for support
+		  AC_CHECK_FUNC(ECDSA_sign, [], [AC_MSG_ERROR([OpenSSL does not support ECDSA: please upgrade or rerun with --disable-ecdsa])])
+		  AC_CHECK_FUNC(SHA384_Init, [], [AC_MSG_ERROR([OpenSSL does not support SHA384: please upgrade or rerun with --disable-ecdsa])])
+		])
 	      AC_CHECK_DECLS([NID_X9_62_prime256v1, NID_secp384r1], [], [AC_MSG_ERROR([OpenSSL does not support the ECDSA curves: please upgrade or rerun with --disable-ecdsa])], [AC_INCLUDES_DEFAULT
 #include <openssl/evp.h>
 	      ])
@@ -1515,6 +1586,76 @@ if test x_$withval = x_yes -o x_$withval != x_no; then
     ])
 fi
 
+# ngtcp2
+AC_ARG_WITH(libngtcp2, AS_HELP_STRING([--with-libngtcp2=path],[specify explicit path for libngtcp2, for QUIC.]),
+    [ ],[ withval="no" ])
+found_libngtcp2="no"
+if test x_$withval = x_yes -o x_$withval != x_no; then
+   AC_MSG_CHECKING(for libngtcp2)
+   if test x_$withval = x_ -o x_$withval = x_yes; then
+            withval="/usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr"
+   fi
+   for dir in $withval ; do
+            if test -f "$dir/include/ngtcp2/ngtcp2.h"; then
+		found_libngtcp2="yes"
+		dnl assume /usr is in default path.
+		if test "$dir" != "/usr"; then
+                    CPPFLAGS="$CPPFLAGS -I$dir/include"
+		    LDFLAGS="$LDFLAGS -L$dir/lib"
+		fi
+		AC_MSG_RESULT(found in $dir)
+		AC_DEFINE([HAVE_NGTCP2], [1], [Define this to use ngtcp2.])
+		LIBS="$LIBS -lngtcp2"
+                break;
+            fi
+    done
+    if test x_$found_libngtcp2 != x_yes; then
+	AC_MSG_ERROR([Could not find libngtcp2, ngtcp2.h])
+    fi
+    AC_CHECK_HEADERS([ngtcp2/ngtcp2.h ngtcp2/ngtcp2_crypto_ossl.h ngtcp2/ngtcp2_crypto_openssl.h ngtcp2/ngtcp2_crypto_quictls.h],,, [AC_INCLUDES_DEFAULT])
+    AC_CHECK_DECLS([ngtcp2_conn_server_new], [], [], [AC_INCLUDES_DEFAULT
+    #include <ngtcp2/ngtcp2.h>
+    ])
+    AC_CHECK_DECLS([ngtcp2_crypto_encrypt_cb], [], [], [AC_INCLUDES_DEFAULT
+    #include <ngtcp2/ngtcp2_crypto.h>
+    ])
+    AC_CHECK_LIB([ngtcp2_crypto_ossl], [ngtcp2_crypto_encrypt_cb], [
+	LIBS="$LIBS -lngtcp2_crypto_ossl"
+	AC_DEFINE(USE_NGTCP2_CRYPTO_OSSL, 1, [Define this to use ngtcp2_crypto_ossl.])
+    ], [
+        AC_CHECK_LIB([ngtcp2_crypto_openssl], [ngtcp2_crypto_encrypt_cb], [ LIBS="$LIBS -lngtcp2_crypto_openssl" ], [
+            AC_CHECK_LIB([ngtcp2_crypto_quictls], [ngtcp2_crypto_encrypt_cb], [ LIBS="$LIBS -lngtcp2_crypto_quictls" ])
+	])
+    ])
+    AC_CHECK_FUNCS([ngtcp2_crypto_encrypt_cb ngtcp2_ccerr_default ngtcp2_conn_in_closing_period ngtcp2_conn_in_draining_period ngtcp2_conn_get_max_local_streams_uni ngtcp2_crypto_quictls_from_ossl_encryption_level ngtcp2_crypto_quictls_configure_server_context ngtcp2_crypto_quictls_configure_client_context ngtcp2_crypto_quictls_init ngtcp2_conn_get_num_scid ngtcp2_conn_tls_early_data_rejected ngtcp2_conn_encode_0rtt_transport_params])
+
+    # these check_funcs need -lssl
+    BAKLIBS="$LIBS"
+    LIBS="-lssl $LIBS"
+    AC_CHECK_FUNCS([SSL_is_quic], [], [AC_MSG_ERROR([No QUIC support detected in OpenSSL. Need OpenSSL version with QUIC support to enable DNS over QUIC with libngtcp2.])])
+    LIBS="$BAKLIBS"
+
+    AC_CHECK_TYPES([struct ngtcp2_version_cid, ngtcp2_encryption_level],,,[AC_INCLUDES_DEFAULT
+    #include <ngtcp2/ngtcp2.h>
+    ])
+    AC_CHECK_MEMBERS([struct ngtcp2_pkt_hd.tokenlen, struct ngtcp2_settings.tokenlen, struct ngtcp2_settings.max_tx_udp_payload_size, struct ngtcp2_transport_params.original_dcid_present],,,[AC_INCLUDES_DEFAULT
+    #include <ngtcp2/ngtcp2.h>
+    ])
+
+    AC_MSG_CHECKING([whether ngtcp2_conn_shutdown_stream has 4 arguments])
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT
+#include <ngtcp2/ngtcp2.h>
+    ],[
+	(void)ngtcp2_conn_shutdown_stream(NULL, 0, 0, 0);
+    ])],[
+	AC_MSG_RESULT(yes)
+	AC_DEFINE(HAVE_NGTCP2_CONN_SHUTDOWN_STREAM4, 1, [Define if ngtcp2_conn_shutdown_stream has 4 arguments.])
+    ],[
+	AC_MSG_RESULT(no)
+    ])
+
+fi
+
 # set static linking for uninstalled libraries if requested
 AC_SUBST(staticexe)
 staticexe=""
@@ -1525,13 +1666,17 @@ if test x_$enable_static_exe = x_yes; then
 	if test "$on_mingw" = yes; then
 		staticexe="-all-static"
 		# for static compile, include gdi32 and zlib here.
-		if echo $LIBS | grep 'lgdi32' >/dev/null; then
+		if echo "$LIBS" | grep 'lgdi32' >/dev/null; then
 			:
 		else
 			LIBS="$LIBS -lgdi32"
 		fi
 		AC_CHECK_LIB([z], [compress], [ LIBS="$LIBS -lz" ])
-		LIBS="$LIBS -l:libssp.a"
+		if echo "$LIBS" | grep -e "libssp.a" -e "lssp" >/dev/null; then
+			:
+		else
+			LIBS="$LIBS -l:libssp.a"
+		fi
 	fi
 fi
 
@@ -1548,7 +1693,11 @@ if test x_$enable_fully_static = x_yes; then
 			LIBS="$LIBS -lgdi32"
 		fi
 		AC_CHECK_LIB([z], [compress], [ LIBS="$LIBS -lz" ])
-		LIBS="$LIBS -l:libssp.a"
+		if echo "$LIBS" | grep -e "libssp.a" -e "lssp" >/dev/null; then
+			:
+		else
+			LIBS="$LIBS -l:libssp.a"
+		fi
 	fi
 fi
 
@@ -1568,7 +1717,11 @@ if test "$USE_WINSOCK" = 1; then
 #include <windows.h>
 	])
 	AC_CHECK_TOOL(WINDRES, windres)
-	LIBS="$LIBS -liphlpapi -lcrypt32"
+	if echo "$LIBS" | grep crypt32 >/dev/null; then
+		LIBS="$LIBS -liphlpapi"
+	else
+		LIBS="$LIBS -liphlpapi -lcrypt32"
+	fi
 	WINAPPS="unbound-service-install.exe unbound-service-remove.exe anchor-update.exe"
 	AC_SUBST(WINAPPS)
 	WIN_DAEMON_SRC="winrc/win_svc.c winrc/w_inst.c"
@@ -1680,6 +1833,25 @@ AC_CHECK_FUNCS([setresgid],,[AC_CHECK_FUNCS([setregid])])
 if echo $host_os | grep darwin8 > /dev/null; then
 	AC_DEFINE(DARWIN_BROKEN_SETREUID, 1, [Define this if on macOSX10.4-darwin8 and setreuid and setregid do not work])
 fi
+
+AC_MSG_CHECKING([for atomic_store])
+AC_LINK_IFELSE([AC_LANG_PROGRAM(AC_INCLUDES_DEFAULT [[
+#ifdef HAVE_STDATOMIC_H
+#include <stdatomic.h>
+#endif
+]], [[
+	int newvar = 5, var = 0;
+	atomic_store((_Atomic int*)&var, newvar);
+	newvar = 0;
+	/* condition to use the variables. */
+	if(var == newvar) return 1;
+]])], [
+	AC_MSG_RESULT(yes)
+	AC_DEFINE(HAVE_LINK_ATOMIC_STORE, 1, [If we have atomic_store])
+], [
+	AC_MSG_RESULT([no])
+])
+
 AC_CHECK_DECLS([inet_pton,inet_ntop], [], [], [
 AC_INCLUDES_DEFAULT
 #ifdef HAVE_NETINET_IN_H
@@ -1818,10 +1990,12 @@ if test x_$enable_lock_checks = x_yes; then
 	UBSYMS="-export-symbols clubsyms.def"
 	cp ${srcdir}/libunbound/ubsyms.def clubsyms.def
 	echo lock_protect >> clubsyms.def
+	echo lock_protect_place >> clubsyms.def
 	echo lock_unprotect >> clubsyms.def
 	echo lock_get_mem >> clubsyms.def
 	echo checklock_start >> clubsyms.def
 	echo checklock_stop >> clubsyms.def
+	echo checklock_set_output_name >> clubsyms.def
 	echo checklock_lock >> clubsyms.def
 	echo checklock_unlock >> clubsyms.def
 	echo checklock_init >> clubsyms.def
@@ -1902,15 +2076,17 @@ case "$enable_ipset" in
 		IPSET_OBJ="ipset.lo"
 		AC_SUBST(IPSET_OBJ)
 
-		# mnl
-		AC_ARG_WITH(libmnl, AS_HELP_STRING([--with-libmnl=path],[specify explicit path for libmnl.]),
+		# BSD's pf
+		AC_CHECK_HEADERS([net/pfvar.h], [], [
+		    # mnl
+		    AC_ARG_WITH(libmnl, AS_HELP_STRING([--with-libmnl=path],[specify explicit path for libmnl.]),
 			[ ],[ withval="yes" ])
-		found_libmnl="no"
-		AC_MSG_CHECKING(for libmnl)
-		if test x_$withval = x_ -o x_$withval = x_yes; then
+		    found_libmnl="no"
+		    AC_MSG_CHECKING(for libmnl)
+		    if test x_$withval = x_ -o x_$withval = x_yes; then
 			withval="/usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr"
-		fi
-		for dir in $withval ; do
+		    fi
+		    for dir in $withval ; do
 			if test -f "$dir/include/libmnl/libmnl.h" -o -f "$dir/include/libmnl/libmnl/libmnl.h"; then
 				found_libmnl="yes"
 				dnl assume /usr is in default path.
@@ -1928,10 +2104,14 @@ case "$enable_ipset" in
 				LIBS="$LIBS -lmnl"
 				break;
 			fi
-		done
-		if test x_$found_libmnl != x_yes; then
-			AC_MSG_ERROR([Could not find libmnl, libmnl.h])
-		fi
+		    done
+		    if test x_$found_libmnl != x_yes; then
+			  AC_MSG_ERROR([Could not find libmnl, libmnl.h])
+		    fi
+		], [
+		  #include <netinet/in.h>
+		  #include <net/if.h>
+		])
 		;;
     no|*)
     	# nothing
@@ -2249,6 +2429,10 @@ struct sockaddr_storage;
 #  define calloc(n,s) unbound_stat_calloc_log(n, s, __FILE__, __LINE__, __func__)
 #  define free(p) unbound_stat_free_log(p, __FILE__, __LINE__, __func__)
 #  define realloc(p,s) unbound_stat_realloc_log(p, s, __FILE__, __LINE__, __func__)
+#  define strdup(s) unbound_stat_strdup_log(s, __FILE__, __LINE__, __func__)
+#ifdef HAVE_REALLOCARRAY
+#  define reallocarray(p,n,s) unbound_stat_reallocarray_log(p, n, s, __FILE__, __LINE__, __func__)
+#endif
 void *unbound_stat_malloc(size_t size);
 void *unbound_stat_calloc(size_t nmemb, size_t size);
 void unbound_stat_free(void *ptr);
@@ -2261,6 +2445,10 @@ void unbound_stat_free_log(void *ptr, const char* file, int line,
 	const char* func);
 void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
 	int line, const char* func);
+void *unbound_stat_reallocarray_log(void *ptr, size_t nmemb, size_t size,
+	const char* file, int line, const char* func);
+char *unbound_stat_strdup_log(const char *s, const char* file, int line,
+	const char* func);
 #elif defined(UNBOUND_ALLOC_LITE)
 #  include "util/alloc.h"
 #endif /* UNBOUND_ALLOC_LITE and UNBOUND_ALLOC_STATS */
@@ -2271,6 +2459,8 @@ void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
 #define UNBOUND_DNS_OVER_TLS_PORT 853
 /** default port for DNS over HTTPS traffic. */
 #define UNBOUND_DNS_OVER_HTTPS_PORT 443
+/** default port for DNS over QUIC traffic. */
+#define UNBOUND_DNS_OVER_QUIC_PORT 853
 /** default port for unbound control traffic, registered port with IANA,
     ub-dns-control  8953/tcp    unbound dns nameserver control */
 #define UNBOUND_CONTROL_PORT 8953
@@ -2282,7 +2472,8 @@ void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
 dnl if we build from source tree, the man pages need @date@ and @version@
 dnl if this is a distro tarball, that was already done by makedist.sh
 AC_SUBST(version, [VERSION_MAJOR.VERSION_MINOR.VERSION_MICRO])
-AC_SUBST(date, [`date +'%b %e, %Y'`])
+AX_BUILD_DATE_EPOCH(date, [[%b %e, %Y]])
+AC_SUBST(date)
 
 AC_CONFIG_FILES([Makefile doc/example.conf doc/libunbound.3 doc/unbound.8 doc/unbound-anchor.8 doc/unbound-checkconf.8 doc/unbound.conf.5 doc/unbound-control.8 doc/unbound-host.1 smallapp/unbound-control-setup.sh dnstap/dnstap_config.h dnscrypt/dnscrypt_config.h contrib/libunbound.pc contrib/unbound.socket contrib/unbound.service contrib/unbound_portable.service])
 AC_CONFIG_HEADERS([config.h])

contrib/aaaa-filter-iterator.patch

@@ -1,10 +1,10 @@
 diff --git a/doc/unbound.conf.5.in b/doc/unbound.conf.5.in
-index 5a75e319..c6c6dbe2 100644
+index 172eb26c..2921c87f 100644
 --- a/doc/unbound.conf.5.in
 +++ b/doc/unbound.conf.5.in
-@@ -970,6 +970,13 @@ potentially broken nameservers. A lot of domains will not be resolvable when
- this option in enabled. Only use if you know what you are doing.
- This option only has effect when qname-minimisation is enabled. Default is no.
+@@ -2146,6 +2146,13 @@ Default: no
+ .UNINDENT
+ .INDENT 0.0
  .TP
 +.B aaaa\-filter: \fI<yes or no>
 +Activate behavior similar to BIND's AAAA-filter.
@@ -13,14 +13,14 @@ index 5a75e319..c6c6dbe2 100644
 +This also causes an additional A query to be sent for each AAAA query.
 +This breaks DNSSEC!
 +.TP
- .B aggressive\-nsec: \fI<yes or no>
- Aggressive NSEC uses the DNSSEC NSEC chain to synthesize NXDOMAIN
- and other denials, using information from previous NXDOMAINs answers.
+ .B aggressive\-nsec: \fI<yes or no>\fP 
+ Aggressive NSEC uses the DNSSEC NSEC chain to synthesize NXDOMAIN and other
+ denials, using information from previous NXDOMAINs answers.
 diff --git a/iterator/iter_scrub.c b/iterator/iter_scrub.c
-index f093c1bf..e55a2246 100644
+index 49a5f5da..fbe434fa 100644
 --- a/iterator/iter_scrub.c
 +++ b/iterator/iter_scrub.c
-@@ -679,6 +679,32 @@ static int sanitize_nsec_is_overreach(sldns_buffer* pkt,
+@@ -849,6 +849,32 @@ scrub_sanitize_rr_length(sldns_buffer* pkt, struct msg_parse* msg,
  	return 0;
  }
  
@@ -53,15 +53,15 @@ index f093c1bf..e55a2246 100644
  /**
   * Given a response event, remove suspect RRsets from the response.
   * "Suspect" rrsets are potentially poison. Note that this routine expects
-@@ -698,6 +724,7 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
+@@ -869,6 +895,7 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
  	struct query_info* qinfo, uint8_t* zonename, struct module_env* env,
- 	struct iter_env* ie)
+ 	struct iter_env* ie, struct module_qstate* qstate)
  {
 +	int found_a_record = 0; /* ASN: do we have a A record? */
  	int del_addi = 0; /* if additional-holding rrsets are deleted, we
  		do not trust the normalized additional-A-AAAA any more */
- 	struct rrset_parse* rrset, *prev;
-@@ -733,6 +760,13 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
+ 	uint8_t* ns_rrset_dname = NULL;
+@@ -906,6 +933,13 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
  		rrset = rrset->rrset_all_next;
  	}
  
@@ -75,9 +75,9 @@ index f093c1bf..e55a2246 100644
  	/* At this point, we brutally remove ALL rrsets that aren't 
  	 * children of the originating zone. The idea here is that, 
  	 * as far as we know, the server that we contacted is ONLY 
-@@ -744,6 +778,24 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
- 	rrset = msg->rrset_first;
- 	while(rrset) {
+@@ -925,6 +959,24 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
+ 				continue;
+ 		}
  
 +		/* ASN: For AAAA records only... */
 +		if((ie->aaaa_filter) && (rrset->type == LDNS_RR_TYPE_AAAA)) {
@@ -101,10 +101,10 @@ index f093c1bf..e55a2246 100644
  		if( (rrset->type == LDNS_RR_TYPE_A || 
  			rrset->type == LDNS_RR_TYPE_AAAA)) {
 diff --git a/iterator/iter_utils.c b/iterator/iter_utils.c
-index 2482a1f4..bd5ba243 100644
+index 1da21896..6583dd0e 100644
 --- a/iterator/iter_utils.c
 +++ b/iterator/iter_utils.c
-@@ -177,6 +177,7 @@ iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
+@@ -250,6 +250,7 @@ iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
  	iter_env->outbound_msg_retry = cfg->outbound_msg_retry;
  	iter_env->max_sent_count = cfg->max_sent_count;
  	iter_env->max_query_restarts = cfg->max_query_restarts;
@@ -113,12 +113,12 @@ index 2482a1f4..bd5ba243 100644
  }
  
 diff --git a/iterator/iterator.c b/iterator/iterator.c
-index 54006940..768fe202 100644
+index 71e64655..735f4ca0 100644
 --- a/iterator/iterator.c
 +++ b/iterator/iterator.c
-@@ -2155,6 +2155,53 @@ processDSNSFind(struct module_qstate* qstate, struct iter_qstate* iq, int id)
- 
- 	return 0;
+@@ -2412,6 +2412,53 @@ check_waiting_queries(struct iter_qstate* iq, struct module_qstate* qstate,
+ 		qstate->ext_state[id] = module_wait_reply;
+ 	}
  }
 +
 +/**
@@ -170,8 +170,8 @@ index 54006940..768fe202 100644
  	
  /** 
   * This is the request event state where the request will be sent to one of
-@@ -2216,6 +2263,13 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq,
- 		return error_response(qstate, id, LDNS_RCODE_SERVFAIL);
+@@ -2554,6 +2601,13 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq,
+ 		}
  	}
  	
 +	/* ASN: If we have a AAAA query, then also query for A records */
@@ -184,7 +184,7 @@ index 54006940..768fe202 100644
  	/* Make sure we have a delegation point, otherwise priming failed
  	 * or another failure occurred */
  	if(!iq->dp) {
-@@ -3648,6 +3702,61 @@ processFinished(struct module_qstate* qstate, struct iter_qstate* iq,
+@@ -4178,6 +4232,61 @@ processFinished(struct module_qstate* qstate, struct iter_qstate* iq,
  	return 0;
  }
  
@@ -236,7 +236,7 @@ index 54006940..768fe202 100644
 +		/* see if the failure did get (parent-lame) info */
 +		if(!cache_fill_missing(super->env,
 +			super_iq->qchase.qclass, super->region,
-+			super_iq->dp))
++			super_iq->dp, 0))
 +		log_err("ASN-AAAA-filter: out of memory adding missing");
 +		dpns->resolved = 1; /* mark as failed */
 +	}
@@ -246,7 +246,7 @@ index 54006940..768fe202 100644
  /*
   * Return priming query results to interested super querystates.
   * 
-@@ -3667,6 +3776,9 @@ iter_inform_super(struct module_qstate* qstate, int id,
+@@ -4197,6 +4306,9 @@ iter_inform_super(struct module_qstate* qstate, int id,
  	else if(super->qinfo.qtype == LDNS_RR_TYPE_DS && ((struct iter_qstate*)
  		super->minfo[id])->state == DSNS_FIND_STATE)
  		processDSNSResponse(qstate, id, super);
@@ -256,7 +256,7 @@ index 54006940..768fe202 100644
  	else if(qstate->return_rcode != LDNS_RCODE_NOERROR)
  		error_supers(qstate, id, super);
  	else if(qstate->is_priming)
-@@ -3704,6 +3816,9 @@ iter_handle(struct module_qstate* qstate, struct iter_qstate* iq,
+@@ -4234,6 +4346,9 @@ iter_handle(struct module_qstate* qstate, struct iter_qstate* iq,
  			case INIT_REQUEST_3_STATE:
  				cont = processInitRequest3(qstate, iq, id);
  				break;
@@ -266,7 +266,7 @@ index 54006940..768fe202 100644
  			case QUERYTARGETS_STATE:
  				cont = processQueryTargets(qstate, iq, ie, id);
  				break;
-@@ -4040,6 +4155,8 @@ iter_state_to_string(enum iter_state state)
+@@ -4578,6 +4693,8 @@ iter_state_to_string(enum iter_state state)
  		return "INIT REQUEST STATE (stage 2)";
  	case INIT_REQUEST_3_STATE:
  		return "INIT REQUEST STATE (stage 3)";
@@ -275,7 +275,7 @@ index 54006940..768fe202 100644
  	case QUERYTARGETS_STATE :
  		return "QUERY TARGETS STATE";
  	case PRIME_RESP_STATE :
-@@ -4064,6 +4181,7 @@ iter_state_is_responsestate(enum iter_state s)
+@@ -4602,6 +4719,7 @@ iter_state_is_responsestate(enum iter_state s)
  		case INIT_REQUEST_STATE :
  		case INIT_REQUEST_2_STATE :
  		case INIT_REQUEST_3_STATE :
@@ -284,10 +284,10 @@ index 54006940..768fe202 100644
  		case COLLECT_CLASS_STATE :
  			return 0;
 diff --git a/iterator/iterator.h b/iterator/iterator.h
-index 8b840528..a61c4195 100644
+index ae4b4e45..a44f9d27 100644
 --- a/iterator/iterator.h
 +++ b/iterator/iterator.h
-@@ -133,6 +133,9 @@ struct iter_env {
+@@ -157,6 +157,9 @@ struct iter_env {
  	 */
  	int* target_fetch_policy;
  
@@ -297,7 +297,7 @@ index 8b840528..a61c4195 100644
  	/** lock on ratelimit counter */
  	lock_basic_type queries_ratelimit_lock;
  	/** number of queries that have been ratelimited */
-@@ -187,6 +190,14 @@ enum iter_state {
+@@ -217,6 +220,14 @@ enum iter_state {
  	 */
  	INIT_REQUEST_3_STATE,
  
@@ -312,9 +312,9 @@ index 8b840528..a61c4195 100644
  	/**
  	 * Each time a delegation point changes for a given query or a 
  	 * query times out and/or wakes up, this state is (re)visited. 
-@@ -376,6 +387,13 @@ struct iter_qstate {
- 	 */
- 	int refetch_glue;
+@@ -434,6 +445,13 @@ struct iter_qstate {
+ 	 * already so that it is accepted later. */
+ 	int empty_nodata_found;
  
 +	/**
 +	 * ASN: This is a flag that, if true, means that this query is
@@ -327,10 +327,10 @@ index 8b840528..a61c4195 100644
  	struct outbound_list outlist;
  
 diff --git a/pythonmod/interface.i b/pythonmod/interface.i
-index 1ca8686a..d91b19ec 100644
+index 2040fb9e..f073c3dc 100644
 --- a/pythonmod/interface.i
 +++ b/pythonmod/interface.i
-@@ -995,6 +995,7 @@ struct config_file {
+@@ -1013,6 +1013,7 @@ struct config_file {
     int harden_dnssec_stripped;
     int harden_referral_path;
     int use_caps_bits_for_id;
@@ -339,23 +339,23 @@ index 1ca8686a..d91b19ec 100644
     struct config_strlist* private_domain;
     size_t unwanted_threshold;
 diff --git a/util/config_file.c b/util/config_file.c
-index 969d664b..8d94b008 100644
+index b1e767b3..5eb3c099 100644
 --- a/util/config_file.c
 +++ b/util/config_file.c
-@@ -231,6 +231,7 @@ config_create(void)
- 	cfg->harden_referral_path = 0;
+@@ -247,6 +247,7 @@ config_create(void)
  	cfg->harden_algo_downgrade = 0;
+ 	cfg->harden_unknown_additional = 0;
  	cfg->use_caps_bits_for_id = 0;
 +	cfg->aaaa_filter = 0; /* ASN: default is disabled */
  	cfg->caps_whitelist = NULL;
  	cfg->private_address = NULL;
  	cfg->private_domain = NULL;
 diff --git a/util/config_file.h b/util/config_file.h
-index c7c9a0a4..e3aa15b0 100644
+index 44ac036b..1e59ab07 100644
 --- a/util/config_file.h
 +++ b/util/config_file.h
-@@ -285,6 +285,8 @@ struct config_file {
- 	int harden_algo_downgrade;
+@@ -311,6 +311,8 @@ struct config_file {
+ 	int harden_unknown_additional;
  	/** use 0x20 bits in query as random ID bits */
  	int use_caps_bits_for_id;
 +	/** ASN: enable AAAA filter? */
@@ -364,10 +364,10 @@ index c7c9a0a4..e3aa15b0 100644
  	struct config_strlist* caps_whitelist;
  	/** strip away these private addrs from answers, no DNS Rebinding */
 diff --git a/util/configlexer.lex b/util/configlexer.lex
-index 34a0e5dd..c890be2a 100644
+index bc258673..76aab170 100644
 --- a/util/configlexer.lex
 +++ b/util/configlexer.lex
-@@ -317,6 +317,7 @@ use-caps-for-id{COLON}		{ YDVAR(1, VAR_USE_CAPS_FOR_ID) }
+@@ -327,6 +327,7 @@ use-caps-for-id{COLON}		{ YDVAR(1, VAR_USE_CAPS_FOR_ID) }
  caps-whitelist{COLON}		{ YDVAR(1, VAR_CAPS_WHITELIST) }
  caps-exempt{COLON}		{ YDVAR(1, VAR_CAPS_WHITELIST) }
  unwanted-reply-threshold{COLON}	{ YDVAR(1, VAR_UNWANTED_REPLY_THRESHOLD) }
@@ -376,10 +376,10 @@ index 34a0e5dd..c890be2a 100644
  private-domain{COLON}		{ YDVAR(1, VAR_PRIVATE_DOMAIN) }
  prefetch-key{COLON}		{ YDVAR(1, VAR_PREFETCH_KEY) }
 diff --git a/util/configparser.y b/util/configparser.y
-index d4f965f9..8cc237c6 100644
+index 82e1d878..dc19bed5 100644
 --- a/util/configparser.y
 +++ b/util/configparser.y
-@@ -97,6 +97,7 @@ extern struct config_parser_state* cfg_parser;
+@@ -100,6 +100,7 @@ extern struct config_parser_state* cfg_parser;
  %token VAR_STATISTICS_CUMULATIVE VAR_OUTGOING_PORT_PERMIT
  %token VAR_OUTGOING_PORT_AVOID VAR_DLV_ANCHOR_FILE VAR_DLV_ANCHOR
  %token VAR_NEG_CACHE_SIZE VAR_HARDEN_REFERRAL_PATH VAR_PRIVATE_ADDRESS
@@ -387,7 +387,7 @@ index d4f965f9..8cc237c6 100644
  %token VAR_PRIVATE_DOMAIN VAR_REMOTE_CONTROL VAR_CONTROL_ENABLE
  %token VAR_CONTROL_INTERFACE VAR_CONTROL_PORT VAR_SERVER_KEY_FILE
  %token VAR_SERVER_CERT_FILE VAR_CONTROL_KEY_FILE VAR_CONTROL_CERT_FILE
-@@ -247,6 +248,7 @@ content_server: server_num_threads | server_verbosity | server_port |
+@@ -276,6 +277,7 @@ content_server: server_num_threads | server_verbosity | server_port |
  	server_dlv_anchor_file | server_dlv_anchor | server_neg_cache_size |
  	server_harden_referral_path | server_private_address |
  	server_private_domain | server_extended_statistics |
@@ -395,7 +395,7 @@ index d4f965f9..8cc237c6 100644
  	server_local_data_ptr | server_jostle_timeout |
  	server_unwanted_reply_threshold | server_log_time_ascii |
  	server_domain_insecure | server_val_sig_skew_min |
-@@ -1754,6 +1756,15 @@ server_caps_whitelist: VAR_CAPS_WHITELIST STRING_ARG
+@@ -1932,6 +1934,15 @@ server_caps_whitelist: VAR_CAPS_WHITELIST STRING_ARG
  			yyerror("out of memory");
  	}
  	;

contrib/android/install_expat.sh

@@ -1,21 +1,24 @@
 #!/usr/bin/env bash
 
+LIBEXPAT_FNAME=expat-2.7.0
+LIBEXPAT_VERSION_DIR=R_2_7_0
+
 echo "Downloading Expat"
-if ! curl -L -k -s -o expat-2.2.9.tar.gz https://github.com/libexpat/libexpat/releases/download/R_2_2_9/expat-2.2.9.tar.gz;
+if ! curl -L -k -s -o $LIBEXPAT_FNAME.tar.gz https://github.com/libexpat/libexpat/releases/download/$LIBEXPAT_VERSION_DIR/$LIBEXPAT_FNAME.tar.gz;
 then
     echo "Failed to download Expat"
     exit 1
 fi
 
 echo "Unpacking Expat"
-rm -rf ./expat-2.2.9
-if ! tar -xf expat-2.2.9.tar.gz;
+rm -rf ./$LIBEXPAT_FNAME
+if ! tar -xf $LIBEXPAT_FNAME.tar.gz;
 then
     echo "Failed to unpack Expat"
     exit 1
 fi
 
-cd expat-2.2.9 || exit 1
+cd $LIBEXPAT_FNAME || exit 1
 
 echo "Configuring Expat"
 if ! ./configure --build="$AUTOTOOLS_BUILD" --host="$AUTOTOOLS_HOST" --prefix="$ANDROID_PREFIX"; then

contrib/ios/install_expat.sh

@@ -1,28 +1,32 @@
 #!/usr/bin/env bash
 
+LIBEXPAT_FNAME=expat-2.7.0
+LIBEXPAT_VERSION_DIR=R_2_7_0
+
 echo "Downloading Expat"
-if ! curl -L -k -s -o expat-2.2.9.tar.gz https://github.com/libexpat/libexpat/releases/download/R_2_2_9/expat-2.2.9.tar.gz;
+if ! curl -L -k -s -o $LIBEXPAT_FNAME.tar.gz https://github.com/libexpat/libexpat/releases/download/$LIBEXPAT_VERSION_DIR/$LIBEXPAT_FNAME.tar.gz;
 then
     echo "Failed to download Expat"
     exit 1
 fi
 
 echo "Unpacking Expat"
-rm -rf ./expat-2.2.9
-if ! tar -xf expat-2.2.9.tar.gz;
+rm -rf ./$LIBEXPAT_FNAME
+if ! tar -xf $LIBEXPAT_FNAME.tar.gz;
 then
     echo "Failed to unpack Expat"
     exit 1
 fi
 
-cd expat-2.2.9 || exit 1
+cd $LIBEXPAT_FNAME || exit 1
 
 export PKG_CONFIG_PATH="$IOS_PREFIX/lib/pkgconfig"
 
 echo "Configuring Expat"
-if ! ./configure \
-       --build="$AUTOTOOLS_BUILD" --host="$AUTOTOOLS_HOST" \
-       --prefix="$IOS_PREFIX" ; then
+if ! ./configure --without-tests				\
+       --build="$AUTOTOOLS_BUILD" --host="$AUTOTOOLS_HOST"	\
+       --prefix="$IOS_PREFIX" ;
+then
     echo "Error: Failed to configure Expat"
     cat config.log
     exit 1

contrib/rc_d_unbound

@@ -22,4 +22,13 @@ pidfile=${unbound_pidfile:-"/usr/local/etc/unbound/unbound.pid"}
 command_args=${unbound_flags:-"-c /usr/local/etc/unbound/unbound.conf"}
 extra_commands="reload"
 
+if test "$1" = "stop" ; then
+	run_rc_command "$1"
+	ret=$?
+	if test $ret -eq 0; then
+		rm -f "$pidfile"
+	fi
+	exit $ret
+fi
+
 run_rc_command "$1"

contrib/unbound.init

@@ -75,6 +75,7 @@ stop() {
     retval=$?
     echo
     [ $retval -eq 0 ] && rm -f $lockfile
+    [ $retval -eq 0 ] && rm -f $pidfile
     if egrep -q '^/[^[:space:]]+[[:space:]]+'${rootdir}'/dev/log' /proc/mounts; then
 	umount ${rootdir}/dev/log >/dev/null 2>&1
     fi;

contrib/unbound.init_fedora

@@ -58,6 +58,7 @@ stop() {
     killproc -p $pidfile unbound
     retval=$?
     [ $retval -eq 0 ] && rm -f $lockfile
+    [ $retval -eq 0 ] && rm -f $pidfile
     for mountfile in /dev/log /dev/urandom /etc/localtime /etc/resolv.conf /var/run/unbound
     do
     if egrep -q '^/[^[:space:]]+[[:space:]]+'${rootdir}''${mountfile}'' /proc/mounts; then

contrib/unbound.init_yocto

@@ -75,6 +75,7 @@ stop() {
     retval=$?
     echo
     [ $retval -eq 0 ] && rm -f $lockfile
+    [ $retval -eq 0 ] && rm -f $pidfile
     if egrep -q '^/[^[:space:]]+[[:space:]]+'${rootdir}'/dev/log' /proc/mounts; then
 	umount ${rootdir}/dev/log >/dev/null 2>&1
     fi;

contrib/unbound.service.in

@@ -38,12 +38,18 @@
 ;   - `LockPersonality=yes` locks down the personality system call so that the
 ;     kernel execution domain may not be changed from the default.
 ;
+;   - With /etc/systemd/network/*.network a setting to make sure the network
+;     is not considered online too early, can reduce network unreachable
+;     errors on server start:
+;     [Link]
+;     RequiredForOnline=routable
 ;
 [Unit]
 Description=Validating, recursive, and caching DNS resolver
 Documentation=man:unbound(8)
-After=network.target
-Before=network-online.target nss-lookup.target
+After=network-online.target
+Wants=network-online.target
+Before=nss-lookup.target
 
 [Install]
 WantedBy=multi-user.target

contrib/unbound_portable.service.in

@@ -14,8 +14,8 @@
 [Unit]
 Description=Validating, recursive, and caching DNS resolver
 Documentation=man:unbound(8)
-After=network.target
-Before=network-online.target nss-lookup.target
+After=network-online.target
+Before=nss-lookup.target
 Wants=nss-lookup.target
 
 [Install]

daemon/acl_list.c

@@ -221,7 +221,9 @@ acl_interface_insert(struct acl_list* acl_interface,
 	struct sockaddr_storage* addr, socklen_t addrlen,
 	enum acl_access control)
 {
-	return acl_find_or_create(acl_interface, addr, addrlen, control);
+	struct acl_addr* node = acl_find_or_create(acl_interface, addr, addrlen, control);
+	node->is_interface = 1;
+	return node;
 }
 
 /** apply acl_tag string */
@@ -551,17 +553,6 @@ acl_list_apply_cfg(struct acl_list* acl, struct config_file* cfg,
 	return 1;
 }
 
-int
-acl_interface_compare(const void* k1, const void* k2)
-{
-	struct addr_tree_node* n1 = (struct addr_tree_node*)k1;
-	struct addr_tree_node* n2 = (struct addr_tree_node*)k2;
-	return sockaddr_cmp(&n1->addr, n1->addrlen, &n2->addr,
-		n2->addrlen);
-	/* We don't care about comparing node->net. All addresses in the
-	 * acl_interface tree have either 32 (ipv4) or 128 (ipv6). */
-}
-
 void
 acl_interface_init(struct acl_list* acl_interface)
 {
@@ -816,10 +807,23 @@ log_acl_action(const char* action, struct sockaddr_storage* addr,
 		addr_to_str(&acladdr->node.addr, acladdr->node.addrlen,
 			n, sizeof(n));
 		verbose(VERB_ALGO, "%s query from %s port %d because of "
-			"%s/%d %s", action, a, (int)port, n, acladdr->node.net,
+			"%s/%d %s%s", action, a, (int)port, n,
+			acladdr->node.net,
+			acladdr->is_interface?"(ACL on interface IP) ":"",
 			acl_access_to_str(acl));
 	} else {
 		verbose(VERB_ALGO, "%s query from %s port %d", action, a,
 			(int)port);
 	}
 }
+
+void acl_list_swap_tree(struct acl_list* acl, struct acl_list* data)
+{
+	/* swap tree and region */
+	rbtree_type oldtree = acl->tree;
+	struct regional* oldregion = acl->region;
+	acl->tree = data->tree;
+	acl->region = data->region;
+	data->tree = oldtree;
+	data->region = oldregion;
+}

daemon/acl_list.h

@@ -107,6 +107,8 @@ struct acl_addr {
 	struct config_strlist** tag_datas;
 	/** size of the tag_datas array */
 	size_t tag_datas_size;
+	/* If the acl node is for an interface */
+	int is_interface;
 	/* view element, NULL if none */
 	struct view* view;
 };
@@ -147,9 +149,6 @@ acl_interface_insert(struct acl_list* acl_interface,
 int acl_list_apply_cfg(struct acl_list* acl, struct config_file* cfg,
 	struct views* v);
 
-/** compare ACL interface "addr_tree" nodes (+port) */
-int acl_interface_compare(const void* k1, const void* k2);
-
 /**
  * Initialise (also clean) the acl_interface struct.
  * @param acl_interface: where to store.
@@ -202,4 +201,12 @@ const char* acl_access_to_str(enum acl_access acl);
 void log_acl_action(const char* action, struct sockaddr_storage* addr,
 	socklen_t addrlen, enum acl_access acl, struct acl_addr* acladdr);
 
+/**
+ * Swap internal tree with preallocated entries.
+ * @param acl: the acl structure.
+ * @param data: the data structure used to take elements from. This contains
+ * 	the old elements on return.
+ */
+void acl_list_swap_tree(struct acl_list* acl, struct acl_list* data);
+
 #endif /* DAEMON_ACL_LIST_H */

daemon/cachedump.c

@@ -62,84 +62,231 @@
 #include "sldns/wire2str.h"
 #include "sldns/str2wire.h"
 
+static void spool_txt_printf(struct config_strlist_head* txt,
+	const char* format, ...) ATTR_FORMAT(printf, 2, 3);
+
+/** Append to strlist at end, and log error if out of memory. */
+static void
+spool_txt_string(struct config_strlist_head* txt, char* str)
+{
+	if(!cfg_strlist_append(txt, strdup(str))) {
+		log_err("out of memory in spool text");
+	}
+}
+
+/** Spool txt to spool list. */
+static void
+spool_txt_vmsg(struct config_strlist_head* txt, const char* format,
+	va_list args)
+{
+	char msg[65535];
+	vsnprintf(msg, sizeof(msg), format, args);
+	spool_txt_string(txt, msg);
+}
+
+/** Print item to spool list. On alloc failure the list is as before. */
+static void
+spool_txt_printf(struct config_strlist_head* txt, const char* format, ...)
+{
+	va_list args;
+	va_start(args, format);
+	spool_txt_vmsg(txt, format, args);
+	va_end(args);
+}
+
 /** dump one rrset zonefile line */
-static int
-dump_rrset_line(RES* ssl, struct ub_packed_rrset_key* k, time_t now, size_t i)
+static void
+dump_rrset_line(struct config_strlist_head* txt, struct ub_packed_rrset_key* k,
+	time_t now, size_t i)
 {
 	char s[65535];
 	if(!packed_rr_to_string(k, i, now, s, sizeof(s))) {
-		return ssl_printf(ssl, "BADRR\n");
+		spool_txt_string(txt, "BADRR\n");
+		return;
 	}
-	return ssl_printf(ssl, "%s", s);
+	spool_txt_string(txt, s);
 }
 
 /** dump rrset key and data info */
-static int
-dump_rrset(RES* ssl, struct ub_packed_rrset_key* k, 
+static void
+dump_rrset(struct config_strlist_head* txt, struct ub_packed_rrset_key* k,
 	struct packed_rrset_data* d, time_t now)
 {
 	size_t i;
 	/* rd lock held by caller */
-	if(!k || !d) return 1;
-	if(k->id == 0) return 1; /* deleted */
-	if(d->ttl < now) return 1; /* expired */
+	if(!k || !d) return;
+	if(k->id == 0) return; /* deleted */
+	if(d->ttl < now) return; /* expired */
 
 	/* meta line */
-	if(!ssl_printf(ssl, ";rrset%s " ARG_LL "d %u %u %d %d\n",
+	spool_txt_printf(txt, ";rrset%s " ARG_LL "d %u %u %d %d\n",
 		(k->rk.flags & PACKED_RRSET_NSEC_AT_APEX)?" nsec_apex":"",
 		(long long)(d->ttl - now),
 		(unsigned)d->count, (unsigned)d->rrsig_count,
 		(int)d->trust, (int)d->security
-		)) 
-		return 0;
+		);
 	for(i=0; i<d->count + d->rrsig_count; i++) {
-		if(!dump_rrset_line(ssl, k, now, i))
+		dump_rrset_line(txt, k, now, i);
+	}
+}
+
+/** Spool strlist to the output. */
+static int
+spool_strlist(RES* ssl, struct config_strlist* list)
+{
+	struct config_strlist* s;
+	for(s=list; s; s=s->next) {
+		if(!ssl_printf(ssl, "%s", s->str))
 			return 0;
 	}
 	return 1;
 }
 
-/** dump lruhash rrset cache */
+/** dump lruhash cache and call callback for every item. */
 static int
-dump_rrset_lruhash(RES* ssl, struct lruhash* h, time_t now)
+dump_lruhash(struct lruhash* table,
+	void (*func)(struct lruhash_entry*, struct config_strlist_head*, void*),
+	RES* ssl, void* arg)
 {
-	struct lruhash_entry* e;
-	/* lruhash already locked by caller */
-	/* walk in order of lru; best first */
-	for(e=h->lru_start; e; e = e->lru_next) {
-		lock_rw_rdlock(&e->lock);
-		if(!dump_rrset(ssl, (struct ub_packed_rrset_key*)e->key,
-			(struct packed_rrset_data*)e->data, now)) {
-			lock_rw_unlock(&e->lock);
+	int just_started = 1;
+	int not_done = 1;
+	hashvalue_type hash;
+	size_t num = 0; /* number of entries processed. */
+	size_t max = 2; /* number of entries after which it unlocks. */
+	struct config_strlist_head txt; /* Text strings spooled. */
+	memset(&txt, 0, sizeof(txt));
+
+	while(not_done) {
+		size_t i; /* hash bin. */
+		/* Process a number of items. */
+		num = 0;
+		lock_quick_lock(&table->lock);
+		if(just_started) {
+			i = 0;
+		} else {
+			i = hash&table->size_mask;
+		}
+		while(num < max) {
+			/* Process bin. */
+			int found = 0;
+			size_t num_bin = 0;
+			struct lruhash_bin* bin = &table->array[i];
+			struct lruhash_entry* e;
+			lock_quick_lock(&bin->lock);
+			for(e = bin->overflow_list; e; e = e->overflow_next) {
+				/* Entry e is locked by the func. */
+				func(e, &txt, arg);
+				num_bin++;
+			}
+			lock_quick_unlock(&bin->lock);
+			/* This addition of bin number of entries may take
+			 * it over the max. */
+			num += num_bin;
+
+			/* Move to next bin. */
+			/* Find one with an entry, with a  hash value, so we
+			 * can continue from the hash value. The hash value
+			 * can be indexed also if the array changes size. */
+			i++;
+			while(i < table->size) {
+				bin = &table->array[i];
+				lock_quick_lock(&bin->lock);
+				if(bin->overflow_list) {
+					hash = bin->overflow_list->hash;
+					lock_quick_unlock(&bin->lock);
+					found = 1;
+					just_started = 0;
+					break;
+				}
+				lock_quick_unlock(&bin->lock);
+				i++;
+			}
+			if(!found) {
+				not_done = 0;
+				break;
+			}
+		}
+		lock_quick_unlock(&table->lock);
+		/* Print the spooled items, that are collected while the
+		 * locks are locked. The print happens while they are not
+		 * locked. */
+		if(txt.first) {
+			if(!spool_strlist(ssl, txt.first)) {
+				config_delstrlist(txt.first);
+				return 0;
+			}
+			config_delstrlist(txt.first);
+			memset(&txt, 0, sizeof(txt));
+		}
+	}
+	/* Print the final spooled items. */
+	if(txt.first) {
+		if(!spool_strlist(ssl, txt.first)) {
+			config_delstrlist(txt.first);
 			return 0;
 		}
-		lock_rw_unlock(&e->lock);
+		config_delstrlist(txt.first);
 	}
 	return 1;
 }
 
+/** dump slabhash cache and call callback for every item. */
+static int
+dump_slabhash(struct slabhash* sh,
+	void (*func)(struct lruhash_entry*, struct config_strlist_head*, void*),
+	RES* ssl, void* arg)
+{
+	/* Process a number of items at a time, then unlock the cache,
+	 * so that ordinary processing can continue. Keep an iteration marker
+	 * to continue the loop. That means the cache can change, items
+	 * could be inserted and deleted. And, for example, the hash table
+	 * can grow. */
+	size_t slab;
+	for(slab=0; slab<sh->size; slab++) {
+		if(!dump_lruhash(sh->array[slab], func, ssl, arg))
+			return 0;
+	}
+	return 1;
+}
+
+/** Struct for dump information. */
+struct dump_info {
+	/** The worker. */
+	struct worker* worker;
+	/** The printout connection. */
+	RES* ssl;
+};
+
+/** Dump the rrset cache entry */
+static void
+dump_rrset_entry(struct lruhash_entry* e, struct config_strlist_head* txt,
+	void* arg)
+{
+	struct dump_info* dump_info = (struct dump_info*)arg;
+	lock_rw_rdlock(&e->lock);
+	dump_rrset(txt, (struct ub_packed_rrset_key*)e->key,
+		(struct packed_rrset_data*)e->data,
+		*dump_info->worker->env.now);
+	lock_rw_unlock(&e->lock);
+}
+
 /** dump rrset cache */
 static int
 dump_rrset_cache(RES* ssl, struct worker* worker)
 {
 	struct rrset_cache* r = worker->env.rrset_cache;
-	size_t slab;
+	struct dump_info dump_info;
+	dump_info.worker = worker;
+	dump_info.ssl = ssl;
 	if(!ssl_printf(ssl, "START_RRSET_CACHE\n")) return 0;
-	for(slab=0; slab<r->table.size; slab++) {
-		lock_quick_lock(&r->table.array[slab]->lock);
-		if(!dump_rrset_lruhash(ssl, r->table.array[slab],
-			*worker->env.now)) {
-			lock_quick_unlock(&r->table.array[slab]->lock);
-			return 0;
-		}
-		lock_quick_unlock(&r->table.array[slab]->lock);
-	}
+	if(!dump_slabhash(&r->table, &dump_rrset_entry, ssl, &dump_info))
+		return 0;
 	return ssl_printf(ssl, "END_RRSET_CACHE\n");
 }
 
 /** dump message to rrset reference */
-static int
-dump_msg_ref(RES* ssl, struct ub_packed_rrset_key* k)
+static void
+dump_msg_ref(struct config_strlist_head* txt, struct ub_packed_rrset_key* k)
 {
 	char* nm, *tp, *cl;
 	nm = sldns_wire2str_dname(k->rk.dname, k->rk.dname_len);
@@ -149,30 +296,25 @@ dump_msg_ref(RES* ssl, struct ub_packed_rrset_key* k)
 		free(nm);
 		free(tp);
 		free(cl);
-		return ssl_printf(ssl, "BADREF\n");
-	}
-	if(!ssl_printf(ssl, "%s %s %s %d\n", nm, cl, tp, (int)k->rk.flags)) {
-		free(nm);
-		free(tp);
-		free(cl);
-		return 0;
+		spool_txt_string(txt, "BADREF\n");
+		return;
 	}
+	spool_txt_printf(txt, "%s %s %s %d\n", nm, cl, tp, (int)k->rk.flags);
 	free(nm);
 	free(tp);
 	free(cl);
-
-	return 1;
 }
 
 /** dump message entry */
-static int
-dump_msg(RES* ssl, struct query_info* k, struct reply_info* d, time_t now)
+static void
+dump_msg(struct config_strlist_head* txt, struct query_info* k,
+	struct reply_info* d, time_t now)
 {
 	size_t i;
 	char* nm, *tp, *cl;
-	if(!k || !d) return 1;
-	if(d->ttl < now) return 1; /* expired */
-	
+	if(!k || !d) return;
+	if(d->ttl < now) return; /* expired */
+
 	nm = sldns_wire2str_dname(k->qname, k->qname_len);
 	tp = sldns_wire2str_type(k->qtype);
 	cl = sldns_wire2str_class(k->qclass);
@@ -180,45 +322,35 @@ dump_msg(RES* ssl, struct query_info* k, struct reply_info* d, time_t now)
 		free(nm);
 		free(tp);
 		free(cl);
-		return 1; /* skip this entry */
+		return; /* skip this entry */
 	}
 	if(!rrset_array_lock(d->ref, d->rrset_count, now)) {
 		/* rrsets have timed out or do not exist */
 		free(nm);
 		free(tp);
 		free(cl);
-		return 1; /* skip this entry */
+		return; /* skip this entry */
 	}
-	
+
 	/* meta line */
-	if(!ssl_printf(ssl, "msg %s %s %s %d %d " ARG_LL "d %d %u %u %u %d %s\n",
-			nm, cl, tp,
-			(int)d->flags, (int)d->qdcount, 
-			(long long)(d->ttl-now), (int)d->security,
-			(unsigned)d->an_numrrsets,
-			(unsigned)d->ns_numrrsets,
-			(unsigned)d->ar_numrrsets,
-			(int)d->reason_bogus,
-			d->reason_bogus_str?d->reason_bogus_str:"")) {
-		free(nm);
-		free(tp);
-		free(cl);
-		rrset_array_unlock(d->ref, d->rrset_count);
-		return 0;
-	}
+	spool_txt_printf(txt,
+		"msg %s %s %s %d %d " ARG_LL "d %d %u %u %u %d %s\n",
+		nm, cl, tp,
+		(int)d->flags, (int)d->qdcount,
+		(long long)(d->ttl-now), (int)d->security,
+		(unsigned)d->an_numrrsets,
+		(unsigned)d->ns_numrrsets,
+		(unsigned)d->ar_numrrsets,
+		(int)d->reason_bogus,
+		d->reason_bogus_str?d->reason_bogus_str:"");
 	free(nm);
 	free(tp);
 	free(cl);
 	
 	for(i=0; i<d->rrset_count; i++) {
-		if(!dump_msg_ref(ssl, d->rrsets[i])) {
-			rrset_array_unlock(d->ref, d->rrset_count);
-			return 0;
-		}
+		dump_msg_ref(txt, d->rrsets[i]);
 	}
 	rrset_array_unlock(d->ref, d->rrset_count);
-
-	return 1;
 }
 
 /** copy msg to worker pad */
@@ -247,49 +379,40 @@ copy_msg(struct regional* region, struct lruhash_entry* e,
 	return (*k)->qname != NULL;
 }
 
-/** dump lruhash msg cache */
-static int
-dump_msg_lruhash(RES* ssl, struct worker* worker, struct lruhash* h)
+/** Dump the msg entry. */
+static void
+dump_msg_entry(struct lruhash_entry* e, struct config_strlist_head* txt,
+	void* arg)
 {
-	struct lruhash_entry* e;
+	struct dump_info* dump_info = (struct dump_info*)arg;
 	struct query_info* k;
 	struct reply_info* d;
 
-	/* lruhash already locked by caller */
-	/* walk in order of lru; best first */
-	for(e=h->lru_start; e; e = e->lru_next) {
-		regional_free_all(worker->scratchpad);
-		lock_rw_rdlock(&e->lock);
-		/* make copy of rrset in worker buffer */
-		if(!copy_msg(worker->scratchpad, e, &k, &d)) {
-			lock_rw_unlock(&e->lock);
-			return 0;
-		}
+	regional_free_all(dump_info->worker->scratchpad);
+	/* Make copy of rrset in worker buffer. */
+	lock_rw_rdlock(&e->lock);
+	if(!copy_msg(dump_info->worker->scratchpad, e, &k, &d)) {
 		lock_rw_unlock(&e->lock);
-		/* release lock so we can lookup the rrset references 
-		 * in the rrset cache */
-		if(!dump_msg(ssl, k, d, *worker->env.now)) {
-			return 0;
-		}
+		log_err("out of memory in dump_msg_entry");
+		return;
 	}
-	return 1;
+	lock_rw_unlock(&e->lock);
+	/* Release lock so we can lookup the rrset references
+	 * in the rrset cache. */
+	dump_msg(txt, k, d, *dump_info->worker->env.now);
 }
 
 /** dump msg cache */
 static int
 dump_msg_cache(RES* ssl, struct worker* worker)
 {
-	struct slabhash* sh = worker->env.msg_cache;
-	size_t slab;
+	struct dump_info dump_info;
+	dump_info.worker = worker;
+	dump_info.ssl = ssl;
 	if(!ssl_printf(ssl, "START_MSG_CACHE\n")) return 0;
-	for(slab=0; slab<sh->size; slab++) {
-		lock_quick_lock(&sh->array[slab]->lock);
-		if(!dump_msg_lruhash(ssl, worker, sh->array[slab])) {
-			lock_quick_unlock(&sh->array[slab]->lock);
-			return 0;
-		}
-		lock_quick_unlock(&sh->array[slab]->lock);
-	}
+	if(!dump_slabhash(worker->env.msg_cache, &dump_msg_entry, ssl,
+		&dump_info))
+		return 0;
 	return ssl_printf(ssl, "END_MSG_CACHE\n");
 }
 
@@ -692,7 +815,7 @@ load_msg(RES* ssl, sldns_buffer* buf, struct worker* worker)
 		return 1; /* skip this one, not all references satisfied */
 
 	if(!dns_cache_store(&worker->env, &qinf, &rep, 0, 0, 0, NULL, flags,
-		*worker->env.now)) {
+		*worker->env.now, 1)) {
 		log_warn("error out of memory");
 		return 0;
 	}
@@ -811,12 +934,18 @@ print_dp_main(RES* ssl, struct delegpt* dp, struct dns_msg* msg)
 		struct ub_packed_rrset_key* k = msg->rep->rrsets[i];
 		struct packed_rrset_data* d = 
 			(struct packed_rrset_data*)k->entry.data;
+		struct config_strlist_head txt;
+		memset(&txt, 0, sizeof(txt));
 		if(d->security == sec_status_bogus) {
 			if(!ssl_printf(ssl, "Address is BOGUS:\n"))
 				return;
 		}
-		if(!dump_rrset(ssl, k, d, 0))
+		dump_rrset(&txt, k, d, 0);
+		if(!spool_strlist(ssl, txt.first)) {
+			config_delstrlist(txt.first);
 			return;
+		}
+		config_delstrlist(txt.first);
 	    }
 	delegpt_count_ns(dp, &n_ns, &n_miss);
 	delegpt_count_addr(dp, &n_addr, &n_res, &n_avail);
@@ -836,9 +965,10 @@ int print_deleg_lookup(RES* ssl, struct worker* worker, uint8_t* nm,
 	struct delegpt* dp;
 	struct dns_msg* msg;
 	struct regional* region = worker->scratchpad;
-	char b[260];
+	char b[LDNS_MAX_DOMAINLEN];
 	struct query_info qinfo;
 	struct iter_hints_stub* stub;
+	int nolock = 0;
 	regional_free_all(region);
 	qinfo.qname = nm;
 	qinfo.qname_len = nmlen;
@@ -850,13 +980,16 @@ int print_deleg_lookup(RES* ssl, struct worker* worker, uint8_t* nm,
 	if(!ssl_printf(ssl, "The following name servers are used for lookup "
 		"of %s\n", b)) 
 		return 0;
-	
-	dp = forwards_lookup(worker->env.fwds, nm, qinfo.qclass);
+
+	dp = forwards_lookup(worker->env.fwds, nm, qinfo.qclass, nolock);
 	if(dp) {
-		if(!ssl_printf(ssl, "forwarding request:\n"))
+		if(!ssl_printf(ssl, "forwarding request:\n")) {
+			lock_rw_unlock(&worker->env.fwds->lock);
 			return 0;
+		}
 		print_dp_main(ssl, dp, NULL);
 		print_dp_details(ssl, worker, dp);
+		lock_rw_unlock(&worker->env.fwds->lock);
 		return 1;
 	}
 	
@@ -892,21 +1025,26 @@ int print_deleg_lookup(RES* ssl, struct worker* worker, uint8_t* nm,
 					return 0;
 				continue;
 			}
-		} 
+		}
 		stub = hints_lookup_stub(worker->env.hints, nm, qinfo.qclass,
-			dp);
+			dp, nolock);
 		if(stub) {
 			if(stub->noprime) {
 				if(!ssl_printf(ssl, "The noprime stub servers "
-					"are used:\n"))
+					"are used:\n")) {
+					lock_rw_unlock(&worker->env.hints->lock);
 					return 0;
+				}
 			} else {
 				if(!ssl_printf(ssl, "The stub is primed "
-						"with servers:\n"))
+						"with servers:\n")) {
+					lock_rw_unlock(&worker->env.hints->lock);
 					return 0;
+				}
 			}
 			print_dp_main(ssl, stub->dp, NULL);
 			print_dp_details(ssl, worker, stub->dp);
+			lock_rw_unlock(&worker->env.hints->lock);
 		} else {
 			print_dp_main(ssl, dp, msg);
 			print_dp_details(ssl, worker, dp);

daemon/daemon.c

@@ -91,6 +91,8 @@
 #include "util/net_help.h"
 #include "sldns/keyraw.h"
 #include "respip/respip.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_hints.h"
 #include <signal.h>
 
 #ifdef HAVE_SYSTEMD
@@ -99,6 +101,9 @@
 #ifdef HAVE_NETDB_H
 #include <netdb.h>
 #endif
+#ifdef USE_CACHEDB
+#include "cachedb/cachedb.h"
+#endif
 
 /** How many quit requests happened. */
 static int sig_record_quit = 0;
@@ -260,6 +265,7 @@ daemon_init(void)
 		free(daemon);
 		return NULL;
 	}
+	daemon->env->modstack = &daemon->mods;
 	/* init edns_known_options */
 	if(!edns_known_options_init(daemon->env)) {
 		free(daemon->env);
@@ -317,21 +323,18 @@ daemon_init(void)
 	return daemon;	
 }
 
-static int setup_acl_for_ports(struct acl_list* list,
-	struct listen_port* port_list)
+int setup_acl_for_ports(struct acl_list* list, struct listen_port* port_list)
 {
 	struct acl_addr* acl_node;
-	struct addrinfo* addr;
 	for(; port_list; port_list=port_list->next) {
 		if(!port_list->socket) {
 			/* This is mainly for testbound where port_list is
 			 * empty. */
 			continue;
 		}
-		addr = port_list->socket->addr;
 		if(!(acl_node = acl_interface_insert(list,
-			(struct sockaddr_storage*)addr->ai_addr,
-			(socklen_t)addr->ai_addrlen,
+			(struct sockaddr_storage*)port_list->socket->addr,
+			port_list->socket->addrlen,
 			acl_refuse))) {
 			return 0;
 		}
@@ -340,7 +343,7 @@ static int setup_acl_for_ports(struct acl_list* list,
 	return 1;
 }
 
-int 
+int
 daemon_open_shared_ports(struct daemon* daemon)
 {
 	log_assert(daemon);
@@ -440,6 +443,19 @@ daemon_open_shared_ports(struct daemon* daemon)
 	return 1;
 }
 
+int
+daemon_privileged(struct daemon* daemon)
+{
+	daemon->env->cfg = daemon->cfg;
+	daemon->env->alloc = &daemon->superalloc;
+	daemon->env->worker = NULL;
+	if(!modstack_call_startup(&daemon->mods, daemon->cfg->module_conf,
+		daemon->env)) {
+		fatal_exit("failed to startup modules");
+	}
+	return 1;
+}
+
 /**
  * Setup modules. setup module stack.
  * @param daemon: the daemon
@@ -449,11 +465,15 @@ static void daemon_setup_modules(struct daemon* daemon)
 	daemon->env->cfg = daemon->cfg;
 	daemon->env->alloc = &daemon->superalloc;
 	daemon->env->worker = NULL;
-	daemon->env->need_to_validate = 0; /* set by module init below */
-	if(!modstack_setup(&daemon->mods, daemon->cfg->module_conf, 
-		daemon->env)) {
-		fatal_exit("failed to setup modules");
+	if(daemon->mods_inited) {
+		modstack_call_deinit(&daemon->mods, daemon->env);
 	}
+	daemon->env->need_to_validate = 0; /* set by module init below */
+	if(!modstack_call_init(&daemon->mods, daemon->cfg->module_conf,
+		daemon->env)) {
+		fatal_exit("failed to init modules");
+	}
+	daemon->mods_inited = 1;
 	log_edns_known_options(VERB_ALGO, daemon->env);
 }
 
@@ -499,7 +519,10 @@ daemon_clear_allocs(struct daemon* daemon)
 {
 	int i;
 
-	for(i=0; i<daemon->num; i++) {
+	/* daemon->num may be different during reloads (after configuration
+	 * read). Use old_num which has the correct value used to setup the
+	 * worker_allocs */
+	for(i=0; i<daemon->old_num; i++) {
 		alloc_clear(daemon->worker_allocs[i]);
 		free(daemon->worker_allocs[i]);
 	}
@@ -533,6 +556,12 @@ daemon_create_workers(struct daemon* daemon)
 		fatal_exit("out of memory during daemon init");
 	numport = daemon_get_shufport(daemon, shufport);
 	verbose(VERB_ALGO, "total of %d outgoing ports available", numport);
+
+#ifdef HAVE_NGTCP2
+	daemon->doq_table = doq_table_create(daemon->cfg, daemon->rand);
+	if(!daemon->doq_table)
+		fatal_exit("could not create doq_table: out of memory");
+#endif
 	
 	daemon->num = (daemon->cfg->num_threads?daemon->cfg->num_threads:1);
 	if(daemon->reuseport && (int)daemon->num < (int)daemon->num_ports) {
@@ -687,16 +716,16 @@ daemon_fork(struct daemon* daemon)
 #endif
 
 	log_assert(daemon);
-	if(!(daemon->views = views_create()))
+	if(!(daemon->env->views = views_create()))
 		fatal_exit("Could not create views: out of memory");
 	/* create individual views and their localzone/data trees */
-	if(!views_apply_cfg(daemon->views, daemon->cfg))
+	if(!views_apply_cfg(daemon->env->views, daemon->cfg))
 		fatal_exit("Could not set up views");
 
-	if(!acl_list_apply_cfg(daemon->acl, daemon->cfg, daemon->views))
+	if(!acl_list_apply_cfg(daemon->acl, daemon->cfg, daemon->env->views))
 		fatal_exit("Could not setup access control list");
 	if(!acl_interface_apply_cfg(daemon->acl_interface, daemon->cfg,
-		daemon->views))
+		daemon->env->views))
 		fatal_exit("Could not setup interface control list");
 	if(!tcl_list_apply_cfg(daemon->tcl, daemon->cfg))
 		fatal_exit("Could not setup TCP connection limits");
@@ -711,22 +740,36 @@ daemon_fork(struct daemon* daemon)
 				   "dnscrypt support");
 #endif
 	}
+	if(daemon->cfg->cookie_secret_file &&
+		daemon->cfg->cookie_secret_file[0]) {
+		if(!(daemon->cookie_secrets = cookie_secrets_create()))
+			fatal_exit("Could not create cookie_secrets: out of memory");
+		if(!cookie_secrets_apply_cfg(daemon->cookie_secrets,
+			daemon->cfg->cookie_secret_file))
+			fatal_exit("Could not setup cookie_secrets");
+	}
 	/* create global local_zones */
 	if(!(daemon->local_zones = local_zones_create()))
 		fatal_exit("Could not create local zones: out of memory");
 	if(!local_zones_apply_cfg(daemon->local_zones, daemon->cfg))
 		fatal_exit("Could not set up local zones");
+	if(!(daemon->env->fwds = forwards_create()) ||
+		!forwards_apply_cfg(daemon->env->fwds, daemon->cfg))
+		fatal_exit("Could not set forward zones");
+	if(!(daemon->env->hints = hints_create()) ||
+		!hints_apply_cfg(daemon->env->hints, daemon->cfg))
+		fatal_exit("Could not set root or stub hints");
 
 	/* process raw response-ip configuration data */
-	if(!(daemon->respip_set = respip_set_create()))
+	if(!(daemon->env->respip_set = respip_set_create()))
 		fatal_exit("Could not create response IP set");
-	if(!respip_global_apply_cfg(daemon->respip_set, daemon->cfg))
+	if(!respip_global_apply_cfg(daemon->env->respip_set, daemon->cfg))
 		fatal_exit("Could not set up response IP set");
-	if(!respip_views_apply_cfg(daemon->views, daemon->cfg,
+	if(!respip_views_apply_cfg(daemon->env->views, daemon->cfg,
 		&have_view_respip_cfg))
 		fatal_exit("Could not set up per-view response IP sets");
-	daemon->use_response_ip = !respip_set_is_empty(daemon->respip_set) ||
-		have_view_respip_cfg;
+	daemon->use_response_ip = !respip_set_is_empty(
+		daemon->env->respip_set) || have_view_respip_cfg;
 
 	/* setup modules */
 	daemon_setup_modules(daemon);
@@ -740,6 +783,10 @@ daemon_fork(struct daemon* daemon)
 	if(!edns_strings_apply_cfg(daemon->env->edns_strings, daemon->cfg))
 		fatal_exit("Could not set up EDNS strings");
 
+#ifdef USE_CACHEDB
+	daemon->env->cachedb_enabled = cachedb_is_enabled(&daemon->mods,
+		daemon->env);
+#endif
 	/* response-ip-xxx options don't work as expected without the respip
 	 * module.  To avoid run-time operational surprise we reject such
 	 * configuration. */
@@ -832,16 +879,24 @@ daemon_cleanup(struct daemon* daemon)
 		slabhash_clear(daemon->env->msg_cache);
 	}
 	daemon->old_num = daemon->num; /* save the current num */
+	forwards_delete(daemon->env->fwds);
+	daemon->env->fwds = NULL;
+	hints_delete(daemon->env->hints);
+	daemon->env->hints = NULL;
 	local_zones_delete(daemon->local_zones);
 	daemon->local_zones = NULL;
-	respip_set_delete(daemon->respip_set);
-	daemon->respip_set = NULL;
-	views_delete(daemon->views);
-	daemon->views = NULL;
+	respip_set_delete(daemon->env->respip_set);
+	daemon->env->respip_set = NULL;
+	views_delete(daemon->env->views);
+	daemon->env->views = NULL;
 	if(daemon->env->auth_zones)
 		auth_zones_cleanup(daemon->env->auth_zones);
-	/* key cache is cleared by module desetup during next daemon_fork() */
+	/* key cache is cleared by module deinit during next daemon_fork() */
 	daemon_remote_clear(daemon->rc);
+	if(daemon->fast_reload_thread)
+		fast_reload_thread_stop(daemon->fast_reload_thread);
+	if(daemon->fast_reload_printq_list)
+		fast_reload_printq_list_delete(daemon->fast_reload_printq_list);
 	for(i=0; i<daemon->num; i++)
 		worker_delete(daemon->workers[i]);
 	free(daemon->workers);
@@ -860,6 +915,10 @@ daemon_cleanup(struct daemon* daemon)
 #ifdef USE_DNSCRYPT
 	dnsc_delete(daemon->dnscenv);
 	daemon->dnscenv = NULL;
+#endif
+#ifdef HAVE_NGTCP2
+	doq_table_delete(daemon->doq_table);
+	daemon->doq_table = NULL;
 #endif
 	daemon->cfg = NULL;
 }
@@ -870,7 +929,9 @@ daemon_delete(struct daemon* daemon)
 	size_t i;
 	if(!daemon)
 		return;
-	modstack_desetup(&daemon->mods, daemon->env);
+	modstack_call_deinit(&daemon->mods, daemon->env);
+	modstack_call_destartup(&daemon->mods, daemon->env);
+	modstack_free(&daemon->mods);
 	daemon_remote_delete(daemon->rc);
 	for(i = 0; i < daemon->num_ports; i++)
 		listening_ports_free(daemon->ports[i]);
@@ -889,14 +950,20 @@ daemon_delete(struct daemon* daemon)
 	acl_list_delete(daemon->acl);
 	acl_list_delete(daemon->acl_interface);
 	tcl_list_delete(daemon->tcl);
+	cookie_secrets_delete(daemon->cookie_secrets);
 	listen_desetup_locks();
 	free(daemon->chroot);
 	free(daemon->pidfile);
+	free(daemon->cfgfile);
 	free(daemon->env);
 #ifdef HAVE_SSL
 	listen_sslctx_delete_ticket_keys();
-	SSL_CTX_free((SSL_CTX*)daemon->listen_sslctx);
-	SSL_CTX_free((SSL_CTX*)daemon->connect_sslctx);
+	SSL_CTX_free((SSL_CTX*)daemon->listen_dot_sslctx);
+	SSL_CTX_free((SSL_CTX*)daemon->listen_doh_sslctx);
+	SSL_CTX_free((SSL_CTX*)daemon->connect_dot_sslctx);
+#endif
+#ifdef HAVE_NGTCP2
+	SSL_CTX_free((SSL_CTX*)daemon->listen_quic_sslctx);
 #endif
 	free(daemon);
 	/* lex cleanup */

daemon/daemon.h

@@ -58,6 +58,10 @@ struct ub_randstate;
 struct daemon_remote;
 struct respip_set;
 struct shm_main_info;
+struct doq_table;
+struct cookie_secrets;
+struct fast_reload_thread;
+struct fast_reload_printq;
 
 #include "dnstap/dnstap_config.h"
 #ifdef USE_DNSTAP
@@ -95,8 +99,14 @@ struct daemon {
 	struct listen_port* rc_ports;
 	/** remote control connections management (for first worker) */
 	struct daemon_remote* rc;
-	/** ssl context for listening to dnstcp over ssl, and connecting ssl */
-	void* listen_sslctx, *connect_sslctx;
+	/** ssl context for listening to dnstcp over ssl */
+	void* listen_dot_sslctx;
+	/** ssl context for connecting to dnstcp over ssl */
+	void* connect_dot_sslctx;
+	/** ssl context for listening to DoH */
+	void* listen_doh_sslctx;
+	/** ssl context for listening to quic */
+	void* listen_quic_sslctx;
 	/** num threads allocated */
 	int num;
 	/** num threads allocated in the previous config or 0 at first */
@@ -115,6 +125,8 @@ struct daemon {
 	struct module_env* env;
 	/** stack of module callbacks */
 	struct module_stack mods;
+	/** The module stack has been inited */
+	int mods_inited;
 	/** access control, which client IPs are allowed to connect */
 	struct acl_list* acl;
 	/** access control, which interfaces are allowed to connect */
@@ -127,15 +139,11 @@ struct daemon {
 	struct timeval time_last_stat;
 	/** time when daemon started */
 	struct timeval time_boot;
-	/** views structure containing view tree */
-	struct views* views;
 #ifdef USE_DNSTAP
 	/** the dnstap environment master value, copied and changed by threads*/
 	struct dt_env* dtenv;
 #endif
 	struct shm_main_info* shm_info;
-	/** response-ip set with associated actions and tags. */
-	struct respip_set* respip_set;
 	/** some response-ip tags or actions are configured if true */
 	int use_response_ip;
 	/** some RPZ policies are configured */
@@ -144,8 +152,23 @@ struct daemon {
 	/** the dnscrypt environment */
 	struct dnsc_env* dnscenv;
 #endif
+	/** the doq connection table */
+	struct doq_table* doq_table;
 	/** reuse existing cache on reload if other conditions allow it. */
 	int reuse_cache;
+	/** the EDNS cookie secrets from the cookie-secret-file */
+	struct cookie_secrets* cookie_secrets;
+	/** the fast reload thread, or NULL */
+	struct fast_reload_thread* fast_reload_thread;
+	/** the fast reload printq list */
+	struct fast_reload_printq* fast_reload_printq_list;
+	/** the fast reload option to drop mesh queries, true if so. */
+	int fast_reload_drop_mesh;
+	/** for fast reload, if the tcl, tcp connection limits, has
+	 * changes for workers */
+	int fast_reload_tcl_has_changes;
+	/** config file name */
+	char* cfgfile;
 };
 
 /**
@@ -162,6 +185,15 @@ struct daemon* daemon_init(void);
  */
 int daemon_open_shared_ports(struct daemon* daemon);
 
+/**
+ * Do daemon setup that needs privileges
+ * like opening privileged ports or opening device files.
+ * The cfg member pointer must have been set for the daemon.
+ * @param daemon: the daemon.
+ * @return: false on error.
+ */
+int daemon_privileged(struct daemon* daemon);
+
 /**
  * Fork workers and start service.
  * When the routine exits, it is no longer forked.
@@ -189,4 +221,12 @@ void daemon_delete(struct daemon* daemon);
  */
 void daemon_apply_cfg(struct daemon* daemon, struct config_file* cfg);
 
+/**
+ * Setup acl list to have entries for the port list.
+ * @param list: the acl interface
+ * @param port_list: list of open ports, or none.
+ * @return false on failure
+ */
+int setup_acl_for_ports(struct acl_list* list, struct listen_port* port_list);
+
 #endif /* DAEMON_H */

daemon/remote.h

@@ -48,6 +48,7 @@
 #ifdef HAVE_OPENSSL_SSL_H
 #include <openssl/ssl.h>
 #endif
+#include "util/locks.h"
 struct config_file;
 struct listen_list;
 struct listen_port;
@@ -55,6 +56,7 @@ struct worker;
 struct comm_reply;
 struct comm_point;
 struct daemon_remote;
+struct config_strlist_head;
 
 /** number of milliseconds timeout on incoming remote control handshake */
 #define REMOTE_CONTROL_TCP_TIMEOUT 120000
@@ -118,6 +120,137 @@ struct remote_stream {
 };
 typedef struct remote_stream RES;
 
+/**
+ * Notification status. This is exchanged between the fast reload thread
+ * and the server thread, over the commpair sockets.
+ */
+enum fast_reload_notification {
+	/** nothing, not used */
+	fast_reload_notification_none = 0,
+	/** the fast reload thread is done */
+	fast_reload_notification_done = 1,
+	/** the fast reload thread is done but with an error, it failed */
+	fast_reload_notification_done_error = 2,
+	/** the fast reload thread is told to exit by the server thread.
+	 * Sent on server quit while the reload is running. */
+	fast_reload_notification_exit = 3,
+	/** the fast reload thread has exited, after being told to exit */
+	fast_reload_notification_exited = 4,
+	/** the fast reload thread has information to print out */
+	fast_reload_notification_printout = 5,
+	/** stop as part of the reload the thread and other threads */
+	fast_reload_notification_reload_stop = 6,
+	/** ack the stop as part of the reload, and also ack start */
+	fast_reload_notification_reload_ack = 7,
+	/** resume from stop as part of the reload */
+	fast_reload_notification_reload_start = 8,
+	/** the fast reload thread wants the mainthread to poll workers,
+	 * after the reload, sent when nopause is used */
+	fast_reload_notification_reload_nopause_poll = 9
+};
+
+/**
+ * Fast reload printout queue. Contains a list of strings, that need to be
+ * printed over the file descriptor.
+ */
+struct fast_reload_printq {
+	/** if this item is in a list, the previous and next */
+	struct fast_reload_printq *prev, *next;
+	/** if this item is in a list, it is true. */
+	int in_list;
+	/** list of strings to printout */
+	struct config_strlist_head* to_print;
+	/** the current item to print. It is malloced. NULL if none. */
+	char* client_item;
+	/** The length, strlen, of the client_item, that has to be sent. */
+	int client_len;
+	/** The number of bytes sent of client_item. */
+	int client_byte_count;
+	/** the comm point for the client connection, the remote control
+	 * client. */
+	struct comm_point* client_cp;
+	/** the remote control connection to print output to. */
+	struct remote_stream remote;
+	/** the worker that the event is added in */
+	struct worker* worker;
+};
+
+/**
+ * Fast reload auth zone change. Keeps track if an auth zone was removed,
+ * added or changed. This is needed because workers can have events for
+ * dealing with auth zones, like transfers, and those have to be removed
+ * too, not just the auth zone structure from the tree. */
+struct fast_reload_auth_change {
+	/** next in the list of auth zone changes. */
+	struct fast_reload_auth_change* next;
+	/** the zone in the old config */
+	struct auth_zone* old_z;
+	/** the zone in the new config */
+	struct auth_zone* new_z;
+	/** if the zone was deleted */
+	int is_deleted;
+	/** if the zone was added */
+	int is_added;
+	/** if the zone has been changed */
+	int is_changed;
+};
+
+/**
+ * Fast reload thread structure
+ */
+struct fast_reload_thread {
+	/** the thread number for the dtio thread,
+	 * must be first to cast thread arg to int* in checklock code. */
+	int threadnum;
+	/** communication socket pair, that sends commands */
+	int commpair[2];
+	/** thread id, of the io thread */
+	ub_thread_type tid;
+	/** if the io processing has started */
+	int started;
+	/** if the thread has to quit */
+	int need_to_quit;
+	/** verbosity of the fast_reload command, the number of +v options */
+	int fr_verb;
+	/** option to not pause threads during reload */
+	int fr_nopause;
+	/** option to drop mesh queries */
+	int fr_drop_mesh;
+
+	/** the event that listens on the remote service worker to the
+	 * commpair, it receives content from the fast reload thread. */
+	void* service_event;
+	/** if the event that listens on the remote service worker has
+	 * been added to the comm base. */
+	int service_event_is_added;
+	/** the service event can read a cmd, nonblocking, so it can
+	 * save the partial read cmd here */
+	uint32_t service_read_cmd;
+	/** the number of bytes in service_read_cmd */
+	int service_read_cmd_count;
+	/** the worker that the service_event is added in */
+	struct worker* worker;
+
+	/** the printout of output to the remote client. */
+	struct fast_reload_printq *printq;
+
+	/** lock on fr_output, to stop race when both remote control thread
+	 * and fast reload thread use fr_output list. */
+	lock_basic_type fr_output_lock;
+	/** list of strings, that the fast reload thread produces that have
+	 * to be printed. The remote control thread can pick them up with
+	 * the lock. */
+	struct config_strlist_head* fr_output;
+
+	/** communication socket pair, to respond to the reload request */
+	int commreload[2];
+
+	/** the list of auth zone changes. */
+	struct fast_reload_auth_change* auth_zone_change_list;
+	/** the old tree of auth zones, to lookup. */
+	struct auth_zones* old_auth_zones;
+};
+
 /**
  * Create new remote control state for the daemon.
  * @param cfg: config file with key file settings.
@@ -203,4 +336,38 @@ int ssl_printf(RES* ssl, const char* format, ...)
 int ssl_read_line(RES* ssl, char* buf, size_t max);
 #endif /* HAVE_SSL */
 
+/**
+ * Start fast reload thread
+ * @param ssl: the RES connection to print to.
+ * @param worker: the remote servicing worker.
+ * @param s: the rc_state that is servicing the remote control connection to
+ *	the remote control client. It needs to be moved away to stay connected
+ *	while the fast reload is running.
+ * @param fr_verb: verbosity to print output at. 0 is nothing, 1 is some
+ *	and 2 is more detail.
+ * @param fr_nopause: option to not pause threads during reload.
+ * @param fr_drop_mesh: option to drop mesh queries.
+ */
+void fast_reload_thread_start(RES* ssl, struct worker* worker,
+	struct rc_state* s, int fr_verb, int fr_nopause, int fr_drop_mesh);
+
+/**
+ * Stop fast reload thread
+ * @param fast_reload_thread: the thread struct.
+ */
+void fast_reload_thread_stop(struct fast_reload_thread* fast_reload_thread);
+
+/** fast reload thread commands to remote service thread event callback */
+void fast_reload_service_cb(int fd, short bits, void* arg);
+
+/** fast reload callback for the remote control client connection */
+int fast_reload_client_callback(struct comm_point* c, void* arg, int err,
+	struct comm_reply* rep);
+
+/** fast reload printq delete list */
+void fast_reload_printq_list_delete(struct fast_reload_printq* list);
+
+/** Pick up per worker changes after a fast reload. */
+void fast_reload_worker_pickup_changes(struct worker* worker);
+
 #endif /* DAEMON_REMOTE_H */

daemon/stats.c

@@ -273,6 +273,7 @@ server_stats_compile(struct worker* worker, struct ub_stats_info* s, int reset)
 	/* add in the values from the mesh */
 	s->svr.ans_secure += (long long)worker->env.mesh->ans_secure;
 	s->svr.ans_bogus += (long long)worker->env.mesh->ans_bogus;
+	s->svr.val_ops += (long long)worker->env.mesh->val_ops;
 	s->svr.ans_rcode_nodata += (long long)worker->env.mesh->ans_nodata;
 	s->svr.ans_expired += (long long)worker->env.mesh->ans_expired;
 	for(i=0; i<UB_STATS_RCODE_NUM; i++)
@@ -281,6 +282,12 @@ server_stats_compile(struct worker* worker, struct ub_stats_info* s, int reset)
 		s->svr.rpz_action[i] += (long long)worker->env.mesh->rpz_action[i];
 	timehist_export(worker->env.mesh->histogram, s->svr.hist,
 		NUM_BUCKETS_HIST);
+	s->svr.num_queries_discard_timeout +=
+		(long long)worker->env.mesh->num_queries_discard_timeout;
+	s->svr.num_queries_wait_limit +=
+		(long long)worker->env.mesh->num_queries_wait_limit;
+	s->svr.num_dns_error_reports +=
+		(long long)worker->env.mesh->num_dns_error_reports;
 	/* values from outside network */
 	s->svr.unwanted_replies = (long long)worker->back->unwanted_replies;
 	s->svr.qtcp_outgoing = (long long)worker->back->num_tcp_outgoing;
@@ -325,20 +332,8 @@ server_stats_compile(struct worker* worker, struct ub_stats_info* s, int reset)
 	s->svr.num_query_dnscrypt_replay = 0;
 #endif /* USE_DNSCRYPT */
 	if(worker->env.auth_zones) {
-		if(reset && !worker->env.cfg->stat_cumulative) {
-			lock_rw_wrlock(&worker->env.auth_zones->lock);
-		} else {
-			lock_rw_rdlock(&worker->env.auth_zones->lock);
-		}
-		s->svr.num_query_authzone_up = (long long)worker->env.
-			auth_zones->num_query_up;
-		s->svr.num_query_authzone_down = (long long)worker->env.
-			auth_zones->num_query_down;
-		if(reset && !worker->env.cfg->stat_cumulative) {
-			worker->env.auth_zones->num_query_up = 0;
-			worker->env.auth_zones->num_query_down = 0;
-		}
-		lock_rw_unlock(&worker->env.auth_zones->lock);
+		s->svr.num_query_authzone_up += (long long)worker->env.mesh->num_query_authzone_up;
+		s->svr.num_query_authzone_down += (long long)worker->env.mesh->num_query_authzone_down;
 	}
 	s->svr.mem_stream_wait =
 		(long long)tcp_req_info_get_stream_buffer_size();
@@ -346,6 +341,12 @@ server_stats_compile(struct worker* worker, struct ub_stats_info* s, int reset)
 		(long long)http2_get_query_buffer_size();
 	s->svr.mem_http2_response_buffer =
 		(long long)http2_get_response_buffer_size();
+#ifdef HAVE_NGTCP2
+	s->svr.mem_quic = (long long)doq_table_quic_size_get(
+		worker->daemon->doq_table);
+#else
+	s->svr.mem_quic = 0;
+#endif /* HAVE_NGTCP2 */
 
 	/* Set neg cache usage numbers */
 	set_neg_cache_stats(worker, &s->svr, reset);
@@ -391,6 +392,13 @@ void server_stats_obtain(struct worker* worker, struct worker* who,
 	else 	worker_send_cmd(who, worker_cmd_stats_noreset);
 	verbose(VERB_ALGO, "wait for stats reply");
 	if(tube_wait_timeout(worker->cmd, STATS_THREAD_WAIT) == 0) {
+#if defined(HAVE_PTHREAD) && defined(SIZEOF_PTHREAD_T) && defined(SIZEOF_UNSIGNED_LONG)
+#  if SIZEOF_PTHREAD_T == SIZEOF_UNSIGNED_LONG
+		unsigned long pthid = 0;
+		if(verbosity >= VERB_OPS)
+			memcpy(&pthid, &who->thr_id, sizeof(unsigned long));
+#  endif
+#endif
 		verbose(VERB_OPS, "no response from thread %d"
 #ifdef HAVE_GETTID
 			" LWP %u"
@@ -407,7 +415,7 @@ void server_stats_obtain(struct worker* worker, struct worker* who,
 #endif
 #if defined(HAVE_PTHREAD) && defined(SIZEOF_PTHREAD_T) && defined(SIZEOF_UNSIGNED_LONG)
 #  if SIZEOF_PTHREAD_T == SIZEOF_UNSIGNED_LONG
-			, (unsigned long)*((unsigned long*)&who->thr_id)
+			, pthid
 #  endif
 #endif
 			);
@@ -438,9 +446,15 @@ void server_stats_add(struct ub_stats_info* total, struct ub_stats_info* a)
 	total->svr.num_queries_cookie_valid += a->svr.num_queries_cookie_valid;
 	total->svr.num_queries_cookie_client += a->svr.num_queries_cookie_client;
 	total->svr.num_queries_cookie_invalid += a->svr.num_queries_cookie_invalid;
+	total->svr.num_queries_discard_timeout +=
+		a->svr.num_queries_discard_timeout;
+	total->svr.num_queries_wait_limit += a->svr.num_queries_wait_limit;
+	total->svr.num_dns_error_reports += a->svr.num_dns_error_reports;
 	total->svr.num_queries_missed_cache += a->svr.num_queries_missed_cache;
 	total->svr.num_queries_prefetch += a->svr.num_queries_prefetch;
 	total->svr.num_queries_timed_out += a->svr.num_queries_timed_out;
+	total->svr.num_query_authzone_up += a->svr.num_query_authzone_up;
+	total->svr.num_query_authzone_down += a->svr.num_query_authzone_down;
 	if (total->svr.max_query_time_us < a->svr.max_query_time_us)
 		total->svr.max_query_time_us = a->svr.max_query_time_us;
 	total->svr.sum_query_list_size += a->svr.sum_query_list_size;
@@ -448,9 +462,9 @@ void server_stats_add(struct ub_stats_info* total, struct ub_stats_info* a)
 #ifdef USE_DNSCRYPT
 	total->svr.num_query_dnscrypt_crypted += a->svr.num_query_dnscrypt_crypted;
 	total->svr.num_query_dnscrypt_cert += a->svr.num_query_dnscrypt_cert;
-	total->svr.num_query_dnscrypt_cleartext += \
+	total->svr.num_query_dnscrypt_cleartext +=
 		a->svr.num_query_dnscrypt_cleartext;
-	total->svr.num_query_dnscrypt_crypted_malformed += \
+	total->svr.num_query_dnscrypt_crypted_malformed +=
 		a->svr.num_query_dnscrypt_crypted_malformed;
 #endif /* USE_DNSCRYPT */
 	/* the max size reached is upped to higher of both */
@@ -467,6 +481,7 @@ void server_stats_add(struct ub_stats_info* total, struct ub_stats_info* a)
 		total->svr.qtls += a->svr.qtls;
 		total->svr.qtls_resume += a->svr.qtls_resume;
 		total->svr.qhttps += a->svr.qhttps;
+		total->svr.qquic += a->svr.qquic;
 		total->svr.qipv6 += a->svr.qipv6;
 		total->svr.qbit_QR += a->svr.qbit_QR;
 		total->svr.qbit_AA += a->svr.qbit_AA;
@@ -481,6 +496,7 @@ void server_stats_add(struct ub_stats_info* total, struct ub_stats_info* a)
 		total->svr.ans_rcode_nodata += a->svr.ans_rcode_nodata;
 		total->svr.ans_secure += a->svr.ans_secure;
 		total->svr.ans_bogus += a->svr.ans_bogus;
+		total->svr.val_ops += a->svr.val_ops;
 		total->svr.unwanted_replies += a->svr.unwanted_replies;
 		total->svr.unwanted_queries += a->svr.unwanted_queries;
 		total->svr.tcp_accept_usage += a->svr.tcp_accept_usage;
@@ -526,7 +542,8 @@ void server_stats_insquery(struct ub_server_stats* stats, struct comm_point* c,
 	else	stats->qclass_big++;
 	stats->qopcode[ LDNS_OPCODE_WIRE(sldns_buffer_begin(c->buffer)) ]++;
 	if(c->type != comm_udp) {
-		stats->qtcp++;
+		if(c->type != comm_doq)
+			stats->qtcp++;
 		if(c->ssl != NULL) {
 			stats->qtls++;
 #ifdef HAVE_SSL
@@ -535,6 +552,10 @@ void server_stats_insquery(struct ub_server_stats* stats, struct comm_point* c,
 #endif
 			if(c->type == comm_http)
 				stats->qhttps++;
+#ifdef HAVE_NGTCP2
+			else if(c->type == comm_doq)
+				stats->qquic++;
+#endif
 		}
 	}
 	if(repinfo && addr_is_ip6(&repinfo->remote_addr, repinfo->remote_addrlen))

daemon/unbound.c

@@ -174,7 +174,7 @@ static void
 checkrlimits(struct config_file* cfg)
 {
 #ifndef S_SPLINT_S
-#ifdef HAVE_GETRLIMIT
+#if defined(HAVE_GETRLIMIT) && !defined(unbound_testbound)
 	/* list has number of ports to listen to, ifs number addresses */
 	int list = ((cfg->do_udp?1:0) + (cfg->do_tcp?1 + 
 			(int)cfg->incoming_num_tcp:0));
@@ -366,9 +366,8 @@ readpid (const char* file)
 /** write pid to file. 
  * @param pidfile: file name of pid file.
  * @param pid: pid to write to file.
- * @return false on failure
  */
-static int
+static void
 writepid (const char* pidfile, pid_t pid)
 {
 	int fd;
@@ -383,7 +382,7 @@ writepid (const char* pidfile, pid_t pid)
 		, 0644)) == -1) {
 		log_err("cannot open pidfile %s: %s", 
 			pidfile, strerror(errno));
-		return 0;
+		return;
 	}
 	while(count < strlen(pidbuf)) {
 		ssize_t r = write(fd, pidbuf+count, strlen(pidbuf)-count);
@@ -393,17 +392,16 @@ writepid (const char* pidfile, pid_t pid)
 			log_err("cannot write to pidfile %s: %s",
 				pidfile, strerror(errno));
 			close(fd);
-			return 0;
+			return;
 		} else if(r == 0) {
 			log_err("cannot write any bytes to pidfile %s: "
 				"write returns 0 bytes written", pidfile);
 			close(fd);
-			return 0;
+			return;
 		}
 		count += r;
 	}
 	close(fd);
-	return 1;
 }
 
 /**
@@ -465,6 +463,62 @@ detach(void)
 #endif /* HAVE_DAEMON */
 }
 
+#ifdef HAVE_SSL
+/* setup a listening ssl context, fatal_exit() on any failure */
+static void
+setup_listen_sslctx(void** ctx, int is_dot, int is_doh, struct config_file* cfg)
+{
+	if(!(*ctx = listen_sslctx_create(
+		cfg->ssl_service_key, cfg->ssl_service_pem, NULL,
+		cfg->tls_ciphers, cfg->tls_ciphersuites,
+		(cfg->tls_session_ticket_keys.first &&
+		cfg->tls_session_ticket_keys.first->str[0] != 0),
+		is_dot, is_doh))) {
+		fatal_exit("could not set up listen SSL_CTX");
+	}
+}
+#endif /* HAVE_SSL */
+
+/* setups the needed ssl contexts, fatal_exit() on any failure */
+static void
+setup_sslctxs(struct daemon* daemon, struct config_file* cfg)
+{
+#ifdef HAVE_SSL
+	if(!(daemon->rc = daemon_remote_create(cfg)))
+		fatal_exit("could not set up remote-control");
+	if(cfg->ssl_service_key && cfg->ssl_service_key[0]) {
+		/* setup the session keys; the callback to use them will be
+		 * attached to each sslctx separately */
+		if(cfg->tls_session_ticket_keys.first &&
+			cfg->tls_session_ticket_keys.first->str[0] != 0) {
+			if(!listen_sslctx_setup_ticket_keys(
+				cfg->tls_session_ticket_keys.first)) {
+				fatal_exit("could not set session ticket SSL_CTX");
+			}
+		}
+		(void)setup_listen_sslctx(&daemon->listen_dot_sslctx, 1, 0, cfg);
+#ifdef HAVE_NGHTTP2_NGHTTP2_H
+		if(cfg_has_https(cfg)) {
+			(void)setup_listen_sslctx(&daemon->listen_doh_sslctx, 0, 1, cfg);
+		}
+#endif
+#ifdef HAVE_NGTCP2
+		if(cfg_has_quic(cfg)) {
+			if(!(daemon->listen_quic_sslctx = quic_sslctx_create(
+				cfg->ssl_service_key, cfg->ssl_service_pem, NULL))) {
+				fatal_exit("could not set up quic SSL_CTX");
+			}
+		}
+#endif /* HAVE_NGTCP2 */
+	}
+	if(!(daemon->connect_dot_sslctx = connect_sslctx_create(NULL, NULL,
+		cfg->tls_cert_bundle, cfg->tls_win_cert)))
+		fatal_exit("could not set up connect SSL_CTX");
+#else /* HAVE_SSL */
+	(void)daemon;(void)cfg;
+#endif /* HAVE_SSL */
+}
+
 /** daemonize, drop user privileges and chroot if needed */
 static void
 perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode,
@@ -475,7 +529,11 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode,
 #endif
 #ifdef HAVE_GETPWNAM
 	struct passwd *pwd = NULL;
+#endif
 
+	if(!daemon_privileged(daemon))
+		fatal_exit("could not do privileged setup");
+#ifdef HAVE_GETPWNAM
 	if(cfg->username && cfg->username[0]) {
 		if((pwd = getpwnam(cfg->username)) == NULL)
 			fatal_exit("user '%s' does not exist.", cfg->username);
@@ -487,36 +545,7 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode,
 #endif
 
 	/* read ssl keys while superuser and outside chroot */
-#ifdef HAVE_SSL
-	if(!(daemon->rc = daemon_remote_create(cfg)))
-		fatal_exit("could not set up remote-control");
-	if(cfg->ssl_service_key && cfg->ssl_service_key[0]) {
-		if(!(daemon->listen_sslctx = listen_sslctx_create(
-			cfg->ssl_service_key, cfg->ssl_service_pem, NULL)))
-			fatal_exit("could not set up listen SSL_CTX");
-		if(cfg->tls_ciphers && cfg->tls_ciphers[0]) {
-			if (!SSL_CTX_set_cipher_list(daemon->listen_sslctx, cfg->tls_ciphers)) {
-				fatal_exit("failed to set tls-cipher %s", cfg->tls_ciphers);
-			}
-		}
-#ifdef HAVE_SSL_CTX_SET_CIPHERSUITES
-		if(cfg->tls_ciphersuites && cfg->tls_ciphersuites[0]) {
-			if (!SSL_CTX_set_ciphersuites(daemon->listen_sslctx, cfg->tls_ciphersuites)) {
-				fatal_exit("failed to set tls-ciphersuites %s", cfg->tls_ciphersuites);
-			}
-		}
-#endif
-		if(cfg->tls_session_ticket_keys.first &&
-			cfg->tls_session_ticket_keys.first->str[0] != 0) {
-			if(!listen_sslctx_setup_ticket_keys(daemon->listen_sslctx, cfg->tls_session_ticket_keys.first)) {
-				fatal_exit("could not set session ticket SSL_CTX");
-			}
-		}
-	}
-	if(!(daemon->connect_sslctx = connect_sslctx_create(NULL, NULL,
-		cfg->tls_cert_bundle, cfg->tls_win_cert)))
-		fatal_exit("could not set up connect SSL_CTX");
-#endif
+	(void)setup_sslctxs(daemon, cfg);
 
 	/* init syslog (as root) if needed, before daemonize, otherwise
 	 * a fork error could not be printed since daemonize closed stderr.*/
@@ -545,7 +574,15 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode,
 				cfg, 1);
 		if(!daemon->pidfile)
 			fatal_exit("pidfile alloc: out of memory");
-		checkoldpid(daemon->pidfile, pidinchroot);
+		/* Check old pid if there is no username configured.
+		 * With a username, the assumption is that the privilege
+		 * drop makes a pidfile not removed when the server stopped
+		 * last time. The server does not chown the pidfile for it,
+		 * because that creates privilege escape problems, with the
+		 * pidfile writable by unprivileged users, but used by
+		 * privileged users. */
+		if(!(cfg->username && cfg->username[0]))
+			checkoldpid(daemon->pidfile, pidinchroot);
 	}
 #endif
 
@@ -557,18 +594,7 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode,
 	/* write new pidfile (while still root, so can be outside chroot) */
 #ifdef HAVE_KILL
 	if(cfg->pidfile && cfg->pidfile[0] && need_pidfile) {
-		if(writepid(daemon->pidfile, getpid())) {
-			if(cfg->username && cfg->username[0] && cfg_uid != (uid_t)-1 &&
-				pidinchroot) {
-#  ifdef HAVE_CHOWN
-				if(chown(daemon->pidfile, cfg_uid, cfg_gid) == -1) {
-					verbose(VERB_QUERY, "cannot chown %u.%u %s: %s",
-						(unsigned)cfg_uid, (unsigned)cfg_gid,
-						daemon->pidfile, strerror(errno));
-				}
-#  endif /* HAVE_CHOWN */
-			}
-		}
+		writepid(daemon->pidfile, getpid());
 	}
 #else
 	(void)daemon;
@@ -682,6 +708,9 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode,
 	 * it would succeed on SIGHUP as well */
 	if(!cfg->use_syslog)
 		log_init(cfg->logfile, cfg->use_syslog, cfg->chrootdir);
+	daemon->cfgfile = strdup(*cfgfile);
+	if(!daemon->cfgfile)
+		fatal_exit("out of memory in daemon cfgfile strdup");
 }
 
 /**
@@ -716,6 +745,7 @@ run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode, int need_pi
 					"the commandline to see more errors, "
 					"or unbound-checkconf", cfgfile);
 			log_warn("Continuing with default config settings");
+			config_auto_slab_values(cfg);
 		}
 		apply_settings(daemon, cfg, cmdline_verbose, debug_mode);
 		if(!done_setup)
@@ -746,7 +776,11 @@ run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode, int need_pi
 	if(daemon->pidfile) {
 		int fd;
 		/* truncate pidfile */
-		fd = open(daemon->pidfile, O_WRONLY | O_TRUNC, 0644);
+		fd = open(daemon->pidfile, O_WRONLY | O_TRUNC
+#ifdef O_NOFOLLOW
+			| O_NOFOLLOW
+#endif
+			, 0644);
 		if(fd != -1)
 			close(fd);
 		/* delete pidfile */

daemon/worker.c

@@ -160,9 +160,11 @@ worker_mem_report(struct worker* ATTR_UNUSED(worker),
 		+ sizeof(worker->rndstate)
 		+ regional_get_mem(worker->scratchpad)
 		+ sizeof(*worker->env.scratch_buffer)
-		+ sldns_buffer_capacity(worker->env.scratch_buffer)
-		+ forwards_get_mem(worker->env.fwds)
-		+ hints_get_mem(worker->env.hints);
+		+ sldns_buffer_capacity(worker->env.scratch_buffer);
+	if(worker->daemon->env->fwds)
+		log_info("forwards=%u", (unsigned)forwards_get_mem(worker->env.fwds));
+	if(worker->daemon->env->hints)
+		log_info("hints=%u", (unsigned)hints_get_mem(worker->env.hints));
 	if(worker->thread_num == 0)
 		me += acl_list_get_mem(worker->daemon->acl);
 	if(cur_serv) {
@@ -369,6 +371,84 @@ worker_check_request(sldns_buffer* pkt, struct worker* worker,
 	return;
 }
 
+/**
+ * Send fast-reload acknowledgement to the mainthread in one byte.
+ * This signals that this worker has received the previous command.
+ * The worker is waiting if that is after a reload_stop command.
+ * Or the worker has briefly processed the event itself, and in doing so
+ * released data pointers to old config, after a reload_poll command.
+ */
+static void
+worker_send_reload_ack(struct worker* worker)
+{
+	/* If this is clipped to 8 bits because thread_num>255, then that
+	 * is not a problem, the receiver counts the number of bytes received.
+	 * The number is informative only. */
+	uint8_t c = (uint8_t)worker->thread_num;
+	ssize_t ret;
+	while(1) {
+		ret = send(worker->daemon->fast_reload_thread->commreload[1],
+			(void*)&c, 1, 0);
+		if(ret == -1) {
+			if(
+#ifndef USE_WINSOCK
+				errno == EINTR || errno == EAGAIN
+#  ifdef EWOULDBLOCK
+				|| errno == EWOULDBLOCK
+#  endif
+#else
+				WSAGetLastError() == WSAEINTR ||
+				WSAGetLastError() == WSAEINPROGRESS ||
+				WSAGetLastError() == WSAEWOULDBLOCK
+#endif
+				)
+				continue; /* Try again. */
+			log_err("worker reload ack reply: send failed: %s",
+				sock_strerror(errno));
+			break;
+		}
+		break;
+	}
+}
+
+/** stop and wait to resume the worker */
+static void
+worker_stop_and_wait(struct worker* worker)
+{
+	uint8_t* buf = NULL;
+	uint32_t len = 0, cmd;
+	worker_send_reload_ack(worker);
+	/* wait for reload */
+	if(!tube_read_msg(worker->cmd, &buf, &len, 0)) {
+		log_err("worker reload read reply failed");
+		return;
+	}
+	if(len != sizeof(uint32_t)) {
+		log_err("worker reload reply, bad control msg length %d",
+			(int)len);
+		free(buf);
+		return;
+	}
+	cmd = sldns_read_uint32(buf);
+	free(buf);
+	if(cmd == worker_cmd_quit) {
+		/* quit anyway */
+		verbose(VERB_ALGO, "reload reply, control cmd quit");
+		comm_base_exit(worker->base);
+		return;
+	}
+	if(cmd != worker_cmd_reload_start) {
+		log_err("worker reload reply, wrong reply command");
+	}
+	if(worker->daemon->fast_reload_drop_mesh) {
+		verbose(VERB_ALGO, "worker: drop mesh queries after reload");
+		mesh_delete_all(worker->env.mesh);
+	}
+	fast_reload_worker_pickup_changes(worker);
+	worker_send_reload_ack(worker);
+	verbose(VERB_ALGO, "worker resume after reload");
+}
+
 void
 worker_handle_control_cmd(struct tube* ATTR_UNUSED(tube), uint8_t* msg,
 	size_t len, int error, void* arg)
@@ -404,6 +484,15 @@ worker_handle_control_cmd(struct tube* ATTR_UNUSED(tube), uint8_t* msg,
 		verbose(VERB_ALGO, "got control cmd remote");
 		daemon_remote_exec(worker);
 		break;
+	case worker_cmd_reload_stop:
+		verbose(VERB_ALGO, "got control cmd reload_stop");
+		worker_stop_and_wait(worker);
+		break;
+	case worker_cmd_reload_poll:
+		verbose(VERB_ALGO, "got control cmd reload_poll");
+		fast_reload_worker_pickup_changes(worker);
+		worker_send_reload_ack(worker);
+		break;
 	default:
 		log_err("bad command %d", (int)cmd);
 		break;
@@ -598,7 +687,8 @@ apply_respip_action(struct worker* worker, const struct query_info* qinfo,
 		return 1;
 
 	if(!respip_rewrite_reply(qinfo, cinfo, rep, encode_repp, &actinfo,
-		alias_rrset, 0, worker->scratchpad, az, NULL))
+		alias_rrset, 0, worker->scratchpad, az, NULL,
+		worker->env.views, worker->env.respip_set))
 		return 0;
 
 	/* xxx_deny actions mean dropping the reply, unless the original reply
@@ -659,17 +749,18 @@ answer_from_cache(struct worker* worker, struct query_info* qinfo,
 	if(rep->ttl < timenow) {
 		/* Check if we need to serve expired now */
 		if(worker->env.cfg->serve_expired &&
-			!worker->env.cfg->serve_expired_client_timeout) {
-				if(worker->env.cfg->serve_expired_ttl &&
-					rep->serve_expired_ttl < timenow)
-					return 0;
-				/* Ignore expired failure answers */
-				if(FLAGS_GET_RCODE(rep->flags) !=
-					LDNS_RCODE_NOERROR &&
-					FLAGS_GET_RCODE(rep->flags) !=
-					LDNS_RCODE_NXDOMAIN &&
-					FLAGS_GET_RCODE(rep->flags) !=
-					LDNS_RCODE_YXDOMAIN)
+			/* if serve-expired-client-timeout is set, serve
+			 * an expired record without attempting recursion
+			 * if the serve_expired_norec_ttl is set for the record
+			 * as we know that recursion is currently failing. */
+			(!worker->env.cfg->serve_expired_client_timeout ||
+			 timenow < rep->serve_expired_norec_ttl)
+#ifdef USE_CACHEDB
+			&& !(worker->env.cachedb_enabled &&
+			  worker->env.cfg->cachedb_check_when_serve_expired)
+#endif
+			) {
+				if(!reply_info_can_answer_expired(rep, timenow))
 					return 0;
 				if(!rrset_array_lock(rep->ref, rep->rrset_count, 0))
 					return 0;
@@ -758,7 +849,8 @@ answer_from_cache(struct worker* worker, struct query_info* qinfo,
 	} else if(partial_rep &&
 		!respip_merge_cname(partial_rep, qinfo, rep, cinfo,
 		must_validate, &encode_rep, worker->scratchpad,
-		worker->env.auth_zones)) {
+		worker->env.auth_zones, worker->env.views,
+		worker->env.respip_set)) {
 		goto bail_out;
 	}
 	if(encode_rep != rep) {
@@ -1079,7 +1171,7 @@ answer_notify(struct worker* w, struct query_info* qinfo,
 
 	if(verbosity >= VERB_DETAIL) {
 		char buf[380];
-		char zname[255+1];
+		char zname[LDNS_MAX_DOMAINLEN];
 		char sr[25];
 		dname_str(qinfo->qname, zname);
 		sr[0]=0;
@@ -1151,7 +1243,7 @@ deny_refuse(struct comm_point* c, enum acl_access acl,
 		log_assert(sldns_buffer_limit(c->buffer) >= LDNS_HEADER_SIZE
 			&& LDNS_QDCOUNT(sldns_buffer_begin(c->buffer)) == 1);
 
-		sldns_buffer_skip(c->buffer, LDNS_HEADER_SIZE); /* skip header */
+		sldns_buffer_set_position(c->buffer, LDNS_HEADER_SIZE); /* skip header */
 
 		/* check additional section is present and that we respond with EDEs */
 		if(LDNS_ARCOUNT(sldns_buffer_begin(c->buffer)) != 1
@@ -1163,6 +1255,7 @@ deny_refuse(struct comm_point* c, enum acl_access acl,
 			LDNS_QR_SET(sldns_buffer_begin(c->buffer));
 			LDNS_RCODE_SET(sldns_buffer_begin(c->buffer),
 				LDNS_RCODE_REFUSED);
+			sldns_buffer_set_position(c->buffer, LDNS_HEADER_SIZE);
 			sldns_buffer_flip(c->buffer);
 			return 1;
 		}
@@ -1326,15 +1419,6 @@ deny_refuse_non_local(struct comm_point* c, enum acl_access acl,
 		worker, repinfo, acladdr, ede, check_result);
 }
 
-/* Returns 1 if the ip rate limit check can happen before EDNS parsing,
- * else 0 */
-static int
-pre_edns_ip_ratelimit_check(enum acl_access acl)
-{
-	if(acl == acl_allow_cookie) return 0;
-	return 1;
-}
-
 /* Check if the query is blocked by source IP rate limiting.
  * Returns 1 if it passes the check, 0 otherwise. */
 static int
@@ -1418,7 +1502,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		return 0;
 	}
 	if(c->dnscrypt && !repinfo->is_dnscrypted) {
-		char buf[LDNS_MAX_DOMAINLEN+1];
+		char buf[LDNS_MAX_DOMAINLEN];
 		/* Check if this is unencrypted and asking for certs */
 		worker_check_request(c->buffer, worker, &check_result);
 		if(check_result.value != 0) {
@@ -1462,8 +1546,8 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 	 */
 	if(worker->dtenv.log_client_query_messages) {
 		log_addr(VERB_ALGO, "request from client", &repinfo->client_addr, repinfo->client_addrlen);
-		log_addr(VERB_ALGO, "to local addr", (void*)repinfo->c->socket->addr->ai_addr, repinfo->c->socket->addr->ai_addrlen);
-		dt_msg_send_client_query(&worker->dtenv, &repinfo->client_addr, (void*)repinfo->c->socket->addr->ai_addr, c->type, c->buffer,
+		log_addr(VERB_ALGO, "to local addr", (void*)repinfo->c->socket->addr, repinfo->c->socket->addrlen);
+		dt_msg_send_client_query(&worker->dtenv, &repinfo->client_addr, (void*)repinfo->c->socket->addr, c->type, c->ssl, c->buffer,
 		((worker->env.cfg->sock_queue_timeout && timeval_isset(&c->recv_tv))?&c->recv_tv:NULL));
 	}
 #endif
@@ -1498,7 +1582,9 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 	}
 
 	worker->stats.num_queries++;
-	pre_edns_ip_ratelimit = pre_edns_ip_ratelimit_check(acl);
+	pre_edns_ip_ratelimit = !worker->env.cfg->do_answer_cookie
+		|| sldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE
+		|| LDNS_ARCOUNT(sldns_buffer_begin(c->buffer)) == 0;
 
 	/* If the IP rate limiting check needs extra EDNS information (e.g.,
 	 * DNS Cookies) postpone the check until after EDNS is parsed. */
@@ -1572,7 +1658,8 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 	if((ret=parse_edns_from_query_pkt(
 			c->buffer, &edns, worker->env.cfg, c, repinfo,
 			(worker->env.now ? *worker->env.now : time(NULL)),
-			worker->scratchpad)) != 0) {
+			worker->scratchpad,
+			worker->daemon->cookie_secrets)) != 0) {
 		struct edns_data reply_edns;
 		verbose(VERB_ALGO, "worker parse edns: formerror.");
 		log_addr(VERB_CLIENT, "from", &repinfo->client_addr,
@@ -1620,6 +1707,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 			repinfo->client_addrlen, edns.cookie_valid,
 			c->buffer)) {
 			worker->stats.num_queries_ip_ratelimited++;
+			regional_free_all(worker->scratchpad);
 			comm_point_drop_reply(repinfo);
 			return 0;
 		}
@@ -1731,8 +1819,9 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		goto send_reply;
 	}
 	if(worker->env.auth_zones &&
-		auth_zones_answer(worker->env.auth_zones, &worker->env,
-		&qinfo, &edns, repinfo, c->buffer, worker->scratchpad)) {
+		auth_zones_downstream_answer(worker->env.auth_zones,
+		&worker->env, &qinfo, &edns, repinfo, c->buffer,
+		worker->scratchpad)) {
 		regional_free_all(worker->scratchpad);
 		if(sldns_buffer_limit(c->buffer) == 0) {
 			comm_point_drop_reply(repinfo);
@@ -1785,20 +1874,11 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 	/* If we've found a local alias, replace the qname with the alias
 	 * target before resolving it. */
 	if(qinfo.local_alias) {
-		struct ub_packed_rrset_key* rrset = qinfo.local_alias->rrset;
-		struct packed_rrset_data* d = rrset->entry.data;
-
-		/* Sanity check: our current implementation only supports
-		 * a single CNAME RRset as a local alias. */
-		if(qinfo.local_alias->next ||
-			rrset->rk.type != htons(LDNS_RR_TYPE_CNAME) ||
-			d->count != 1) {
-			log_err("assumption failure: unexpected local alias");
+		if(!local_alias_shallow_copy_qname(qinfo.local_alias, &qinfo.qname,
+			&qinfo.qname_len)) {
 			regional_free_all(worker->scratchpad);
 			return 0; /* drop it */
 		}
-		qinfo.qname = d->rr_data[0] + 2;
-		qinfo.qname_len = d->rr_len[0] - 2;
 	}
 
 	/* If we may apply IP-based actions to the answer, build the client
@@ -1815,7 +1895,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		cinfo_tmp.tag_datas = acladdr->tag_datas;
 		cinfo_tmp.tag_datas_size = acladdr->tag_datas_size;
 		cinfo_tmp.view = acladdr->view;
-		cinfo_tmp.respip_set = worker->daemon->respip_set;
+		cinfo_tmp.view_name = NULL;
 		cinfo = &cinfo_tmp;
 	}
 
@@ -1847,10 +1927,10 @@ lookup_cache:
 				 * its qname must be that used for cache
 				 * lookup. */
 				if((worker->env.cfg->prefetch &&
-					*worker->env.now >= rep->prefetch_ttl) ||
+					rep->prefetch_ttl <= *worker->env.now) ||
 					(worker->env.cfg->serve_expired &&
-					*worker->env.now > rep->ttl)) {
-
+					rep->ttl < *worker->env.now  &&
+					!(*worker->env.now < rep->serve_expired_norec_ttl))) {
 					time_t leeway = rep->ttl - *worker->env.now;
 					if(rep->ttl < *worker->env.now)
 						leeway = 0;
@@ -1949,10 +2029,10 @@ send_reply_rc:
 	/*
 	 * sending src (client)/dst (local service) addresses over DNSTAP from send_reply code label (when we serviced local zone for ex.)
 	 */
-	if(worker->dtenv.log_client_response_messages) {
-		log_addr(VERB_ALGO, "from local addr", (void*)repinfo->c->socket->addr->ai_addr, repinfo->c->socket->addr->ai_addrlen);
+	if(worker->dtenv.log_client_response_messages && rc !=0) {
+		log_addr(VERB_ALGO, "from local addr", (void*)repinfo->c->socket->addr, repinfo->c->socket->addrlen);
 		log_addr(VERB_ALGO, "response to client", &repinfo->client_addr, repinfo->client_addrlen);
-		dt_msg_send_client_response(&worker->dtenv, &repinfo->client_addr, (void*)repinfo->c->socket->addr->ai_addr, c->type, c->buffer);
+		dt_msg_send_client_response(&worker->dtenv, &repinfo->client_addr, (void*)repinfo->c->socket->addr, c->type, c->ssl, c->buffer);
 	}
 #endif
 	if(worker->env.cfg->log_replies)
@@ -1966,11 +2046,15 @@ send_reply_rc:
 			qinfo.qname = qinfo.local_alias->rrset->rk.dname;
 			log_reply_info(NO_VERBOSE, &qinfo,
 				&repinfo->client_addr, repinfo->client_addrlen,
-				tv, 1, c->buffer);
+				tv, 1, c->buffer,
+				(worker->env.cfg->log_destaddr?(void*)repinfo->c->socket->addr:NULL),
+				c->type, c->ssl);
 		} else {
 			log_reply_info(NO_VERBOSE, &qinfo,
 				&repinfo->client_addr, repinfo->client_addrlen,
-				tv, 1, c->buffer);
+				tv, 1, c->buffer,
+				(worker->env.cfg->log_destaddr?(void*)repinfo->c->socket->addr:NULL),
+				c->type, c->ssl);
 		}
 	}
 #ifdef USE_DNSCRYPT
@@ -2171,8 +2255,11 @@ worker_init(struct worker* worker, struct config_file *cfg,
 			: cfg->tcp_idle_timeout,
 		cfg->harden_large_queries, cfg->http_max_streams,
 		cfg->http_endpoint, cfg->http_notls_downstream,
-		worker->daemon->tcl, worker->daemon->listen_sslctx,
-		dtenv, worker_handle_request, worker);
+		worker->daemon->tcl, worker->daemon->listen_dot_sslctx,
+		worker->daemon->listen_doh_sslctx,
+		worker->daemon->listen_quic_sslctx,
+		dtenv, worker->daemon->doq_table, worker->env.rnd,
+		cfg, worker_handle_request, worker);
 	if(!worker->front) {
 		log_err("could not create listening sockets");
 		worker_delete(worker);
@@ -2187,7 +2274,7 @@ worker_init(struct worker* worker, struct config_file *cfg,
 		cfg->unwanted_threshold, cfg->outgoing_tcp_mss,
 		&worker_alloc_cleanup, worker,
 		cfg->do_udp || cfg->udp_upstream_without_downstream,
-		worker->daemon->connect_sslctx, cfg->delay_close,
+		worker->daemon->connect_dot_sslctx, cfg->delay_close,
 		cfg->tls_use_sni, dtenv, cfg->udp_connect,
 		cfg->max_reuse_tcp_queries, cfg->tcp_reuse_timeout,
 		cfg->tcp_auth_query_timeout);
@@ -2263,18 +2350,6 @@ worker_init(struct worker* worker, struct config_file *cfg,
 		worker_delete(worker);
 		return 0;
 	}
-	if(!(worker->env.fwds = forwards_create()) ||
-		!forwards_apply_cfg(worker->env.fwds, cfg)) {
-		log_err("Could not set forward zones");
-		worker_delete(worker);
-		return 0;
-	}
-	if(!(worker->env.hints = hints_create()) ||
-		!hints_apply_cfg(worker->env.hints, cfg)) {
-		log_err("Could not set root or stub hints");
-		worker_delete(worker);
-		return 0;
-	}
 	/* one probe timer per process -- if we have 5011 anchors */
 	if(autr_get_num_anchors(worker->env.anchors) > 0
 #ifndef THREADS_DISABLED
@@ -2347,8 +2422,6 @@ worker_delete(struct worker* worker)
 	outside_network_quit_prepare(worker->back);
 	mesh_delete(worker->env.mesh);
 	sldns_buffer_free(worker->env.scratch_buffer);
-	forwards_delete(worker->env.fwds);
-	hints_delete(worker->env.hints);
 	listen_delete(worker->front);
 	outside_network_delete(worker->back);
 	comm_signal_delete(worker->comsig);
@@ -2520,3 +2593,19 @@ void dtio_mainfdcallback(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
 	log_assert(0);
 }
 #endif
+
+#ifdef HAVE_NGTCP2
+void doq_client_event_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+#endif
+
+#ifdef HAVE_NGTCP2
+void doq_client_timer_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+#endif

daemon/worker.h

@@ -72,7 +72,13 @@ enum worker_commands {
 	/** obtain statistics without statsclear */
 	worker_cmd_stats_noreset,
 	/** execute remote control command */
-	worker_cmd_remote
+	worker_cmd_remote,
+	/** for fast-reload, perform stop */
+	worker_cmd_reload_stop,
+	/** for fast-reload, start again */
+	worker_cmd_reload_start,
+	/** for fast-reload, poll to make sure worker has released data */
+	worker_cmd_reload_poll
 };
 
 /**

dns64/dns64.c

@@ -631,7 +631,7 @@ handle_event_moddone(struct module_qstate* qstate, int id)
 
 	/* When an AAAA query completes check if we want to perform DNS64
 	 * synthesis. We skip queries with DNSSEC enabled (!CD) and
-	 * ones generated by us to retrive the A/PTR record to use for
+	 * ones generated by us to retrieve the A/PTR record to use for
 	 * synth. */
 	int could_synth =
 		qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA &&
@@ -657,8 +657,9 @@ handle_event_moddone(struct module_qstate* qstate, int id)
 		qstate->return_msg->rep &&
 		!dns_cache_store(
 			qstate->env, &qstate->qinfo, qstate->return_msg->rep,
-			0, 0, 0, NULL,
-			qstate->query_flags, qstate->qstarttime))
+			0, qstate->prefetch_leeway, 0, NULL,
+			qstate->query_flags, qstate->qstarttime,
+			qstate->is_valrec))
 		log_err("out of memory");
 
 	/* do nothing */
@@ -701,6 +702,7 @@ dns64_operate(struct module_qstate* qstate, enum module_ev event, int id,
 			iq->state = DNS64_NEW_QUERY;
 			iq->started_no_cache_store = qstate->no_cache_store;
 			qstate->no_cache_store = 1;
+			ATTR_FALLTHROUGH
   			/* fallthrough */
 		case module_event_pass:
 			qstate->ext_state[id] = handle_event_pass(qstate, id);
@@ -846,6 +848,7 @@ dns64_adjust_a(int id, struct module_qstate* super, struct module_qstate* qstate
 	 */
 	cp = construct_reply_info_base(super->region, rep->flags, rep->qdcount,
 		rep->ttl, rep->prefetch_ttl, rep->serve_expired_ttl,
+		rep->serve_expired_norec_ttl,
 		rep->an_numrrsets, rep->ns_numrrsets, rep->ar_numrrsets,
 		rep->rrset_count, rep->security, LDNS_EDE_NONE);
 	if(!cp)
@@ -1006,7 +1009,8 @@ dns64_inform_super(struct module_qstate* qstate, int id,
 	/* Store the generated response in cache. */
 	if ( (!super_dq || !super_dq->started_no_cache_store) &&
 		!dns_cache_store(super->env, &super->qinfo, super->return_msg->rep,
-		0, 0, 0, NULL, super->query_flags, qstate->qstarttime))
+		0, super->prefetch_leeway, 0, NULL, super->query_flags,
+		qstate->qstarttime, qstate->is_valrec))
 		log_err("out of memory");
 }
 
@@ -1044,8 +1048,8 @@ dns64_get_mem(struct module_env* env, int id)
  */
 static struct module_func_block dns64_block = {
 	"dns64",
-	&dns64_init, &dns64_deinit, &dns64_operate, &dns64_inform_super,
-	&dns64_clear, &dns64_get_mem
+	NULL, NULL, &dns64_init, &dns64_deinit, &dns64_operate,
+	&dns64_inform_super, &dns64_clear, &dns64_get_mem
 };
 
 /**

dnstap/dnstap.c

@@ -86,6 +86,31 @@ dt_pack(const Dnstap__Dnstap *d, void **buf, size_t *sz)
 	return 1;
 }
 
+/** See if the message is sent due to dnstap sample rate */
+static int
+dt_sample_rate_limited(struct dt_env* env)
+{
+	lock_basic_lock(&env->sample_lock);
+	/* Sampling is every [n] packets. Where n==1, every packet is sent */
+	if(env->sample_rate > 1) {
+		int submit = 0;
+		/* if sampling is engaged... */
+		if (env->sample_rate_count > env->sample_rate) {
+			/* once the count passes the limit */
+			/* submit the message */
+			submit = 1;
+			/* and reset the count */
+			env->sample_rate_count = 0;
+		}
+		/* increment count regardless */
+		env->sample_rate_count++;
+		lock_basic_unlock(&env->sample_lock);
+		return !submit;
+	}
+	lock_basic_unlock(&env->sample_lock);
+	return 0;
+}
+
 static void
 dt_send(const struct dt_env *env, void *buf, size_t len_buf)
 {
@@ -146,6 +171,7 @@ dt_create(struct config_file* cfg)
 	env = (struct dt_env *) calloc(1, sizeof(struct dt_env));
 	if (!env)
 		return NULL;
+	lock_basic_init(&env->sample_lock);
 
 	env->dtio = dt_io_thread_create();
 	if(!env->dtio) {
@@ -166,8 +192,11 @@ static void
 dt_apply_identity(struct dt_env *env, struct config_file *cfg)
 {
 	char buf[MAXHOSTNAMELEN+1];
-	if (!cfg->dnstap_send_identity)
+	if (!cfg->dnstap_send_identity) {
+		free(env->identity);
+		env->identity = NULL;
 		return;
+	}
 	free(env->identity);
 	if (cfg->dnstap_identity == NULL || cfg->dnstap_identity[0] == 0) {
 		if (gethostname(buf, MAXHOSTNAMELEN) == 0) {
@@ -189,8 +218,11 @@ dt_apply_identity(struct dt_env *env, struct config_file *cfg)
 static void
 dt_apply_version(struct dt_env *env, struct config_file *cfg)
 {
-	if (!cfg->dnstap_send_version)
+	if (!cfg->dnstap_send_version) {
+		free(env->version);
+		env->version = NULL;
 		return;
+	}
 	free(env->version);
 	if (cfg->dnstap_version == NULL || cfg->dnstap_version[0] == 0)
 		env->version = strdup(PACKAGE_STRING);
@@ -204,13 +236,8 @@ dt_apply_version(struct dt_env *env, struct config_file *cfg)
 }
 
 void
-dt_apply_cfg(struct dt_env *env, struct config_file *cfg)
+dt_apply_logcfg(struct dt_env *env, struct config_file *cfg)
 {
-	if (!cfg->dnstap)
-		return;
-
-	dt_apply_identity(env, cfg);
-	dt_apply_version(env, cfg);
 	if ((env->log_resolver_query_messages = (unsigned int)
 	     cfg->dnstap_log_resolver_query_messages))
 	{
@@ -241,6 +268,23 @@ dt_apply_cfg(struct dt_env *env, struct config_file *cfg)
 	{
 		verbose(VERB_OPS, "dnstap Message/FORWARDER_RESPONSE enabled");
 	}
+	lock_basic_lock(&env->sample_lock);
+	if((env->sample_rate = (unsigned int)cfg->dnstap_sample_rate))
+	{
+		verbose(VERB_OPS, "dnstap SAMPLE_RATE enabled and set to \"%d\"", (int)env->sample_rate);
+	}
+	lock_basic_unlock(&env->sample_lock);
+}
+
+void
+dt_apply_cfg(struct dt_env *env, struct config_file *cfg)
+{
+	if (!cfg->dnstap)
+		return;
+
+	dt_apply_identity(env, cfg);
+	dt_apply_version(env, cfg);
+	dt_apply_logcfg(env, cfg);
 }
 
 int
@@ -273,6 +317,7 @@ dt_delete(struct dt_env *env)
 	if (!env)
 		return;
 	dt_io_thread_delete(env->dtio);
+	lock_basic_destroy(&env->sample_lock);
 	free(env->identity);
 	free(env->version);
 	free(env);
@@ -305,6 +350,7 @@ dt_msg_fill_net(struct dt_msg *dm,
 		struct sockaddr_storage *qs,
 		struct sockaddr_storage *rs,
 		enum comm_point_type cptype,
+		void *cpssl,
 		ProtobufCBinaryData *qaddr, protobuf_c_boolean *has_qaddr,
 		uint32_t *qport, protobuf_c_boolean *has_qport,
 		ProtobufCBinaryData *raddr, protobuf_c_boolean *has_raddr,
@@ -371,13 +417,26 @@ dt_msg_fill_net(struct dt_msg *dm,
                 *has_rport = 1;
         }
 
-	log_assert(cptype == comm_udp || cptype == comm_tcp);
 	if (cptype == comm_udp) {
 		/* socket_protocol */
 		dm->m.socket_protocol = DNSTAP__SOCKET_PROTOCOL__UDP;
 		dm->m.has_socket_protocol = 1;
 	} else if (cptype == comm_tcp) {
+		if (cpssl == NULL) {
+			/* socket_protocol */
+			dm->m.socket_protocol = DNSTAP__SOCKET_PROTOCOL__TCP;
+			dm->m.has_socket_protocol = 1;
+		} else {
+			/* socket_protocol */
+			dm->m.socket_protocol = DNSTAP__SOCKET_PROTOCOL__DOT;
+			dm->m.has_socket_protocol = 1;
+		}
+	} else if (cptype == comm_http) {
 		/* socket_protocol */
+		dm->m.socket_protocol = DNSTAP__SOCKET_PROTOCOL__DOH;
+		dm->m.has_socket_protocol = 1;
+	} else {
+		/* other socket protocol */
 		dm->m.socket_protocol = DNSTAP__SOCKET_PROTOCOL__TCP;
 		dm->m.has_socket_protocol = 1;
 	}
@@ -388,12 +447,16 @@ dt_msg_send_client_query(struct dt_env *env,
 			 struct sockaddr_storage *qsock,
 			 struct sockaddr_storage *rsock,
 			 enum comm_point_type cptype,
+			 void *cpssl,
 			 sldns_buffer *qmsg,
 			 struct timeval* tstamp)
 {
 	struct dt_msg dm;
 	struct timeval qtime;
 
+	if(dt_sample_rate_limited(env))
+		return;
+
 	if(tstamp)
 		memcpy(&qtime, tstamp, sizeof(qtime));
 	else 	gettimeofday(&qtime, NULL);
@@ -410,8 +473,7 @@ dt_msg_send_client_query(struct dt_env *env,
 	dt_fill_buffer(qmsg, &dm.m.query_message, &dm.m.has_query_message);
 
 	/* socket_family, socket_protocol, query_address, query_port, response_address, response_port */
-	log_assert(cptype == comm_udp || cptype == comm_tcp);
-	dt_msg_fill_net(&dm, qsock, rsock, cptype,
+	dt_msg_fill_net(&dm, qsock, rsock, cptype, cpssl,
 			&dm.m.query_address, &dm.m.has_query_address,
 			&dm.m.query_port, &dm.m.has_query_port,
 			&dm.m.response_address, &dm.m.has_response_address,
@@ -427,11 +489,15 @@ dt_msg_send_client_response(struct dt_env *env,
 			    struct sockaddr_storage *qsock,
 			    struct sockaddr_storage *rsock,
 			    enum comm_point_type cptype,
+			    void *cpssl,
 			    sldns_buffer *rmsg)
 {
 	struct dt_msg dm;
 	struct timeval rtime;
 
+	if(dt_sample_rate_limited(env))
+		return;
+
 	gettimeofday(&rtime, NULL);
 
 	/* type */
@@ -446,8 +512,7 @@ dt_msg_send_client_response(struct dt_env *env,
 	dt_fill_buffer(rmsg, &dm.m.response_message, &dm.m.has_response_message);
 
 	/* socket_family, socket_protocol, query_address, query_port, response_address, response_port */
-	log_assert(cptype == comm_udp || cptype == comm_tcp);
-	dt_msg_fill_net(&dm, qsock, rsock, cptype,
+	dt_msg_fill_net(&dm, qsock, rsock, cptype, cpssl,
 			&dm.m.query_address, &dm.m.has_query_address,
 			&dm.m.query_port, &dm.m.has_query_port,
                         &dm.m.response_address, &dm.m.has_response_address,
@@ -462,6 +527,7 @@ dt_msg_send_outside_query(struct dt_env *env,
 			  struct sockaddr_storage *rsock,
 			  struct sockaddr_storage *qsock,
 			  enum comm_point_type cptype,
+			  void *cpssl,
 			  uint8_t *zone, size_t zone_len,
 			  sldns_buffer *qmsg)
 {
@@ -469,11 +535,14 @@ dt_msg_send_outside_query(struct dt_env *env,
 	struct timeval qtime;
 	uint16_t qflags;
 
+	if(dt_sample_rate_limited(env))
+		return;
+
 	gettimeofday(&qtime, NULL);
 	qflags = sldns_buffer_read_u16_at(qmsg, 2);
 
 	/* type */
-	if (qflags & BIT_RD) {
+	if ((qflags & BIT_RD)) {
 		if (!env->log_forwarder_query_messages)
 			return;
 		dt_msg_init(env, &dm, DNSTAP__MESSAGE__TYPE__FORWARDER_QUERY);
@@ -497,8 +566,7 @@ dt_msg_send_outside_query(struct dt_env *env,
 	dt_fill_buffer(qmsg, &dm.m.query_message, &dm.m.has_query_message);
 
 	/* socket_family, socket_protocol, response_address, response_port, query_address, query_port */
-	log_assert(cptype == comm_udp || cptype == comm_tcp);
-	dt_msg_fill_net(&dm, rsock, qsock, cptype,
+	dt_msg_fill_net(&dm, rsock, qsock, cptype, cpssl,
 			&dm.m.response_address, &dm.m.has_response_address,
 			&dm.m.response_port, &dm.m.has_response_port,
 			&dm.m.query_address, &dm.m.has_query_address,
@@ -513,6 +581,7 @@ dt_msg_send_outside_response(struct dt_env *env,
 	struct sockaddr_storage *rsock,
 	struct sockaddr_storage *qsock,
 	enum comm_point_type cptype,
+	void *cpssl,
 	uint8_t *zone, size_t zone_len,
 	uint8_t *qbuf, size_t qbuf_len,
 	const struct timeval *qtime,
@@ -522,12 +591,15 @@ dt_msg_send_outside_response(struct dt_env *env,
 	struct dt_msg dm;
 	uint16_t qflags;
 
+	if(dt_sample_rate_limited(env))
+		return;
+
 	(void)qbuf_len; log_assert(qbuf_len >= sizeof(qflags));
 	memcpy(&qflags, qbuf, sizeof(qflags));
 	qflags = ntohs(qflags);
 
 	/* type */
-	if (qflags & BIT_RD) {
+	if ((qflags & BIT_RD)) {
 		if (!env->log_forwarder_response_messages)
 			return;
 		dt_msg_init(env, &dm, DNSTAP__MESSAGE__TYPE__FORWARDER_RESPONSE);
@@ -556,8 +628,7 @@ dt_msg_send_outside_response(struct dt_env *env,
 	dt_fill_buffer(rmsg, &dm.m.response_message, &dm.m.has_response_message);
 
 	/* socket_family, socket_protocol, response_address, response_port, query_address, query_port */
-	log_assert(cptype == comm_udp || cptype == comm_tcp);
-	dt_msg_fill_net(&dm, rsock, qsock, cptype,
+	dt_msg_fill_net(&dm, rsock, qsock, cptype, cpssl,
 			&dm.m.response_address, &dm.m.has_response_address,
 			&dm.m.response_port, &dm.m.has_response_port,
 			&dm.m.query_address, &dm.m.has_query_address,

dnstap/dnstap.h

@@ -39,6 +39,7 @@
 
 #ifdef USE_DNSTAP
 
+#include "util/locks.h"
 struct config_file;
 struct sldns_buffer;
 struct dt_msg_queue;
@@ -75,6 +76,13 @@ struct dt_env {
 	unsigned log_forwarder_query_messages : 1;
 	/** whether to log Message/FORWARDER_RESPONSE */
 	unsigned log_forwarder_response_messages : 1;
+
+	/** lock on sample count */
+	lock_basic_type sample_lock;
+	/** rate limit value from config, samples 1/N messages */
+	unsigned int sample_rate;
+	/** rate limit counter */
+	unsigned int sample_rate_count;
 };
 
 /**
@@ -98,6 +106,13 @@ dt_create(struct config_file* cfg);
 void
 dt_apply_cfg(struct dt_env *env, struct config_file *cfg);
 
+/**
+ * Apply config settings for log enable for message types.
+ * @param env: dnstap environment object.
+ * @param cfg: new config settings.
+ */
+void dt_apply_logcfg(struct dt_env *env, struct config_file *cfg);
+
 /**
  * Initialize per-worker state in dnstap environment object.
  * @param env: dnstap environment object to initialize, created with dt_create().
@@ -133,6 +148,7 @@ dt_msg_send_client_query(struct dt_env *env,
 			 struct sockaddr_storage *qsock,
 			 struct sockaddr_storage *rsock,
 			 enum comm_point_type cptype,
+			 void *cpssl,
 			 struct sldns_buffer *qmsg,
 			 struct timeval* tstamp);
 
@@ -149,6 +165,7 @@ dt_msg_send_client_response(struct dt_env *env,
 			    struct sockaddr_storage *qsock,
 			    struct sockaddr_storage *rsock,
 			    enum comm_point_type cptype,
+			    void *cpssl,
 			    struct sldns_buffer *rmsg);
 
 /**
@@ -168,6 +185,7 @@ dt_msg_send_outside_query(struct dt_env *env,
 			  struct sockaddr_storage *rsock,
 			  struct sockaddr_storage *qsock,
 			  enum comm_point_type cptype,
+			  void *cpssl,
 			  uint8_t *zone, size_t zone_len,
 			  struct sldns_buffer *qmsg);
 
@@ -192,6 +210,7 @@ dt_msg_send_outside_response(struct dt_env *env,
 			     struct sockaddr_storage *rsock,
 			     struct sockaddr_storage *qsock,
 			     enum comm_point_type cptype,
+			     void *cpssl,
 			     uint8_t *zone, size_t zone_len,
 			     uint8_t *qbuf, size_t qbuf_len,
 			     const struct timeval *qtime,

dnstap/dnstap.m4

@@ -5,45 +5,104 @@
 # Check for required dnstap libraries and add dnstap configure args.
 AC_DEFUN([dt_DNSTAP],
 [
-  AC_ARG_ENABLE([dnstap],
-    AS_HELP_STRING([--enable-dnstap],
-                   [Enable dnstap support (requires protobuf-c)]),
-    [opt_dnstap=$enableval], [opt_dnstap=no])
+    AC_ARG_ENABLE([dnstap],
+        AS_HELP_STRING([--enable-dnstap],
+            [Enable dnstap support (requires protobuf-c)]),
+        [opt_dnstap=$enableval],
+        [opt_dnstap=no])
 
-  AC_ARG_WITH([dnstap-socket-path],
-    AS_HELP_STRING([--with-dnstap-socket-path=pathname],
-                   [set default dnstap socket path]),
-    [opt_dnstap_socket_path=$withval], [opt_dnstap_socket_path="$1"])
+    AC_ARG_WITH([dnstap-socket-path],
+        AS_HELP_STRING([--with-dnstap-socket-path=pathname],
+            [set default dnstap socket path]),
+        [opt_dnstap_socket_path=$withval],
+        [opt_dnstap_socket_path="$1"])
 
-  if test "x$opt_dnstap" != "xno"; then
-    AC_PATH_PROG([PROTOC_C], [protoc-c])
-    if test -z "$PROTOC_C"; then
-      AC_MSG_ERROR([The protoc-c program was not found. Please install protobuf-c!])
+    if test "x$opt_dnstap" != "xno"; then
+	AC_PATH_PROG([PROTOC], [protoc])
+	# 'protoc-c' is deprecated. We use 'protoc' instead. If it can not be
+	# found, try 'protoc-c'.
+	if test -z "$PROTOC"; then
+	    AC_PATH_PROG([PROTOC_C], [protoc-c])
+	else
+	    PROTOC_C="$PROTOC"
+	fi
+	if test -z "$PROTOC_C"; then
+	  AC_MSG_ERROR([[The protoc or protoc-c program was not found. It is needed for dnstap, use --disable-dnstap, or install protobuf-c to provide protoc or protoc-c]])
+	fi
+
+	# Check for protoc-gen-c plugin
+	AC_PATH_PROG([PROTOC_GEN_C], [protoc-gen-c])
+	if test -z "$PROTOC_GEN_C"; then
+	  AC_MSG_ERROR([[The protoc-gen-c plugin was not found. It is needed for dnstap, use --disable-dnstap, or install protobuf-c-compiler to provide protoc-gen-c]])
+	fi
+
+	# Test that protoc-gen-c actually works
+	AC_MSG_CHECKING([if protoc-gen-c plugin works])
+	cat > conftest.proto << EOF
+syntax = "proto2";
+message TestMessage {
+  optional string test_field = 1;
+}
+EOF
+	if $PROTOC_C --c_out=. conftest.proto >/dev/null 2>&1; then
+	  AC_MSG_RESULT([yes])
+	  rm -f conftest.proto conftest.pb-c.c conftest.pb-c.h
+	else
+	  AC_MSG_RESULT([no])
+	  rm -f conftest.proto conftest.pb-c.c conftest.pb-c.h
+	  AC_MSG_ERROR([[The protoc-gen-c plugin is not working properly. Please ensure protobuf-c-compiler is properly installed]])
+	fi
+
+        AC_ARG_WITH([protobuf-c],
+            AS_HELP_STRING([--with-protobuf-c=path], [Path where protobuf-c is installed, for dnstap]),
+            [
+                # workaround for protobuf-c includes at old dir before protobuf-c-1.0.0
+                if test -f $withval/include/google/protobuf-c/protobuf-c.h; then
+                    CFLAGS="$CFLAGS -I$withval/include/google"
+                else
+                    CFLAGS="$CFLAGS -I$withval/include"
+                fi
+                LDFLAGS="$LDFLAGS -L$withval/lib"
+            ],
+            [
+                if test -n "$PKG_CONFIG"; then
+                    PKG_CHECK_MODULES([PROTOBUFC], [libprotobuf-c],
+                        [
+                            CFLAGS="$CFLAGS $PROTOBUFC_CFLAGS"
+                            LIBS="$LIBS $PROTOBUFC_LIBS"
+                        ],
+                        [
+                            # pkg-config failed; try falling back to known values
+                            # workaround for protobuf-c includes at old dir before protobuf-c-1.0.0
+                            if test -f /usr/include/google/protobuf-c/protobuf-c.h; then
+                                CFLAGS="$CFLAGS -I/usr/include/google"
+                            else
+                                if test -f /usr/local/include/google/protobuf-c/protobuf-c.h; then
+                                    CFLAGS="$CFLAGS -I/usr/local/include/google"
+                                    LDFLAGS="$LDFLAGS -L/usr/local/lib"
+                                else
+                                    AC_MSG_ERROR([The protobuf-c package was not found with pkg-config. Please install protobuf-c!])
+                                fi
+                            fi
+                        ]
+                    )
+                else
+                    # workaround for protobuf-c includes at old dir before protobuf-c-1.0.0
+                    if test -f /usr/include/google/protobuf-c/protobuf-c.h; then
+                        CFLAGS="$CFLAGS -I/usr/include/google"
+                    else
+                        if test -f /usr/local/include/google/protobuf-c/protobuf-c.h; then
+                            CFLAGS="$CFLAGS -I/usr/local/include/google"
+                            LDFLAGS="$LDFLAGS -L/usr/local/lib"
+                        fi
+                    fi
+                fi
+            ]
+        )
+        AC_SEARCH_LIBS([protobuf_c_message_pack], [protobuf-c], [],
+            AC_MSG_ERROR([The protobuf-c library was not found. Please install the development libraries for protobuf-c!]))
+        $2
+    else
+        $3
     fi
-    AC_ARG_WITH([protobuf-c], AS_HELP_STRING([--with-protobuf-c=path],
-    	[Path where protobuf-c is installed, for dnstap]), [
-	  # workaround for protobuf-c includes at old dir before protobuf-c-1.0.0
-	  if test -f $withval/include/google/protobuf-c/protobuf-c.h; then
-	    CFLAGS="$CFLAGS -I$withval/include/google"
-	  else
-	    CFLAGS="$CFLAGS -I$withval/include"
-	  fi
-	  LDFLAGS="$LDFLAGS -L$withval/lib"
-	], [
-	  # workaround for protobuf-c includes at old dir before protobuf-c-1.0.0
-	  if test -f /usr/include/google/protobuf-c/protobuf-c.h; then
-	    CFLAGS="$CFLAGS -I/usr/include/google"
-	  else
-	    if test -f /usr/local/include/google/protobuf-c/protobuf-c.h; then
-	      CFLAGS="$CFLAGS -I/usr/local/include/google"
-	      LDFLAGS="$LDFLAGS -L/usr/local/lib"
-	    fi
-	  fi
-    ])
-    AC_SEARCH_LIBS([protobuf_c_message_pack], [protobuf-c], [],
-      AC_MSG_ERROR([The protobuf-c library was not found. Please install the development libraries for protobuf-c!]))
-    $2
-  else
-    $3
-  fi
 ])

dnstap/dnstap.proto

@@ -13,8 +13,8 @@
 // with this file. If not, see:
 //
 // <http://creativecommons.org/publicdomain/zero/1.0/>.
-syntax = "proto2";
 
+syntax = "proto2";
 package dnstap;
 
 // "Dnstap": this is the top-level dnstap type, which is a "union" type that
@@ -56,11 +56,60 @@ enum SocketFamily {
     INET6 = 2;  // IPv6 (RFC 2460)
 }
 
-// SocketProtocol: the transport protocol of a socket. This specifies how to
-// interpret "transport port" fields.
+// SocketProtocol: the protocol used to transport a DNS message.
 enum SocketProtocol {
-    UDP = 1;    // User Datagram Protocol (RFC 768)
-    TCP = 2;    // Transmission Control Protocol (RFC 793)
+    UDP = 1;         // DNS over UDP transport (RFC 1035 section 4.2.1)
+    TCP = 2;         // DNS over TCP transport (RFC 1035 section 4.2.2)
+    DOT = 3;         // DNS over TLS (RFC 7858)
+    DOH = 4;         // DNS over HTTPS (RFC 8484)
+    DNSCryptUDP = 5; // DNSCrypt over UDP (https://dnscrypt.info/protocol)
+    DNSCryptTCP = 6; // DNSCrypt over TCP (https://dnscrypt.info/protocol)
+    DOQ = 7;         // DNS over QUIC (RFC 9250)
+}
+
+// Policy: information about any name server operator policy
+// applied to the processing of a DNS message.
+message Policy {
+
+    // Match: what aspect of the message or message exchange
+    // triggered the application of the Policy.
+    enum Match {
+        QNAME = 1;       // Name in question section of query
+        CLIENT_IP = 2;   // Client IP address
+        RESPONSE_IP = 3; // Address in A/AAAA RRSet
+        NS_NAME = 4;     // Authoritative name server, by name
+        NS_IP = 5;       // Authoritative name server, by IP address
+    }
+
+    // The Action taken to implement the Policy.
+    enum Action {
+        NXDOMAIN = 1;   // Respond with NXDOMAIN
+        NODATA = 2;     // Respond with empty answer section
+        PASS = 3;       // Do not alter the response (passthrough)
+        DROP = 4;       // Do not respond.
+        TRUNCATE = 5;   // Truncate UDP response, forcing TCP retry
+        LOCAL_DATA = 6; // Respond with local data from policy
+    }
+
+    // type: the type of policy applied, e.g. "RPZ" for a
+    // policy from a Response Policy Zone.
+    optional string type = 1;
+
+    // rule: the rule matched by the message.
+    //
+    // In a RPZ context, this is the owner name of the rule in
+    // the Response Policy Zone in wire format.
+    optional bytes rule = 2;
+
+    // action: the policy action taken in response to the
+    // rule match.
+    optional Action action = 3;
+
+    // match: the feature of the message exchange which matched the rule.
+    optional Match match = 4;
+
+    // The matched value. Format depends on the matched feature .
+    optional bytes value = 5;
 }
 
 // Message: a wire-format (RFC 1035 section 4) DNS message and associated
@@ -158,6 +207,24 @@ message Message {
         // STUB_RESPONSE is a DNS response message sent from a DNS server to a
         // stub resolver, from the perspective of the stub resolver.
         STUB_RESPONSE = 10;
+
+        // TOOL_QUERY is a DNS query message sent from a DNS software tool to a
+        // DNS server, from the perspective of the tool.
+        TOOL_QUERY = 11;
+
+        // TOOL_RESPONSE is a DNS response message received by a DNS software
+        // tool from a DNS server, from the perspective of the tool.
+        TOOL_RESPONSE = 12;
+
+        // UPDATE_QUERY is a Dynamic DNS Update request (RFC 2136) received
+        // by an authoritative name server, from the perspective of the
+        // authoritative name server.
+        UPDATE_QUERY = 13;
+
+        // UPDATE_RESPONSE is a Dynamic DNS Update response (RFC 2136) sent
+        // from an authoritative name server, from the perspective of the
+        // authoritative name server.
+        UPDATE_RESPONSE = 14;
     }
 
     // One of the Type values described above.
@@ -215,6 +282,9 @@ message Message {
 
     // The responder's original wire-format DNS response message, verbatim.
     optional bytes              response_message = 14;
+
+    // Operator policy applied to the processing of this message, if any.
+    optional Policy             policy = 15;
 }
 
 // All fields except for 'type' in the Message schema are optional.
@@ -236,7 +306,6 @@ message Message {
 
 // RESOLVER_QUERY:
 //      socket_family, socket_protocol
-//      query_name, query_type, query_class
 //      query_message
 //      query_time_sec, query_time_nsec
 //      query_zone
@@ -244,7 +313,6 @@ message Message {
 
 // RESOLVER_RESPONSE:
 //      socket_family, socket_protocol
-//      query_name, query_type, query_class
 //      query_time_sec, query_time_nsec
 //      query_zone
 //      response_address, response_port

dnstap/dtstream.c

@@ -176,10 +176,7 @@ void
 mq_wakeup_cb(void* arg)
 {
 	struct dt_msg_queue* mq = (struct dt_msg_queue*)arg;
-	/* even if the dtio is already active, because perhaps much
-	 * traffic suddenly, we leave the timer running to save on
-	 * managing it, the once a second timer is less work then
-	 * starting and stopping the timer frequently */
+
 	lock_basic_lock(&mq->dtio->wakeup_timer_lock);
 	mq->dtio->wakeup_timer_enabled = 0;
 	lock_basic_unlock(&mq->dtio->wakeup_timer_lock);
@@ -210,6 +207,8 @@ dt_msg_queue_start_timer(struct dt_msg_queue* mq, int wakeupnow)
 	lock_basic_lock(&mq->dtio->wakeup_timer_lock);
 	if(mq->dtio->wakeup_timer_enabled) {
 		if(wakeupnow) {
+			tv.tv_sec = 0;
+			tv.tv_usec = 0;
 			comm_timer_set(mq->wakeup_timer, &tv);
 		}
 		lock_basic_unlock(&mq->dtio->wakeup_timer_lock);
@@ -221,8 +220,14 @@ dt_msg_queue_start_timer(struct dt_msg_queue* mq, int wakeupnow)
 	if(!wakeupnow) {
 		tv.tv_sec = 1;
 		tv.tv_usec = 0;
+		/* If it is already set, keep it running. */
+		if(!comm_timer_is_set(mq->wakeup_timer))
+			comm_timer_set(mq->wakeup_timer, &tv);
+	} else {
+		tv.tv_sec = 0;
+		tv.tv_usec = 0;
+		comm_timer_set(mq->wakeup_timer, &tv);
 	}
-	comm_timer_set(mq->wakeup_timer, &tv);
 	lock_basic_unlock(&mq->dtio->wakeup_timer_lock);
 }
 
@@ -260,8 +265,9 @@ dt_msg_queue_submit(struct dt_msg_queue* mq, void* buf, size_t len)
 
 	/* acquire lock */
 	lock_basic_lock(&mq->lock);
-	/* if list was empty, start timer for (eventual) wakeup */
-	if(mq->first == NULL)
+	/* if list was empty, start timer for (eventual) wakeup,
+	 * or if dtio is not writing now an eventual wakeup is needed. */
+	if(mq->first == NULL || !mq->dtio->event_added_is_write)
 		wakeupstarttimer = 1;
 	/* if list contains more than wakeupnum elements, wakeup now,
 	 * or if list is (going to be) almost full */
@@ -1259,6 +1265,13 @@ static void dtio_sleep(struct dt_io_thread* dtio)
 	/* unregister the event polling for write, because there is
 	 * nothing to be written */
 	(void)dtio_add_output_event_read(dtio);
+
+	/* Set wakeuptimer enabled off; so that the next worker thread that
+	 * wants to log starts a timer if needed, since the writer thread
+	 * has gone to sleep. */
+	lock_basic_lock(&dtio->wakeup_timer_lock);
+	dtio->wakeup_timer_enabled = 0;
+	lock_basic_unlock(&dtio->wakeup_timer_lock);
 }
 
 #ifdef HAVE_SSL
@@ -1322,7 +1335,11 @@ static int dtio_ssl_check_peer(struct dt_io_thread* dtio)
 	if((SSL_get_verify_mode(dtio->ssl)&SSL_VERIFY_PEER)) {
 		/* verification */
 		if(SSL_get_verify_result(dtio->ssl) == X509_V_OK) {
+#ifdef HAVE_SSL_GET1_PEER_CERTIFICATE
+			X509* x = SSL_get1_peer_certificate(dtio->ssl);
+#else
 			X509* x = SSL_get_peer_certificate(dtio->ssl);
+#endif
 			if(!x) {
 				verbose(VERB_ALGO, "dnstap io, %s, SSL "
 					"connection failed no certificate",
@@ -1347,7 +1364,11 @@ static int dtio_ssl_check_peer(struct dt_io_thread* dtio)
 #endif
 			X509_free(x);
 		} else {
+#ifdef HAVE_SSL_GET1_PEER_CERTIFICATE
+			X509* x = SSL_get1_peer_certificate(dtio->ssl);
+#else
 			X509* x = SSL_get_peer_certificate(dtio->ssl);
+#endif
 			if(x) {
 				log_cert(VERB_ALGO, "dnstap io, peer "
 					"certificate", x);
@@ -1488,9 +1509,11 @@ void dtio_output_cb(int ATTR_UNUSED(fd), short bits, void* arg)
 	}
 #endif
 
-	if((bits&UB_EV_READ || dtio->ssl_brief_write)) {
+	if((bits&UB_EV_READ) || dtio->ssl_brief_write) {
+#ifdef HAVE_SSL
 		if(dtio->ssl_brief_write)
 			(void)dtio_disable_brief_write(dtio);
+#endif
 		if(dtio->ready_frame_sent && !dtio->accept_frame_received) {
 			if(dtio_read_accept_frame(dtio) <= 0)
 				return;
@@ -1513,8 +1536,22 @@ void dtio_output_cb(int ATTR_UNUSED(fd), short bits, void* arg)
 					/* no messages on the first iteration,
 					 * the queues are all empty */
 					dtio_sleep(dtio);
+					/* After putting to sleep, see if
+					 * a message is in a message queue,
+					 * if so, resume service. Stops a
+					 * race condition where a thread could
+					 * have one message but the dtio
+					 * also just went to sleep. With the
+					 * message queued between the
+					 * dtio_find_msg and dtio_sleep
+					 * calls. */
+					if(dtio_find_msg(dtio)) {
+						if(!dtio_add_output_event_write(dtio))
+							return;
+					}
 				}
-				return; /* nothing to do */
+				if(!dtio->cur_msg)
+					return; /* nothing to do */
 			}
 		}
 

dnstap/unbound-dnstap-socket.c

@@ -75,17 +75,18 @@
 static void usage(char* argv[])
 {
 	printf("usage: %s [options]\n", argv[0]);
-	printf(" 	Listen to dnstap messages\n");
+	printf("	Listen to dnstap messages\n");
 	printf("stdout has dnstap log, stderr has verbose server log\n");
-	printf("-u <socketpath> listen to unix socket with this file name\n");
-	printf("-s <serverip[@port]> listen for TCP on the IP and port\n");
-	printf("-t <serverip[@port]> listen for TLS on IP and port\n");
-	printf("-x <server.key> server key file for TLS service\n");
-	printf("-y <server.pem> server cert file for TLS service\n");
-	printf("-z <verify.pem> cert file to verify client connections\n");
-	printf("-l 		long format for DNS printout\n");
-	printf("-v 		more verbose log output\n");
-	printf("-h 		this help text\n");
+	printf("-u <socketpath>		listen to unix socket with this file name\n");
+	printf("-s <serverip[@port]>	listen for TCP on the IP and port\n");
+	printf("-t <serverip[@port]>	listen for TLS on IP and port\n");
+	printf("-x <server.key>		server key file for TLS service\n");
+	printf("-y <server.pem>		server cert file for TLS service\n");
+	printf("-z <verify.pem>		cert file to verify client connections\n");
+	printf("-l			long format for DNS printout\n");
+	printf("-v			more verbose log output\n");
+	printf("-c			internal unit test and exit\n");
+	printf("-h			this help text\n");
 	exit(1);
 }
 
@@ -102,6 +103,14 @@ struct main_tap_data {
 	struct tap_socket_list* acceptlist;
 };
 
+/* list of data */
+struct tap_data_list {
+	/** next in list */
+	struct tap_data_list* next;
+	/** the data */
+	struct tap_data* d;
+};
+
 /** tap callback variables */
 struct tap_data {
 	/** the fd */
@@ -128,6 +137,10 @@ struct tap_data {
 	uint8_t* frame;
 	/** length of this frame */
 	size_t len;
+	/** back pointer to the tap_data_list entry;
+	 * used to NULL the forward pointer to this data
+	 * when this data is freed. */
+	struct tap_data_list* data_list;
 };
 
 /** list of sockets */
@@ -156,8 +169,89 @@ struct tap_socket {
 	char* ip;
 	/** for a TLS socket, the tls context */
 	SSL_CTX* sslctx;
+	/** dumb way to deal with memory leaks:
+	 * tap_data was only freed on errors and not during exit leading to
+	 * false positives when testing for memory leaks. */
+	struct tap_data_list* data_list;
 };
 
+/** try to delete tail entries from the list if all of them have no data */
+static void tap_data_list_try_to_free_tail(struct tap_data_list* list)
+{
+	struct tap_data_list* current = list;
+	log_assert(!list->d);
+	if(!list->next) /* we are the last, we can't remove ourselves */
+		return;
+	list = list->next;
+	while(list) {
+		if(list->d) /* a tail entry still has data; return */
+			return;
+		list = list->next;
+	}
+	/* keep the next */
+	list = current->next;
+	/* the tail will be removed; but not ourselves */
+	current->next = NULL;
+	while(list) {
+		current = list;
+		list = list->next;
+		free(current);
+	}
+}
+
+/** delete the tap structure */
+static void tap_data_free(struct tap_data* data, int free_tail)
+{
+	if(!data)
+		return;
+	if(data->ev) {
+		ub_event_del(data->ev);
+		ub_event_free(data->ev);
+	}
+#ifdef HAVE_SSL
+	SSL_free(data->ssl);
+#endif
+	sock_close(data->fd);
+	free(data->id);
+	free(data->frame);
+	if(data->data_list) {
+		data->data_list->d = NULL;
+		if(free_tail)
+			tap_data_list_try_to_free_tail(data->data_list);
+	}
+	free(data);
+}
+
+/** insert tap_data in the tap_data_list */
+static int tap_data_list_insert(struct tap_data_list** liststart,
+	struct tap_data* d)
+{
+	struct tap_data_list* entry = (struct tap_data_list*)
+		malloc(sizeof(*entry));
+	if(!entry)
+		return 0;
+	entry->next = *liststart;
+	entry->d = d;
+	d->data_list = entry;
+	*liststart = entry;
+	return 1;
+}
+
+/** delete the tap_data_list and free any remaining tap_data */
+static void tap_data_list_delete(struct tap_data_list* list)
+{
+	struct tap_data_list* e = list, *next;
+	while(e) {
+		next = e->next;
+		if(e->d) {
+			tap_data_free(e->d, 0);
+			e->d = NULL;
+		}
+		free(e);
+		e = next;
+	}
+}
+
 /** del the tap event */
 static void tap_socket_delev(struct tap_socket* s)
 {
@@ -173,7 +267,7 @@ static void tap_socket_close(struct tap_socket* s)
 {
 	if(!s) return;
 	if(s->fd == -1) return;
-	close(s->fd);
+	sock_close(s->fd);
 	s->fd = -1;
 }
 
@@ -184,6 +278,7 @@ static void tap_socket_delete(struct tap_socket* s)
 #ifdef HAVE_SSL
 	SSL_CTX_free(s->sslctx);
 #endif
+	tap_data_list_delete(s->data_list);
 	ub_event_free(s->ev);
 	free(s->socketpath);
 	free(s->ip);
@@ -251,7 +346,8 @@ static struct tap_socket* tap_socket_new_tlsaccept(char* ip,
 	s->fd = -1;
 	s->ev_cb = ev_cb;
 	s->data = data;
-	s->sslctx = listen_sslctx_create(server_key, server_cert, verifypem);
+	s->sslctx = listen_sslctx_create(server_key, server_cert, verifypem,
+		NULL, NULL, 0, 0, 0);
 	if(!s->sslctx) {
 		log_err("could not create ssl context");
 		free(s->ip);
@@ -728,27 +824,12 @@ static ssize_t tap_receive(struct tap_data* data, void* buf, size_t len)
 	return receive_bytes(data, data->fd, buf, len);
 }
 
-/** delete the tap structure */
-static void tap_data_free(struct tap_data* data)
-{
-	ub_event_del(data->ev);
-	ub_event_free(data->ev);
-#ifdef HAVE_SSL
-	SSL_free(data->ssl);
-#endif
-	close(data->fd);
-	free(data->id);
-	free(data->frame);
-	free(data);
-}
-
 /** reply with ACCEPT control frame to bidirectional client,
  * returns 0 on error */
 static int reply_with_accept(struct tap_data* data)
 {
 #ifdef USE_DNSTAP
 	/* len includes the escape and framelength */
-	int r;
 	size_t len = 0;
 	void* acceptframe = fstrm_create_control_frame_accept(
 		DNSTAP_CONTENT_TYPE, &len);
@@ -759,6 +840,8 @@ static int reply_with_accept(struct tap_data* data)
 
 	fd_set_block(data->fd);
 	if(data->ssl) {
+#ifdef HAVE_SSL
+		int r;
 		if((r=SSL_write(data->ssl, acceptframe, len)) <= 0) {
 			int r2;
 			if((r2=SSL_get_error(data->ssl, r)) == SSL_ERROR_ZERO_RETURN)
@@ -769,6 +852,7 @@ static int reply_with_accept(struct tap_data* data)
 			free(acceptframe);
 			return 0;
 		}
+#endif
 	} else {
 		if(send(data->fd, acceptframe, len, 0) == -1) {
 			log_err("send failed: %s", sock_strerror(errno));
@@ -804,6 +888,7 @@ static int reply_with_finish(struct tap_data* data)
 
 	fd_set_block(data->fd);
 	if(data->ssl) {
+#ifdef HAVE_SSL
 		int r;
 		if((r=SSL_write(data->ssl, finishframe, len)) <= 0) {
 			int r2;
@@ -815,6 +900,7 @@ static int reply_with_finish(struct tap_data* data)
 			free(finishframe);
 			return 0;
 		}
+#endif
 	} else {
 		if(send(data->fd, finishframe, len, 0) == -1) {
 			log_err("send failed: %s", sock_strerror(errno));
@@ -842,7 +928,11 @@ static int tap_check_peer(struct tap_data* data)
 	if((SSL_get_verify_mode(data->ssl)&SSL_VERIFY_PEER)) {
 		/* verification */
 		if(SSL_get_verify_result(data->ssl) == X509_V_OK) {
+#ifdef HAVE_SSL_GET1_PEER_CERTIFICATE
+			X509* x = SSL_get1_peer_certificate(data->ssl);
+#else
 			X509* x = SSL_get_peer_certificate(data->ssl);
+#endif
 			if(!x) {
 				if(verbosity) log_info("SSL connection %s"
 					" failed no certificate", data->id);
@@ -864,7 +954,11 @@ static int tap_check_peer(struct tap_data* data)
 #endif
 			X509_free(x);
 		} else {
+#ifdef HAVE_SSL_GET1_PEER_CERTIFICATE
+			X509* x = SSL_get1_peer_certificate(data->ssl);
+#else
 			X509* x = SSL_get_peer_certificate(data->ssl);
+#endif
 			if(x) {
 				if(verbosity)
 					log_cert(VERB_ALGO, "peer certificate", x);
@@ -906,7 +1000,7 @@ static int tap_handshake(struct tap_data* data)
 			return 0;
 		} else if(r == 0) {
 			/* closed */
-			tap_data_free(data);
+			tap_data_free(data, 1);
 			return 0;
 		} else if(want == SSL_ERROR_SYSCALL) {
 			/* SYSCALL and errno==0 means closed uncleanly */
@@ -924,7 +1018,7 @@ static int tap_handshake(struct tap_data* data)
 			if(!silent)
 				log_err("SSL_handshake syscall: %s",
 					strerror(errno));
-			tap_data_free(data);
+			tap_data_free(data, 1);
 			return 0;
 		} else {
 			unsigned long err = ERR_get_error();
@@ -934,7 +1028,7 @@ static int tap_handshake(struct tap_data* data)
 				verbose(VERB_OPS, "ssl handshake failed "
 					"from %s", data->id);
 			}
-			tap_data_free(data);
+			tap_data_free(data, 1);
 			return 0;
 		}
 	}
@@ -942,7 +1036,7 @@ static int tap_handshake(struct tap_data* data)
 	data->ssl_handshake_done = 1;
 	if(!tap_check_peer(data)) {
 		/* closed */
-		tap_data_free(data);
+		tap_data_free(data, 1);
 		return 0;
 	}
 	return 1;
@@ -968,7 +1062,7 @@ void dtio_tap_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(bits), void* arg)
 		if(verbosity>=4) log_info("s recv %d", (int)ret);
 		if(ret == 0) {
 			/* closed or error */
-			tap_data_free(data);
+			tap_data_free(data, 1);
 			return;
 		} else if(ret == -1) {
 			/* continue later */
@@ -990,7 +1084,7 @@ void dtio_tap_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(bits), void* arg)
 			data->frame = calloc(1, data->len);
 			if(!data->frame) {
 				log_err("out of memory");
-				tap_data_free(data);
+				tap_data_free(data, 1);
 				return;
 			}
 		}
@@ -1003,7 +1097,7 @@ void dtio_tap_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(bits), void* arg)
 		if(verbosity>=4) log_info("f recv %d", (int)r);
 		if(r == 0) {
 			/* closed or error */
-			tap_data_free(data);
+			tap_data_free(data, 1);
 			return;
 		} else if(r == -1) {
 			/* continue later */
@@ -1028,13 +1122,13 @@ void dtio_tap_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(bits), void* arg)
 		data->is_bidirectional = 1;
 		if(verbosity) log_info("bidirectional stream");
 		if(!reply_with_accept(data)) {
-			tap_data_free(data);
+			tap_data_free(data, 1);
 			return;
 		}
 	} else if(data->len >= 4 && sldns_read_uint32(data->frame) ==
 		FSTRM_CONTROL_FRAME_STOP && data->is_bidirectional) {
 		if(!reply_with_finish(data)) {
-			tap_data_free(data);
+			tap_data_free(data, 1);
 			return;
 		}
 	}
@@ -1046,7 +1140,6 @@ void dtio_tap_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(bits), void* arg)
 	data->len = 0;
 	data->len_done = 0;
 	data->data_done = 0;
-
 }
 
 /** callback for main listening file descriptor */
@@ -1059,7 +1152,9 @@ void dtio_mainfdcallback(int fd, short ATTR_UNUSED(bits), void* arg)
 	char* id = NULL;
 	struct sockaddr_storage addr;
 	socklen_t addrlen = (socklen_t)sizeof(addr);
-	int s = accept(fd, (struct sockaddr*)&addr, &addrlen);
+	int s;
+	memset(&addr, 0, sizeof(addr));
+	s = accept(fd, (struct sockaddr*)&addr, &addrlen);
 	if(s == -1) {
 #ifndef USE_WINSOCK
 		/* EINTR is signal interrupt. others are closed connection. */
@@ -1129,6 +1224,8 @@ void dtio_mainfdcallback(int fd, short ATTR_UNUSED(bits), void* arg)
 		&dtio_tap_callback, data);
 	if(!data->ev) fatal_exit("could not ub_event_new");
 	if(ub_event_add(data->ev, NULL) != 0) fatal_exit("could not ub_event_add");
+	if(!tap_data_list_insert(&tap_sock->data_list, data))
+		fatal_exit("could not tap_data_list_insert");
 }
 
 /** setup local accept sockets */
@@ -1243,6 +1340,114 @@ setup_and_run(struct config_strlist_head* local_list,
 	free(maindata);
 }
 
+/* internal unit tests */
+static int internal_unittest()
+{
+	/* unit test tap_data_list_try_to_free_tail() */
+#define unit_tap_datas_max 5
+	struct tap_data* datas[unit_tap_datas_max];
+	struct tap_data_list* list;
+	struct tap_socket* socket = calloc(1, sizeof(*socket));
+	size_t i = 0;
+	log_assert(socket);
+	log_assert(unit_tap_datas_max>2); /* needed for the test */
+	for(i=0; i<unit_tap_datas_max; i++) {
+		datas[i] = calloc(1, sizeof(struct tap_data));
+		log_assert(datas[i]);
+		log_assert(tap_data_list_insert(&socket->data_list, datas[i]));
+	}
+	/* sanity base check */
+	list = socket->data_list;
+	for(i=0; list; i++) list = list->next;
+	log_assert(i==unit_tap_datas_max);
+
+	/* Free the last data, tail cannot be erased */
+	list = socket->data_list;
+	while(list->next) list = list->next;
+	free(list->d);
+	list->d = NULL;
+	tap_data_list_try_to_free_tail(list);
+	list = socket->data_list;
+	for(i=0; list; i++) list = list->next;
+	log_assert(i==unit_tap_datas_max);
+
+	/* Free the third to last data, tail cannot be erased */
+	list = socket->data_list;
+	for(i=0; i<unit_tap_datas_max-3; i++) list = list->next;
+	free(list->d);
+	list->d = NULL;
+	tap_data_list_try_to_free_tail(list);
+	list = socket->data_list;
+	for(i=0; list; i++) list = list->next;
+	log_assert(i==unit_tap_datas_max);
+
+	/* Free the second to last data, try to remove tail from the third
+	 * again, tail (last 2) should be removed */
+	list = socket->data_list;
+	for(i=0; i<unit_tap_datas_max-2; i++) list = list->next;
+	free(list->d);
+	list->d = NULL;
+	list = socket->data_list;
+	while(list->d) list = list->next;
+	tap_data_list_try_to_free_tail(list);
+	list = socket->data_list;
+	for(i=0; list; i++) list = list->next;
+	log_assert(i==unit_tap_datas_max-2);
+
+	/* Free all the remaining data, try to remove tail from the start,
+	 * only the start should remain */
+	list = socket->data_list;
+	while(list) {
+		free(list->d);
+		list->d = NULL;
+		list = list->next;
+	}
+	tap_data_list_try_to_free_tail(socket->data_list);
+	list = socket->data_list;
+	for(i=0; list; i++) list = list->next;
+	log_assert(i==1);
+
+	/* clean up */
+	tap_data_list_delete(socket->data_list);
+	free(socket);
+
+	/* Start again. Add two elements */
+	socket = calloc(1, sizeof(*socket));
+	log_assert(socket);
+	for(i=0; i<2; i++) {
+		datas[i] = calloc(1, sizeof(struct tap_data));
+		log_assert(datas[i]);
+		log_assert(tap_data_list_insert(&socket->data_list, datas[i]));
+	}
+	/* sanity base check */
+	list = socket->data_list;
+	for(i=0; list; i++) list = list->next;
+	log_assert(i==2);
+
+	/* Free the last data, tail cannot be erased */
+	list = socket->data_list;
+	while(list->next) list = list->next;
+	free(list->d);
+	list->d = NULL;
+	tap_data_list_try_to_free_tail(list);
+	list = socket->data_list;
+	for(i=0; list; i++) list = list->next;
+	log_assert(i==2);
+
+	/* clean up */
+	tap_data_list_delete(socket->data_list);
+	free(socket);
+
+	if(log_get_lock()) {
+		lock_basic_destroy((lock_basic_type*)log_get_lock());
+	}
+	checklock_stop();
+#ifdef USE_WINSOCK
+	WSACleanup();
+#endif
+	return 0;
+}
+
 /** getopt global, in case header files fail to declare it. */
 extern int optind;
 /** getopt global, in case header files fail to declare it. */
@@ -1293,7 +1498,7 @@ int main(int argc, char** argv)
 #endif
 
 	/* command line options */
-	while( (c=getopt(argc, argv, "hls:t:u:vx:y:z:")) != -1) {
+	while( (c=getopt(argc, argv, "hcls:t:u:vx:y:z:")) != -1) {
 		switch(c) {
 			case 'u':
 				if(!cfg_strlist_append(&local_list,
@@ -1329,14 +1534,20 @@ int main(int argc, char** argv)
 			case 'v':
 				verbosity++;
 				break;
+			case 'c':
+#ifndef UNBOUND_DEBUG
+				fatal_exit("-c option needs compilation with "
+					"--enable-debug");
+#endif
+				return internal_unittest();
 			case 'h':
 			case '?':
 			default:
 				usage(argv);
 		}
 	}
-	argc -= optind;
-	argv += optind;
+	/* argc -= optind; not using further arguments */
+	/* argv += optind; not using further arguments */
 
 	if(usessl) {
 #ifdef HAVE_SSL
@@ -1365,6 +1576,9 @@ int main(int argc, char** argv)
 	config_delstrlist(tcp_list.first);
 	config_delstrlist(tls_list.first);
 
+	if(log_get_lock()) {
+		lock_basic_destroy((lock_basic_type*)log_get_lock());
+	}
 	checklock_stop();
 #ifdef USE_WINSOCK
 	WSACleanup();
@@ -1572,3 +1786,33 @@ void remote_get_opt_ssl(char* ATTR_UNUSED(str), void* ATTR_UNUSED(arg))
 {
         log_assert(0);
 }
+
+void fast_reload_service_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+int fast_reload_client_callback(struct comm_point* ATTR_UNUSED(c),
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(repinfo))
+{
+	log_assert(0);
+	return 0;
+}
+
+#ifdef HAVE_NGTCP2
+void doq_client_event_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+#endif
+
+#ifdef HAVE_NGTCP2
+void doq_client_timer_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+#endif

doc/README

@@ -19,11 +19,10 @@ The DNSTAP code has BSD license in dnstap/dnstap.c.
 * Make and install: ./configure; make; make install
   * --with-libevent=/path/to/libevent
   	Can be set to either the system install or the build directory.
-	--with-libevent=no (default) gives a builtin alternative 
-	implementation. libevent is useful when having many (thousands) 
-	of outgoing ports. This improves randomization and spoof 
-	resistance. For the default of 16 ports the builtin alternative 
-	works well and is a little faster.
+	--with-libevent=no gives a builtin alternative implementation.
+	Libevent is enabled by default, it is useful when having many
+	(thousands) of outgoing ports. This improves randomization and spoof
+	resistance. It also allows a higher number of outgoing queries.
   * --with-libexpat=/path/to/libexpat
   	Can be set to the install directory of libexpat.
   * --without-pthreads 

doc/README.man

@@ -0,0 +1,16 @@
+After Unbound 1.23.0, the source of the man pages is in reStructuredText format.
+
+This helps with the online documentation at https://unbound.docs.nlnetlabs.nl
+and makes it easier to maintain and contribute to the documentation.
+
+The templated man pages (*.in) are still part of the code repository as to not
+alter current procedures that could be in place by users/packagers.
+
+The templated man pages (*.in) are generated by Sphinx (used for the online
+documentation).
+The online documentation has its own repository at
+https://github.com/NLnetLabs/unbound-manual.
+
+In the README.md there (branch test-auto for now), there are further simple
+instructions on how to generate the templated man pages there and update them
+in this repository.

doc/example.conf.in

@@ -116,8 +116,8 @@ server:
 	# so-rcvbuf: 0
 
 	# buffer size for UDP port 53 outgoing (SO_SNDBUF socket option).
-	# 0 is system default.  Use 4m to handle spikes on very busy servers.
-	# so-sndbuf: 0
+	# 0 is system default. Set larger to handle spikes on very busy servers.
+	# so-sndbuf: 4m
 
 	# use SO_REUSEPORT to distribute queries over threads.
 	# at extreme load it could be better to turn it off to distribute even.
@@ -163,7 +163,7 @@ server:
 	# msg-cache-slabs: 4
 
 	# the number of queries that a thread gets to service.
-	# num-queries-per-thread: 1024
+	# num-queries-per-thread: 2048
 
 	# if very busy, 50% queries run to completion, 50% get timeout in msec
 	# jostle-timeout: 200
@@ -187,10 +187,44 @@ server:
 	# query upon encountering a CNAME record.
 	# max-query-restarts: 11
 
+	# Limit on number of NS records in NS RRset for incoming packets.
+	# iter-scrub-ns: 20
+
+	# Limit on number of CNAME, DNAME records for incoming packets.
+	# iter-scrub-cname: 11
+
+	# Limit on upstream queries for an incoming query and its recursion.
+	# max-global-quota: 200
+
+	# Should the scrubber remove promiscuous NS from positive answers,
+	# protects against poison attempts.
+	# iter-scrub-promiscuous: yes
+
 	# msec for waiting for an unknown server to reply.  Increase if you
 	# are behind a slow satellite link, to eg. 1128.
 	# unknown-server-time-limit: 376
 
+	# msec before recursion replies are dropped. The work item continues.
+	# discard-timeout: 1900
+
+	# Max number of replies waiting for recursion per IP address.
+	# wait-limit: 1000
+
+	# Max replies waiting for recursion for IP address with cookie.
+	# wait-limit-cookie: 10000
+
+	# Apart from the default, the wait limit can be set for a netblock.
+	# wait-limit-netblock: 192.0.2.0/24 50000
+
+	# Apart from the default, the wait limit with cookie can be adjusted.
+	# wait-limit-cookie-netblock: 192.0.2.0/24 50000
+
+	# Defaults for loopback, it has no wait limit.
+	# wait-limit-netblock: 127.0.0.0/8 -1
+	# wait-limit-netblock: ::1/128 -1
+	# wait-limit-cookie-netblock: 127.0.0.0/8 -1
+	# wait-limit-cookie-netblock: ::1/128 -1
+
 	# the amount of memory to use for the RRset cache.
 	# plain value in bytes or you can append k, m or G. default is "4Mb".
 	# rrset-cache-size: 4m
@@ -211,6 +245,11 @@ server:
 	# the time to live (TTL) value cap for negative responses in the cache
 	# cache-max-negative-ttl: 3600
 
+	# the time to live (TTL) value lower bound, in seconds. Default 0.
+	# For negative responses in the cache. If disabled, default,
+	# cache-min-ttl applies if configured.
+	# cache-min-negative-ttl: 0
+
 	# the time to live (TTL) value for cached roundtrip times, lameness and
 	# EDNS version information for hosts. In seconds.
 	# infra-host-ttl: 900
@@ -244,7 +283,7 @@ server:
 	# do-ip6: yes
 
 	# If running unbound on an IPv6-only host, domains that only have
-	# IPv4 servers would become unresolveable.  If NAT64 is available in
+	# IPv4 servers would become unresolvable.  If NAT64 is available in
 	# the network, unbound can use NAT64 to reach these servers with
 	# the following option.  This is NOT needed for enabling DNS64 on a
 	# system that has IPv4 connectivity.
@@ -283,7 +322,8 @@ server:
 	# Enable EDNS TCP keepalive option.
 	# edns-tcp-keepalive: no
 
-	# Timeout for EDNS TCP keepalive, in msec.
+	# Timeout for EDNS TCP keepalive, in msec. Overrides tcp-idle-timeout
+	# if edns-tcp-keepalive is set.
 	# edns-tcp-keepalive-timeout: 120000
 
 	# UDP queries that have waited in the socket buffer for a long time
@@ -303,6 +343,7 @@ server:
 	# Choose deny (drop message), refuse (polite error reply),
 	# allow (recursive ok), allow_setrd (recursive ok, rd bit is forced on),
 	# allow_snoop (recursive and nonrecursive ok)
+	# allow_cookie (allow UDP with valid cookie or stateful transport)
 	# deny_non_local (drop queries unless can be answered from local-data)
 	# refuse_non_local (like deny_non_local but polite error reply).
 	# access-control: 127.0.0.0/8 allow
@@ -430,6 +471,10 @@ server:
 	# print UTC timestamp in ascii to logfile, default is epoch in seconds.
 	# log-time-ascii: no
 
+	# log timestamp in ISO8601 format if also log-time-ascii is enabled.
+	# (y-m-dTh:m:s.msec[+-]tzhours:tzminutes)
+	# log-time-iso: no
+
 	# print one line with time, IP, name, type, class for every query.
 	# log-queries: no
 
@@ -441,6 +486,9 @@ server:
 	# filtering log-queries and log-replies from the log.
 	# log-tag-queryreply: no
 
+	# log with destination address, port and type for log-replies.
+	# log-destaddr: no
+
 	# log the local-zone actions, like local-zone type inform is enabled
 	# also for the other local zone types.
 	# log-local-actions: no
@@ -499,6 +547,9 @@ server:
 	# Harden against out of zone rrsets, to avoid spoofing attempts.
 	# harden-glue: yes
 
+	# Harden against unverified (outside-zone, including sibling zone) glue rrsets
+	# harden-unverified-glue: no
+
 	# Harden against receiving dnssec-stripped data. If you turn it
 	# off, failing to validate dnskey data for a trustanchor will
 	# trigger insecure mode for that zone (like without a trustanchor).
@@ -515,8 +566,9 @@ server:
 	# harden-referral-path: no
 
 	# Harden against algorithm downgrade when multiple algorithms are
-	# advertised in the DS record.  If no, allows the weakest algorithm
-	# to validate the zone.
+	# advertised in the DS record.  If no, allows any algorithm
+	# to validate the zone which is the standard behavior for validators.
+	# Check the manpage for detailed information.
 	# harden-algo-downgrade: no
 
 	# Harden against unknown records in the authority section and the
@@ -689,12 +741,13 @@ server:
 	# disable-edns-do: no
 
 	# Serve expired responses from cache, with serve-expired-reply-ttl in
-	# the response, and then attempt to fetch the data afresh.
+	# the response. By default it first tries to refresh an expired answer.
+	# Can be configured with serve-expired-client-timeout.
 	# serve-expired: no
 	#
 	# Limit serving of expired responses to configured seconds after
 	# expiration. 0 disables the limit.
-	# serve-expired-ttl: 0
+	# serve-expired-ttl: 86400
 	#
 	# Set the TTL of expired records to the serve-expired-ttl value after a
 	# failed attempt to retrieve the record from upstream. This makes sure
@@ -707,10 +760,9 @@ server:
 	#
 	# Time in milliseconds before replying to the client with expired data.
 	# This essentially enables the serve-stale behavior as specified in
-	# RFC 8767 that first tries to resolve before
-	# immediately responding with expired data.  0 disables this behavior.
-	# A recommended value is 1800.
-	# serve-expired-client-timeout: 0
+	# RFC 8767 that first tries to resolve before immediately responding
+	# with expired data.  0 disables this behavior.
+	# serve-expired-client-timeout: 1800
 
 	# Return the original TTL as received from the upstream name server rather
 	# than the decrementing TTL as stored in the cache.  Enabling this feature
@@ -769,6 +821,8 @@ server:
 	# local-zone: "127.in-addr.arpa." nodefault
 	# local-zone: "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa." nodefault
 	# local-zone: "home.arpa." nodefault
+	# local-zone: "resolver.arpa." nodefault
+	# local-zone: "service.arpa." nodefault
 	# local-zone: "onion." nodefault
 	# local-zone: "test." nodefault
 	# local-zone: "invalid." nodefault
@@ -879,6 +933,7 @@ server:
 	# tls-service-pem: "path/to/publiccertfile.pem"
 	# tls-port: 853
 	# https-port: 443
+	# quic-port: 853
 
 	# cipher setting for TLSv1.2
 	# tls-ciphers: "DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256"
@@ -943,6 +998,9 @@ server:
 	# Disable TLS for DNS-over-HTTP downstream service.
 	# http-notls-downstream: no
 
+	# Maximum number of bytes used for QUIC buffers.
+	# quic-size: 8m
+
 	# The interfaces that use these listed port numbers will support and
 	# expect PROXYv2. For UDP and TCP/TLS interfaces.
 	# proxy-protocol-port: portno for each of the port numbers.
@@ -983,6 +1041,13 @@ server:
 	# if 0(default) it is disabled, otherwise states qps allowed per ip address
 	# ip-ratelimit: 0
 
+	# global query ratelimit for all ip addresses with a valid DNS Cookie.
+	# feature is experimental.
+	# if 0(default) it is disabled, otherwise states qps allowed per ip address
+	# useful in combination with 'allow_cookie'.
+	# If used, suggested to be higher than ip-ratelimit, tenfold.
+	# ip-ratelimit-cookie: 0
+
 	# ip ratelimits are tracked in a cache, size in bytes of cache (or k,m).
 	# ip-ratelimit-size: 4m
 	# ip ratelimit cache slabs, reduces lock contention if equal to cpucount.
@@ -1004,6 +1069,19 @@ server:
 	# the number of servers that will be used in the fast server selection.
 	# fast-server-num: 3
 
+	# reply to requests containing DNS Cookies as specified in RFC 7873 and RFC 9018.
+	# answer-cookie: no
+
+	# secret for DNS Cookie generation.
+	# useful for anycast deployments.
+	# example value "000102030405060708090a0b0c0d0e0f".
+	# cookie-secret: <128 bit random hex string>
+
+	# File with cookie secrets, the 'cookie-secret:' option is ignored
+	# and the file can be managed to have staging and active secrets
+	# with remote control commands. Disabled with "". Default is "".
+	# cookie-secret-file: "/usr/local/etc/unbound_cookiesecrets.txt"
+
 	# Enable to attach Extended DNS Error codes (RFC8914) to responses.
 	# ede: no
 
@@ -1012,6 +1090,11 @@ server:
 	# Note that the ede option above needs to be enabled for this to work.
 	# ede-serve-expired: no
 
+	# Enable DNS Error Reporting (RFC9567).
+	# qname-minimisation is advised to be turned on as well to increase
+	# privacy on the outgoing reports.
+	# dns-error-reporting: no
+
 	# Specific options for ipsecmod. Unbound needs to be configured with
 	# --enable-ipsecmod for these to take effect.
 	#
@@ -1150,7 +1233,7 @@ remote-control:
 # sources of notifies.
 # auth-zone:
 #	name: "."
-#	primary: 199.9.14.201         # b.root-servers.net
+#	primary: 170.247.170.2        # b.root-servers.net
 #	primary: 192.33.4.12          # c.root-servers.net
 #	primary: 199.7.91.13          # d.root-servers.net
 #	primary: 192.5.5.241          # f.root-servers.net
@@ -1158,7 +1241,7 @@ remote-control:
 #	primary: 193.0.14.129         # k.root-servers.net
 #	primary: 192.0.47.132         # xfr.cjr.dns.icann.org
 #	primary: 192.0.32.132         # xfr.lax.dns.icann.org
-#	primary: 2001:500:200::b      # b.root-servers.net
+#	primary: 2801:1b8:10::b       # b.root-servers.net
 #	primary: 2001:500:2::c        # c.root-servers.net
 #	primary: 2001:500:2d::d       # d.root-servers.net
 #	primary: 2001:500:2f::f       # f.root-servers.net
@@ -1228,6 +1311,9 @@ remote-control:
 #     secret-seed: "default"
 #     # if the backend should be read from, but not written to.
 #     cachedb-no-store: no
+#     # if the cachedb should be checked before a serve-expired response is
+#     # given, when serve-expired is enabled.
+#     cachedb-check-when-serve-expired: yes
 #
 #     # For "redis" backend:
 #     # (to enable, use --with-libhiredis to configure before compiling)
@@ -1236,15 +1322,35 @@ remote-control:
 #     # redis server's TCP port
 #     redis-server-port: 6379
 #     # if the server uses a unix socket, set its path, or "" when not used.
-#     # redis-server-path: "/var/lib/redis/redis-server.sock"
+#     redis-server-path: "/var/lib/redis/redis-server.sock"
 #     # if the server uses an AUTH password, specify here, or "" when not used.
-#     # redis-server-password: ""
+#     redis-server-password: ""
 #     # timeout (in ms) for communication with the redis server
 #     redis-timeout: 100
+#     # timeout (in ms) for commands, if 0, uses redis-timeout.
+#     redis-command-timeout: 0
+#     # timeout (in ms) for connection set up, if 0, uses redis-timeout.
+#     redis-connect-timeout: 0
 #     # set timeout on redis records based on DNS response TTL
 #     redis-expire-records: no
 #     # redis logical database to use, 0 is the default database.
 #     redis-logical-db: 0
+#     # redis replica server's IP address or host name
+#     redis-replica-server-host: 127.0.0.1
+#     # redis replica server's TCP port
+#     redis-replica-server-port: 6379
+#     # if the replica server uses a unix socket, set its path, or "" when not used.
+#     redis-replica-server-path: "/var/lib/redis/redis-server.sock"
+#     # if the replica server uses an AUTH password, specify here, or "" when not used.
+#     redis-replica-server-password: ""
+#     # timeout (in ms) for communication with the redis replica server
+#     redis-replica-timeout: 100
+#     # timeout (in ms) for redis replica commands, if 0, uses redis-replica-timeout.
+#     redis-replica-command-timeout: 0
+#     # timeout (in ms) for redis replica connection set up, if 0, uses redis-replica-timeout.
+#     redis-replica-connect-timeout: 0
+#     # redis logical database to use for the replica server, 0 is the default database.
+#     redis-replica-logical-db: 0
 
 # IPSet
 # Add specify domain into set via ipset.
@@ -1286,6 +1392,8 @@ remote-control:
 # 	dnstap-identity: ""
 # 	# if "" it uses the package version.
 # 	dnstap-version: ""
+# 	# log only 1/N messages, if 0 it is disabled. default 0.
+# 	dnstap-sample-rate: 0
 # 	dnstap-log-resolver-query-messages: no
 # 	dnstap-log-resolver-response-messages: no
 # 	dnstap-log-client-query-messages: no
@@ -1294,7 +1402,8 @@ remote-control:
 # 	dnstap-log-forwarder-response-messages: no
 
 # Response Policy Zones
-# RPZ policies. Applied in order of configuration. QNAME, Response IP
+# RPZ policies. Applied in order of configuration. Any match from an earlier
+# RPZ zone will terminate the RPZ lookup. QNAME, Response IP
 # Address, nsdname, nsip and clientip triggers are supported. Supported
 # actions are: NXDOMAIN, NODATA, PASSTHRU, DROP, Local Data, tcp-only
 # and drop.  Policies can be loaded from a file, or using zone

doc/libunbound.3.in

@@ -1,335 +1,306 @@
-.TH "libunbound" "3" "@date@" "NLnet Labs" "unbound @version@"
-.\"
-.\" libunbound.3 -- unbound library functions manual
-.\"
-.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
-.\"
-.\" See LICENSE for the license.
-.\"
-.\"
-.SH "NAME"
-.B libunbound,
-.B unbound.h,
-.B ub_ctx,
-.B ub_result,
-.B ub_callback_type,
-.B ub_ctx_create,
-.B ub_ctx_delete,
-.B ub_ctx_set_option,
-.B ub_ctx_get_option,
-.B ub_ctx_config,
-.B ub_ctx_set_fwd,
-.B ub_ctx_set_stub,
-.B ub_ctx_set_tls,
-.B ub_ctx_resolvconf,
-.B ub_ctx_hosts,
-.B ub_ctx_add_ta,
-.B ub_ctx_add_ta_autr,
-.B ub_ctx_add_ta_file,
-.B ub_ctx_trustedkeys,
-.B ub_ctx_debugout,
-.B ub_ctx_debuglevel,
-.B ub_ctx_async,
-.B ub_poll,
-.B ub_wait,
-.B ub_fd,
-.B ub_process,
-.B ub_resolve,
-.B ub_resolve_async,
-.B ub_cancel,
-.B ub_resolve_free,
-.B ub_strerror,
-.B ub_ctx_print_local_zones,
-.B ub_ctx_zone_add,
-.B ub_ctx_zone_remove,
-.B ub_ctx_data_add,
-.B ub_ctx_data_remove
-\- Unbound DNS validating resolver @version@ functions.
-.SH "SYNOPSIS"
-.B #include <unbound.h>
-.LP
-\fIstruct ub_ctx *\fR
-\fBub_ctx_create\fR(\fIvoid\fR);
-.LP
-\fIvoid\fR
-\fBub_ctx_delete\fR(\fIstruct ub_ctx*\fR ctx);
-.LP
-\fIint\fR
-\fBub_ctx_set_option\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR opt, \fIchar*\fR val);
-.LP
-\fIint\fR
-\fBub_ctx_get_option\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR opt, \fIchar**\fR val);
-.LP
-\fIint\fR
-\fBub_ctx_config\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR fname);
-.LP
-\fIint\fR
-\fBub_ctx_set_fwd\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR addr);
-.LP
-\fIint\fR
-\fBub_ctx_set_stub\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR zone,
-\fIchar*\fR addr,
-.br
-		\fIint\fR isprime);
-.LP
-\fIint\fR
-\fBub_ctx_set_tls\fR(\fIstruct ub_ctx*\fR ctx, \fIint\fR tls);
-.LP
-\fIint\fR
-\fBub_ctx_resolvconf\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR fname);
-.LP
-\fIint\fR
-\fBub_ctx_hosts\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR fname);
-.LP
-\fIint\fR
-\fBub_ctx_add_ta\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR ta);
-.LP
-\fIint\fR
-\fBub_ctx_add_ta_autr\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR fname);
-.LP
-\fIint\fR
-\fBub_ctx_add_ta_file\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR fname);
-.LP
-\fIint\fR
-\fBub_ctx_trustedkeys\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR fname);
-.LP
-\fIint\fR
-\fBub_ctx_debugout\fR(\fIstruct ub_ctx*\fR ctx, \fIFILE*\fR out);
-.LP
-\fIint\fR
-\fBub_ctx_debuglevel\fR(\fIstruct ub_ctx*\fR ctx, \fIint\fR d);
-.LP
-\fIint\fR
-\fBub_ctx_async\fR(\fIstruct ub_ctx*\fR ctx, \fIint\fR dothread);
-.LP
-\fIint\fR
-\fBub_poll\fR(\fIstruct ub_ctx*\fR ctx);
-.LP
-\fIint\fR
-\fBub_wait\fR(\fIstruct ub_ctx*\fR ctx);
-.LP
-\fIint\fR
-\fBub_fd\fR(\fIstruct ub_ctx*\fR ctx);
-.LP
-\fIint\fR
-\fBub_process\fR(\fIstruct ub_ctx*\fR ctx);
-.LP
-\fIint\fR
-\fBub_resolve\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR name, 
-.br
-           \fIint\fR rrtype, \fIint\fR rrclass, \fIstruct ub_result**\fR result);
-.LP
-\fIint\fR
-\fBub_resolve_async\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR name, 
-.br
-                 \fIint\fR rrtype, \fIint\fR rrclass, \fIvoid*\fR mydata, 
-.br
-                 \fIub_callback_type\fR callback, \fIint*\fR async_id);
-.LP
-\fIint\fR
-\fBub_cancel\fR(\fIstruct ub_ctx*\fR ctx, \fIint\fR async_id);
-.LP
-\fIvoid\fR
-\fBub_resolve_free\fR(\fIstruct ub_result*\fR result);
-.LP
-\fIconst char *\fR
-\fBub_strerror\fR(\fIint\fR err);
-.LP
-\fIint\fR
-\fBub_ctx_print_local_zones\fR(\fIstruct ub_ctx*\fR ctx);
-.LP
-\fIint\fR
-\fBub_ctx_zone_add\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR zone_name, \fIchar*\fR zone_type);
-.LP
-\fIint\fR
-\fBub_ctx_zone_remove\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR zone_name);
-.LP
-\fIint\fR
-\fBub_ctx_data_add\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR data);
-.LP
-\fIint\fR
-\fBub_ctx_data_remove\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR data);
-.SH "DESCRIPTION"
-.B Unbound 
-is an implementation of a DNS resolver, that does caching and 
-DNSSEC validation. This is the library API, for using the \-lunbound library.
-The server daemon is described in \fIunbound\fR(8).
-The library works independent from a running unbound server, and
-can be used to convert hostnames to ip addresses, and back,
-and obtain other information from the DNS. The library performs public\-key
-validation of results with DNSSEC.
-.P
-The library uses a variable of type \fIstruct ub_ctx\fR to keep context
-between calls. The user must maintain it, creating it with
-.B ub_ctx_create
-and deleting it with
-.B ub_ctx_delete\fR.
-It can be created and deleted at any time. Creating it anew removes any 
-previous configuration (such as trusted keys) and clears any cached results.
-.P
-The functions are thread\-safe, and a context can be used in a threaded (as 
-well as in a non\-threaded) environment. Also resolution (and validation) 
-can be performed blocking and non\-blocking (also called asynchronous). 
-The async method returns from the call immediately, so that processing 
-can go on, while the results become available later. 
-.P
+.\" Man page generated from reStructuredText.
+.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.TH "LIBUNBOUND" "3" "@date@" "@version@" "Unbound"
+.SH NAME
+libunbound \- Unbound DNS validating resolver @version@ functions.
+.SH SYNOPSIS
+.sp
+\fB#include <unbound.h>\fP
+.sp
+struct ub_ctx * \fBub_ctx_create\fP(void);
+.sp
+void \fBub_ctx_delete\fP(struct ub_ctx* ctx);
+.sp
+int \fBub_ctx_set_option\fP(struct ub_ctx* ctx, char* opt, char* val);
+.sp
+int \fBub_ctx_get_option\fP(struct ub_ctx* ctx, char* opt, char** val);
+.sp
+int \fBub_ctx_config\fP(struct ub_ctx* ctx, char* fname);
+.sp
+int \fBub_ctx_set_fwd\fP(struct ub_ctx* ctx, char* addr);
+.INDENT 0.0
+.TP
+int \fBub_ctx_set_stub\fP(struct ub_ctx* ctx, char* zone, char* addr,
+int isprime);
+.UNINDENT
+.sp
+int \fBub_ctx_set_tls\fP(struct ub_ctx* ctx, int tls);
+.sp
+int \fBub_ctx_resolvconf\fP(struct ub_ctx* ctx, char* fname);
+.sp
+int \fBub_ctx_hosts\fP(struct ub_ctx* ctx, char* fname);
+.sp
+int \fBub_ctx_add_ta\fP(struct ub_ctx* ctx, char* ta);
+.sp
+int \fBub_ctx_add_ta_autr\fP(struct ub_ctx* ctx, char* fname);
+.sp
+int \fBub_ctx_add_ta_file\fP(struct ub_ctx* ctx, char* fname);
+.sp
+int \fBub_ctx_trustedkeys\fP(struct ub_ctx* ctx, char* fname);
+.sp
+int \fBub_ctx_debugout\fP(struct ub_ctx* ctx, FILE* out);
+.sp
+int \fBub_ctx_debuglevel\fP(struct ub_ctx* ctx, int d);
+.sp
+int \fBub_ctx_async\fP(struct ub_ctx* ctx, int dothread);
+.sp
+int \fBub_poll\fP(struct ub_ctx* ctx);
+.sp
+int \fBub_wait\fP(struct ub_ctx* ctx);
+.sp
+int \fBub_fd\fP(struct ub_ctx* ctx);
+.sp
+int \fBub_process\fP(struct ub_ctx* ctx);
+.INDENT 0.0
+.TP
+int \fBub_resolve\fP(struct ub_ctx* ctx, char* name,
+int rrtype, int rrclass, struct ub_result** result);
+.TP
+int \fBub_resolve_async\fP(struct ub_ctx* ctx, char* name,
+int rrtype, int rrclass, void* mydata,
+ub_callback_type* callback, int* async_id);
+.UNINDENT
+.sp
+int \fBub_cancel\fP(struct ub_ctx* ctx, int async_id);
+.sp
+void \fBub_resolve_free\fP(struct ub_result* result);
+.sp
+const char * \fBub_strerror\fP(int err);
+.sp
+int \fBub_ctx_print_local_zones\fP(struct ub_ctx* ctx);
+.sp
+int \fBub_ctx_zone_add\fP(struct ub_ctx* ctx, char* zone_name, char* zone_type);
+.sp
+int \fBub_ctx_zone_remove\fP(struct ub_ctx* ctx, char* zone_name);
+.sp
+int \fBub_ctx_data_add\fP(struct ub_ctx* ctx, char* data);
+.sp
+int \fBub_ctx_data_remove\fP(struct ub_ctx* ctx, char* data);
+.SH DESCRIPTION
+.sp
+Unbound is an implementation of a DNS resolver, that does caching and DNSSEC
+validation.
+This is the library API, for using the \fB\-lunbound\fP library.
+The server daemon is described in \fI\%unbound(8)\fP\&.
+The library works independent from a running unbound server, and can be used to
+convert hostnames to ip addresses, and back, and obtain other information from
+the DNS.
+The library performs public\-key validation of results with DNSSEC.
+.sp
+The library uses a variable of type \fIstruct ub_ctx\fP to keep context between
+calls.
+The user must maintain it, creating it with \fBub_ctx_create\fP and deleting it
+with \fBub_ctx_delete\fP\&.
+It can be created and deleted at any time.
+Creating it anew removes any previous configuration (such as trusted keys) and
+clears any cached results.
+.sp
+The functions are thread\-safe, and a context can be used in a threaded (as well
+as in a non\-threaded) environment.
+Also resolution (and validation) can be performed blocking and non\-blocking
+(also called asynchronous).
+The async method returns from the call immediately, so that processing can go
+on, while the results become available later.
+.sp
 The functions are discussed in turn below.
-.SH "FUNCTIONS"
-.TP 
+.SH FUNCTIONS
+.INDENT 0.0
+.TP
 .B ub_ctx_create
 Create a new context, initialised with defaults.
-The information from /etc/resolv.conf and /etc/hosts is not utilised 
-by default. Use 
-.B ub_ctx_resolvconf
-and
-.B ub_ctx_hosts
-to read them.
-Before you call this, use the openssl functions CRYPTO_set_id_callback and
-CRYPTO_set_locking_callback to set up asynchronous operation if you use
-lib openssl (the application calls these functions once for initialisation).
-Openssl 1.0.0 or later uses the CRYPTO_THREADID_set_callback function.
+The information from \fB/etc/resolv.conf\fP and \fB/etc/hosts\fP is
+not utilised by default.
+Use \fBub_ctx_resolvconf\fP and \fBub_ctx_hosts\fP to read them.
+Before you call this, use the openssl functions
+\fBCRYPTO_set_id_callback\fP and \fBCRYPTO_set_locking_callback\fP to set
+up asynchronous operation if you use lib openssl (the application calls
+these functions once for initialisation).
+Openssl 1.0.0 or later uses the \fBCRYPTO_THREADID_set_callback\fP
+function.
 .TP
 .B ub_ctx_delete
 Delete validation context and free associated resources.
-Outstanding async queries are killed and callbacks are not called for them.
+Outstanding async queries are killed and callbacks are not called for
+them.
 .TP
 .B ub_ctx_set_option
-A power\-user interface that lets you specify one of the options from the
-config file format, see \fIunbound.conf\fR(5). Not all options are
-relevant. For some specific options, such as adding trust anchors, special
-routines exist. Pass the option name with the trailing ':'.
+A power\-user interface that lets you specify one of the options from
+the config file format, see \fI\%unbound.conf(5)\fP\&.
+Not all options are relevant.
+For some specific options, such as adding trust anchors, special
+routines exist.
+Pass the option name with the trailing \fB\(aq:\(aq\fP\&.
 .TP
 .B ub_ctx_get_option
-A power\-user interface that gets an option value.  Some options cannot be
-gotten, and others return a newline separated list.  Pass the option name
-without trailing ':'.  The returned value must be free(2)d by the caller.
+A power\-user interface that gets an option value.
+Some options cannot be gotten, and others return a newline separated
+list.
+Pass the option name without trailing \fB\(aq:\(aq\fP\&.
+The returned value must be free(2)d by the caller.
 .TP
 .B ub_ctx_config
-A power\-user interface that lets you specify an unbound config file, see
-\fIunbound.conf\fR(5), which is read for configuration. Not all options are
-relevant. For some specific options, such as adding trust anchors, special
-routines exist.  This function is thread\-safe only if a single instance of
-ub_ctx* exists in the application.  If several instances exist the
-application has to ensure that ub_ctx_config is not called in parallel by
-the different instances.
+A power\-user interface that lets you specify an unbound config file,
+see \fI\%unbound.conf(5)\fP, which is read for
+configuration.
+Not all options are relevant.
+For some specific options, such as adding trust anchors, special
+routines exist.
+This function is thread\-safe only if a single instance of \fBub_ctx\fP*
+exists in the application.
+If several instances exist the application has to ensure that
+\fBub_ctx_config\fP is not called in parallel by the different instances.
 .TP
 .B ub_ctx_set_fwd
-Set machine to forward DNS queries to, the caching resolver to use. 
-IP4 or IP6 address. Forwards all DNS requests to that machine, which 
-is expected to run a recursive resolver. If the proxy is not 
-DNSSEC capable, validation may fail. Can be called several times, in 
-that case the addresses are used as backup servers.
-At this time it is only possible to set configuration before the
-first resolve is done.
+Set machine to forward DNS queries to, the caching resolver to use.
+IP4 or IP6 address.
+Forwards all DNS requests to that machine, which is expected to run a
+recursive resolver.
+If the proxy is not DNSSEC capable, validation may fail.
+Can be called several times, in that case the addresses are used as
+backup servers.
+At this time it is only possible to set configuration before the first
+resolve is done.
 .TP
 .B ub_ctx_set_stub
-Set a stub zone, authoritative dns servers to use for a particular zone.
-IP4 or IP6 address.  If the address is NULL the stub entry is removed.
-Set isprime true if you configure root hints with it.  Otherwise similar to
-the stub zone item from unbound's config file.  Can be called several times,
-for different zones, or to add multiple addresses for a particular zone.
-At this time it is only possible to set configuration before the
-first resolve is done.
+Set a stub zone, authoritative dns servers to use for a particular
+zone.
+IP4 or IP6 address.
+If the address is NULL the stub entry is removed.
+Set isprime true if you configure root hints with it.
+Otherwise similar to the stub zone item from unbound\(aqs config file.
+Can be called several times, for different zones, or to add multiple
+addresses for a particular zone.
+At this time it is only possible to set configuration before the first
+resolve is done.
 .TP
 .B ub_ctx_set_tls
-Enable DNS over TLS (DoT) for machines set with 
-.B ub_ctx_set_fwd.
-At this time it is only possible to set configuration before the
-first resolve is done.
+Enable DNS over TLS (DoT) for machines set with \fBub_ctx_set_fwd\fP\&.
+At this time it is only possible to set configuration before the first
+resolve is done.
 .TP
 .B ub_ctx_resolvconf
-By default the root servers are queried and full resolver mode is used, but
-you can use this call to read the list of nameservers to use from the
-filename given.
-Usually "/etc/resolv.conf". Uses those nameservers as caching proxies.
+By default the root servers are queried and full resolver mode is used,
+but you can use this call to read the list of nameservers to use from
+the filename given.
+Usually \fB\(dq/etc/resolv.conf\(dq\fP\&.
+Uses those nameservers as caching proxies.
 If they do not support DNSSEC, validation may fail.
 Only nameservers are picked up, the searchdomain, ndots and other
-settings from \fIresolv.conf\fR(5) are ignored.
-If fname NULL is passed, "/etc/resolv.conf" is used (if on Windows, 
-the system\-wide configured nameserver is picked instead).
-At this time it is only possible to set configuration before the
-first resolve is done.
+settings from \fIresolv.conf(5)\fP are ignored.
+If fname NULL is passed, \fB\(dq/etc/resolv.conf\(dq\fP is used (if on
+Windows, the system\-wide configured nameserver is picked instead).
+At this time it is only possible to set configuration before the first
+resolve is done.
 .TP
 .B ub_ctx_hosts
 Read list of hosts from the filename given.
-Usually "/etc/hosts". When queried for, these addresses are not marked 
-DNSSEC secure. If fname NULL is passed, "/etc/hosts" is used 
-(if on Windows, etc/hosts from WINDIR is picked instead).
-At this time it is only possible to set configuration before the
-first resolve is done.
+Usually \fB\(dq/etc/hosts\(dq\fP\&.
+When queried for, these addresses are not marked DNSSEC secure.
+If fname NULL is passed, \fB\(dq/etc/hosts\(dq\fP is used (if on Windows,
+\fBetc/hosts\fP from WINDIR is picked instead).
+At this time it is only possible to set configuration before the first
+resolve is done.
 .TP
-.B
-ub_ctx_add_ta
+.B ub_ctx_add_ta
 Add a trust anchor to the given context.
-At this time it is only possible to add trusted keys before the
-first resolve is done.
+At this time it is only possible to add trusted keys before the first
+resolve is done.
 The format is a string, similar to the zone\-file format,
-[domainname] [type] [rdata contents]. Both DS and DNSKEY records are accepted.
+\fB[domainname]\fP \fB[type]\fP \fB[rdata contents]\fP\&.
+Both DS and DNSKEY records are accepted.
 .TP
 .B ub_ctx_add_ta_autr
-Add filename with automatically tracked trust anchor to the given context.
-Pass name of a file with the managed trust anchor.  You can create this
-file with \fIunbound\-anchor\fR(8) for the root anchor.  You can also
-create it with an initial file with one line with a DNSKEY or DS record.
+Add filename with automatically tracked trust anchor to the given
+context.
+Pass name of a file with the managed trust anchor.
+You can create this file with
+\fI\%unbound\-anchor(8)\fP for the root anchor.
+You can also create it with an initial file with one line with a DNSKEY
+or DS record.
 If the file is writable, it is updated when the trust anchor changes.
-At this time it is only possible to add trusted keys before the
-first resolve is done.
+At this time it is only possible to add trusted keys before the first
+resolve is done.
 .TP
 .B ub_ctx_add_ta_file
 Add trust anchors to the given context.
 Pass name of a file with DS and DNSKEY records in zone file format.
-At this time it is only possible to add trusted keys before the
-first resolve is done.
+At this time it is only possible to add trusted keys before the first
+resolve is done.
 .TP
 .B ub_ctx_trustedkeys
 Add trust anchors to the given context.
-Pass the name of a bind\-style config file with trusted\-keys{}.
-At this time it is only possible to add trusted keys before the
-first resolve is done.
+Pass the name of a bind\-style config file with \fBtrusted\-keys{}\fP\&.
+At this time it is only possible to add trusted keys before the first
+resolve is done.
 .TP
 .B ub_ctx_debugout
-Set debug and error log output to the given stream. Pass NULL to disable
-output. Default is stderr. File\-names or using syslog can be enabled
-using config options, this routine is for using your own stream.
+Set debug and error log output to the given stream.
+Pass NULL to disable output.
+Default is stderr.
+File\-names or using syslog can be enabled using config options, this
+routine is for using your own stream.
 .TP
 .B ub_ctx_debuglevel
-Set debug verbosity for the context. Output is directed to stderr.
+Set debug verbosity for the context.
+Output is directed to stderr.
 Higher debug level gives more output.
 .TP
 .B ub_ctx_async
 Set a context behaviour for asynchronous action.
-if set to true, enables threading and a call to 
-.B ub_resolve_async 
+if set to true, enables threading and a call to \fBub_resolve_async\fP
 creates a thread to handle work in the background.
 If false, a process is forked to handle work in the background.
-Changes to this setting after 
-.B ub_resolve_async 
-calls have been made have no effect (delete and re\-create the context 
-to change).
+Changes to this setting after \fBub_resolve_async\fP calls have been made
+have no effect (delete and re\-create the context to change).
 .TP
 .B ub_poll
 Poll a context to see if it has any new results.
-Do not poll in a loop, instead extract the fd below to poll for readiness,
-and then check, or wait using the wait routine.
+Do not poll in a loop, instead extract the \fBfd\fP below to poll for
+readiness, and then check, or wait using the wait routine.
 Returns 0 if nothing to read, or nonzero if a result is available.
-If nonzero, call 
-.B ub_process 
-to do callbacks.
+If nonzero, call \fBub_process\fP to do callbacks.
 .TP
 .B ub_wait
-Wait for a context to finish with results. Calls 
-.B ub_process 
-after the wait for you. After the wait, there are no more outstanding 
-asynchronous queries.
+Wait for a context to finish with results.
+Calls \fBub_process\fP after the wait for you.
+After the wait, there are no more outstanding asynchronous queries.
 .TP
 .B ub_fd
-Get file descriptor. Wait for it to become readable, at this point
-answers are returned from the asynchronous validating resolver.
-Then call the \fBub_process\fR to continue processing.
+Get file descriptor.
+Wait for it to become readable, at this point answers are returned from
+the asynchronous validating resolver.
+Then call the \fBub_process\fP to continue processing.
 .TP
 .B ub_process
 Call this routine to continue processing results from the validating
-resolver (when the fd becomes readable).
+resolver (when the \fBfd\fP becomes readable).
 Will perform necessary callbacks.
 .TP
 .B ub_resolve
@@ -340,95 +311,111 @@ The result structure is newly allocated with the resulting data.
 .TP
 .B ub_resolve_async
 Perform asynchronous resolution and validation of the target name.
-Arguments mean the same as for \fBub_resolve\fR except no
-data is returned immediately, instead a callback is called later.
-The callback receives a copy of the mydata pointer, that you can use to pass
-information to the callback. The callback type is a function pointer to
-a function declared as
-.IP
-void my_callback_function(void* my_arg, int err, 
-.br
-                  struct ub_result* result);
-.IP
-The async_id is returned so you can (at your option) decide to track it
-and cancel the request if needed.  If you pass a NULL pointer the async_id
-is not returned. 
+Arguments mean the same as for \fBub_resolve\fP except no data is
+returned immediately, instead a callback is called later.
+The callback receives a copy of the mydata pointer, that you can use to
+pass information to the callback.
+The callback type is a function pointer to a function declared as:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+void my_callback_function(void* my_arg, int err,
+                struct ub_result* result);
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+The \fBasync_id\fP is returned so you can (at your option) decide to
+track it and cancel the request if needed.
+If you pass a NULL pointer the \fBasync_id\fP is not returned.
 .TP
 .B ub_cancel
-Cancel an async query in progress.  This may return an error if the query
-does not exist, or the query is already being delivered, in that case you 
-may still get a callback for the query.
+Cancel an async query in progress.
+This may return an error if the query does not exist, or the query is
+already being delivered, in that case you may still get a callback for
+the query.
 .TP
 .B ub_resolve_free
-Free struct ub_result contents after use.
+Free struct \fBub_result\fP contents after use.
 .TP
 .B ub_strerror
-Convert error value from one of the unbound library functions 
-to a human readable string.
+Convert error value from one of the unbound library functions to a
+human readable string.
 .TP
 .B ub_ctx_print_local_zones
 Debug printout the local authority information to debug output.
 .TP
 .B ub_ctx_zone_add
-Add new zone to local authority info, like local\-zone \fIunbound.conf\fR(5) 
-statement.
+Add new zone to local authority info, like local\-zone
+\fI\%unbound.conf(5)\fP statement.
 .TP
 .B ub_ctx_zone_remove
 Delete zone from local authority info.
 .TP
 .B ub_ctx_data_add
 Add resource record data to local authority info, like local\-data
-\fIunbound.conf\fR(5) statement.
+\fI\%unbound.conf(5)\fP statement.
 .TP
 .B ub_ctx_data_remove
 Delete local authority data from the name given.
-.SH "RESULT DATA STRUCTURE"
-The result of the DNS resolution and validation is returned as 
-\fIstruct ub_result\fR. The result structure contains the following entries.
-.P
+.UNINDENT
+.SH RESULT DATA STRUCTURE
+.sp
+The result of the DNS resolution and validation is returned as \fIstruct
+ub_result\fP\&.
+The result structure contains the following entries:
+.INDENT 0.0
+.INDENT 3.5
+.sp
 .nf
-	struct ub_result {
-		char* qname; /* text string, original question */
-		int qtype;   /* type code asked for */
-		int qclass;  /* class code asked for */
-		char** data; /* array of rdata items, NULL terminated*/
-		int* len;    /* array with lengths of rdata items */
-		char* canonname; /* canonical name of result */
-		int rcode;   /* additional error code in case of no data */
-		void* answer_packet; /* full network format answer packet */
-		int answer_len;  /* length of packet in octets */
-		int havedata; /* true if there is data */
-		int nxdomain; /* true if nodata because name does not exist */
-		int secure;   /* true if result is secure */
-		int bogus;    /* true if a security failure happened */
-		char* why_bogus; /* string with error if bogus */
-		int was_ratelimited; /* true if the query was ratelimited (SERVFAIL) by unbound */
-		int ttl;     /* number of seconds the result is valid */
-	};
+.ft C
+struct ub_result {
+     char* qname;         /* text string, original question */
+     int qtype;           /* type code asked for */
+     int qclass;          /* class code asked for */
+     char** data;         /* array of rdata items, NULL terminated*/
+     int* len;            /* array with lengths of rdata items */
+     char* canonname;     /* canonical name of result */
+     int rcode;           /* additional error code in case of no data */
+     void* answer_packet; /* full network format answer packet */
+     int answer_len;      /* length of packet in octets */
+     int havedata;        /* true if there is data */
+     int nxdomain;        /* true if nodata because name does not exist */
+     int secure;          /* true if result is secure */
+     int bogus;           /* true if a security failure happened */
+     char* why_bogus;     /* string with error if bogus */
+     int was_ratelimited; /* true if the query was ratelimited (SERVFAIL) by unbound */
+     int ttl;             /* number of seconds the result is valid */
+};
+.ft P
 .fi
-.P
-If both secure and bogus are false, security was not enabled for the 
-domain of the query.  Else, they are not both true, one of them is true.
-.SH "RETURN VALUES"
-Many routines return an error code. The value 0 (zero) denotes no error
-happened. Other values can be passed to
-.B ub_strerror
-to obtain a readable error string.
-.B ub_strerror
-returns a zero terminated string.
-.B ub_ctx_create
-returns NULL on an error (a malloc failure).
-.B ub_poll
-returns true if some information may be available, false otherwise.
-.B ub_fd
-returns a file descriptor or \-1 on error.
-.B ub_ctx_config
-and
-.B ub_ctx_resolvconf
-attempt to leave errno informative on a function return with file read failure.
-.SH "SEE ALSO"
-\fIunbound.conf\fR(5), 
-\fIunbound\fR(8).
-.SH "AUTHORS"
-.B Unbound
-developers are mentioned in the CREDITS file in the distribution.
+.UNINDENT
+.UNINDENT
+.sp
+If both secure and bogus are false, security was not enabled for the domain of
+the query.
+Else, they are not both true, one of them is true.
+.SH RETURN VALUES
+.sp
+Many routines return an error code.
+The value 0 (zero) denotes no error happened.
+Other values can be passed to \fBub_strerror\fP to obtain a readable error
+string.
+\fBub_strerror\fP returns a zero terminated string.
+\fBub_ctx_create\fP returns NULL on an error (a malloc failure).
+\fBub_poll\fP returns true if some information may be available, false otherwise.
+\fBub_fd\fP returns a file descriptor or \-1 on error.
+\fBub_ctx_config\fP and \fBub_ctx_resolvconf\fP attempt to leave errno informative
+on a function return with file read failure.
+.SH SEE ALSO
+.sp
+\fI\%unbound.conf(5)\fP, \fI\%unbound(8)\fP\&.
+.SH AUTHOR
+Unbound developers are mentioned in the CREDITS file in the distribution.
+.SH COPYRIGHT
+1999-2025, NLnet Labs
+.\" Generated by docutils manpage writer.
+.

doc/libunbound.rst

@@ -0,0 +1,491 @@
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+libunbound(3)
+=============
+
+Synopsis
+--------
+
+.. only:: html
+
+    .. code-block:: c
+
+        #include <unbound.h>
+
+        struct ub_ctx * ub_ctx_create(void);
+
+        void ub_ctx_delete(struct ub_ctx* ctx);
+
+        int ub_ctx_set_option(struct ub_ctx* ctx, char* opt, char* val);
+
+        int ub_ctx_get_option(struct ub_ctx* ctx, char* opt, char** val);
+
+        int ub_ctx_config(struct ub_ctx* ctx, char* fname);
+
+        int ub_ctx_set_fwd(struct ub_ctx* ctx, char* addr);
+
+        int ub_ctx_set_stub(struct ub_ctx* ctx, char* zone, char* addr,
+                            int isprime);
+
+        int ub_ctx_set_tls(struct ub_ctx* ctx, int tls);
+
+        int ub_ctx_resolvconf(struct ub_ctx* ctx, char* fname);
+
+        int ub_ctx_hosts(struct ub_ctx* ctx, char* fname);
+
+        int ub_ctx_add_ta(struct ub_ctx* ctx, char* ta);
+
+        int ub_ctx_add_ta_autr(struct ub_ctx* ctx, char* fname);
+
+        int ub_ctx_add_ta_file(struct ub_ctx* ctx, char* fname);
+
+        int ub_ctx_trustedkeys(struct ub_ctx* ctx, char* fname);
+
+        int ub_ctx_debugout(struct ub_ctx* ctx, FILE* out);
+
+        int ub_ctx_debuglevel(struct ub_ctx* ctx, int d);
+
+        int ub_ctx_async(struct ub_ctx* ctx, int dothread);
+
+        int ub_poll(struct ub_ctx* ctx);
+
+        int ub_wait(struct ub_ctx* ctx);
+
+        int ub_fd(struct ub_ctx* ctx);
+
+        int ub_process(struct ub_ctx* ctx);
+
+        int ub_resolve(struct ub_ctx* ctx, char* name, int rrtype,
+                       int rrclass, struct ub_result** result);
+
+        int ub_resolve_async(struct ub_ctx* ctx, char* name, int rrtype,
+                             int rrclass, void* mydata, ub_callback_type callback,
+                             int* async_id);
+
+        int ub_cancel(struct ub_ctx* ctx, int async_id);
+
+        void ub_resolve_free(struct ub_result* result);
+
+        const char * ub_strerror(int err);
+
+        int ub_ctx_print_local_zones(struct ub_ctx* ctx);
+
+        int ub_ctx_zone_add(struct ub_ctx* ctx, char* zone_name, char* zone_type);
+
+        int ub_ctx_zone_remove(struct ub_ctx* ctx, char* zone_name);
+
+        int ub_ctx_data_add(struct ub_ctx* ctx, char* data);
+
+        int ub_ctx_data_remove(struct ub_ctx* ctx, char* data);
+
+.. only:: man
+
+    **#include <unbound.h>**
+
+    struct ub_ctx \* **ub_ctx_create**\ (void);
+
+    void **ub_ctx_delete**\ (struct ub_ctx\* ctx);
+
+    int **ub_ctx_set_option**\ (struct ub_ctx\* ctx, char\* opt, char\* val);
+
+    int **ub_ctx_get_option**\ (struct ub_ctx\* ctx, char\* opt, char\*\* val);
+
+    int **ub_ctx_config**\ (struct ub_ctx\* ctx, char* fname);
+
+    int **ub_ctx_set_fwd**\ (struct ub_ctx\* ctx, char\* addr);
+
+    int **ub_ctx_set_stub**\ (struct ub_ctx\* ctx, char\* zone, char\* addr,
+        int isprime);
+
+    int **ub_ctx_set_tls**\ (struct ub_ctx\* ctx, int tls);
+
+    int **ub_ctx_resolvconf**\ (struct ub_ctx\* ctx, char\* fname);
+
+    int **ub_ctx_hosts**\ (struct ub_ctx\* ctx, char\* fname);
+
+    int **ub_ctx_add_ta**\ (struct ub_ctx\* ctx, char\* ta);
+
+    int **ub_ctx_add_ta_autr**\ (struct ub_ctx\* ctx, char\* fname);
+
+    int **ub_ctx_add_ta_file**\ (struct ub_ctx\* ctx, char\* fname);
+
+    int **ub_ctx_trustedkeys**\ (struct ub_ctx\* ctx, char\* fname);
+
+    int **ub_ctx_debugout**\ (struct ub_ctx\* ctx, FILE\* out);
+
+    int **ub_ctx_debuglevel**\ (struct ub_ctx\* ctx, int d);
+
+    int **ub_ctx_async**\ (struct ub_ctx\* ctx, int dothread);
+
+    int **ub_poll**\ (struct ub_ctx\* ctx);
+
+    int **ub_wait**\ (struct ub_ctx\* ctx);
+
+    int **ub_fd**\ (struct ub_ctx\* ctx);
+
+    int **ub_process**\ (struct ub_ctx\* ctx);
+
+    int **ub_resolve**\ (struct ub_ctx\* ctx, char\* name,
+        int rrtype, int rrclass, struct ub_result\*\* result);
+
+    int **ub_resolve_async**\ (struct ub_ctx\* ctx, char\* name,
+        int rrtype, int rrclass, void\* mydata,
+        ub_callback_type\* callback, int\* async_id);
+
+    int **ub_cancel**\ (struct ub_ctx\* ctx, int async_id);
+
+    void **ub_resolve_free**\ (struct ub_result\* result);
+
+    const char \* **ub_strerror**\ (int err);
+
+    int **ub_ctx_print_local_zones**\ (struct ub_ctx\* ctx);
+
+    int **ub_ctx_zone_add**\ (struct ub_ctx\* ctx, char\* zone_name, char\* zone_type);
+
+    int **ub_ctx_zone_remove**\ (struct ub_ctx\* ctx, char\* zone_name);
+
+    int **ub_ctx_data_add**\ (struct ub_ctx\* ctx, char\* data);
+
+    int **ub_ctx_data_remove**\ (struct ub_ctx\* ctx, char\* data);
+
+Description
+-----------
+
+Unbound is an implementation of a DNS resolver, that does caching and DNSSEC
+validation.
+This is the library API, for using the ``-lunbound`` library.
+The server daemon is described in :doc:`unbound(8)</manpages/unbound>`.
+The library works independent from a running unbound server, and can be used to
+convert hostnames to ip addresses, and back, and obtain other information from
+the DNS.
+The library performs public-key validation of results with DNSSEC.
+
+The library uses a variable of type *struct ub_ctx* to keep context between
+calls.
+The user must maintain it, creating it with **ub_ctx_create** and deleting it
+with **ub_ctx_delete**.
+It can be created and deleted at any time.
+Creating it anew removes any previous configuration (such as trusted keys) and
+clears any cached results.
+
+The functions are thread-safe, and a context can be used in a threaded (as well
+as in a non-threaded) environment.
+Also resolution (and validation) can be performed blocking and non-blocking
+(also called asynchronous).
+The async method returns from the call immediately, so that processing can go
+on, while the results become available later.
+
+The functions are discussed in turn below.
+
+Functions
+---------
+
+.. glossary::
+
+    ub_ctx_create
+        Create a new context, initialised with defaults.
+        The information from :file:`/etc/resolv.conf` and :file:`/etc/hosts` is
+        not utilised by default.
+        Use **ub_ctx_resolvconf** and **ub_ctx_hosts** to read them.
+        Before you call this, use the openssl functions
+        **CRYPTO_set_id_callback** and **CRYPTO_set_locking_callback** to set
+        up asynchronous operation if you use lib openssl (the application calls
+        these functions once for initialisation).
+        Openssl 1.0.0 or later uses the **CRYPTO_THREADID_set_callback**
+        function.
+
+    ub_ctx_delete
+        Delete validation context and free associated resources.
+        Outstanding async queries are killed and callbacks are not called for
+        them.
+
+    ub_ctx_set_option
+        A power-user interface that lets you specify one of the options from
+        the config file format, see :doc:`unbound.conf(5)</manpages/unbound.conf>`.
+        Not all options are relevant.
+        For some specific options, such as adding trust anchors, special
+        routines exist.
+        Pass the option name with the trailing ``':'``.
+
+    ub_ctx_get_option
+        A power-user interface that gets an option value.
+        Some options cannot be gotten, and others return a newline separated
+        list.
+        Pass the option name without trailing ``':'``.
+        The returned value must be free(2)d by the caller.
+
+    ub_ctx_config
+        A power-user interface that lets you specify an unbound config file,
+        see :doc:`unbound.conf(5)</manpages/unbound.conf>`, which is read for
+        configuration.
+        Not all options are relevant.
+        For some specific options, such as adding trust anchors, special
+        routines exist.
+        This function is thread-safe only if a single instance of **ub_ctx**\*
+        exists in the application.
+        If several instances exist the application has to ensure that
+        **ub_ctx_config** is not called in parallel by the different instances.
+
+    ub_ctx_set_fwd
+        Set machine to forward DNS queries to, the caching resolver to use.
+        IP4 or IP6 address.
+        Forwards all DNS requests to that machine, which is expected to run a
+        recursive resolver.
+        If the proxy is not DNSSEC capable, validation may fail.
+        Can be called several times, in that case the addresses are used as
+        backup servers.
+        At this time it is only possible to set configuration before the first
+        resolve is done.
+
+    ub_ctx_set_stub
+        Set a stub zone, authoritative dns servers to use for a particular
+        zone.
+        IP4 or IP6 address.
+        If the address is NULL the stub entry is removed.
+        Set isprime true if you configure root hints with it.
+        Otherwise similar to the stub zone item from unbound's config file.
+        Can be called several times, for different zones, or to add multiple
+        addresses for a particular zone.
+        At this time it is only possible to set configuration before the first
+        resolve is done.
+
+    ub_ctx_set_tls
+        Enable DNS over TLS (DoT) for machines set with **ub_ctx_set_fwd**.
+        At this time it is only possible to set configuration before the first
+        resolve is done.
+
+    ub_ctx_resolvconf
+        By default the root servers are queried and full resolver mode is used,
+        but you can use this call to read the list of nameservers to use from
+        the filename given.
+        Usually :file:`"/etc/resolv.conf"`.
+        Uses those nameservers as caching proxies.
+        If they do not support DNSSEC, validation may fail.
+        Only nameservers are picked up, the searchdomain, ndots and other
+        settings from *resolv.conf(5)* are ignored.
+        If fname NULL is passed, :file:`"/etc/resolv.conf"` is used (if on
+        Windows, the system-wide configured nameserver is picked instead).
+        At this time it is only possible to set configuration before the first
+        resolve is done.
+
+    ub_ctx_hosts
+        Read list of hosts from the filename given.
+        Usually :file:`"/etc/hosts"`.
+        When queried for, these addresses are not marked DNSSEC secure.
+        If fname NULL is passed, :file:`"/etc/hosts"` is used (if on Windows,
+        :file:`etc/hosts` from WINDIR is picked instead).
+        At this time it is only possible to set configuration before the first
+        resolve is done.
+
+    ub_ctx_add_ta
+        Add a trust anchor to the given context.
+        At this time it is only possible to add trusted keys before the first
+        resolve is done.
+        The format is a string, similar to the zone-file format,
+        **[domainname]** **[type]** **[rdata contents]**.
+        Both DS and DNSKEY records are accepted.
+
+    ub_ctx_add_ta_autr
+        Add filename with automatically tracked trust anchor to the given
+        context.
+        Pass name of a file with the managed trust anchor.
+        You can create this file with
+        :doc:`unbound-anchor(8)</manpages/unbound-anchor>` for the root anchor.
+        You can also create it with an initial file with one line with a DNSKEY
+        or DS record.
+        If the file is writable, it is updated when the trust anchor changes.
+        At this time it is only possible to add trusted keys before the first
+        resolve is done.
+
+    ub_ctx_add_ta_file
+        Add trust anchors to the given context.
+        Pass name of a file with DS and DNSKEY records in zone file format.
+        At this time it is only possible to add trusted keys before the first
+        resolve is done.
+
+    ub_ctx_trustedkeys
+        Add trust anchors to the given context.
+        Pass the name of a bind-style config file with ``trusted-keys{}``.
+        At this time it is only possible to add trusted keys before the first
+        resolve is done.
+
+    ub_ctx_debugout
+        Set debug and error log output to the given stream.
+        Pass NULL to disable output.
+        Default is stderr.
+        File-names or using syslog can be enabled using config options, this
+        routine is for using your own stream.
+
+    ub_ctx_debuglevel
+        Set debug verbosity for the context.
+        Output is directed to stderr.
+        Higher debug level gives more output.
+
+    ub_ctx_async
+        Set a context behaviour for asynchronous action.
+        if set to true, enables threading and a call to **ub_resolve_async**
+        creates a thread to handle work in the background.
+        If false, a process is forked to handle work in the background.
+        Changes to this setting after **ub_resolve_async** calls have been made
+        have no effect (delete and re-create the context to change).
+
+    ub_poll
+        Poll a context to see if it has any new results.
+        Do not poll in a loop, instead extract the **fd** below to poll for
+        readiness, and then check, or wait using the wait routine.
+        Returns 0 if nothing to read, or nonzero if a result is available.
+        If nonzero, call **ub_process** to do callbacks.
+
+    ub_wait
+        Wait for a context to finish with results.
+        Calls **ub_process** after the wait for you.
+        After the wait, there are no more outstanding asynchronous queries.
+
+    ub_fd 
+        Get file descriptor.
+        Wait for it to become readable, at this point answers are returned from
+        the asynchronous validating resolver.
+        Then call the **ub_process** to continue processing.
+
+    ub_process
+        Call this routine to continue processing results from the validating
+        resolver (when the **fd** becomes readable).
+        Will perform necessary callbacks.
+
+    ub_resolve
+        Perform resolution and validation of the target name.
+        The name is a domain name in a zero terminated text string.
+        The rrtype and rrclass are DNS type and class codes.
+        The result structure is newly allocated with the resulting data.
+
+    ub_resolve_async
+        Perform asynchronous resolution and validation of the target name.
+        Arguments mean the same as for **ub_resolve** except no data is
+        returned immediately, instead a callback is called later.
+        The callback receives a copy of the mydata pointer, that you can use to
+        pass information to the callback.
+        The callback type is a function pointer to a function declared as:
+
+        .. code-block:: c
+
+            void my_callback_function(void* my_arg, int err,
+                            struct ub_result* result);
+
+        The **async_id** is returned so you can (at your option) decide to
+        track it and cancel the request if needed.
+        If you pass a NULL pointer the **async_id** is not returned.
+
+    ub_cancel
+        Cancel an async query in progress.
+        This may return an error if the query does not exist, or the query is
+        already being delivered, in that case you may still get a callback for
+        the query.
+
+    ub_resolve_free
+        Free struct **ub_result** contents after use.
+
+    ub_strerror
+        Convert error value from one of the unbound library functions to a
+        human readable string.
+
+    ub_ctx_print_local_zones
+        Debug printout the local authority information to debug output.
+
+    ub_ctx_zone_add
+        Add new zone to local authority info, like local-zone
+        :doc:`unbound.conf(5)</manpages/unbound.conf>` statement.
+
+    ub_ctx_zone_remove
+        Delete zone from local authority info.
+
+    ub_ctx_data_add
+        Add resource record data to local authority info, like local-data
+        :doc:`unbound.conf(5)</manpages/unbound.conf>` statement.
+
+    ub_ctx_data_remove
+        Delete local authority data from the name given.
+
+Result Data structure
+---------------------
+
+The result of the DNS resolution and validation is returned as *struct
+ub_result*.
+The result structure contains the following entries:
+
+.. code-block:: c
+
+    struct ub_result {
+         char* qname;         /* text string, original question */
+         int qtype;           /* type code asked for */
+         int qclass;          /* class code asked for */
+         char** data;         /* array of rdata items, NULL terminated*/
+         int* len;            /* array with lengths of rdata items */
+         char* canonname;     /* canonical name of result */
+         int rcode;           /* additional error code in case of no data */
+         void* answer_packet; /* full network format answer packet */
+         int answer_len;      /* length of packet in octets */
+         int havedata;        /* true if there is data */
+         int nxdomain;        /* true if nodata because name does not exist */
+         int secure;          /* true if result is secure */
+         int bogus;           /* true if a security failure happened */
+         char* why_bogus;     /* string with error if bogus */
+         int was_ratelimited; /* true if the query was ratelimited (SERVFAIL) by unbound */
+         int ttl;             /* number of seconds the result is valid */
+    };
+
+If both secure and bogus are false, security was not enabled for the domain of
+the query.
+Else, they are not both true, one of them is true.
+
+Return Values
+-------------
+
+Many routines return an error code.
+The value 0 (zero) denotes no error happened.
+Other values can be passed to **ub_strerror** to obtain a readable error
+string.
+**ub_strerror** returns a zero terminated string.
+**ub_ctx_create** returns NULL on an error (a malloc failure).
+**ub_poll** returns true if some information may be available, false otherwise.
+**ub_fd** returns a file descriptor or -1 on error.
+**ub_ctx_config** and **ub_ctx_resolvconf** attempt to leave errno informative
+on a function return with file read failure.
+
+See Also
+--------
+
+:doc:`unbound.conf(5)</manpages/unbound.conf>`, :doc:`unbound(8)</manpages/unbound>`.

doc/unbound-anchor.8.in

@@ -1,189 +1,300 @@
-.TH "unbound-anchor" "8" "@date@" "NLnet Labs" "unbound @version@"
-.\"
-.\" unbound-anchor.8 -- unbound anchor maintenance utility manual
-.\"
-.\" Copyright (c) 2008, NLnet Labs. All rights reserved.
-.\"
-.\" See LICENSE for the license.
-.\"
-.\"
-.SH "NAME"
-.B unbound\-anchor
-\- Unbound anchor utility.
-.SH "SYNOPSIS"
-.B unbound\-anchor
-.RB [ opts ]
-.SH "DESCRIPTION"
-.B Unbound\-anchor
-performs setup or update of the root trust anchor for DNSSEC validation.
-The program fetches the trust anchor with the method from RFC7958 when
-regular RFC5011 update fails to bring it up to date.
+.\" Man page generated from reStructuredText.
+.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.TH "UNBOUND-ANCHOR" "8" "@date@" "@version@" "Unbound"
+.SH NAME
+unbound-anchor \- Unbound @version@ anchor utility.
+.SH SYNOPSIS
+.sp
+\fBunbound\-anchor\fP [\fBopts\fP]
+.SH DESCRIPTION
+.sp
+\fBunbound\-anchor\fP performs setup or update of the root trust anchor for DNSSEC
+validation.
+The program fetches the trust anchor with the method from \fI\%RFC 7958\fP when
+regular \fI\%RFC 5011\fP update fails to bring it up to date.
 It can be run (as root) from the commandline, or run as part of startup
-scripts.  Before you start the \fIunbound\fR(8) DNS server.
-.P
+scripts.
+Before you start the \fI\%unbound(8)\fP DNS server.
+.sp
 Suggested usage:
-.P
+.INDENT 0.0
+.INDENT 3.5
+.sp
 .nf
-	# in the init scripts.
-	# provide or update the root anchor (if necessary)
-	unbound-anchor \-a "@UNBOUND_ROOTKEY_FILE@"
-	# Please note usage of this root anchor is at your own risk
-	# and under the terms of our LICENSE (see source).
-	#
-	# start validating resolver
-	# the unbound.conf contains:
-	#   auto-trust-anchor-file: "@UNBOUND_ROOTKEY_FILE@"
-	unbound \-c unbound.conf
+.ft C
+# in the init scripts.
+# provide or update the root anchor (if necessary)
+unbound\-anchor \-a \(dq@UNBOUND_ROOTKEY_FILE@\(dq
+# Please note usage of this root anchor is at your own risk
+# and under the terms of our LICENSE (see source).
+#
+# start validating resolver
+# the unbound.conf contains:
+# auto\-trust\-anchor\-file: \(dq@UNBOUND_ROOTKEY_FILE@\(dq
+unbound \-c unbound.conf
+.ft P
 .fi
-.P
-This tool provides builtin default contents for the root anchor and root
-update certificate files.
-.P
+.UNINDENT
+.UNINDENT
+.sp
+This tool provides builtin default contents for the root anchor and root update
+certificate files.
+.sp
 It tests if the root anchor file works, and if not, and an update is possible,
 attempts to update the root anchor using the root update certificate.
-It performs a https fetch of root-anchors.xml and checks the results (RFC7958),
-if all checks are successful, it updates the root anchor file.  Otherwise
-the root anchor file is unchanged.  It performs RFC5011 tracking if the
-DNSSEC information available via the DNS makes that possible.
-.P
-It does not perform an update if the certificate is expired, if the network
-is down or other errors occur.
-.P
+It performs a https fetch of
+\fI\%root\-anchors.xml\fP
+and checks the results (\fI\%RFC 7958\fP); if all checks are successful, it updates
+the root anchor file.
+Otherwise the root anchor file is unchanged.
+It performs \fI\%RFC 5011\fP tracking if the DNSSEC information available via the
+DNS makes that possible.
+.sp
+It does not perform an update if the certificate is expired, if the network is
+down or other errors occur.
+.sp
 The available options are:
+.INDENT 0.0
 .TP
-.B \-a \fIfile
+.B \-a <file>
 The root anchor key file, that is read in and written out.
-Default is @UNBOUND_ROOTKEY_FILE@.
-If the file does not exist, or is empty, a builtin root key is written to it.
+Default is \fB@UNBOUND_ROOTKEY_FILE@\fP\&.
+If the file does not exist, or is empty, a builtin root key is written
+to it.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-c \fIfile
+.B \-c <file>
 The root update certificate file, that is read in.
-Default is @UNBOUND_ROOTCERT_FILE@.
+Default is \fB@UNBOUND_ROOTCERT_FILE@\fP\&.
 If the file does not exist, or is empty, a builtin certificate is used.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-l
 List the builtin root key and builtin root update certificate on stdout.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-u \fIname
-The server name, it connects to https://name.  Specify without https:// prefix.
-The default is "data.iana.org".  It connects to the port specified with \-P.
+.B \-u <name>
+The server name, it connects to \fBhttps://name\fP\&.
+Specify without \fBhttps://\fP prefix.
+The default is \fB\(dqdata.iana.org\(dq\fP\&.
+It connects to the port specified with \fI\%\-P\fP\&.
 You can pass an IPv4 address or IPv6 address (no brackets) if you want.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-S
-Do not use SNI for the HTTPS connection.  Default is to use SNI.
+Do not use SNI for the HTTPS connection.
+Default is to use SNI.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-b \fIaddress
-The source address to bind to for domain resolution and contacting the server
-on https.  May be either an IPv4 address or IPv6 address (no brackets).
+.B \-b <address>
+The source address to bind to for domain resolution and contacting the
+server on https.
+May be either an IPv4 address or IPv6 address (no brackets).
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-x \fIpath
-The pathname to the root\-anchors.xml file on the server. (forms URL with \-u).
-The default is /root\-anchors/root\-anchors.xml.
+.B \-x <path>
+The pathname to the root\-anchors.xml file on the server.
+(forms URL with \fI\%\-u\fP).
+The default is \fB/root\-anchors/root\-anchors.xml\fP\&.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-s \fIpath
-The pathname to the root\-anchors.p7s file on the server. (forms URL with \-u).
-The default is /root\-anchors/root\-anchors.p7s.  This file has to be a PKCS7
-signature over the xml file, using the pem file (\-c) as trust anchor.
+.B \-s <path>
+The pathname to the root\-anchors.p7s file on the server.
+(forms URL with \fI\%\-u\fP).
+The default is \fB/root\-anchors/root\-anchors.p7s\fP\&.
+This file has to be a PKCS7 signature over the xml file, using the pem
+file (\fI\%\-c\fP) as trust anchor.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-n \fIname
-The emailAddress for the Subject of the signer's certificate from the p7s
-signature file.  Only signatures from this name are allowed.  default is
-dnssec@iana.org.  If you pass "" then the emailAddress is not checked.
+.B \-n <name>
+The emailAddress for the Subject of the signer\(aqs certificate from the
+p7s signature file.
+Only signatures from this name are allowed.
+The default is \fBdnssec@iana.org\fP\&.
+If you pass \fB\(dq\(dq\fP then the emailAddress is not checked.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-4
-Use IPv4 for domain resolution and contacting the server on https.  Default is
-to use IPv4 and IPv6 where appropriate.
+Use IPv4 for domain resolution and contacting the server on
+https.
+Default is to use IPv4 and IPv6 where appropriate.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-6
-Use IPv6 for domain resolution and contacting the server on https.  Default is
-to use IPv4 and IPv6 where appropriate.
+Use IPv6 for domain resolution and contacting the server on https.
+Default is to use IPv4 and IPv6 where appropriate.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-f \fIresolv.conf
-Use the given resolv.conf file.  Not enabled by default, but you could try to
-pass /etc/resolv.conf on some systems.  It contains the IP addresses of the
-recursive nameservers to use.  However, since this tool could be used to
-bootstrap that very recursive nameserver, it would not be useful (since
-that server is not up yet, since we are bootstrapping it).  It could be
-useful in a situation where you know an upstream cache is deployed (and
-running) and in captive portal situations.
+.B \-f <resolv.conf>
+Use the given resolv.conf file.
+Not enabled by default, but you could try to pass
+\fB/etc/resolv.conf\fP on some systems.
+It contains the IP addresses of the recursive nameservers to use.
+However, since this tool could be used to bootstrap that very recursive
+nameserver, it would not be useful (since that server is not up yet,
+since we are bootstrapping it).
+It could be useful in a situation where you know an upstream cache is
+deployed (and running) and in captive portal situations.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-r \fIroot.hints
-Use the given root.hints file (same syntax as the BIND and Unbound root hints
-file) to bootstrap domain resolution.  By default a list of builtin root
-hints is used.  Unbound\-anchor goes to the network itself for these roots,
-to resolve the server (\-u option) and to check the root DNSKEY records.
+.B \-r <root.hints>
+Use the given root.hints file (same syntax as the BIND and Unbound root
+hints file) to bootstrap domain resolution.
+By default a list of builtin root hints is used.
+unbound\-anchor goes to the network itself for these roots, to resolve
+the server (\fI\%\-u\fP option) and to check the root DNSKEY records.
 It does so, because the tool when used for bootstrapping the recursive
-resolver, cannot use that recursive resolver itself because it is bootstrapping
-that server.
+resolver, cannot use that recursive resolver itself because it is
+bootstrapping that server.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-R
-Allow fallback from \-f resolv.conf file to direct root servers query.
-It allows you to prefer local resolvers, but fallback automatically
-to direct root query if they do not respond or do not support DNSSEC.
+Allow fallback from \fI\%\-f\fP \fB<resolv.conf>\fP file to direct root
+servers query.
+It allows you to prefer local resolvers, but fallback automatically to
+direct root query if they do not respond or do not support DNSSEC.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-v
-More verbose. Once prints informational messages, multiple times may enable
-large debug amounts (such as full certificates or byte\-dumps of downloaded
-files).  By default it prints almost nothing.  It also prints nothing on
-errors by default; in that case the original root anchor file is simply
-left undisturbed, so that a recursive server can start right after it.
+More verbose.
+Once prints informational messages, multiple times may enable large
+debug amounts (such as full certificates or byte\-dumps of downloaded
+files).
+By default it prints almost nothing.
+It also prints nothing on errors by default; in that case the original
+root anchor file is simply left undisturbed, so that a recursive server
+can start right after it.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-C \fIunbound.conf
-Debug option to read unbound.conf into the resolver process used.
+.B \-C <unbound.conf>
+Debug option to read \fB<unbound.conf>\fP into the resolver process
+used.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-P \fIport
-Set the port number to use for the https connection.  The default is 443.
+.B \-P <port>
+Set the port number to use for the https connection.
+The default is 443.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-F
-Debug option to force update of the root anchor through downloading the xml
-file and verifying it with the certificate.  By default it first tries to
-update by contacting the DNS, which uses much less bandwidth, is much
-faster (200 msec not 2 sec), and is nicer to the deployed infrastructure.
-With this option, it still attempts to do so (and may verbosely tell you),
-but then ignores the result and goes on to use the xml fallback method.
+Debug option to force update of the root anchor through downloading the
+xml file and verifying it with the certificate.
+By default it first tries to update by contacting the DNS, which uses
+much less bandwidth, is much faster (200 msec not 2 sec), and is nicer
+to the deployed infrastructure.
+With this option, it still attempts to do so (and may verbosely tell
+you), but then ignores the result and goes on to use the xml fallback
+method.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-h
 Show the version and commandline option help.
-.SH "EXIT CODE"
+.UNINDENT
+.SH EXIT CODE
+.sp
 This tool exits with value 1 if the root anchor was updated using the
-certificate or if the builtin root-anchor was used.  It exits with code
-0 if no update was necessary, if the update was possible with RFC5011
-tracking, or if an error occurred.
-.P
+certificate or if the builtin root\-anchor was used.
+It exits with code 0 if no update was necessary, if the update was possible
+with \fI\%RFC 5011\fP tracking, or if an error occurred.
+.sp
 You can check the exit value in this manner:
+.INDENT 0.0
+.INDENT 3.5
+.sp
 .nf
-	unbound-anchor \-a "root.key" || logger "Please check root.key"
+.ft C
+unbound\-anchor \-a \(dqroot.key\(dq || logger \(dqPlease check root.key\(dq
+.ft P
 .fi
+.UNINDENT
+.UNINDENT
+.sp
 Or something more suitable for your operational environment.
-.SH "TRUST"
-The root keys and update certificate included in this tool
-are provided for convenience and under the terms of our
-license (see the LICENSE file in the source distribution or
-https://github.com/NLnetLabs/unbound/blob/master/LICENSE) and might be stale or
-not suitable to your purpose.
-.P
-By running "unbound\-anchor \-l" the  keys and certificate that are
+.SH TRUST
+.sp
+The root keys and update certificate included in this tool are provided for
+convenience and under the terms of our license (see the LICENSE file in the
+source distribution or \fI\%https://github.com/NLnetLabs/unbound/blob/master/LICENSE\fP
+and might be stale or not suitable to your purpose.
+.sp
+By running \fI\%unbound\-anchor \-l\fP the keys and certificate that are
 configured in the code are printed for your convenience.
-.P
-The build\-in configuration can be overridden by providing a root\-cert
-file and a rootkey file.
-.SH "FILES"
+.sp
+The built\-in configuration can be overridden by providing a root\-cert file and
+a rootkey file.
+.SH FILES
+.INDENT 0.0
 .TP
-.I @UNBOUND_ROOTKEY_FILE@
-The root anchor file, updated with 5011 tracking, and read and written to.
+.B @UNBOUND_ROOTKEY_FILE@
+The root anchor file, updated with 5011 tracking, and read and written
+to.
 The file is created if it does not exist.
 .TP
-.I @UNBOUND_ROOTCERT_FILE@
-The trusted self\-signed certificate that is used to verify the downloaded
-DNSSEC root trust anchor.  You can update it by fetching it from
-https://data.iana.org/root\-anchors/icannbundle.pem (and validate it).
+.B @UNBOUND_ROOTCERT_FILE@
+The trusted self\-signed certificate that is used to verify the
+downloaded DNSSEC root trust anchor.
+You can update it by fetching it from
+\fI\%https://data.iana.org/root\-anchors/icannbundle.pem\fP (and validate it).
 If the file does not exist or is empty, a builtin version is used.
 .TP
-.I https://data.iana.org/root\-anchors/root\-anchors.xml
+.B \fI\%https://data.iana.org/root\-anchors/root\-anchors.xml\fP
 Source for the root key information.
 .TP
-.I https://data.iana.org/root\-anchors/root\-anchors.p7s
+.B \fI\%https://data.iana.org/root\-anchors/root\-anchors.p7s\fP
 Signature on the root key information.
-.SH "SEE ALSO"
-\fIunbound.conf\fR(5),
-\fIunbound\fR(8).
+.UNINDENT
+.SH SEE ALSO
+.sp
+\fI\%unbound.conf(5)\fP,
+\fI\%unbound(8)\fP\&.
+.SH AUTHOR
+Unbound developers are mentioned in the CREDITS file in the distribution.
+.SH COPYRIGHT
+1999-2025, NLnet Labs
+.\" Generated by docutils manpage writer.
+.

doc/unbound-anchor.rst

@@ -0,0 +1,281 @@
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+.. program:: unbound-anchor
+
+unbound-anchor(8)
+=================
+
+Synopsis
+--------
+
+**unbound-anchor** [``opts``]
+
+Description
+-----------
+
+``unbound-anchor`` performs setup or update of the root trust anchor for DNSSEC
+validation.
+The program fetches the trust anchor with the method from :rfc:`7958` when
+regular :rfc:`5011` update fails to bring it up to date.
+It can be run (as root) from the commandline, or run as part of startup
+scripts.
+Before you start the :doc:`unbound(8)</manpages/unbound>` DNS server.
+
+Suggested usage:
+
+.. code-block:: text
+
+   # in the init scripts.
+   # provide or update the root anchor (if necessary)
+   unbound-anchor -a "@UNBOUND_ROOTKEY_FILE@"
+   # Please note usage of this root anchor is at your own risk
+   # and under the terms of our LICENSE (see source).
+   #
+   # start validating resolver
+   # the unbound.conf contains:
+   # auto-trust-anchor-file: "@UNBOUND_ROOTKEY_FILE@"
+   unbound -c unbound.conf
+
+This tool provides builtin default contents for the root anchor and root update
+certificate files.
+
+It tests if the root anchor file works, and if not, and an update is possible,
+attempts to update the root anchor using the root update certificate.
+It performs a https fetch of
+`root-anchors.xml <http://data.iana.org/root-anchors/root-anchors.xml>`__
+and checks the results (:rfc:`7958`); if all checks are successful, it updates
+the root anchor file.
+Otherwise the root anchor file is unchanged.
+It performs :rfc:`5011` tracking if the DNSSEC information available via the
+DNS makes that possible.
+
+It does not perform an update if the certificate is expired, if the network is
+down or other errors occur.
+
+The available options are:
+
+.. option:: -a <file>
+
+       The root anchor key file, that is read in and written out.
+       Default is :file:`@UNBOUND_ROOTKEY_FILE@`.
+       If the file does not exist, or is empty, a builtin root key is written
+       to it.
+
+.. option:: -c <file>
+
+       The root update certificate file, that is read in.
+       Default is :file:`@UNBOUND_ROOTCERT_FILE@`.
+       If the file does not exist, or is empty, a builtin certificate is used.
+
+.. option:: -l
+
+       List the builtin root key and builtin root update certificate on stdout.
+
+.. option:: -u <name>
+
+       The server name, it connects to ``https://name``.
+       Specify without ``https://`` prefix.
+       The default is ``"data.iana.org"``.
+       It connects to the port specified with :option:`-P`.
+       You can pass an IPv4 address or IPv6 address (no brackets) if you want.
+
+.. option:: -S
+
+       Do not use SNI for the HTTPS connection.
+       Default is to use SNI.
+
+.. option:: -b <address>
+
+       The source address to bind to for domain resolution and contacting the
+       server on https.
+       May be either an IPv4 address or IPv6 address (no brackets).
+
+.. option:: -x <path>
+
+       The pathname to the root-anchors.xml file on the server.
+       (forms URL with :option:`-u`).
+       The default is :file:`/root-anchors/root-anchors.xml`.
+
+.. option:: -s <path>
+
+       The pathname to the root-anchors.p7s file on the server.
+       (forms URL with :option:`-u`).
+       The default is :file:`/root-anchors/root-anchors.p7s`.
+       This file has to be a PKCS7 signature over the xml file, using the pem
+       file (:option:`-c`) as trust anchor.
+
+.. option:: -n <name>
+
+       The emailAddress for the Subject of the signer's certificate from the
+       p7s signature file.
+       Only signatures from this name are allowed.
+       The default is ``dnssec@iana.org``.
+       If you pass ``""`` then the emailAddress is not checked.
+
+.. option:: -4
+
+       Use IPv4 for domain resolution and contacting the server on
+       https.
+       Default is to use IPv4 and IPv6 where appropriate.
+
+.. option:: -6
+
+       Use IPv6 for domain resolution and contacting the server on https.
+       Default is to use IPv4 and IPv6 where appropriate.
+
+.. option:: -f <resolv.conf>
+
+       Use the given resolv.conf file.
+       Not enabled by default, but you could try to pass
+       :file:`/etc/resolv.conf` on some systems.
+       It contains the IP addresses of the recursive nameservers to use.
+       However, since this tool could be used to bootstrap that very recursive
+       nameserver, it would not be useful (since that server is not up yet,
+       since we are bootstrapping it).
+       It could be useful in a situation where you know an upstream cache is
+       deployed (and running) and in captive portal situations.
+
+.. option:: -r <root.hints>
+
+       Use the given root.hints file (same syntax as the BIND and Unbound root
+       hints file) to bootstrap domain resolution.
+       By default a list of builtin root hints is used.
+       unbound-anchor goes to the network itself for these roots, to resolve
+       the server (:option:`-u` option) and to check the root DNSKEY records.
+       It does so, because the tool when used for bootstrapping the recursive
+       resolver, cannot use that recursive resolver itself because it is
+       bootstrapping that server.
+
+.. option:: -R
+
+       Allow fallback from :option:`-f` ``<resolv.conf>`` file to direct root
+       servers query.
+       It allows you to prefer local resolvers, but fallback automatically to
+       direct root query if they do not respond or do not support DNSSEC.
+
+.. option:: -v
+
+       More verbose.
+       Once prints informational messages, multiple times may enable large
+       debug amounts (such as full certificates or byte-dumps of downloaded
+       files).
+       By default it prints almost nothing.
+       It also prints nothing on errors by default; in that case the original
+       root anchor file is simply left undisturbed, so that a recursive server
+       can start right after it.
+
+.. option:: -C <unbound.conf>
+
+       Debug option to read :file:`<unbound.conf>` into the resolver process
+       used.
+
+.. option:: -P <port>
+
+       Set the port number to use for the https connection.
+       The default is 443.
+
+.. option:: -F
+
+       Debug option to force update of the root anchor through downloading the
+       xml file and verifying it with the certificate.
+       By default it first tries to update by contacting the DNS, which uses
+       much less bandwidth, is much faster (200 msec not 2 sec), and is nicer
+       to the deployed infrastructure.
+       With this option, it still attempts to do so (and may verbosely tell
+       you), but then ignores the result and goes on to use the xml fallback
+       method.
+
+.. option:: -h
+
+       Show the version and commandline option help.
+
+Exit Code
+---------
+
+This tool exits with value 1 if the root anchor was updated using the
+certificate or if the builtin root-anchor was used.
+It exits with code 0 if no update was necessary, if the update was possible
+with :rfc:`5011` tracking, or if an error occurred.
+
+You can check the exit value in this manner:
+
+.. code-block:: text
+
+       unbound-anchor -a "root.key" || logger "Please check root.key"
+
+Or something more suitable for your operational environment.
+
+Trust
+-----
+
+The root keys and update certificate included in this tool are provided for
+convenience and under the terms of our license (see the LICENSE file in the
+source distribution or https://github.com/NLnetLabs/unbound/blob/master/LICENSE
+and might be stale or not suitable to your purpose.
+
+By running :option:`unbound-anchor -l` the keys and certificate that are
+configured in the code are printed for your convenience.
+
+The built-in configuration can be overridden by providing a root-cert file and
+a rootkey file.
+
+Files
+-----
+
+@UNBOUND_ROOTKEY_FILE@
+       The root anchor file, updated with 5011 tracking, and read and written
+       to.
+       The file is created if it does not exist.
+
+@UNBOUND_ROOTCERT_FILE@
+       The trusted self-signed certificate that is used to verify the
+       downloaded DNSSEC root trust anchor.
+       You can update it by fetching it from
+       https://data.iana.org/root-anchors/icannbundle.pem (and validate it).
+       If the file does not exist or is empty, a builtin version is used.
+
+https://data.iana.org/root-anchors/root-anchors.xml
+       Source for the root key information.
+
+https://data.iana.org/root-anchors/root-anchors.p7s
+       Signature on the root key information.
+
+See Also
+--------
+
+:doc:`unbound.conf(5)</manpages/unbound.conf>`,
+:doc:`unbound(8)</manpages/unbound>`.

doc/unbound-checkconf.8.in

@@ -1,52 +1,93 @@
-.TH "unbound-checkconf" "8" "@date@" "NLnet Labs" "unbound @version@"
-.\"
-.\" unbound-checkconf.8 -- unbound configuration checker manual
-.\"
-.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
-.\"
-.\" See LICENSE for the license.
-.\"
-.\"
-.SH "NAME"
-unbound\-checkconf
-\- Check Unbound configuration file for errors.
-.SH "SYNOPSIS"
-.B unbound\-checkconf
-.RB [ \-h ]
-.RB [ \-f ]
-.RB [ \-o
-.IR option ]
-.RI [ cfgfile ]
-.SH "DESCRIPTION"
-.B Unbound\-checkconf
-checks the configuration file for the
-\fIunbound\fR(8)
-DNS resolver for syntax and other errors.
+.\" Man page generated from reStructuredText.
+.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.TH "UNBOUND-CHECKCONF" "8" "@date@" "@version@" "Unbound"
+.SH NAME
+unbound-checkconf \- Check Unbound @version@ configuration file for errors.
+.SH SYNOPSIS
+.sp
+\fBunbound\-checkconf\fP [\fB\-hf\fP] [\fB\-o option\fP] [cfgfile]
+.SH DESCRIPTION
+.sp
+\fBunbound\-checkconf\fP checks the configuration file for the
+\fI\%unbound(8)\fP DNS resolver for syntax and other errors.
 The config file syntax is described in
-\fIunbound.conf\fR(5).
-.P
+\fI\%unbound.conf(5)\fP\&.
+.sp
 The available options are:
+.INDENT 0.0
 .TP
 .B \-h
 Show the version and commandline option help.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-f
-Print full pathname, with chroot applied to it.  Use with the \-o option.
+Print full pathname, with chroot applied to it.
+Use with the \fI\%\-o\fP option.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-o\fI option
+.B \-q
+Make the operation quiet, suppress output on success.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-o <option>
 If given, after checking the config file the value of this option is
-printed to stdout.  For "" (disabled) options an empty line is printed.
+printed to stdout.
+For \fB\(dq\(dq\fP (disabled) options an empty line is printed.
+.UNINDENT
+.INDENT 0.0
 .TP
-.I cfgfile
-The config file to read with settings for Unbound. It is checked.
+.B cfgfile
+The config file to read with settings for Unbound.
+It is checked.
 If omitted, the config file at the default location is checked.
-.SH "EXIT CODE"
-The unbound\-checkconf program exits with status code 1 on error,
-0 for a correct config file.
-.SH "FILES"
+.UNINDENT
+.SH EXIT CODE
+.sp
+The \fBunbound\-checkconf\fP program exits with status code 1 on error, 0 for a
+correct config file.
+.SH FILES
+.INDENT 0.0
 .TP
-.I @ub_conf_file@
+.B @ub_conf_file@
 Unbound configuration file.
-.SH "SEE ALSO"
-\fIunbound.conf\fR(5),
-\fIunbound\fR(8).
+.UNINDENT
+.SH SEE ALSO
+.sp
+\fI\%unbound.conf(5)\fP,
+\fI\%unbound(8)\fP\&.
+.SH AUTHOR
+Unbound developers are mentioned in the CREDITS file in the distribution.
+.SH COPYRIGHT
+1999-2025, NLnet Labs
+.\" Generated by docutils manpage writer.
+.

doc/unbound-checkconf.rst

@@ -0,0 +1,98 @@
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+.. program:: unbound-checkconf
+
+unbound-checkconf(8)
+====================
+
+Synopsis
+--------
+
+**unbound-checkconf** [``-hf``] [``-o option``] [cfgfile]
+
+Description
+-----------
+
+``unbound-checkconf`` checks the configuration file for the
+:doc:`unbound(8)</manpages/unbound>` DNS resolver for syntax and other errors.
+The config file syntax is described in
+:doc:`unbound.conf(5)</manpages/unbound.conf>`.
+
+The available options are:
+
+.. option:: -h
+
+    Show the version and commandline option help.
+
+.. option:: -f
+
+    Print full pathname, with chroot applied to it.
+    Use with the :option:`-o` option.
+
+.. option:: -q
+
+    Make the operation quiet, suppress output on success.
+
+.. option:: -o <option>
+
+    If given, after checking the config file the value of this option is
+    printed to stdout.
+    For ``""`` (disabled) options an empty line is printed.
+
+.. option:: cfgfile
+
+    The config file to read with settings for Unbound.
+    It is checked.
+    If omitted, the config file at the default location is checked.
+
+Exit Code
+---------
+
+The ``unbound-checkconf`` program exits with status code 1 on error, 0 for a
+correct config file.
+
+Files
+-----
+
+@ub_conf_file@
+    Unbound configuration file.
+
+See Also
+--------
+
+:doc:`unbound.conf(5)</manpages/unbound.conf>`,
+:doc:`unbound(8)</manpages/unbound>`.

doc/unbound-host.1.in

@@ -1,118 +1,190 @@
-.TH "unbound\-host" "1" "@date@" "NLnet Labs" "unbound @version@"
-.\"
-.\" unbound-host.1 -- unbound DNS lookup utility
-.\"
-.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
-.\"
-.\" See LICENSE for the license.
-.\"
-.\"
-.SH "NAME"
-.B unbound\-host
-\- unbound DNS lookup utility
-.SH "SYNOPSIS"
-.B unbound\-host
-.RB [ \-C
-.IR configfile ]
-.RB [ \-vdhr46D ]
-.RB [ \-c
-.IR class ]
-.RB [ \-t
-.IR type ]
-.RB [ \-y
-.IR key ]
-.RB [ \-f
-.IR keyfile ]
-.RB [ \-F
-.IR namedkeyfile ]
-.I hostname
-.SH "DESCRIPTION"
-.B Unbound\-host
-uses the Unbound validating resolver to query for the hostname and display
-results. With the \fB\-v\fR option it displays validation
-status: secure, insecure, bogus (security failure).
-.P
-By default it reads no configuration file whatsoever.  It attempts to reach
-the internet root servers.  With \fB\-C\fR an Unbound config file and with
-\fB\-r\fR resolv.conf can be read.
-.P
+.\" Man page generated from reStructuredText.
+.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.TH "UNBOUND-HOST" "1" "@date@" "@version@" "Unbound"
+.SH NAME
+unbound-host \- Unbound @version@ DNS lookup utility.
+.SH SYNOPSIS
+.sp
+\fBunbound\-host\fP [\fB\-C configfile\fP] [\fB\-vdhr46D\fP] [\fB\-c class\fP]
+[\fB\-t type\fP] [\fB\-y key\fP] [\fB\-f keyfile\fP] [\fB\-F namedkeyfile\fP] hostname
+.SH DESCRIPTION
+.sp
+\fBunbound\-host\fP uses the Unbound validating resolver to query for the hostname
+and display results.
+With the \fI\%\-v\fP option it displays validation status: secure, insecure,
+bogus (security failure).
+.sp
+By default it reads no configuration file whatsoever.
+It attempts to reach the internet root servers.
+With \fI\%\-C\fP an unbound config file and with \fI\%\-r\fP \fBresolv.conf\fP
+can be read.
+.sp
 The available options are:
+.INDENT 0.0
 .TP
-.I hostname
+.B hostname
 This name is resolved (looked up in the DNS).
 If a IPv4 or IPv6 address is given, a reverse lookup is performed.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-h
 Show the version and commandline option help.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-v
 Enable verbose output and it shows validation results, on every line.
-Secure means that the NXDOMAIN (no such domain name), nodata (no such data)
-or positive data response validated correctly with one of the keys.
+Secure means that the NXDOMAIN (no such domain name), nodata (no such
+data) or positive data response validated correctly with one of the
+keys.
 Insecure means that that domain name has no security set up for it.
-Bogus (security failure) means that the response failed one or more checks,
-it is likely wrong, outdated, tampered with, or broken.
+Bogus (security failure) means that the response failed one or more
+checks, it is likely wrong, outdated, tampered with, or broken.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-d
-Enable debug output to stderr. One \-d shows what the resolver and validator
-are doing and may tell you what is going on. More times, \-d \-d, gives a
-lot of output, with every packet sent and received.
+Enable debug output to stderr.
+One \fI\%\-d\fP shows what the resolver and validator are doing and may
+tell you what is going on.
+More times, \fI\%\-d\fP \fI\%\-d\fP, gives a lot of output, with every
+packet sent and received.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-c \fIclass
-Specify the class to lookup for, the default is IN the internet class.
+.B \-c <class>
+Specify the class to lookup for, the default is IN the internet
+class.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-t \fItype
-Specify the type of data to lookup. The default looks for IPv4, IPv6 and
-mail handler data, or domain name pointers for reverse queries.
+.B \-t <type>
+Specify the type of data to lookup.
+The default looks for IPv4, IPv6 and mail handler data, or domain name
+pointers for reverse queries.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-y \fIkey
-Specify a public key to use as trust anchor. This is the base for a chain
-of trust that is built up from the trust anchor to the response, in order
-to validate the response message. Can be given as a DS or DNSKEY record.
-For example \-y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD".
+.B \-y <key>
+Specify a public key to use as trust anchor.
+This is the base for a chain of trust that is built up from the trust
+anchor to the response, in order to validate the response message.
+Can be given as a DS or DNSKEY record.
+For example:
+.INDENT 7.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+\-y \(dqexample.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD\(dq
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-D
-Enables DNSSEC validation.  Reads the root anchor from the default configured
-root anchor at the default location, \fI@UNBOUND_ROOTKEY_FILE@\fR.
+Enables DNSSEC validation.
+Reads the root anchor from the default configured root anchor at the
+default location, \fB@UNBOUND_ROOTKEY_FILE@\fP\&.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-f \fIkeyfile
-Reads keys from a file. Every line has a DS or DNSKEY record, in the format
-as for \-y. The zone file format, the same as dig and drill produce.
+.B \-f <keyfile>
+Reads keys from a file.
+Every line has a DS or DNSKEY record, in the format as for \fI\%\-y\fP\&.
+The zone file format, the same as \fBdig\fP and \fBdrill\fP produce.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-F \fInamedkeyfile
-Reads keys from a BIND\-style named.conf file. Only the trusted\-key {}; entries
-are read.
+.B \-F <namedkeyfile>
+Reads keys from a BIND\-style \fBnamed.conf\fP file.
+Only the \fBtrusted\-key {};\fP entries are read.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-C \fIconfigfile
-Uses the specified unbound.conf to prime
-.IR libunbound (3).
+.B \-C <configfile>
+Uses the specified unbound.conf to prime \fI\%libunbound(3)\fP\&.
 Pass it as first argument if you want to override some options from the
 config file with further arguments on the commandline.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-r
-Read /etc/resolv.conf, and use the forward DNS servers from there (those could
-have been set by DHCP).  More info in
-.IR resolv.conf (5).
+Read \fB/etc/resolv.conf\fP, and use the forward DNS servers from
+there (those could have been set by DHCP).
+More info in \fIresolv.conf(5)\fP\&.
 Breaks validation if those servers do not support DNSSEC.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-4
 Use solely the IPv4 network for sending packets.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-6
 Use solely the IPv6 network for sending packets.
-.SH "EXAMPLES"
-Some examples of use. The keys shown below are fakes, thus a security failure
-is encountered.
-.P
+.UNINDENT
+.SH EXAMPLES
+.sp
+Some examples of use.
+The keys shown below are fakes, thus a security failure is encountered.
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
 $ unbound\-host www.example.com
-.P
-$ unbound\-host \-v \-y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD" www.example.com
-.P
-$ unbound\-host \-v \-y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD" 192.0.2.153
-.SH "EXIT CODE"
-The unbound\-host program exits with status code 1 on error,
-0 on no error. The data may not be available on exit code 0, exit code 1
-means the lookup encountered a fatal error.
-.SH "SEE ALSO"
-\fIunbound.conf\fR(5),
-\fIunbound\fR(8).
+
+$ unbound\-host \-v \-y \(dqexample.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD\(dq www.example.com
+
+$ unbound\-host \-v \-y \(dqexample.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD\(dq 192.0.2.153
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.SH EXIT CODE
+.sp
+The \fBunbound\-host\fP program exits with status code 1 on error, 0 on no error.
+The data may not be available on exit code 0, exit code 1 means the lookup
+encountered a fatal error.
+.SH SEE ALSO
+.sp
+\fI\%unbound.conf(5)\fP,
+\fI\%unbound(8)\fP\&.
+.SH AUTHOR
+Unbound developers are mentioned in the CREDITS file in the distribution.
+.SH COPYRIGHT
+1999-2025, NLnet Labs
+.\" Generated by docutils manpage writer.
+.

doc/unbound-host.rst

@@ -0,0 +1,176 @@
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+.. program:: unbound-host
+
+unbound-host(1)
+===============
+
+Synopsis
+--------
+
+**unbound-host** [``-C configfile``] [``-vdhr46D``] [``-c class``]
+[``-t type``] [``-y key``] [``-f keyfile``] [``-F namedkeyfile``] hostname
+
+Description
+-----------
+
+``unbound-host`` uses the Unbound validating resolver to query for the hostname
+and display results.
+With the :option:`-v` option it displays validation status: secure, insecure,
+bogus (security failure).
+
+By default it reads no configuration file whatsoever.
+It attempts to reach the internet root servers.
+With :option:`-C` an unbound config file and with :option:`-r` ``resolv.conf``
+can be read.
+
+The available options are:
+
+.. option:: hostname
+
+       This name is resolved (looked up in the DNS).
+       If a IPv4 or IPv6 address is given, a reverse lookup is performed.
+
+.. option:: -h
+
+       Show the version and commandline option help.
+
+.. option:: -v
+
+       Enable verbose output and it shows validation results, on every line.
+       Secure means that the NXDOMAIN (no such domain name), nodata (no such
+       data) or positive data response validated correctly with one of the
+       keys.
+       Insecure means that that domain name has no security set up for it.
+       Bogus (security failure) means that the response failed one or more
+       checks, it is likely wrong, outdated, tampered with, or broken.
+
+.. option:: -d
+
+       Enable debug output to stderr.
+       One :option:`-d` shows what the resolver and validator are doing and may
+       tell you what is going on.
+       More times, :option:`-d` :option:`-d`, gives a lot of output, with every
+       packet sent and received.
+
+.. option:: -c <class>
+
+       Specify the class to lookup for, the default is IN the internet
+       class.
+
+.. option:: -t <type>
+
+       Specify the type of data to lookup.
+       The default looks for IPv4, IPv6 and mail handler data, or domain name
+       pointers for reverse queries.
+
+.. option:: -y <key>
+
+       Specify a public key to use as trust anchor.
+       This is the base for a chain of trust that is built up from the trust
+       anchor to the response, in order to validate the response message.
+       Can be given as a DS or DNSKEY record.
+       For example:
+
+       .. code-block:: text
+
+            -y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD"
+
+.. option:: -D
+
+       Enables DNSSEC validation.
+       Reads the root anchor from the default configured root anchor at the
+       default location, :file:`@UNBOUND_ROOTKEY_FILE@`.
+
+.. option:: -f <keyfile>
+
+       Reads keys from a file.
+       Every line has a DS or DNSKEY record, in the format as for :option:`-y`.
+       The zone file format, the same as ``dig`` and ``drill`` produce.
+
+.. option:: -F <namedkeyfile>
+
+       Reads keys from a BIND-style :file:`named.conf` file.
+       Only the ``trusted-key {};`` entries are read.
+
+.. option:: -C <configfile>
+
+       Uses the specified unbound.conf to prime :doc:`libunbound(3)</manpages/libunbound>`.
+       Pass it as first argument if you want to override some options from the
+       config file with further arguments on the commandline.
+
+.. option:: -r
+
+       Read :file:`/etc/resolv.conf`, and use the forward DNS servers from
+       there (those could have been set by DHCP).
+       More info in *resolv.conf(5)*.
+       Breaks validation if those servers do not support DNSSEC.
+
+.. option:: -4
+
+       Use solely the IPv4 network for sending packets.
+
+.. option:: -6
+
+       Use solely the IPv6 network for sending packets.
+
+Examples
+--------
+
+Some examples of use.
+The keys shown below are fakes, thus a security failure is encountered.
+
+.. code-block:: text
+
+       $ unbound-host www.example.com
+
+       $ unbound-host -v -y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD" www.example.com
+
+       $ unbound-host -v -y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD" 192.0.2.153
+
+Exit Code
+---------
+
+The ``unbound-host`` program exits with status code 1 on error, 0 on no error.
+The data may not be available on exit code 0, exit code 1 means the lookup
+encountered a fatal error.
+
+See Also
+--------
+
+:doc:`unbound.conf(5)</manpages/unbound.conf>`,
+:doc:`unbound(8)</manpages/unbound>`.

doc/unbound.8.in

@@ -1,88 +1,123 @@
-.TH "unbound" "8" "@date@" "NLnet Labs" "unbound @version@"
-.\"
-.\" unbound.8 -- unbound manual
-.\"
-.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
-.\"
-.\" See LICENSE for the license.
-.\"
-.\"
-.SH "NAME"
-.B unbound
-\- Unbound DNS validating resolver @version@.
-.SH "SYNOPSIS"
-.B unbound
-.RB [ \-h ]
-.RB [ \-d ]
-.RB [ \-p ]
-.RB [ \-v ]
-.RB [ \-c
-.IR cfgfile ]
-.SH "DESCRIPTION"
-.B Unbound
-is a caching DNS resolver.
-.P
-It uses a built in list of authoritative nameservers for the root zone (.),
+.\" Man page generated from reStructuredText.
+.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.TH "UNBOUND" "8" "@date@" "@version@" "Unbound"
+.SH NAME
+unbound \- Unbound DNS validating resolver @version@.
+.SH SYNOPSIS
+.sp
+\fBunbound\fP [\fB\-hdpv\fP] [\fB\-c <cfgfile>\fP]
+.SH DESCRIPTION
+.sp
+\fBunbound\fP is a caching DNS resolver.
+.sp
+It uses a built in list of authoritative nameservers for the root zone (\fB\&.\fP),
 the so called root hints.
-On receiving a DNS query it will ask the root nameservers for
-an answer and will in almost all cases receive a delegation to a top level
-domain (TLD) authoritative nameserver.
+On receiving a DNS query it will ask the root nameservers for an answer and
+will in almost all cases receive a delegation to a top level domain (TLD)
+authoritative nameserver.
 It will then ask that nameserver for an answer.
-It will recursively continue until an answer is found or no answer is
-available (NXDOMAIN).
-For performance and efficiency reasons that answer is cached for a
-certain time (the answer's time\-to\-live or TTL).
+It will recursively continue until an answer is found or no answer is available
+(NXDOMAIN).
+For performance and efficiency reasons that answer is cached for a certain time
+(the answer\(aqs time\-to\-live or TTL).
 A second query for the same name will then be answered from the cache.
 Unbound can also do DNSSEC validation.
-.P
-To use a locally running
-.B Unbound
-for resolving put
 .sp
-.RS 6n
+To use a locally running Unbound for resolving put:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
 nameserver 127.0.0.1
-.RE
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+into \fIresolv.conf(5)\fP\&.
+.sp
+If authoritative DNS is needed as well using \fI\%nsd(8)\fP,
+careful setup is required because authoritative nameservers and resolvers are
+using the same port number (53).
 .sp
-into
-.IR resolv.conf (5).
-.P
-If authoritative DNS is needed as well using
-.IR nsd (8),
-careful setup is required because authoritative nameservers and
-resolvers are using the same port number (53).
-.P
 The available options are:
+.INDENT 0.0
 .TP
 .B \-h
 Show the version number and commandline option help, and exit.
+.UNINDENT
+.INDENT 0.0
 .TP
-.B \-c\fI cfgfile
-Set the config file with settings for Unbound to read instead of reading the
-file at the default location, @ub_conf_file@. The syntax is
-described in \fIunbound.conf\fR(5).
+.B \-c <cfgfile>
+Set the config file with settings for unbound to read instead of reading the
+file at the default location, \fB@ub_conf_file@\fP\&.
+The syntax is described in \fI\%unbound.conf(5)\fP\&.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-d
-Debug flag: do not fork into the background, but stay attached to
-the console.  This flag will also delay writing to the log file until
-the thread\-spawn time, so that most config and setup errors appear on
-stderr. If given twice or more, logging does not switch to the log file
-or to syslog, but the log messages are printed to stderr all the time.
+Debug flag: do not fork into the background, but stay attached to the
+console.
+This flag will also delay writing to the log file until the thread\-spawn
+time, so that most config and setup errors appear on stderr.
+If given twice or more, logging does not switch to the log file or to
+syslog, but the log messages are printed to stderr all the time.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-p
-Don't use a pidfile.  This argument should only be used by supervision
-systems which can ensure that only one instance of Unbound will run
-concurrently.
+Don\(aqt use a pidfile.
+This argument should only be used by supervision systems which can ensure
+that only one instance of Unbound will run concurrently.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-v
-Increase verbosity. If given multiple times, more information is logged.
-This is added to the verbosity (if any) from the config file.
+Increase verbosity.
+If given multiple times, more information is logged.
+This is in addition to the verbosity (if any) from the config file.
+.UNINDENT
+.INDENT 0.0
 .TP
 .B \-V
 Show the version number and build options, and exit.
-.SH "SEE ALSO"
-\fIunbound.conf\fR(5),
-\fIunbound\-checkconf\fR(8),
-\fInsd\fR(8).
-.SH "AUTHORS"
-.B Unbound
-developers are mentioned in the CREDITS file in the distribution.
+.UNINDENT
+.SH SEE ALSO
+.sp
+\fI\%unbound.conf(5)\fP,
+\fI\%unbound\-checkconf(8)\fP,
+\fI\%nsd(8)\fP\&.
+.SH AUTHOR
+Unbound developers are mentioned in the CREDITS file in the distribution.
+.SH COPYRIGHT
+1999-2025, NLnet Labs
+.\" Generated by docutils manpage writer.
+.

doc/unbound.rst

@@ -0,0 +1,119 @@
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+..
+    WHEN EDITING MAKE SURE EACH SENTENCE STARTS ON A NEW LINE
+
+..
+    IT HELPS RENDERERS TO DO THE RIGHT THING WRT SPACE
+
+..
+    IT HELPS PEOPLE DIFFING THE CHANGES
+
+.. program:: unbound
+
+unbound(8)
+==========
+
+Synopsis
+--------
+
+**unbound** [``-hdpv``] [``-c <cfgfile>``]
+
+Description
+-----------
+
+``unbound`` is a caching DNS resolver.
+
+It uses a built in list of authoritative nameservers for the root zone (``.``),
+the so called root hints.
+On receiving a DNS query it will ask the root nameservers for an answer and
+will in almost all cases receive a delegation to a top level domain (TLD)
+authoritative nameserver.
+It will then ask that nameserver for an answer.
+It will recursively continue until an answer is found or no answer is available
+(NXDOMAIN).
+For performance and efficiency reasons that answer is cached for a certain time
+(the answer's time-to-live or TTL).
+A second query for the same name will then be answered from the cache.
+Unbound can also do DNSSEC validation.
+
+To use a locally running Unbound for resolving put:
+
+.. code-block:: text
+
+   nameserver 127.0.0.1
+
+into *resolv.conf(5)*.
+
+If authoritative DNS is needed as well using :external+nsd:doc:`manpages/nsd`,
+careful setup is required because authoritative nameservers and resolvers are
+using the same port number (53).
+
+The available options are:
+
+.. option:: -h
+
+ Show the version number and commandline option help, and exit.
+
+.. option:: -c <cfgfile>
+
+   Set the config file with settings for unbound to read instead of reading the
+   file at the default location, :file:`@ub_conf_file@`.
+   The syntax is described in :doc:`unbound.conf(5)</manpages/unbound.conf>`.
+
+.. option:: -d
+
+   Debug flag: do not fork into the background, but stay attached to the
+   console.
+   This flag will also delay writing to the log file until the thread-spawn
+   time, so that most config and setup errors appear on stderr.
+   If given twice or more, logging does not switch to the log file or to
+   syslog, but the log messages are printed to stderr all the time.
+
+.. option:: -p
+
+   Don't use a pidfile.
+   This argument should only be used by supervision systems which can ensure
+   that only one instance of Unbound will run concurrently.
+
+.. option:: -v
+
+   Increase verbosity.
+   If given multiple times, more information is logged.
+   This is in addition to the verbosity (if any) from the config file.
+
+.. option:: -V
+
+   Show the version number and build options, and exit.
+
+See Also
+--------
+
+:doc:`unbound.conf(5)</manpages/unbound.conf>`,
+:doc:`unbound-checkconf(8)</manpages/unbound-checkconf>`,
+:external+nsd:doc:`manpages/nsd`.

dynlibmod/dynlibmod.c

@@ -297,8 +297,8 @@ inplace_cb_delete_wrapped(struct module_env* env, enum inplace_cb_list_type type
  */
 static struct module_func_block dynlibmod_block = {
    "dynlib",
-   &dynlibmod_init, &dynlibmod_deinit, &dynlibmod_operate, &dynlibmod_inform_super,
-   &dynlibmod_clear, &dynlibmod_get_mem
+   NULL, NULL, &dynlibmod_init, &dynlibmod_deinit, &dynlibmod_operate,
+   &dynlibmod_inform_super, &dynlibmod_clear, &dynlibmod_get_mem
 };
 
 struct module_func_block* dynlibmod_get_funcblock(void)

edns-subnet/addrtree.h

@@ -116,7 +116,7 @@ struct addredge {
 	addrlen_t len;
 	/** child node this edge is connected to */
 	struct addrnode *node;
-	/** Parent node this ege is connected to */
+	/** Parent node this edge is connected to */
 	struct addrnode *parent_node;
 	/** Index of this edge in parent_node */
 	int parent_index;

edns-subnet/subnetmod.c

@@ -51,12 +51,16 @@
 #include "services/cache/dns.h"
 #include "util/module.h"
 #include "util/regional.h"
+#include "util/fptr_wlist.h"
 #include "util/storage/slabhash.h"
 #include "util/config_file.h"
 #include "util/data/msgreply.h"
 #include "sldns/sbuffer.h"
 #include "sldns/wire2str.h"
 #include "iterator/iter_utils.h"
+#ifdef USE_CACHEDB
+#include "cachedb/cachedb.h"
+#endif
 
 /** externally called */
 void 
@@ -150,9 +154,25 @@ int ecs_whitelist_check(struct query_info* qinfo,
 		return 1;
 	sn_env = (struct subnet_env*)qstate->env->modinfo[id];
 
+	if(sq->is_subquery_nonsubnet) {
+		if(sq->is_subquery_scopezero) {
+			/* Check if the result can be stored in the global cache,
+			 * this is okay if the address and name are not configured
+			 * as subnet address and subnet zone. */
+			if(!ecs_is_whitelisted(sn_env->whitelist,
+				addr, addrlen, qinfo->qname, qinfo->qname_len,
+				qinfo->qclass)) {
+				verbose(VERB_ALGO, "subnet store subquery global, name and addr have no subnet treatment.");
+				qstate->no_cache_store = 0;
+			}
+		}
+		return 1;
+	}
+
 	/* Cache by default, might be disabled after parsing EDNS option
 	 * received from nameserver. */
-	if(!iter_stub_fwd_no_cache(qstate, &qstate->qinfo, NULL, NULL)) {
+	if(!iter_stub_fwd_no_cache(qstate, &qstate->qinfo, NULL, NULL, NULL, 0)
+		&& sq->ecs_client_in.subnet_validdata) {
 		qstate->no_cache_store = 0;
 	}
 
@@ -229,13 +249,13 @@ subnetmod_init(struct module_env *env, int id)
 		HASH_DEFAULT_STARTARRAY, env->cfg->msg_cache_size,
 		msg_cache_sizefunc, query_info_compare, query_entry_delete,
 		subnet_data_delete, NULL);
-	slabhash_setmarkdel(sn_env->subnet_msg_cache, &subnet_markdel);
 	if(!sn_env->subnet_msg_cache) {
 		log_err("subnetcache: could not create cache");
 		free(sn_env);
 		env->modinfo[id] = NULL;
 		return 0;
 	}
+	slabhash_setmarkdel(sn_env->subnet_msg_cache, &subnet_markdel);
 	/* whitelist for edns subnet capable servers */
 	sn_env->whitelist = ecs_whitelist_create();
 	if(!sn_env->whitelist ||
@@ -310,9 +330,18 @@ delfunc(void *envptr, void *elemptr) {
 static size_t
 sizefunc(void *elemptr) {
 	struct reply_info *elem  = (struct reply_info *)elemptr;
-	return sizeof (struct reply_info) - sizeof (struct rrset_ref)
+	size_t s = sizeof (struct reply_info) - sizeof (struct rrset_ref)
 		+ elem->rrset_count * sizeof (struct rrset_ref)
 		+ elem->rrset_count * sizeof (struct ub_packed_rrset_key *);
+	size_t i;
+	for (i = 0; i < elem->rrset_count; i++) {
+		struct ub_packed_rrset_key *key = elem->rrsets[i];
+		struct packed_rrset_data *data = key->entry.data;
+		s += ub_rrset_sizefunc(key, data);
+	}
+	if(elem->reason_bogus_str)
+		s += strlen(elem->reason_bogus_str)+1;
+	return s;
 }
 
 /**
@@ -352,7 +381,7 @@ update_cache(struct module_qstate *qstate, int id)
 	struct slabhash *subnet_msg_cache = sne->subnet_msg_cache;
 	struct ecs_data *edns = &sq->ecs_client_in;
 	size_t i;
-	int only_match_scope_zero;
+	int only_match_scope_zero, diff_size;
 
 	/* We already calculated hash upon lookup (lookup_and_reply) if we were
 	 * allowed to look in the ECS cache */
@@ -412,19 +441,25 @@ update_cache(struct module_qstate *qstate, int id)
 		rep->ref[i].id = rep->rrsets[i]->id;
 	}
 	reply_info_set_ttls(rep, *qstate->env->now);
+	reply_info_sortref(rep);
 	rep->flags |= (BIT_RA | BIT_QR); /* fix flags to be sensible for */
 	rep->flags &= ~(BIT_AA | BIT_CD);/* a reply based on the cache   */
 	if(edns->subnet_source_mask == 0 && edns->subnet_scope_mask == 0)
 		only_match_scope_zero = 1;
 	else only_match_scope_zero = 0;
+	diff_size = (int)tree->size_bytes;
 	addrtree_insert(tree, (addrkey_t*)edns->subnet_addr, 
 		edns->subnet_source_mask, sq->max_scope, rep,
 		rep->ttl, *qstate->env->now, only_match_scope_zero);
+	diff_size = (int)tree->size_bytes - diff_size;
 
 	lock_rw_unlock(&lru_entry->lock);
 	if (need_to_insert) {
 		slabhash_insert(subnet_msg_cache, h, lru_entry, lru_entry->data,
 			NULL);
+	} else {
+		slabhash_update_space_used(subnet_msg_cache, h, NULL,
+			diff_size);
 	}
 }
 
@@ -504,6 +539,83 @@ common_prefix(uint8_t *a, uint8_t *b, uint8_t net)
 	return !memcmp(a, b, n) && ((net % 8) == 0 || a[n] == b[n]);
 }
 
+/**
+ * Create sub request that looks up the query.
+ * @param qstate: query state
+ * @param id: module id.
+ * @param sq: subnet qstate
+ * @return false on failure.
+ */
+static int
+generate_sub_request(struct module_qstate *qstate, int id, struct subnet_qstate* sq)
+{
+	struct module_qstate* subq = NULL;
+	uint16_t qflags = 0; /* OPCODE QUERY, no flags */
+	int prime = 0;
+	int valrec = 0;
+	struct query_info qinf;
+	qinf.qname = qstate->qinfo.qname;
+	qinf.qname_len = qstate->qinfo.qname_len;
+	qinf.qtype = qstate->qinfo.qtype;
+	qinf.qclass = qstate->qinfo.qclass;
+	qinf.local_alias = NULL;
+
+	qflags |= BIT_RD;
+	if((qstate->query_flags & BIT_CD)!=0) {
+		qflags |= BIT_CD;
+		valrec = 1;
+	}
+
+	fptr_ok(fptr_whitelist_modenv_attach_sub(qstate->env->attach_sub));
+	if(!(*qstate->env->attach_sub)(qstate, &qinf, qflags, prime, valrec,
+		&subq)) {
+		return 0;
+	}
+	if(subq) {
+		/* It is possible to access the subquery module state. */
+		struct subnet_qstate* subsq;
+		if(!subnet_new_qstate(subq, id)) {
+			verbose(VERB_ALGO, "Could not allocate new subnet qstate");
+			return 0;
+		}
+		subsq = (struct subnet_qstate*)subq->minfo[id];
+		subsq->is_subquery_nonsubnet = 1;
+
+		/* When the client asks 0.0.0.0/0 and the name is not treated
+		 * as subnet, it is to be stored in the global cache.
+		 * Store that the client asked for that, if so. */
+		if(sq->ecs_client_in.subnet_source_mask == 0 &&
+			edns_opt_list_find(qstate->edns_opts_front_in,
+				qstate->env->cfg->client_subnet_opcode)) {
+			subq->no_cache_store = 1;
+			subsq->is_subquery_scopezero = 1;
+		}
+	}
+	return 1;
+}
+
+/**
+ * Perform the query without subnet
+ * @param qstate: query state
+ * @param id: module id.
+ * @param sq: subnet qstate
+ * @return module state
+ */
+static enum module_ext_state
+generate_lookup_without_subnet(struct module_qstate *qstate, int id,
+	struct subnet_qstate* sq)
+{
+	verbose(VERB_ALGO, "subnetcache: make subquery to look up without subnet");
+	if(!generate_sub_request(qstate, id, sq)) {
+		verbose(VERB_ALGO, "Could not generate sub query");
+		qstate->return_rcode = LDNS_RCODE_SERVFAIL;
+		qstate->return_msg = NULL;
+		return module_finished;
+	}
+	sq->wait_subquery = 1;
+	return module_wait_subquery;
+}
+
 static enum module_ext_state
 eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 {
@@ -539,14 +651,7 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 		 * is still useful to put it in the edns subnet cache for
 		 * when a client explicitly asks for subnet specific answer. */
 		verbose(VERB_QUERY, "subnetcache: Authority indicates no support");
-		if(!sq->started_no_cache_store) {
-			lock_rw_wrlock(&sne->biglock);
-			update_cache(qstate, id);
-			lock_rw_unlock(&sne->biglock);
-		}
-		if (sq->subnet_downstream)
-			cp_edns_bad_response(c_out, c_in);
-		return module_finished;
+		return generate_lookup_without_subnet(qstate, id, sq);
 	}
 
 	/* Purposefully there was no sent subnet, and there is consequently
@@ -571,14 +676,14 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 		!common_prefix(s_out->subnet_addr, s_in->subnet_addr, 
 			s_out->subnet_source_mask))
 	{
-		/* we can not accept, restart query without option */
+		/* we can not accept, perform query without option */
 		verbose(VERB_QUERY, "subnetcache: forged data");
 		s_out->subnet_validdata = 0;
 		(void)edns_opt_list_remove(&qstate->edns_opts_back_out,
 			qstate->env->cfg->client_subnet_opcode);
 		sq->subnet_sent = 0;
 		sq->subnet_sent_no_subnet = 0;
-		return module_restart_next;
+		return generate_lookup_without_subnet(qstate, id, sq);
 	}
 
 	lock_rw_wrlock(&sne->biglock);
@@ -587,7 +692,21 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 	}
 	sne->num_msg_nocache++;
 	lock_rw_unlock(&sne->biglock);
-	
+
+	/* If there is an expired answer in the global cache, remove that,
+	 * because expired answers would otherwise resurface once the ecs data
+	 * expires, giving once in a while global data responses for ecs
+	 * domains, with serve expired enabled. */
+	if(qstate->env->cfg->serve_expired) {
+		msg_cache_remove(qstate->env, qstate->qinfo.qname,
+			qstate->qinfo.qname_len, qstate->qinfo.qtype,
+			qstate->qinfo.qclass, 0);
+#ifdef USE_CACHEDB
+		if(qstate->env->cachedb_enabled)
+			cachedb_msg_remove(qstate);
+#endif
+	}
+
 	if (sq->subnet_downstream) {
 		/* Client wants to see the answer, echo option back
 		 * and adjust the scope. */
@@ -763,6 +882,9 @@ ecs_edns_back_parsed(struct module_qstate* qstate, int id,
 	} else if(sq->subnet_sent_no_subnet) {
 		/* The answer can be stored as scope 0, not in global cache. */
 		qstate->no_cache_store = 1;
+	} else if(sq->subnet_sent) {
+		/* Need another query to be able to store in global cache. */
+		qstate->no_cache_store = 1;
 	}
 
 	return 1;
@@ -780,6 +902,32 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 		strmodulevent(event));
 	log_query_info(VERB_QUERY, "subnetcache operate: query", &qstate->qinfo);
 
+	if(sq && sq->wait_subquery_done) {
+		/* The subquery lookup returned. */
+		if(sq->ecs_client_in.subnet_source_mask == 0 &&
+			edns_opt_list_find(qstate->edns_opts_front_in,
+				qstate->env->cfg->client_subnet_opcode)) {
+			if(!sq->started_no_cache_store &&
+				qstate->return_msg) {
+				lock_rw_wrlock(&sne->biglock);
+				update_cache(qstate, id);
+				lock_rw_unlock(&sne->biglock);
+			}
+			if (sq->subnet_downstream)
+				cp_edns_bad_response(&sq->ecs_client_out,
+					&sq->ecs_client_in);
+			/* It is a scope zero lookup, append edns subnet
+			 * option to the querier. */
+			subnet_ecs_opt_list_append(&sq->ecs_client_out,
+				&qstate->edns_opts_front_out, qstate,
+				qstate->region);
+		}
+		sq->wait_subquery_done = 0;
+		qstate->ext_state[id] = module_finished;
+		qstate->no_cache_store = sq->started_no_cache_store;
+		qstate->no_cache_lookup = sq->started_no_cache_lookup;
+		return;
+	}
 	if((event == module_event_new || event == module_event_pass) &&
 		sq == NULL) {
 		struct edns_option* ecs_opt;
@@ -790,6 +938,8 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 		}
 
 		sq = (struct subnet_qstate*)qstate->minfo[id];
+		if(sq->wait_subquery)
+			return; /* Wait for that subquery to return */
 
 		if((ecs_opt = edns_opt_list_find(
 			qstate->edns_opts_front_in,
@@ -819,6 +969,14 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 			/* No clients are interested in result or we could not
 			 * parse it, we don't do client subnet */
 			sq->ecs_server_out.subnet_validdata = 0;
+			if(edns_opt_list_find(qstate->edns_opts_front_in,
+				qstate->env->cfg->client_subnet_opcode)) {
+				/* aggregated this deaggregated state */
+				qstate->ext_state[id] =
+					generate_lookup_without_subnet(
+					qstate, id, sq);
+				return;
+			}
 			verbose(VERB_ALGO, "subnetcache: pass to next module");
 			qstate->ext_state[id] = module_wait_module;
 			return;
@@ -859,6 +1017,14 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 			}
 			lock_rw_unlock(&sne->biglock);
 		}
+		if(sq->ecs_client_in.subnet_source_mask == 0 &&
+			edns_opt_list_find(qstate->edns_opts_front_in,
+				qstate->env->cfg->client_subnet_opcode)) {
+			/* client asked for resolution without edns subnet */
+			qstate->ext_state[id] = generate_lookup_without_subnet(
+				qstate, id, sq);
+			return;
+		}
 		
 		sq->ecs_server_out.subnet_addr_fam =
 			sq->ecs_client_in.subnet_addr_fam;
@@ -895,6 +1061,8 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 		qstate->ext_state[id] = module_wait_module;
 		return;
 	}
+	if(sq && sq->wait_subquery)
+		return; /* Wait for that subquery to return */
 	/* Query handed back by next module, we have a 'final' answer */
 	if(sq && event == module_event_moddone) {
 		qstate->ext_state[id] = eval_response(qstate, id, sq);
@@ -943,10 +1111,27 @@ subnetmod_clear(struct module_qstate *ATTR_UNUSED(qstate),
 }
 
 void
-subnetmod_inform_super(struct module_qstate *ATTR_UNUSED(qstate),
-	int ATTR_UNUSED(id), struct module_qstate *ATTR_UNUSED(super))
+subnetmod_inform_super(struct module_qstate *qstate, int id,
+	struct module_qstate *super)
 {
-	/* Not used */
+	struct subnet_qstate* super_sq =
+		(struct subnet_qstate*)super->minfo[id];
+	log_query_info(VERB_ALGO, "subnetcache inform_super: query",
+		&super->qinfo);
+	super_sq->wait_subquery = 0;
+	super_sq->wait_subquery_done = 1;
+	if(qstate->return_rcode != LDNS_RCODE_NOERROR ||
+		!qstate->return_msg) {
+		super->return_msg = NULL;
+		super->return_rcode = LDNS_RCODE_SERVFAIL;
+		return;
+	}
+	super->return_rcode = LDNS_RCODE_NOERROR;
+	super->return_msg = dns_copy_msg(qstate->return_msg, super->region);
+	if(!super->return_msg) {
+		log_err("subnetcache: copy response, out of memory");
+		super->return_rcode = LDNS_RCODE_SERVFAIL;
+	}
 }
 
 size_t
@@ -963,7 +1148,8 @@ subnetmod_get_mem(struct module_env *env, int id)
  * The module function block 
  */
 static struct module_func_block subnetmod_block = {
-	"subnetcache", &subnetmod_init, &subnetmod_deinit, &subnetmod_operate,
+	"subnetcache",
+	NULL, NULL, &subnetmod_init, &subnetmod_deinit, &subnetmod_operate,
 	&subnetmod_inform_super, &subnetmod_clear, &subnetmod_get_mem
 };
 

edns-subnet/subnetmod.h

@@ -102,6 +102,14 @@ struct subnet_qstate {
 	int started_no_cache_store;
 	/** has the subnet module been started with no_cache_lookup? */
 	int started_no_cache_lookup;
+	/** Wait for subquery that has been started for nonsubnet lookup. */
+	int wait_subquery;
+	/** The subquery waited for is done. */
+	int wait_subquery_done;
+	/** The subnet state is a subquery state for nonsubnet lookup. */
+	int is_subquery_nonsubnet;
+	/** This is a subquery, and it is made due to a scope zero request. */
+	int is_subquery_scopezero;
 };
 
 void subnet_data_delete(void* d, void* ATTR_UNUSED(arg));

ipsecmod/ipsecmod.c

@@ -456,7 +456,8 @@ ipsecmod_handle_query(struct module_qstate* qstate,
 	/* Store A/AAAA in cache. */
 	if(!dns_cache_store(qstate->env, &qstate->qinfo,
 		qstate->return_msg->rep, 0, qstate->prefetch_leeway,
-		0, qstate->region, qstate->query_flags, qstate->qstarttime)) {
+		0, qstate->region, qstate->query_flags, qstate->qstarttime,
+		qstate->is_valrec)) {
 		log_err("ipsecmod: out of memory caching record");
 	}
 	qstate->ext_state[id] = module_finished;
@@ -615,7 +616,7 @@ ipsecmod_get_mem(struct module_env* env, int id)
  */
 static struct module_func_block ipsecmod_block = {
 	"ipsecmod",
-	&ipsecmod_init, &ipsecmod_deinit, &ipsecmod_operate,
+	NULL, NULL, &ipsecmod_init, &ipsecmod_deinit, &ipsecmod_operate,
 	&ipsecmod_inform_super, &ipsecmod_clear, &ipsecmod_get_mem
 };
 

ipset/ipset.c

@@ -17,9 +17,19 @@
 #include "sldns/wire2str.h"
 #include "sldns/parseutil.h"
 
+#ifdef HAVE_NET_PFVAR_H
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <netinet/in.h>
+#include <net/if.h>
+#include <net/pfvar.h>
+typedef intptr_t filter_dev;
+#else
 #include <libmnl/libmnl.h>
 #include <linux/netfilter/nfnetlink.h>
 #include <linux/netfilter/ipset/ip_set.h>
+typedef struct mnl_socket * filter_dev;
+#endif
 
 #define BUFF_LEN 256
 
@@ -41,24 +51,95 @@ static int error_response(struct module_qstate* qstate, int id, int rcode) {
 	return 0;
 }
 
-static struct mnl_socket * open_mnl_socket() {
-	struct mnl_socket *mnl;
+#ifdef HAVE_NET_PFVAR_H
+static void * open_filter() {
+	filter_dev dev;
 
-	mnl = mnl_socket_open(NETLINK_NETFILTER);
-	if (!mnl) {
+	dev = open("/dev/pf", O_RDWR);
+	if (dev == -1) {
+		log_err("open(\"/dev/pf\") failed: %s", strerror(errno));
+		return NULL;
+	}
+	else
+		return (void *)dev;
+}
+#else
+static void * open_filter() {
+	filter_dev dev;
+
+	dev = mnl_socket_open(NETLINK_NETFILTER);
+	if (!dev) {
 		log_err("ipset: could not open netfilter.");
 		return NULL;
 	}
 
-	if (mnl_socket_bind(mnl, 0, MNL_SOCKET_AUTOPID) < 0) {
-		mnl_socket_close(mnl);
+	if (mnl_socket_bind(dev, 0, MNL_SOCKET_AUTOPID) < 0) {
+		mnl_socket_close(dev);
 		log_err("ipset: could not bind netfilter.");
 		return NULL;
 	}
-	return mnl;
+	return (void *)dev;
 }
+#endif
 
-static int add_to_ipset(struct mnl_socket *mnl, const char *setname, const void *ipaddr, int af) {
+#ifdef HAVE_NET_PFVAR_H
+static int add_to_ipset(filter_dev dev, const char *setname, const void *ipaddr, int af) {
+	struct pfioc_table io;
+	struct pfr_addr addr;
+	const char *p;
+	int i;
+
+	bzero(&io, sizeof(io));
+	bzero(&addr, sizeof(addr));
+
+	p = strrchr(setname, '/');
+	if (p) {
+		i = p - setname;
+		if (i >= PATH_MAX) {
+			errno = ENAMETOOLONG;
+			return -1;
+		}
+		memcpy(io.pfrio_table.pfrt_anchor, setname, i);
+		if (i < PATH_MAX)
+			io.pfrio_table.pfrt_anchor[i] = '\0';
+		p++;
+	}
+	else
+		p = setname;
+
+	if (strlen(p) >= PF_TABLE_NAME_SIZE) {
+		errno = ENAMETOOLONG;
+		return -1;
+	}
+	strlcpy(io.pfrio_table.pfrt_name, p, PF_TABLE_NAME_SIZE);
+
+	io.pfrio_buffer = &addr;
+	io.pfrio_size = 1;
+	io.pfrio_esize = sizeof(addr);
+
+	switch (af) {
+		case AF_INET:
+			addr.pfra_ip4addr = *(struct in_addr *)ipaddr;
+			addr.pfra_net = 32;
+			break;
+		case AF_INET6:
+			addr.pfra_ip6addr = *(struct in6_addr *)ipaddr;
+			addr.pfra_net = 128;
+			break;
+		default:
+		errno = EAFNOSUPPORT;
+		return -1;
+	}
+	addr.pfra_af = af;
+
+	if (ioctl(dev, DIOCRADDADDRS, &io) == -1) {
+		log_err("ioctl failed: %s", strerror(errno));
+		return -1;
+	}
+	return 0;
+}
+#else
+static int add_to_ipset(filter_dev dev, const char *setname, const void *ipaddr, int af) {
 	struct nlmsghdr *nlh;
 	struct nfgenmsg *nfg;
 	struct nlattr *nested[2];
@@ -91,14 +172,15 @@ static int add_to_ipset(struct mnl_socket *mnl, const char *setname, const void
 	mnl_attr_nest_end(nlh, nested[1]);
 	mnl_attr_nest_end(nlh, nested[0]);
 
-	if (mnl_socket_sendto(mnl, nlh, nlh->nlmsg_len) < 0) {
+	if (mnl_socket_sendto(dev, nlh, nlh->nlmsg_len) < 0) {
 		return -1;
 	}
 	return 0;
 }
+#endif
 
 static void
-ipset_add_rrset_data(struct ipset_env *ie, struct mnl_socket *mnl,
+ipset_add_rrset_data(struct ipset_env *ie,
 	struct packed_rrset_data *d, const char* setname, int af,
 	const char* dname)
 {
@@ -123,12 +205,16 @@ ipset_add_rrset_data(struct ipset_env *ie, struct mnl_socket *mnl,
 					snprintf(ip, sizeof(ip), "(inet_ntop_error)");
 				verbose(VERB_QUERY, "ipset: add %s to %s for %s", ip, setname, dname);
 			}
-			ret = add_to_ipset(mnl, setname, rr_data + 2, af);
+			ret = add_to_ipset((filter_dev)ie->dev, setname, rr_data + 2, af);
 			if (ret < 0) {
 				log_err("ipset: could not add %s into %s", dname, setname);
 
-				mnl_socket_close(mnl);
-				ie->mnl = NULL;
+#if HAVE_NET_PFVAR_H
+				/* don't close as we might not be able to open again due to dropped privs */
+#else
+				mnl_socket_close((filter_dev)ie->dev);
+				ie->dev = NULL;
+#endif
 				break;
 			}
 		}
@@ -137,8 +223,8 @@ ipset_add_rrset_data(struct ipset_env *ie, struct mnl_socket *mnl,
 
 static int
 ipset_check_zones_for_rrset(struct module_env *env, struct ipset_env *ie,
-	struct mnl_socket *mnl, struct ub_packed_rrset_key *rrset,
-	const char *qname, const int qlen, const char *setname, int af)
+	struct ub_packed_rrset_key *rrset, const char *qname, int qlen,
+	const char *setname, int af)
 {
 	static char dname[BUFF_LEN];
 	const char *ds, *qs;
@@ -152,11 +238,20 @@ ipset_check_zones_for_rrset(struct module_env *env, struct ipset_env *ie,
 		log_err("bad domain name");
 		return -1;
 	}
+	if (dname[dlen - 1] == '.') {
+		dlen--;
+	}
+	if (qname[qlen - 1] == '.') {
+		qlen--;
+	}
 
 	for (p = env->cfg->local_zones_ipset; p; p = p->next) {
 		ds = NULL;
 		qs = NULL;
 		plen = strlen(p->str);
+		if (p->str[plen - 1] == '.') {
+			plen--;
+		}
 
 		if (dlen == plen || (dlen > plen && dname[dlen - plen - 1] == '.' )) {
 			ds = dname + (dlen - plen);
@@ -167,8 +262,7 @@ ipset_check_zones_for_rrset(struct module_env *env, struct ipset_env *ie,
 		if ((ds && strncasecmp(p->str, ds, plen) == 0)
 			|| (qs && strncasecmp(p->str, qs, plen) == 0)) {
 			d = (struct packed_rrset_data*)rrset->entry.data;
-			ipset_add_rrset_data(ie, mnl, d, setname,
-				af, dname);
+			ipset_add_rrset_data(ie, d, setname, af, dname);
 			break;
 		}
 	}
@@ -178,7 +272,6 @@ ipset_check_zones_for_rrset(struct module_env *env, struct ipset_env *ie,
 static int ipset_update(struct module_env *env, struct dns_msg *return_msg,
 	struct query_info qinfo, struct ipset_env *ie)
 {
-	struct mnl_socket *mnl;
 	size_t i;
 	const char *setname;
 	struct ub_packed_rrset_key *rrset;
@@ -186,15 +279,17 @@ static int ipset_update(struct module_env *env, struct dns_msg *return_msg,
 	static char qname[BUFF_LEN];
 	int qlen;
 
-	mnl = (struct mnl_socket *)ie->mnl;
-	if (!mnl) {
+#ifdef HAVE_NET_PFVAR_H
+#else
+	if (!ie->dev) {
 		/* retry to create mnl socket */
-		mnl = open_mnl_socket();
-		if (!mnl) {
+		ie->dev = open_filter();
+		if (!ie->dev) {
+			log_warn("ipset open_filter failed");
 			return -1;
 		}
-		ie->mnl = mnl;
 	}
+#endif
 
 	qlen = sldns_wire2str_dname_buf(qinfo.qname, qinfo.qname_len,
 		qname, BUFF_LEN);
@@ -217,8 +312,8 @@ static int ipset_update(struct module_env *env, struct dns_msg *return_msg,
 		}
 
 		if (setname) {
-			if(ipset_check_zones_for_rrset(env, ie, mnl, rrset,
-				qname, qlen, setname, af) == -1)
+			if(ipset_check_zones_for_rrset(env, ie, rrset, qname,
+				qlen, setname, af) == -1)
 				return -1;
 		}
 	}
@@ -226,7 +321,7 @@ static int ipset_update(struct module_env *env, struct dns_msg *return_msg,
 	return 0;
 }
 
-int ipset_init(struct module_env* env, int id) {
+int ipset_startup(struct module_env* env, int id) {
 	struct ipset_env *ipset_env;
 
 	ipset_env = (struct ipset_env *)calloc(1, sizeof(struct ipset_env));
@@ -237,7 +332,43 @@ int ipset_init(struct module_env* env, int id) {
 
 	env->modinfo[id] = (void *)ipset_env;
 
-	ipset_env->mnl = NULL;
+#ifdef HAVE_NET_PFVAR_H
+	ipset_env->dev = open_filter();
+	if (!ipset_env->dev) {
+		log_err("ipset open_filter failed");
+		return 0;
+	}
+#else
+	ipset_env->dev = NULL;
+#endif
+	return 1;
+}
+
+void ipset_destartup(struct module_env* env, int id) {
+	filter_dev dev;
+	struct ipset_env *ipset_env;
+
+	if (!env || !env->modinfo[id]) {
+		return;
+	}
+	ipset_env = (struct ipset_env*)env->modinfo[id];
+
+	dev = (filter_dev)ipset_env->dev;
+	if (dev) {
+#if HAVE_NET_PFVAR_H
+		close(dev);
+#else
+		mnl_socket_close(dev);
+#endif
+		ipset_env->dev = NULL;
+	}
+
+	free(ipset_env);
+	env->modinfo[id] = NULL;
+}
+
+int ipset_init(struct module_env* env, int id) {
+	struct ipset_env *ipset_env = env->modinfo[id];
 
 	ipset_env->name_v4 = env->cfg->ipset_name_v4;
 	ipset_env->name_v6 = env->cfg->ipset_name_v6;
@@ -253,24 +384,8 @@ int ipset_init(struct module_env* env, int id) {
 	return 1;
 }
 
-void ipset_deinit(struct module_env *env, int id) {
-	struct mnl_socket *mnl;
-	struct ipset_env *ipset_env;
-
-	if (!env || !env->modinfo[id]) {
-		return;
-	}
-
-	ipset_env = (struct ipset_env *)env->modinfo[id];
-
-	mnl = (struct mnl_socket *)ipset_env->mnl;
-	if (mnl) {
-		mnl_socket_close(mnl);
-		ipset_env->mnl = NULL;
-	}
-
-	free(ipset_env);
-	env->modinfo[id] = NULL;
+void ipset_deinit(struct module_env *ATTR_UNUSED(env), int ATTR_UNUSED(id)) {
+	/* nothing */
 }
 
 static int ipset_new(struct module_qstate* qstate, int id) {
@@ -376,8 +491,8 @@ size_t ipset_get_mem(struct module_env *env, int id) {
  */
 static struct module_func_block ipset_block = {
 	"ipset",
-	&ipset_init, &ipset_deinit, &ipset_operate,
-	&ipset_inform_super, &ipset_clear, &ipset_get_mem
+	&ipset_startup, &ipset_destartup, &ipset_init, &ipset_deinit,
+	&ipset_operate, &ipset_inform_super, &ipset_clear, &ipset_get_mem
 };
 
 struct module_func_block * ipset_get_funcblock(void) {

ipset/ipset.h

@@ -37,7 +37,7 @@ extern "C" {
 #endif
 
 struct ipset_env {
-    void* mnl;
+	void* dev;
 
 	int v4_enabled;
 	int v6_enabled;
@@ -50,6 +50,10 @@ struct ipset_qstate {
 	int dummy;
 };
 
+/** Startup the ipset module */
+int ipset_startup(struct module_env* env, int id);
+/** Destartup the ipset module */
+void ipset_destartup(struct module_env* env, int id);
 /** Init the ipset module */
 int ipset_init(struct module_env* env, int id);
 /** Deinit the ipset module */

iterator/iter_delegpt.c

@@ -278,7 +278,7 @@ delegpt_count_addr(struct delegpt* dp, size_t* numaddr, size_t* numres,
 
 void delegpt_log(enum verbosity_value v, struct delegpt* dp)
 {
-	char buf[LDNS_MAX_DOMAINLEN+1];
+	char buf[LDNS_MAX_DOMAINLEN];
 	struct delegpt_ns* ns;
 	struct delegpt_addr* a;
 	size_t missing=0, numns=0, numaddr=0, numres=0, numavail=0;

iterator/iter_delegpt.h

@@ -79,6 +79,16 @@ struct delegpt {
 	 * Also true if the delegationpoint was created from a delegation
 	 * message and thus contains the parent-side-info already. */
 	uint8_t has_parent_side_NS;
+	/** if true, the delegation point has reached last resort processing
+	 *  and the parent side information has been possibly added to the
+	 *  delegation point.
+	 *  For now this signals that further target lookups will ignore
+	 *  the configured target-fetch-policy and only resolve on
+	 *  demand to try and avoid triggering limits at this stage (.i.e, it
+	 *  is very likely that the A/AAAA queries for the newly added name
+	 *  servers will not yield new IP addresses and trigger NXNS
+	 *  countermeasures. */
+	uint8_t fallback_to_parent_side_NS;
 	/** for assertions on type of delegpt */
 	uint8_t dp_type_mlc;
 	/** use SSL for upstream query */

iterator/iter_fwd.c

@@ -71,6 +71,7 @@ forwards_create(void)
 		sizeof(struct iter_forwards));
 	if(!fwd)
 		return NULL;
+	lock_rw_init(&fwd->lock);
 	return fwd;
 }
 
@@ -100,6 +101,7 @@ forwards_delete(struct iter_forwards* fwd)
 {
 	if(!fwd) 
 		return;
+	lock_rw_destroy(&fwd->lock);
 	fwd_del_tree(fwd);
 	free(fwd);
 }
@@ -127,7 +129,7 @@ forwards_insert_data(struct iter_forwards* fwd, uint16_t c, uint8_t* nm,
 	node->namelabs = nmlabs;
 	node->dp = dp;
 	if(!rbtree_insert(fwd->tree, &node->node)) {
-		char buf[257];
+		char buf[LDNS_MAX_DOMAINLEN];
 		dname_str(nm, buf);
 		log_err("duplicate forward zone %s ignored.", buf);
 		delegpt_free_mlc(dp);
@@ -137,6 +139,17 @@ forwards_insert_data(struct iter_forwards* fwd, uint16_t c, uint8_t* nm,
 	return 1;
 }
 
+static struct iter_forward_zone*
+fwd_zone_find(struct iter_forwards* fwd, uint16_t c, uint8_t* nm)
+{
+	struct iter_forward_zone key;
+	key.node.key = &key;
+	key.dclass = c;
+	key.name = nm;
+	key.namelabs = dname_count_size_labels(nm, &key.namelen);
+	return (struct iter_forward_zone*)rbtree_search(fwd->tree, &key);
+}
+
 /** insert new info into forward structure given dp */
 static int
 forwards_insert(struct iter_forwards* fwd, uint16_t c, struct delegpt* dp)
@@ -319,6 +332,40 @@ make_stub_holes(struct iter_forwards* fwd, struct config_file* cfg)
 			log_err("cannot parse stub name '%s'", s->name);
 			return 0;
 		}
+		if(fwd_zone_find(fwd, LDNS_RR_CLASS_IN, dname) != NULL) {
+			/* Already a forward zone there. */
+			free(dname);
+			continue;
+		}
+		if(!fwd_add_stub_hole(fwd, LDNS_RR_CLASS_IN, dname)) {
+			free(dname);
+			log_err("out of memory");
+			return 0;
+		}
+		free(dname);
+	}
+	return 1;
+}
+
+/** make NULL entries for auths */
+static int
+make_auth_holes(struct iter_forwards* fwd, struct config_file* cfg)
+{
+	struct config_auth* a;
+	uint8_t* dname;
+	size_t dname_len;
+	for(a = cfg->auths; a; a = a->next) {
+		if(!a->name) continue;
+		dname = sldns_str2wire_dname(a->name, &dname_len);
+		if(!dname) {
+			log_err("cannot parse auth name '%s'", a->name);
+			return 0;
+		}
+		if(fwd_zone_find(fwd, LDNS_RR_CLASS_IN, dname) != NULL) {
+			/* Already a forward zone there. */
+			free(dname);
+			continue;
+		}
 		if(!fwd_add_stub_hole(fwd, LDNS_RR_CLASS_IN, dname)) {
 			free(dname);
 			log_err("out of memory");
@@ -332,45 +379,74 @@ make_stub_holes(struct iter_forwards* fwd, struct config_file* cfg)
 int 
 forwards_apply_cfg(struct iter_forwards* fwd, struct config_file* cfg)
 {
+	if(fwd->tree) {
+		lock_unprotect(&fwd->lock, fwd->tree);
+	}
 	fwd_del_tree(fwd);
 	fwd->tree = rbtree_create(fwd_cmp);
 	if(!fwd->tree)
 		return 0;
+	lock_protect(&fwd->lock, fwd->tree, sizeof(*fwd->tree));
 
+	lock_rw_wrlock(&fwd->lock);
 	/* read forward zones */
-	if(!read_forwards(fwd, cfg))
+	if(!read_forwards(fwd, cfg)) {
+		lock_rw_unlock(&fwd->lock);
 		return 0;
-	if(!make_stub_holes(fwd, cfg))
+	}
+	if(!make_stub_holes(fwd, cfg)) {
+		lock_rw_unlock(&fwd->lock);
 		return 0;
+	}
+	/* TODO: Now we punch holes for auth zones as well so that in
+	 *       iterator:forward_request() we see the configured
+	 *       delegation point, but code flow/naming is hard to follow.
+	 *       Consider having a single tree with configured
+	 *       delegation points for all categories
+	 *       (stubs, forwards, auths). */
+	if(!make_auth_holes(fwd, cfg)) {
+		lock_rw_unlock(&fwd->lock);
+		return 0;
+	}
 	fwd_init_parents(fwd);
+	lock_rw_unlock(&fwd->lock);
 	return 1;
 }
 
 struct delegpt* 
-forwards_find(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass)
+forwards_find(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass,
+	int nolock)
 {
-	rbnode_type* res = NULL;
+	struct iter_forward_zone* res;
 	struct iter_forward_zone key;
+	int has_dp;
 	key.node.key = &key;
 	key.dclass = qclass;
 	key.name = qname;
 	key.namelabs = dname_count_size_labels(qname, &key.namelen);
-	res = rbtree_search(fwd->tree, &key);
-	if(res) return ((struct iter_forward_zone*)res)->dp;
-	return NULL;
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_rdlock(&fwd->lock); }
+	res = (struct iter_forward_zone*)rbtree_search(fwd->tree, &key);
+	has_dp = res && res->dp;
+	if(!has_dp && !nolock) { lock_rw_unlock(&fwd->lock); }
+	return has_dp?res->dp:NULL;
 }
 
 struct delegpt* 
-forwards_lookup(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass)
+forwards_lookup(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass,
+	int nolock)
 {
 	/* lookup the forward zone in the tree */
 	rbnode_type* res = NULL;
 	struct iter_forward_zone *result;
 	struct iter_forward_zone key;
+	int has_dp;
 	key.node.key = &key;
 	key.dclass = qclass;
 	key.name = qname;
 	key.namelabs = dname_count_size_labels(qname, &key.namelen);
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_rdlock(&fwd->lock); }
 	if(rbtree_find_less_equal(fwd->tree, &key, &res)) {
 		/* exact */
 		result = (struct iter_forward_zone*)res;
@@ -378,8 +454,10 @@ forwards_lookup(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass)
 		/* smaller element (or no element) */
 		int m;
 		result = (struct iter_forward_zone*)res;
-		if(!result || result->dclass != qclass)
+		if(!result || result->dclass != qclass) {
+			if(!nolock) { lock_rw_unlock(&fwd->lock); }
 			return NULL;
+		}
 		/* count number of labels matched */
 		(void)dname_lab_cmp(result->name, result->namelabs, key.name,
 			key.namelabs, &m);
@@ -389,20 +467,22 @@ forwards_lookup(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass)
 			result = result->parent;
 		}
 	}
-	if(result)
-		return result->dp;
-	return NULL;
+	has_dp = result && result->dp;
+	if(!has_dp && !nolock) { lock_rw_unlock(&fwd->lock); }
+	return has_dp?result->dp:NULL;
 }
 
 struct delegpt* 
-forwards_lookup_root(struct iter_forwards* fwd, uint16_t qclass)
+forwards_lookup_root(struct iter_forwards* fwd, uint16_t qclass, int nolock)
 {
 	uint8_t root = 0;
-	return forwards_lookup(fwd, &root, qclass);
+	return forwards_lookup(fwd, &root, qclass, nolock);
 }
 
-int
-forwards_next_root(struct iter_forwards* fwd, uint16_t* dclass)
+/* Finds next root item in forwards lookup tree.
+ * Caller needs to handle locking of the forwards structure. */
+static int
+next_root_locked(struct iter_forwards* fwd, uint16_t* dclass)
 {
 	struct iter_forward_zone key;
 	rbnode_type* n;
@@ -419,7 +499,7 @@ forwards_next_root(struct iter_forwards* fwd, uint16_t* dclass)
 		}
 		/* root not first item? search for higher items */
 		*dclass = p->dclass + 1;
-		return forwards_next_root(fwd, dclass);
+		return next_root_locked(fwd, dclass);
 	}
 	/* find class n in tree, we may get a direct hit, or if we don't
 	 * this is the last item of the previous class so rbtree_next() takes
@@ -447,10 +527,21 @@ forwards_next_root(struct iter_forwards* fwd, uint16_t* dclass)
 		}
 		/* not a root node, return next higher item */
 		*dclass = p->dclass+1;
-		return forwards_next_root(fwd, dclass);
+		return next_root_locked(fwd, dclass);
 	}
 }
 
+int
+forwards_next_root(struct iter_forwards* fwd, uint16_t* dclass, int nolock)
+{
+	int ret;
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_rdlock(&fwd->lock); }
+	ret = next_root_locked(fwd, dclass);
+	if(!nolock) { lock_rw_unlock(&fwd->lock); }
+	return ret;
+}
+
 size_t 
 forwards_get_mem(struct iter_forwards* fwd)
 {
@@ -458,69 +549,104 @@ forwards_get_mem(struct iter_forwards* fwd)
 	size_t s;
 	if(!fwd)
 		return 0;
+	lock_rw_rdlock(&fwd->lock);
 	s = sizeof(*fwd) + sizeof(*fwd->tree);
 	RBTREE_FOR(p, struct iter_forward_zone*, fwd->tree) {
 		s += sizeof(*p) + p->namelen + delegpt_get_mem(p->dp);
 	}
+	lock_rw_unlock(&fwd->lock);
 	return s;
 }
 
-static struct iter_forward_zone*
-fwd_zone_find(struct iter_forwards* fwd, uint16_t c, uint8_t* nm)
-{
-	struct iter_forward_zone key;
-	key.node.key = &key;
-	key.dclass = c;
-	key.name = nm;
-	key.namelabs = dname_count_size_labels(nm, &key.namelen);
-	return (struct iter_forward_zone*)rbtree_search(fwd->tree, &key);
-}
-
 int 
-forwards_add_zone(struct iter_forwards* fwd, uint16_t c, struct delegpt* dp)
+forwards_add_zone(struct iter_forwards* fwd, uint16_t c, struct delegpt* dp,
+	int nolock)
 {
 	struct iter_forward_zone *z;
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_wrlock(&fwd->lock); }
 	if((z=fwd_zone_find(fwd, c, dp->name)) != NULL) {
 		(void)rbtree_delete(fwd->tree, &z->node);
 		fwd_zone_free(z);
 	}
-	if(!forwards_insert(fwd, c, dp))
+	if(!forwards_insert(fwd, c, dp)) {
+		if(!nolock) { lock_rw_unlock(&fwd->lock); }
 		return 0;
+	}
 	fwd_init_parents(fwd);
+	if(!nolock) { lock_rw_unlock(&fwd->lock); }
 	return 1;
 }
 
 void 
-forwards_delete_zone(struct iter_forwards* fwd, uint16_t c, uint8_t* nm)
+forwards_delete_zone(struct iter_forwards* fwd, uint16_t c, uint8_t* nm,
+	int nolock)
 {
 	struct iter_forward_zone *z;
-	if(!(z=fwd_zone_find(fwd, c, nm)))
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_wrlock(&fwd->lock); }
+	if(!(z=fwd_zone_find(fwd, c, nm))) {
+		if(!nolock) { lock_rw_unlock(&fwd->lock); }
 		return; /* nothing to do */
+	}
 	(void)rbtree_delete(fwd->tree, &z->node);
 	fwd_zone_free(z);
 	fwd_init_parents(fwd);
+	if(!nolock) { lock_rw_unlock(&fwd->lock); }
 }
 
 int
-forwards_add_stub_hole(struct iter_forwards* fwd, uint16_t c, uint8_t* nm)
+forwards_add_stub_hole(struct iter_forwards* fwd, uint16_t c, uint8_t* nm,
+	int nolock)
 {
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_wrlock(&fwd->lock); }
+	if(fwd_zone_find(fwd, c, nm) != NULL) {
+		if(!nolock) { lock_rw_unlock(&fwd->lock); }
+		return 1; /* already a stub zone there */
+	}
 	if(!fwd_add_stub_hole(fwd, c, nm)) {
+		if(!nolock) { lock_rw_unlock(&fwd->lock); }
 		return 0;
 	}
 	fwd_init_parents(fwd);
+	if(!nolock) { lock_rw_unlock(&fwd->lock); }
 	return 1;
 }
 
 void
-forwards_delete_stub_hole(struct iter_forwards* fwd, uint16_t c, uint8_t* nm)
+forwards_delete_stub_hole(struct iter_forwards* fwd, uint16_t c,
+	uint8_t* nm, int nolock)
 {
 	struct iter_forward_zone *z;
-	if(!(z=fwd_zone_find(fwd, c, nm)))
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_wrlock(&fwd->lock); }
+	if(!(z=fwd_zone_find(fwd, c, nm))) {
+		if(!nolock) { lock_rw_unlock(&fwd->lock); }
 		return; /* nothing to do */
-	if(z->dp != NULL)
+	}
+	if(z->dp != NULL) {
+		if(!nolock) { lock_rw_unlock(&fwd->lock); }
 		return; /* not a stub hole */
+	}
 	(void)rbtree_delete(fwd->tree, &z->node);
 	fwd_zone_free(z);
 	fwd_init_parents(fwd);
+	if(!nolock) { lock_rw_unlock(&fwd->lock); }
 }
 
+void
+forwards_swap_tree(struct iter_forwards* fwd, struct iter_forwards* data)
+{
+	rbtree_type* oldtree = fwd->tree;
+	if(oldtree) {
+		lock_unprotect(&fwd->lock, oldtree);
+	}
+	if(data->tree) {
+		lock_unprotect(&data->lock, data->tree);
+	}
+	fwd->tree = data->tree;
+	data->tree = oldtree;
+	lock_protect(&fwd->lock, fwd->tree, sizeof(*fwd->tree));
+	lock_protect(&data->lock, data->tree, sizeof(*data->tree));
+}

iterator/iter_fwd.h

@@ -43,6 +43,7 @@
 #ifndef ITERATOR_ITER_FWD_H
 #define ITERATOR_ITER_FWD_H
 #include "util/rbtree.h"
+#include "util/locks.h"
 struct config_file;
 struct delegpt;
 
@@ -50,6 +51,11 @@ struct delegpt;
  * Iterator forward zones structure
  */
 struct iter_forwards {
+	/** lock on the forwards tree.
+	 * When grabbing both this lock and the anchors.lock, this lock
+	 * is grabbed first. When grabbing both this lock and the hints.lock
+	 * this lock is grabbed first. */
+	lock_rw_type lock;
 	/** 
 	 * Zones are stored in this tree. Sort order is specially chosen.
 	 * first sorted on qclass. Then on dname in nsec-like order, so that
@@ -106,47 +112,65 @@ int forwards_apply_cfg(struct iter_forwards* fwd, struct config_file* cfg);
 
 /**
  * Find forward zone exactly by name
+ * The return value is contents of the forwards structure.
+ * Caller should lock and unlock a readlock on the forwards structure if nolock
+ * is set.
+ * Otherwise caller should unlock the readlock on the forwards structure if a
+ * value was returned.
  * @param fwd: forward storage.
  * @param qname: The qname of the query.
  * @param qclass: The qclass of the query.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return: A delegation point or null.
  */
 struct delegpt* forwards_find(struct iter_forwards* fwd, uint8_t* qname,
-	uint16_t qclass);
+	uint16_t qclass, int nolock);
 
 /**
  * Find forward zone information
  * For this qname/qclass find forward zone information, returns delegation
  * point with server names and addresses, or NULL if no forwarding is needed.
+ * The return value is contents of the forwards structure.
+ * Caller should lock and unlock a readlock on the forwards structure if nolock
+ * is set.
+ * Otherwise caller should unlock the readlock on the forwards structure if a
+ * value was returned.
  *
  * @param fwd: forward storage.
  * @param qname: The qname of the query.
  * @param qclass: The qclass of the query.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return: A delegation point if the query has to be forwarded to that list,
  *         otherwise null.
  */
-struct delegpt* forwards_lookup(struct iter_forwards* fwd, 
-	uint8_t* qname, uint16_t qclass);
+struct delegpt* forwards_lookup(struct iter_forwards* fwd,
+	uint8_t* qname, uint16_t qclass, int nolock);
 
 /**
  * Same as forwards_lookup, but for the root only
  * @param fwd: forward storage.
  * @param qclass: The qclass of the query.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return: A delegation point if root forward exists, otherwise null.
  */
-struct delegpt* forwards_lookup_root(struct iter_forwards* fwd, 
-	uint16_t qclass);
+struct delegpt* forwards_lookup_root(struct iter_forwards* fwd,
+	uint16_t qclass, int nolock);
 
 /**
  * Find next root item in forwards lookup tree.
+ * Handles its own locking unless nolock is set. In that case the caller
+ * should lock and unlock a readlock on the forwards structure.
  * @param fwd: the forward storage
  * @param qclass: class to look at next, or higher.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return false if none found, or if true stored in qclass.
  */
-int forwards_next_root(struct iter_forwards* fwd, uint16_t* qclass);
+int forwards_next_root(struct iter_forwards* fwd, uint16_t* qclass,
+	int nolock);
 
 /**
  * Get memory in use by forward storage
+ * Locks and unlocks the structure.
  * @param fwd: forward storage.
  * @return bytes in use
  */
@@ -158,42 +182,65 @@ int fwd_cmp(const void* k1, const void* k2);
 /**
  * Add zone to forward structure. For external use since it recalcs 
  * the tree parents.
+ * Handles its own locking unless nolock is set. In that case the caller
+ * should lock and unlock a writelock on the forwards structure.
  * @param fwd: the forward data structure
  * @param c: class of zone
  * @param dp: delegation point with name and target nameservers for new
  *	forward zone. malloced.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return false on failure (out of memory);
  */
-int forwards_add_zone(struct iter_forwards* fwd, uint16_t c, 
-	struct delegpt* dp);
+int forwards_add_zone(struct iter_forwards* fwd, uint16_t c,
+	struct delegpt* dp, int nolock);
 
 /**
  * Remove zone from forward structure. For external use since it 
  * recalcs the tree parents.
+ * Handles its own locking unless nolock is set. In that case the caller
+ * should lock and unlock a writelock on the forwards structure.
  * @param fwd: the forward data structure
  * @param c: class of zone
  * @param nm: name of zone (in uncompressed wireformat).
+ * @param nolock: Skip locking, locking is handled by the caller.
  */
-void forwards_delete_zone(struct iter_forwards* fwd, uint16_t c, uint8_t* nm);
+void forwards_delete_zone(struct iter_forwards* fwd, uint16_t c,
+	uint8_t* nm, int nolock);
 
 /**
  * Add stub hole (empty entry in forward table, that makes resolution skip
  * a forward-zone because the stub zone should override the forward zone).
  * Does not add one if not necessary.
+ * Handles its own locking unless nolock is set. In that case the caller
+ * should lock and unlock a writelock on the forwards structure.
  * @param fwd: the forward data structure
  * @param c: class of zone
  * @param nm: name of zone (in uncompressed wireformat).
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return false on failure (out of memory);
  */
-int forwards_add_stub_hole(struct iter_forwards* fwd, uint16_t c, uint8_t* nm);
+int forwards_add_stub_hole(struct iter_forwards* fwd, uint16_t c,
+	uint8_t* nm, int nolock);
 
 /**
  * Remove stub hole, if one exists.
+ * Handles its own locking unless nolock is set. In that case the caller
+ * should lock and unlock a writelock on the forwards structure.
  * @param fwd: the forward data structure
  * @param c: class of zone
  * @param nm: name of zone (in uncompressed wireformat).
+ * @param nolock: Skip locking, locking is handled by the caller.
  */
 void forwards_delete_stub_hole(struct iter_forwards* fwd, uint16_t c,
-	uint8_t* nm);
+	uint8_t* nm, int nolock);
+
+/**
+ * Swap internal tree with preallocated entries. Caller should manage
+ * the locks.
+ * @param fwd: the forward data structure.
+ * @param data: the data structure used to take elements from. This contains
+ * 	the old elements on return.
+ */
+void forwards_swap_tree(struct iter_forwards* fwd, struct iter_forwards* data);
 
 #endif /* ITERATOR_ITER_FWD_H */

iterator/iter_hints.c

@@ -57,6 +57,8 @@ hints_create(void)
 		sizeof(struct iter_hints));
 	if(!hints)
 		return NULL;
+	lock_rw_init(&hints->lock);
+	lock_protect(&hints->lock, &hints->tree, sizeof(hints->tree));
 	return hints;
 }
 
@@ -83,6 +85,7 @@ hints_delete(struct iter_hints* hints)
 {
 	if(!hints) 
 		return;
+	lock_rw_destroy(&hints->lock);
 	hints_del_tree(hints);
 	free(hints);
 }
@@ -129,7 +132,7 @@ compile_time_root_prime(int do_ip4, int do_ip6)
 	dp->has_parent_side_NS = 1;
       if(do_ip4) {
 	if(!ah(dp, "A.ROOT-SERVERS.NET.", "198.41.0.4"))	goto failed;
-	if(!ah(dp, "B.ROOT-SERVERS.NET.", "199.9.14.201")) goto failed;
+	if(!ah(dp, "B.ROOT-SERVERS.NET.", "170.247.170.2"))	goto failed;
 	if(!ah(dp, "C.ROOT-SERVERS.NET.", "192.33.4.12"))	goto failed;
 	if(!ah(dp, "D.ROOT-SERVERS.NET.", "199.7.91.13"))	goto failed;
 	if(!ah(dp, "E.ROOT-SERVERS.NET.", "192.203.230.10")) goto failed;
@@ -144,7 +147,7 @@ compile_time_root_prime(int do_ip4, int do_ip6)
       }
       if(do_ip6) {
 	if(!ah(dp, "A.ROOT-SERVERS.NET.", "2001:503:ba3e::2:30")) goto failed;
-	if(!ah(dp, "B.ROOT-SERVERS.NET.", "2001:500:200::b")) goto failed;
+	if(!ah(dp, "B.ROOT-SERVERS.NET.", "2801:1b8:10::b")) goto failed;
 	if(!ah(dp, "C.ROOT-SERVERS.NET.", "2001:500:2::c")) goto failed;
 	if(!ah(dp, "D.ROOT-SERVERS.NET.", "2001:500:2d::d")) goto failed;
 	if(!ah(dp, "E.ROOT-SERVERS.NET.", "2001:500:a8::e")) goto failed;
@@ -178,7 +181,7 @@ hints_insert(struct iter_hints* hints, uint16_t c, struct delegpt* dp,
 	node->noprime = (uint8_t)noprime;
 	if(!name_tree_insert(&hints->tree, &node->node, dp->name, dp->namelen,
 		dp->namelabs, c)) {
-		char buf[257];
+		char buf[LDNS_MAX_DOMAINLEN];
 		dname_str(dp->name, buf);
 		log_err("second hints for zone %s ignored.", buf);
 		delegpt_free_mlc(dp);
@@ -438,47 +441,70 @@ read_root_hints_list(struct iter_hints* hints, struct config_file* cfg)
 int 
 hints_apply_cfg(struct iter_hints* hints, struct config_file* cfg)
 {
+	int nolock = 1;
+	lock_rw_wrlock(&hints->lock);
 	hints_del_tree(hints);
 	name_tree_init(&hints->tree);
-	
+
 	/* read root hints */
-	if(!read_root_hints_list(hints, cfg))
+	if(!read_root_hints_list(hints, cfg)) {
+		lock_rw_unlock(&hints->lock);
 		return 0;
+	}
 
 	/* read stub hints */
-	if(!read_stubs(hints, cfg))
+	if(!read_stubs(hints, cfg)) {
+		lock_rw_unlock(&hints->lock);
 		return 0;
+	}
 
 	/* use fallback compiletime root hints */
-	if(!hints_lookup_root(hints, LDNS_RR_CLASS_IN)) {
+	if(!hints_find_root(hints, LDNS_RR_CLASS_IN, nolock)) {
 		struct delegpt* dp = compile_time_root_prime(cfg->do_ip4,
 			cfg->do_ip6);
 		verbose(VERB_ALGO, "no config, using builtin root hints.");
-		if(!dp) 
+		if(!dp) {
+			lock_rw_unlock(&hints->lock);
 			return 0;
-		if(!hints_insert(hints, LDNS_RR_CLASS_IN, dp, 0))
+		}
+		if(!hints_insert(hints, LDNS_RR_CLASS_IN, dp, 0)) {
+			lock_rw_unlock(&hints->lock);
 			return 0;
+		}
 	}
 
 	name_tree_init_parents(&hints->tree);
+	lock_rw_unlock(&hints->lock);
 	return 1;
 }
 
-struct delegpt* 
-hints_lookup_root(struct iter_hints* hints, uint16_t qclass)
+struct delegpt*
+hints_find(struct iter_hints* hints, uint8_t* qname, uint16_t qclass,
+	int nolock)
+{
+	struct iter_hints_stub *stub;
+	size_t len;
+	int has_dp;
+	int labs = dname_count_size_labels(qname, &len);
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_rdlock(&hints->lock); }
+	stub = (struct iter_hints_stub*)name_tree_find(&hints->tree,
+		qname, len, labs, qclass);
+	has_dp = stub && stub->dp;
+	if(!has_dp && !nolock) { lock_rw_unlock(&hints->lock); }
+	return has_dp?stub->dp:NULL;
+}
+
+struct delegpt*
+hints_find_root(struct iter_hints* hints, uint16_t qclass, int nolock)
 {
 	uint8_t rootlab = 0;
-	struct iter_hints_stub *stub;
-	stub = (struct iter_hints_stub*)name_tree_find(&hints->tree,
-		&rootlab, 1, 1, qclass);
-	if(!stub)
-		return NULL;
-	return stub->dp;
+	return hints_find(hints, &rootlab, qclass, nolock);
 }
 
 struct iter_hints_stub* 
-hints_lookup_stub(struct iter_hints* hints, uint8_t* qname, 
-	uint16_t qclass, struct delegpt* cache_dp)
+hints_lookup_stub(struct iter_hints* hints, uint8_t* qname,
+	uint16_t qclass, struct delegpt* cache_dp, int nolock)
 {
 	size_t len;
 	int labs;
@@ -486,14 +512,20 @@ hints_lookup_stub(struct iter_hints* hints, uint8_t* qname,
 
 	/* first lookup the stub */
 	labs = dname_count_size_labels(qname, &len);
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_rdlock(&hints->lock); }
 	r = (struct iter_hints_stub*)name_tree_lookup(&hints->tree, qname,
 		len, labs, qclass);
-	if(!r) return NULL;
+	if(!r) {
+		if(!nolock) { lock_rw_unlock(&hints->lock); }
+		return NULL;
+	}
 
 	/* If there is no cache (root prime situation) */
 	if(cache_dp == NULL) {
 		if(r->dp->namelabs != 1)
 			return r; /* no cache dp, use any non-root stub */
+		if(!nolock) { lock_rw_unlock(&hints->lock); }
 		return NULL;
 	}
 
@@ -510,12 +542,18 @@ hints_lookup_stub(struct iter_hints* hints, uint8_t* qname,
 	if(dname_strict_subdomain(r->dp->name, r->dp->namelabs,
 		cache_dp->name, cache_dp->namelabs))
 		return r; /* need to prime this stub */
+	if(!nolock) { lock_rw_unlock(&hints->lock); }
 	return NULL;
 }
 
-int hints_next_root(struct iter_hints* hints, uint16_t* qclass)
+int hints_next_root(struct iter_hints* hints, uint16_t* qclass, int nolock)
 {
-	return name_tree_next_root(&hints->tree, qclass);
+	int ret;
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_rdlock(&hints->lock); }
+	ret = name_tree_next_root(&hints->tree, qclass);
+	if(!nolock) { lock_rw_unlock(&hints->lock); }
+	return ret;
 }
 
 size_t 
@@ -524,40 +562,63 @@ hints_get_mem(struct iter_hints* hints)
 	size_t s;
 	struct iter_hints_stub* p;
 	if(!hints) return 0;
+	lock_rw_rdlock(&hints->lock);
 	s = sizeof(*hints);
 	RBTREE_FOR(p, struct iter_hints_stub*, &hints->tree) {
 		s += sizeof(*p) + delegpt_get_mem(p->dp);
 	}
+	lock_rw_unlock(&hints->lock);
 	return s;
 }
 
 int 
 hints_add_stub(struct iter_hints* hints, uint16_t c, struct delegpt* dp,
-	int noprime)
+	int noprime, int nolock)
 {
 	struct iter_hints_stub *z;
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_wrlock(&hints->lock); }
 	if((z=(struct iter_hints_stub*)name_tree_find(&hints->tree,
 		dp->name, dp->namelen, dp->namelabs, c)) != NULL) {
 		(void)rbtree_delete(&hints->tree, &z->node);
 		hints_stub_free(z);
 	}
-	if(!hints_insert(hints, c, dp, noprime))
+	if(!hints_insert(hints, c, dp, noprime)) {
+		if(!nolock) { lock_rw_unlock(&hints->lock); }
 		return 0;
+	}
 	name_tree_init_parents(&hints->tree);
+	if(!nolock) { lock_rw_unlock(&hints->lock); }
 	return 1;
 }
 
 void 
-hints_delete_stub(struct iter_hints* hints, uint16_t c, uint8_t* nm)
+hints_delete_stub(struct iter_hints* hints, uint16_t c, uint8_t* nm,
+	int nolock)
 {
 	struct iter_hints_stub *z;
 	size_t len;
 	int labs = dname_count_size_labels(nm, &len);
+	/* lock_() calls are macros that could be nothing, surround in {} */
+	if(!nolock) { lock_rw_wrlock(&hints->lock); }
 	if(!(z=(struct iter_hints_stub*)name_tree_find(&hints->tree,
-		nm, len, labs, c)))
+		nm, len, labs, c))) {
+		if(!nolock) { lock_rw_unlock(&hints->lock); }
 		return; /* nothing to do */
+	}
 	(void)rbtree_delete(&hints->tree, &z->node);
 	hints_stub_free(z);
 	name_tree_init_parents(&hints->tree);
+	if(!nolock) { lock_rw_unlock(&hints->lock); }
 }
 
+void
+hints_swap_tree(struct iter_hints* hints, struct iter_hints* data)
+{
+	rbnode_type* oldroot = hints->tree.root;
+	size_t oldcount = hints->tree.count;
+	hints->tree.root = data->tree.root;
+	hints->tree.count = data->tree.count;
+	data->tree.root = oldroot;
+	data->tree.count = oldcount;
+}

iterator/iter_hints.h

@@ -43,6 +43,7 @@
 #ifndef ITERATOR_ITER_HINTS_H
 #define ITERATOR_ITER_HINTS_H
 #include "util/storage/dnstree.h"
+#include "util/locks.h"
 struct iter_env;
 struct config_file;
 struct delegpt;
@@ -51,6 +52,10 @@ struct delegpt;
  * Iterator hints structure
  */
 struct iter_hints {
+	/** lock on the forwards tree.
+	 * When grabbing both this lock and the anchors.lock, this lock
+	 * is grabbed first. */
+	lock_rw_type lock;
 	/** 
 	 * Hints are stored in this tree. Sort order is specially chosen.
 	 * first sorted on qclass. Then on dname in nsec-like order, so that
@@ -95,42 +100,70 @@ void hints_delete(struct iter_hints* hints);
 int hints_apply_cfg(struct iter_hints* hints, struct config_file* cfg);
 
 /**
- * Find root hints for the given class.
+ * Find hints for the given class.
+ * The return value is contents of the hints structure.
+ * Caller should lock and unlock a readlock on the hints structure if nolock
+ * is set.
+ * Otherwise caller should unlock the readlock on the hints structure if a
+ * value was returned.
  * @param hints: hint storage.
+ * @param qname: the qname that generated the delegation point.
  * @param qclass: class for which root hints are requested. host order.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return: NULL if no hints, or a ptr to stored hints.
  */
-struct delegpt* hints_lookup_root(struct iter_hints* hints, uint16_t qclass);
+struct delegpt* hints_find(struct iter_hints* hints, uint8_t* qname,
+	uint16_t qclass, int nolock);
+
+/**
+ * Same as hints_lookup, but for the root only.
+ * @param hints: hint storage.
+ * @param qclass: class for which root hints are requested. host order.
+ * @param nolock: Skip locking, locking is handled by the caller.
+ * @return: NULL if no hints, or a ptr to stored hints.
+ */
+struct delegpt* hints_find_root(struct iter_hints* hints,
+	uint16_t qclass, int nolock);
 
 /**
  * Find next root hints (to cycle through all root hints).
+ * Handles its own locking unless nolock is set. In that case the caller
+ * should lock and unlock a readlock on the hints structure.
  * @param hints: hint storage
  * @param qclass: class for which root hints are sought.
  * 	0 means give the first available root hints class.
  * 	x means, give class x or a higher class if any.
  * 	returns the found class in this variable.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return true if a root hint class is found.
  * 	false if not root hint class is found (qclass may have been changed).
  */
-int hints_next_root(struct iter_hints* hints, uint16_t* qclass);
+int hints_next_root(struct iter_hints* hints, uint16_t* qclass, int nolock);
 
 /**
  * Given a qname/qclass combination, and the delegation point from the cache
  * for this qname/qclass, determine if this combination indicates that a
  * stub hint exists and must be primed.
+ * The return value is contents of the hints structure.
+ * Caller should lock and unlock a readlock on the hints structure if nolock
+ * is set.
+ * Otherwise caller should unlock the readlock on the hints structure if a
+ * value was returned.
  *
  * @param hints: hint storage.
  * @param qname: The qname that generated the delegation point.
  * @param qclass: The qclass that generated the delegation point.
  * @param dp: The cache generated delegation point.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return: A priming delegation point if there is a stub hint that must
  *         be primed, otherwise null.
  */
-struct iter_hints_stub* hints_lookup_stub(struct iter_hints* hints, 
-	uint8_t* qname, uint16_t qclass, struct delegpt* dp);
+struct iter_hints_stub* hints_lookup_stub(struct iter_hints* hints,
+	uint8_t* qname, uint16_t qclass, struct delegpt* dp, int nolock);
 
 /**
  * Get memory in use by hints
+ * Locks and unlocks the structure.
  * @param hints: hint storage.
  * @return bytes in use
  */
@@ -139,23 +172,39 @@ size_t hints_get_mem(struct iter_hints* hints);
 /**
  * Add stub to hints structure. For external use since it recalcs 
  * the tree parents.
+ * Handles its own locking unless nolock is set. In that case the caller
+ * should lock and unlock a writelock on the hints structure.
  * @param hints: the hints data structure
  * @param c: class of zone
  * @param dp: delegation point with name and target nameservers for new
  *	hints stub. malloced.
  * @param noprime: set noprime option to true or false on new hint stub.
+ * @param nolock: Skip locking, locking is handled by the caller.
  * @return false on failure (out of memory);
  */
 int hints_add_stub(struct iter_hints* hints, uint16_t c, struct delegpt* dp,
-	int noprime);
+	int noprime, int nolock);
 
 /**
  * Remove stub from hints structure. For external use since it 
  * recalcs the tree parents.
+ * Handles its own locking unless nolock is set. In that case the caller
+ * should lock and unlock a writelock on the hints structure.
  * @param hints: the hints data structure
  * @param c: class of stub zone
  * @param nm: name of stub zone (in uncompressed wireformat).
+ * @param nolock: Skip locking, locking is handled by the caller.
  */
-void hints_delete_stub(struct iter_hints* hints, uint16_t c, uint8_t* nm);
+void hints_delete_stub(struct iter_hints* hints, uint16_t c,
+	uint8_t* nm, int nolock);
+
+/**
+ * Swap internal tree with preallocated entries. Caller should manage
+ * the locks.
+ * @param hints: the hints data structure.
+ * @param data: the data structure used to take elements from. This contains
+ * 	the old elements on return.
+ */
+void hints_swap_tree(struct iter_hints* hints, struct iter_hints* data);
 
 #endif /* ITERATOR_ITER_HINTS_H */

iterator/iter_scrub.c

@@ -283,7 +283,8 @@ synth_cname_rrset(uint8_t** sname, size_t* snamelen, uint8_t* alias,
 		sizeof(uint32_t)+sizeof(uint16_t)+aliaslen);
 	if(!cn->rr_first->ttl_data)
 		return NULL;
-	sldns_write_uint32(cn->rr_first->ttl_data, 0); /* TTL = 0 */
+	memmove(cn->rr_first->ttl_data, rrset->rr_first->ttl_data,
+		sizeof(uint32_t)); /* RFC6672: synth CNAME TTL == DNAME TTL */
 	sldns_write_uint16(cn->rr_first->ttl_data+4, aliaslen);
 	memmove(cn->rr_first->ttl_data+6, alias, aliaslen);
 	cn->rr_first->size = sizeof(uint16_t)+aliaslen;
@@ -366,6 +367,47 @@ type_allowed_in_additional_section(uint16_t tp)
 	return 0;
 }
 
+/** Shorten RRset */
+static void
+shorten_rrset(sldns_buffer* pkt, struct rrset_parse* rrset, int count)
+{
+	/* The too large NS RRset is shortened. This is so that too large
+	 * content does not overwhelm the cache. It may make the rrset
+	 * bogus if it was signed, and then the domain is not resolved any
+	 * more, that is okay, the NS RRset was too large. During a referral
+	 * it can be shortened and then the first part of the list could
+	 * be used to resolve. The scrub continues to disallow glue for the
+	 * removed nameserver RRs and removes that too. Because the glue
+	 * is not marked as okay, since the RRs have been removed here. */
+	int i;
+	struct rr_parse* rr = rrset->rr_first, *prev = NULL;
+	if(!rr)
+		return;
+	for(i=0; i<count; i++) {
+		prev = rr;
+		rr = rr->next;
+		if(!rr)
+			return; /* The RRset is already short. */
+	}
+	if(verbosity >= VERB_QUERY
+		&& rrset->dname_len <= LDNS_MAX_DOMAINLEN) {
+		uint8_t buf[LDNS_MAX_DOMAINLEN+1];
+		dname_pkt_copy(pkt, buf, rrset->dname);
+		log_nametypeclass(VERB_QUERY, "normalize: shorten RRset:", buf,
+			rrset->type, ntohs(rrset->rrset_class));
+	}
+	/* remove further rrs */
+	rrset->rr_last = prev;
+	rrset->rr_count = count;
+	while(rr) {
+		rrset->size -= rr->size;
+		rr = rr->next;
+	}
+	if(rrset->rr_last)
+		rrset->rr_last->next = NULL;
+	else	rrset->rr_first = NULL;
+}
+
 /**
  * This routine normalizes a response. This includes removing "irrelevant"
  * records from the answer and additional sections and (re)synthesizing
@@ -386,6 +428,7 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 	uint8_t* sname = qinfo->qname;
 	size_t snamelen = qinfo->qname_len;
 	struct rrset_parse* rrset, *prev, *nsset=NULL;
+	int cname_length = 0; /* number of CNAMEs, or DNAMEs */
 
 	if(FLAGS_GET_RCODE(msg->flags) != LDNS_RCODE_NOERROR &&
 		FLAGS_GET_RCODE(msg->flags) != LDNS_RCODE_NXDOMAIN)
@@ -400,6 +443,16 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 	prev = NULL;
 	rrset = msg->rrset_first;
 	while(rrset && rrset->section == LDNS_SECTION_ANSWER) {
+		if(cname_length > env->cfg->iter_scrub_cname) {
+			/* Too many CNAMEs, or DNAMEs, from the authority
+			 * server, scrub down the length to something
+			 * shorter. This deletes everything after the limit
+			 * is reached. The iterator is going to look up
+			 * the content one by one anyway. */
+			remove_rrset("normalize: removing because too many cnames:",
+				pkt, msg, prev, &rrset);
+			continue;
+		}
 		if(rrset->type == LDNS_RR_TYPE_DNAME && 
 			pkt_strict_sub(pkt, sname, rrset->dname)) {
 			/* check if next rrset is correct CNAME. else,
@@ -419,6 +472,7 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 					"too long");
 				return 0;
 			}
+			cname_length++;
 			if(nx && nx->type == LDNS_RR_TYPE_CNAME && 
 			   dname_pkt_compare(pkt, sname, nx->dname) == 0) {
 				/* check next cname */
@@ -459,6 +513,7 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 		if(rrset->type == LDNS_RR_TYPE_CNAME) {
 			struct rrset_parse* nx = rrset->rrset_all_next;
 			uint8_t* oldsname = sname;
+			cname_length++;
 			/* see if the next one is a DNAME, if so, swap them */
 			if(nx && nx->section == LDNS_SECTION_ANSWER &&
 				nx->type == LDNS_RR_TYPE_DNAME &&
@@ -506,6 +561,10 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 					LDNS_SECTION_ANSWER &&
 					dname_pkt_compare(pkt, oldsname,
 					rrset->dname) == 0) {
+					if(rrset->type == LDNS_RR_TYPE_NS &&
+						rrset->rr_count > env->cfg->iter_scrub_ns) {
+						shorten_rrset(pkt, rrset, env->cfg->iter_scrub_ns);
+					}
 					prev = rrset;
 					rrset = rrset->rrset_all_next;
 				}
@@ -521,6 +580,11 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 			continue;
 		}
 
+		if(rrset->type == LDNS_RR_TYPE_NS &&
+			rrset->rr_count > env->cfg->iter_scrub_ns) {
+			shorten_rrset(pkt, rrset, env->cfg->iter_scrub_ns);
+		}
+
 		/* Mark the additional names from relevant rrset as OK. */
 		/* only for RRsets that match the query name, other ones
 		 * will be removed by sanitize, so no additional for them */
@@ -570,6 +634,22 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 					"RRset:", pkt, msg, prev, &rrset);
 				continue;
 			}
+			/* If the NS set is a promiscuous NS set, scrub that
+			 * to remove potential for poisonous contents that
+			 * affects other names in the same zone. Remove
+			 * promiscuous NS sets in positive answers, that
+			 * thus have records in the answer section. Nodata
+			 * and nxdomain promiscuous NS sets have been removed
+			 * already. Since the NS rrset is scrubbed, its
+			 * address records are also not marked to be allowed
+			 * and are removed later. */
+			if(FLAGS_GET_RCODE(msg->flags) == LDNS_RCODE_NOERROR &&
+				msg->an_rrsets != 0 &&
+				env->cfg->iter_scrub_promiscuous) {
+				remove_rrset("normalize: removing promiscuous "
+					"RRset:", pkt, msg, prev, &rrset);
+				continue;
+			}
 			if(nsset == NULL) {
 				nsset = rrset;
 			} else {
@@ -577,6 +657,25 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 					"RRset:", pkt, msg, prev, &rrset);
 				continue;
 			}
+			if(rrset->rr_count > env->cfg->iter_scrub_ns) {
+				/* If this is not a referral, and the NS RRset
+				 * is signed, then remove it entirely, so
+				 * that when it becomes bogus it does not
+				 * make the message that is otherwise fine
+				 * into a bogus message. */
+				if(!(msg->an_rrsets == 0 &&
+					FLAGS_GET_RCODE(msg->flags) ==
+					LDNS_RCODE_NOERROR &&
+					!soa_in_auth(msg) &&
+					!(msg->flags & BIT_AA)) &&
+					rrset->rrsig_count != 0) {
+					remove_rrset("normalize: removing too large NS "
+						"RRset:", pkt, msg, prev, &rrset);
+					continue;
+				} else {
+					shorten_rrset(pkt, rrset, env->cfg->iter_scrub_ns);
+				}
+			}
 		}
 		/* if this is type DS and we query for type DS we just got
 		 * a referral answer for our type DS query, fix packet */
@@ -788,6 +887,7 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
 {
 	int del_addi = 0; /* if additional-holding rrsets are deleted, we
 		do not trust the normalized additional-A-AAAA any more */
+	uint8_t* ns_rrset_dname = NULL;
 	int added_rrlen_ede = 0;
 	struct rrset_parse* rrset, *prev;
 	prev = NULL;
@@ -893,6 +993,16 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
 				continue;
 			}
 		}
+		if(rrset->type == LDNS_RR_TYPE_NS &&
+			(rrset->section == LDNS_SECTION_AUTHORITY ||
+			rrset->section == LDNS_SECTION_ANSWER)) {
+			/* If the type is NS, and we're in the
+			 * answer or authority section, then
+			 * store the dname so we can check
+			 * against the glue records
+			 * further down	*/
+			ns_rrset_dname = rrset->dname;
+		}
 		if(del_addi && rrset->section == LDNS_SECTION_ADDITIONAL) {
 			remove_rrset("sanitize: removing potential "
 			"poison reference RRset:", pkt, msg, prev, &rrset);
@@ -905,6 +1015,26 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
 				"RRset:", pkt, msg, prev, &rrset);
 			continue;
 		}
+		if(env->cfg->harden_unverified_glue && ns_rrset_dname &&
+			rrset->section == LDNS_SECTION_ADDITIONAL &&
+			(rrset->type == LDNS_RR_TYPE_A || rrset->type == LDNS_RR_TYPE_AAAA) &&
+			!pkt_strict_sub(pkt, rrset->dname, ns_rrset_dname)) {
+			/* We're in the additional section, looking
+			 * at an A/AAAA rrset, have a previous
+			 * delegation point and we notice that
+			 * the glue records are NOT for strict
+			 * subdomains of the delegation. So set a
+			 * flag, recompute the hash for the rrset
+			 * and write the A/AAAA record to cache.
+			 * It'll be retrieved if we can't separately
+			 * resolve the glue	*/
+			rrset->flags = PACKED_RRSET_UNVERIFIED_GLUE;
+			rrset->hash = pkt_hash_rrset(pkt, rrset->dname, rrset->type, rrset->rrset_class, rrset->flags);
+			store_rrset(pkt, msg, env, rrset);
+			remove_rrset("sanitize: storing potential "
+			"unverified glue reference RRset:", pkt, msg, prev, &rrset);
+			continue;
+		}
 		prev = rrset;
 		rrset = rrset->rrset_all_next;
 	}

iterator/iter_utils.c

@@ -77,41 +77,73 @@
 static const char DEFAULT_NAT64_PREFIX[] = "64:ff9b::/96";
 
 /** fillup fetch policy array */
-static void
-fetch_fill(struct iter_env* ie, const char* str)
+static int
+fetch_fill(int* target_fetch_policy, int max_dependency_depth, const char* str)
 {
 	char* s = (char*)str, *e;
 	int i;
-	for(i=0; i<ie->max_dependency_depth+1; i++) {
-		ie->target_fetch_policy[i] = strtol(s, &e, 10);
-		if(s == e)
-			fatal_exit("cannot parse fetch policy number %s", s);
+	for(i=0; i<max_dependency_depth+1; i++) {
+		target_fetch_policy[i] = strtol(s, &e, 10);
+		if(s == e) {
+			log_err("cannot parse fetch policy number %s", s);
+			return 0;
+		}
 		s = e;
 	}
+	return 1;
 }
 
 /** Read config string that represents the target fetch policy */
-static int
-read_fetch_policy(struct iter_env* ie, const char* str)
+int
+read_fetch_policy(int** target_fetch_policy, int* max_dependency_depth,
+	const char* str)
 {
 	int count = cfg_count_numbers(str);
 	if(count < 1) {
 		log_err("Cannot parse target fetch policy: \"%s\"", str);
 		return 0;
 	}
-	ie->max_dependency_depth = count - 1;
-	ie->target_fetch_policy = (int*)calloc(
-		(size_t)ie->max_dependency_depth+1, sizeof(int));
-	if(!ie->target_fetch_policy) {
+	*max_dependency_depth = count - 1;
+	*target_fetch_policy = (int*)calloc(
+		(size_t)(*max_dependency_depth)+1, sizeof(int));
+	if(!*target_fetch_policy) {
 		log_err("alloc fetch policy: out of memory");
 		return 0;
 	}
-	fetch_fill(ie, str);
+	if(!fetch_fill(*target_fetch_policy, *max_dependency_depth, str))
+		return 0;
 	return 1;
 }
 
-/** apply config caps whitelist items to name tree */
-static int
+struct rbtree_type*
+caps_white_create(void)
+{
+	struct rbtree_type* caps_white = rbtree_create(name_tree_compare);
+	if(!caps_white)
+		log_err("out of memory");
+	return caps_white;
+}
+
+/** delete caps_whitelist element */
+static void
+caps_free(struct rbnode_type* n, void* ATTR_UNUSED(d))
+{
+	if(n) {
+		free(((struct name_tree_node*)n)->name);
+		free(n);
+	}
+}
+
+void
+caps_white_delete(struct rbtree_type* caps_white)
+{
+	if(!caps_white)
+		return;
+	traverse_postorder(caps_white, caps_free, NULL);
+	free(caps_white);
+}
+
+int
 caps_white_apply_cfg(rbtree_type* ntree, struct config_file* cfg)
 {
 	struct config_strlist* p;
@@ -145,12 +177,41 @@ caps_white_apply_cfg(rbtree_type* ntree, struct config_file* cfg)
 }
 
 int
-iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
+nat64_apply_cfg(struct iter_nat64* nat64, struct config_file* cfg)
 {
 	const char *nat64_prefix;
+
+	nat64_prefix = cfg->nat64_prefix;
+	if(!nat64_prefix)
+		nat64_prefix = cfg->dns64_prefix;
+	if(!nat64_prefix)
+		nat64_prefix = DEFAULT_NAT64_PREFIX;
+	if(!netblockstrtoaddr(nat64_prefix, 0, &nat64->nat64_prefix_addr,
+		&nat64->nat64_prefix_addrlen, &nat64->nat64_prefix_net)) {
+		log_err("cannot parse nat64-prefix netblock: %s", nat64_prefix);
+		return 0;
+	}
+	if(!addr_is_ip6(&nat64->nat64_prefix_addr,
+		nat64->nat64_prefix_addrlen)) {
+		log_err("nat64-prefix is not IPv6: %s", cfg->nat64_prefix);
+		return 0;
+	}
+	if(!prefixnet_is_nat64(nat64->nat64_prefix_net)) {
+		log_err("nat64-prefix length it not 32, 40, 48, 56, 64 or 96: %s",
+			nat64_prefix);
+		return 0;
+	}
+	nat64->use_nat64 = cfg->do_nat64;
+	return 1;
+}
+
+int
+iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
+{
 	int i;
 	/* target fetch policy */
-	if(!read_fetch_policy(iter_env, cfg->target_fetch_policy))
+	if(!read_fetch_policy(&iter_env->target_fetch_policy,
+		&iter_env->max_dependency_depth, cfg->target_fetch_policy))
 		return 0;
 	for(i=0; i<iter_env->max_dependency_depth+1; i++)
 		verbose(VERB_QUERY, "target fetch policy for level %d is %d",
@@ -170,7 +231,7 @@ iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
 	}
 	if(cfg->caps_whitelist) {
 		if(!iter_env->caps_white)
-			iter_env->caps_white = rbtree_create(name_tree_compare);
+			iter_env->caps_white = caps_white_create();
 		if(!iter_env->caps_white || !caps_white_apply_cfg(
 			iter_env->caps_white, cfg)) {
 			log_err("Could not set capsforid whitelist");
@@ -179,31 +240,13 @@ iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
 
 	}
 
-	nat64_prefix = cfg->nat64_prefix;
-	if(!nat64_prefix)
-		nat64_prefix = cfg->dns64_prefix;
-	if(!nat64_prefix)
-		nat64_prefix = DEFAULT_NAT64_PREFIX;
-	if(!netblockstrtoaddr(nat64_prefix, 0, &iter_env->nat64_prefix_addr,
-		&iter_env->nat64_prefix_addrlen,
-		&iter_env->nat64_prefix_net)) {
-		log_err("cannot parse nat64-prefix netblock: %s", nat64_prefix);
-		return 0;
-	}
-	if(!addr_is_ip6(&iter_env->nat64_prefix_addr,
-		iter_env->nat64_prefix_addrlen)) {
-		log_err("nat64-prefix is not IPv6: %s", cfg->nat64_prefix);
-		return 0;
-	}
-	if(!prefixnet_is_nat64(iter_env->nat64_prefix_net)) {
-		log_err("nat64-prefix length it not 32, 40, 48, 56, 64 or 96: %s",
-			nat64_prefix);
+	if(!nat64_apply_cfg(&iter_env->nat64, cfg)) {
+		log_err("Could not setup nat64");
 		return 0;
 	}
 
 	iter_env->supports_ipv6 = cfg->do_ip6;
 	iter_env->supports_ipv4 = cfg->do_ip4;
-	iter_env->use_nat64 = cfg->do_nat64;
 	iter_env->outbound_msg_retry = cfg->outbound_msg_retry;
 	iter_env->max_sent_count = cfg->max_sent_count;
 	iter_env->max_query_restarts = cfg->max_query_restarts;
@@ -270,7 +313,7 @@ iter_filter_unsuitable(struct iter_env* iter_env, struct module_env* env,
 	if(!iter_env->supports_ipv6 && addr_is_ip6(&a->addr, a->addrlen)) {
 		return -1; /* there is no ip6 available */
 	}
-	if(!iter_env->supports_ipv4 && !iter_env->use_nat64 &&
+	if(!iter_env->supports_ipv4 && !iter_env->nat64.use_nat64 &&
 	   !addr_is_ip6(&a->addr, a->addrlen)) {
 		return -1; /* there is no ip4 available */
 	}
@@ -279,9 +322,10 @@ iter_filter_unsuitable(struct iter_env* iter_env, struct module_env* env,
 		name, namelen, qtype, &lame, &dnsseclame, &reclame,
 		&rtt, now)) {
 		log_addr(VERB_ALGO, "servselect", &a->addr, a->addrlen);
-		verbose(VERB_ALGO, "   rtt=%d%s%s%s%s", rtt,
+		verbose(VERB_ALGO, "   rtt=%d%s%s%s%s%s", rtt,
 			lame?" LAME":"",
 			dnsseclame?" DNSSEC_LAME":"",
+			a->dnsseclame?" ADDR_DNSSEC_LAME":"",
 			reclame?" REC_LAME":"",
 			a->lame?" ADDR_LAME":"");
 		if(lame)
@@ -692,10 +736,11 @@ dns_copy_msg(struct dns_msg* from, struct regional* region)
 void
 iter_dns_store(struct module_env* env, struct query_info* msgqinf,
 	struct reply_info* msgrep, int is_referral, time_t leeway, int pside,
-	struct regional* region, uint16_t flags, time_t qstarttime)
+	struct regional* region, uint16_t flags, time_t qstarttime,
+	int is_valrec)
 {
 	if(!dns_cache_store(env, msgqinf, msgrep, is_referral, leeway,
-		pside, region, flags, qstarttime))
+		pside, region, flags, qstarttime, is_valrec))
 		log_err("out of memory: cannot store data in cache");
 }
 
@@ -1284,8 +1329,17 @@ iter_get_next_root(struct iter_hints* hints, struct iter_forwards* fwd,
 	uint16_t* c)
 {
 	uint16_t c1 = *c, c2 = *c;
-	int r1 = hints_next_root(hints, &c1);
-	int r2 = forwards_next_root(fwd, &c2);
+	int r1, r2;
+	int nolock = 1;
+
+	/* prelock both forwards and hints for atomic read. */
+	lock_rw_rdlock(&fwd->lock);
+	lock_rw_rdlock(&hints->lock);
+	r1 = hints_next_root(hints, &c1, nolock);
+	r2 = forwards_next_root(fwd, &c2, nolock);
+	lock_rw_unlock(&fwd->lock);
+	lock_rw_unlock(&hints->lock);
+
 	if(!r1 && !r2) /* got none, end of list */
 		return 0;
 	else if(!r1) /* got one, return that */
@@ -1450,15 +1504,21 @@ int iter_dp_cangodown(struct query_info* qinfo, struct delegpt* dp)
 
 int
 iter_stub_fwd_no_cache(struct module_qstate *qstate, struct query_info *qinf,
-	uint8_t** retdpname, size_t* retdpnamelen)
+	uint8_t** retdpname, size_t* retdpnamelen, uint8_t* dpname_storage,
+	size_t dpname_storage_len)
 {
 	struct iter_hints_stub *stub;
 	struct delegpt *dp;
+	int nolock = 1;
 
 	/* Check for stub. */
+	/* Lock both forwards and hints for atomic read. */
+	lock_rw_rdlock(&qstate->env->fwds->lock);
+	lock_rw_rdlock(&qstate->env->hints->lock);
 	stub = hints_lookup_stub(qstate->env->hints, qinf->qname,
-	    qinf->qclass, NULL);
-	dp = forwards_lookup(qstate->env->fwds, qinf->qname, qinf->qclass);
+	    qinf->qclass, NULL, nolock);
+	dp = forwards_lookup(qstate->env->fwds, qinf->qname, qinf->qclass,
+		nolock);
 
 	/* see if forward or stub is more pertinent */
 	if(stub && stub->dp && dp) {
@@ -1472,35 +1532,62 @@ iter_stub_fwd_no_cache(struct module_qstate *qstate, struct query_info *qinf,
 
 	/* check stub */
 	if (stub != NULL && stub->dp != NULL) {
-		if(stub->dp->no_cache) {
-			char qname[255+1];
-			char dpname[255+1];
+		enum verbosity_value level = VERB_ALGO;
+		int stub_no_cache = stub->dp->no_cache;
+		lock_rw_unlock(&qstate->env->fwds->lock);
+		if(verbosity >= level && stub_no_cache) {
+			char qname[LDNS_MAX_DOMAINLEN];
+			char dpname[LDNS_MAX_DOMAINLEN];
 			dname_str(qinf->qname, qname);
 			dname_str(stub->dp->name, dpname);
-			verbose(VERB_ALGO, "stub for %s %s has no_cache", qname, dpname);
+			verbose(level, "stub for %s %s has no_cache", qname, dpname);
 		}
 		if(retdpname) {
-			*retdpname = stub->dp->name;
+			if(stub->dp->namelen > dpname_storage_len) {
+				verbose(VERB_ALGO, "no cache stub dpname too long");
+				lock_rw_unlock(&qstate->env->hints->lock);
+				*retdpname = NULL;
+				*retdpnamelen = 0;
+				return stub_no_cache;
+			}
+			memmove(dpname_storage, stub->dp->name,
+				stub->dp->namelen);
+			*retdpname = dpname_storage;
 			*retdpnamelen = stub->dp->namelen;
 		}
-		return (stub->dp->no_cache);
+		lock_rw_unlock(&qstate->env->hints->lock);
+		return stub_no_cache;
 	}
 
 	/* Check for forward. */
 	if (dp) {
-		if(dp->no_cache) {
-			char qname[255+1];
-			char dpname[255+1];
+		enum verbosity_value level = VERB_ALGO;
+		int dp_no_cache = dp->no_cache;
+		lock_rw_unlock(&qstate->env->hints->lock);
+		if(verbosity >= level && dp_no_cache) {
+			char qname[LDNS_MAX_DOMAINLEN];
+			char dpname[LDNS_MAX_DOMAINLEN];
 			dname_str(qinf->qname, qname);
 			dname_str(dp->name, dpname);
-			verbose(VERB_ALGO, "forward for %s %s has no_cache", qname, dpname);
+			verbose(level, "forward for %s %s has no_cache", qname, dpname);
 		}
 		if(retdpname) {
-			*retdpname = dp->name;
+			if(dp->namelen > dpname_storage_len) {
+				verbose(VERB_ALGO, "no cache dpname too long");
+				lock_rw_unlock(&qstate->env->fwds->lock);
+				*retdpname = NULL;
+				*retdpnamelen = 0;
+				return dp_no_cache;
+			}
+			memmove(dpname_storage, dp->name, dp->namelen);
+			*retdpname = dpname_storage;
 			*retdpnamelen = dp->namelen;
 		}
-		return (dp->no_cache);
+		lock_rw_unlock(&qstate->env->fwds->lock);
+		return dp_no_cache;
 	}
+	lock_rw_unlock(&qstate->env->fwds->lock);
+	lock_rw_unlock(&qstate->env->hints->lock);
 	if(retdpname) {
 		*retdpname = NULL;
 		*retdpnamelen = 0;
@@ -1523,3 +1610,54 @@ void iterator_set_ip46_support(struct module_stack* mods,
 	if(outnet->num_ip6 == 0)
 		ie->supports_ipv6 = 0;
 }
+
+void
+limit_nsec_ttl(struct dns_msg* msg)
+{
+	/* Limit NSEC and NSEC3 TTL in response, RFC9077 */
+	size_t i;
+	int found = 0;
+	time_t soa_ttl = 0;
+	/* Limit the NSEC and NSEC3 TTL values to the SOA TTL and SOA minimum
+	 * TTL. That has already been applied to the SOA record ttl. */
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_SOA) {
+			struct packed_rrset_data* soadata = (struct packed_rrset_data*)s->entry.data;
+			found = 1;
+			soa_ttl = soadata->ttl;
+			break;
+		}
+	}
+	if(!found)
+		return;
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_NSEC ||
+			ntohs(s->rk.type) == LDNS_RR_TYPE_NSEC3) {
+			struct packed_rrset_data* data = (struct packed_rrset_data*)s->entry.data;
+			/* Limit the negative TTL. */
+			if(data->ttl > soa_ttl) {
+				if(verbosity >= VERB_ALGO) {
+					char buf[256];
+					snprintf(buf, sizeof(buf),
+						"limiting TTL %d of %s record to the SOA TTL of %d for",
+						(int)data->ttl, ((ntohs(s->rk.type) == LDNS_RR_TYPE_NSEC)?"NSEC":"NSEC3"), (int)soa_ttl);
+					log_nametypeclass(VERB_ALGO, buf,
+						s->rk.dname, ntohs(s->rk.type),
+						ntohs(s->rk.rrset_class));
+				}
+				data->ttl = soa_ttl;
+			}
+		}
+	}
+}
+
+void
+iter_make_minimal(struct reply_info* rep)
+{
+	size_t rem = rep->ns_numrrsets + rep->ar_numrrsets;
+	rep->ns_numrrsets = 0;
+	rep->ar_numrrsets = 0;
+	rep->rrset_count -= rem;
+}

iterator/iter_utils.h

@@ -61,6 +61,7 @@ struct sock_list;
 struct ub_packed_rrset_key;
 struct module_stack;
 struct outside_network;
+struct iter_nat64;
 
 /* max number of lookups in the cache for target nameserver names.
  * This stops, for large delegations, N*N lookups in the cache. */
@@ -142,6 +143,7 @@ struct dns_msg* dns_copy_msg(struct dns_msg* from, struct regional* regional);
  * @param region: to copy modified (cache is better) rrs back to.
  * @param flags: with BIT_CD for dns64 AAAA translated queries.
  * @param qstarttime: time of query start.
+ * @param is_valrec: if the query is validation recursion and does not get
  * return void, because we are not interested in alloc errors,
  * 	the iterator and validator can operate on the results in their
  * 	scratch space (the qstate.region) and are not dependent on the cache.
@@ -150,7 +152,8 @@ struct dns_msg* dns_copy_msg(struct dns_msg* from, struct regional* regional);
  */
 void iter_dns_store(struct module_env* env, struct query_info* qinf,
 	struct reply_info* rep, int is_referral, time_t leeway, int pside,
-	struct regional* region, uint16_t flags, time_t qstarttime);
+	struct regional* region, uint16_t flags, time_t qstarttime,
+	int is_valrec);
 
 /**
  * Select randomly with n/m probability.
@@ -407,10 +410,14 @@ int iter_dp_cangodown(struct query_info* qinfo, struct delegpt* dp);
  * 	Used for NXDOMAIN checks, above that it is an nxdomain from a
  * 	different server and zone. You can pass NULL to not get it.
  * @param retdpnamelen: returns the length of the dpname.
+ * @param dpname_storage: this is where the dpname buf is stored, if any.
+ * 	So that caller can manage the buffer.
+ * @param dpname_storage_len: size of dpname_storage buffer.
  * @return true if no_cache is set in stub or fwd.
  */
 int iter_stub_fwd_no_cache(struct module_qstate *qstate,
-	struct query_info *qinf, uint8_t** retdpname, size_t* retdpnamelen);
+	struct query_info *qinf, uint8_t** retdpname, size_t* retdpnamelen,
+	uint8_t* dpname_storage, size_t dpname_storage_len);
 
 /**
  * Set support for IP4 and IP6 depending on outgoing interfaces
@@ -424,4 +431,55 @@ int iter_stub_fwd_no_cache(struct module_qstate *qstate,
 void iterator_set_ip46_support(struct module_stack* mods,
 	struct module_env* env, struct outside_network* outnet);
 
+/**
+ * Read config string that represents the target fetch policy.
+ * @param target_fetch_policy: alloced on return.
+ * @param max_dependency_depth: set on return.
+ * @param str: the config string
+ * @return false on failure.
+ */
+int read_fetch_policy(int** target_fetch_policy, int* max_dependency_depth,
+	const char* str);
+
+/**
+ * Create caps exempt data structure.
+ * @return NULL on failure.
+ */
+struct rbtree_type* caps_white_create(void);
+
+/**
+ * Delete caps exempt data structure.
+ * @param caps_white: caps exempt tree.
+ */
+void caps_white_delete(struct rbtree_type* caps_white);
+
+/**
+ * Apply config caps whitelist items to name tree
+ * @param ntree: caps exempt tree.
+ * @param cfg: config with options.
+ */
+int caps_white_apply_cfg(struct rbtree_type* ntree, struct config_file* cfg);
+
+/**
+ * Apply config for nat64
+ * @param nat64: the nat64 state.
+ * @param cfg: config with options.
+ * @return false on failure.
+ */
+int nat64_apply_cfg(struct iter_nat64* nat64, struct config_file* cfg);
+
+/**
+ * Limit NSEC and NSEC3 TTL in response, RFC9077
+ * @param msg: dns message, the SOA record ttl is used to restrict ttls
+ *	of NSEC and NSEC3 RRsets. If no SOA record, nothing happens.
+ */
+void limit_nsec_ttl(struct dns_msg* msg);
+
+/**
+ * Make the response minimal. Removed authority and additional section,
+ * that works when there is an answer in the answer section.
+ * @param rep: reply to modify.
+ */
+void iter_make_minimal(struct reply_info* rep);
+
 #endif /* ITERATOR_ITER_UTILS_H */

iterator/iterator.h

@@ -46,8 +46,6 @@
 #include "util/data/msgreply.h"
 #include "util/module.h"
 struct delegpt;
-struct iter_hints;
-struct iter_forwards;
 struct iter_donotq;
 struct iter_prep_list;
 struct iter_priv;
@@ -55,6 +53,9 @@ struct rbtree_type;
 
 /** max number of targets spawned for a query and its subqueries */
 #define MAX_TARGET_COUNT	64
+/** max number of upstream queries for a query and its subqueries, it is
+ * never reset. */
+extern int MAX_GLOBAL_QUOTA;
 /** max number of target lookups per qstate, per delegation point */
 #define MAX_DP_TARGET_COUNT	16
 /** max number of nxdomains allowed for target lookups for a query and
@@ -105,15 +106,9 @@ extern int BLACKLIST_PENALTY;
 #define EMPTY_NODATA_RETRY_COUNT 2
 
 /**
- * Global state for the iterator.
+ * Iterator global state for nat64.
  */
-struct iter_env {
-	/** A flag to indicate whether or not we have an IPv6 route */
-	int supports_ipv6;
-
-	/** A flag to indicate whether or not we have an IPv4 route */
-	int supports_ipv4;
-
+struct iter_nat64 {
 	/** A flag to locally apply NAT64 to make IPv4 addrs into IPv6 */
 	int use_nat64;
 
@@ -125,6 +120,20 @@ struct iter_env {
 
 	/** CIDR mask length of NAT64 prefix */
 	int nat64_prefix_net;
+};
+
+/**
+ * Global state for the iterator.
+ */
+struct iter_env {
+	/** A flag to indicate whether or not we have an IPv6 route */
+	int supports_ipv6;
+
+	/** A flag to indicate whether or not we have an IPv4 route */
+	int supports_ipv4;
+
+	/** State for nat64 */
+	struct iter_nat64 nat64;
 
 	/** A set of inetaddrs that should never be queried. */
 	struct iter_donotq* donotq;
@@ -248,6 +257,9 @@ enum target_count_variables {
 	TARGET_COUNT_QUERIES,
 	/** Number of nxdomain responses encountered. */
 	TARGET_COUNT_NX,
+	/** Global quota on number of queries to upstream servers per
+	 * client request, that is never reset. */
+	TARGET_COUNT_GLOBAL_QUOTA,
 
 	/** This should stay last here, it is used for the allocation */
 	TARGET_COUNT_MAX,

libunbound/context.c

@@ -53,6 +53,8 @@
 #include "util/storage/slabhash.h"
 #include "util/edns.h"
 #include "sldns/sbuffer.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_hints.h"
 
 int 
 context_finalize(struct ub_ctx* ctx)
@@ -73,7 +75,9 @@ context_finalize(struct ub_ctx* ctx)
 	ctx->pipe_pid = getpid();
 	cfg_apply_local_port_policy(cfg, 65536);
 	config_apply(cfg);
-	if(!modstack_setup(&ctx->mods, cfg->module_conf, ctx->env))
+	if(!modstack_call_startup(&ctx->mods, cfg->module_conf, ctx->env))
+		return UB_INITFAIL;
+	if(!modstack_call_init(&ctx->mods, cfg->module_conf, ctx->env))
 		return UB_INITFAIL;
 	listen_setup_locks();
 	log_edns_known_options(VERB_ALGO, ctx->env);
@@ -85,6 +89,12 @@ context_finalize(struct ub_ctx* ctx)
 	if(!auth_zones_apply_cfg(ctx->env->auth_zones, cfg, 1, &is_rpz,
 		ctx->env, &ctx->mods))
 		return UB_INITFAIL;
+	if(!(ctx->env->fwds = forwards_create()) ||
+		!forwards_apply_cfg(ctx->env->fwds, cfg))
+		return UB_INITFAIL;
+	if(!(ctx->env->hints = hints_create()) ||
+		!hints_apply_cfg(ctx->env->hints, cfg))
+		return UB_INITFAIL;
 	if(!edns_strings_apply_cfg(ctx->env->edns_strings, cfg))
 		return UB_INITFAIL;
 	if(!slabhash_is_size(ctx->env->msg_cache, cfg->msg_cache_size,
@@ -385,7 +395,7 @@ context_serialize_cancel(struct ctx_query* q, uint32_t* len)
 	/* format of cancel:
 	 * 	o uint32 cmd
 	 * 	o uint32 async-id */
-	uint8_t* p = (uint8_t*)reallocarray(NULL, sizeof(uint32_t), 2);
+	uint8_t* p = (uint8_t*)reallocarray(NULL, 2, sizeof(uint32_t));
 	if(!p) return NULL;
 	*len = 2*sizeof(uint32_t);
 	sldns_write_uint32(p, UB_LIBCMD_CANCEL);

libunbound/libunbound.c

@@ -66,6 +66,8 @@
 #include "services/authzone.h"
 #include "services/listen_dnsport.h"
 #include "sldns/sbuffer.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_hints.h"
 #ifdef HAVE_PTHREAD
 #include <signal.h>
 #endif
@@ -171,6 +173,7 @@ static struct ub_ctx* ub_ctx_create_nopipe(void)
 	ctx->env->worker = NULL;
 	ctx->env->need_to_validate = 0;
 	modstack_init(&ctx->mods);
+	ctx->env->modstack = &ctx->mods;
 	rbtree_init(&ctx->queries, &context_query_cmp);
 	return ctx;
 }
@@ -185,7 +188,9 @@ ub_ctx_create(void)
 		int e = errno;
 		ub_randfree(ctx->seed_rnd);
 		config_delete(ctx->env->cfg);
-		modstack_desetup(&ctx->mods, ctx->env);
+		modstack_call_deinit(&ctx->mods, ctx->env);
+		modstack_call_destartup(&ctx->mods, ctx->env);
+		modstack_free(&ctx->mods);
 		listen_desetup_locks();
 		edns_known_options_delete(ctx->env);
 		edns_strings_delete(ctx->env->edns_strings);
@@ -199,7 +204,9 @@ ub_ctx_create(void)
 		tube_delete(ctx->qq_pipe);
 		ub_randfree(ctx->seed_rnd);
 		config_delete(ctx->env->cfg);
-		modstack_desetup(&ctx->mods, ctx->env);
+		modstack_call_deinit(&ctx->mods, ctx->env);
+		modstack_call_destartup(&ctx->mods, ctx->env);
+		modstack_free(&ctx->mods);
 		listen_desetup_locks();
 		edns_known_options_delete(ctx->env);
 		edns_strings_delete(ctx->env->edns_strings);
@@ -357,7 +364,9 @@ ub_ctx_delete(struct ub_ctx* ctx)
 	}
 	libworker_delete_event(ctx->event_worker);
 
-	modstack_desetup(&ctx->mods, ctx->env);
+	modstack_call_deinit(&ctx->mods, ctx->env);
+	modstack_call_destartup(&ctx->mods, ctx->env);
+	modstack_free(&ctx->mods);
 	a = ctx->alloc_list;
 	while(a) {
 		na = a->super;
@@ -379,6 +388,8 @@ ub_ctx_delete(struct ub_ctx* ctx)
 		config_delete(ctx->env->cfg);
 		edns_known_options_delete(ctx->env);
 		edns_strings_delete(ctx->env->edns_strings);
+		forwards_delete(ctx->env->fwds);
+		hints_delete(ctx->env->hints);
 		auth_zones_delete(ctx->env->auth_zones);
 		free(ctx->env);
 	}
@@ -976,7 +987,8 @@ ub_ctx_set_fwd(struct ub_ctx* ctx, const char* addr)
 	if(!addr) {
 		/* disable fwd mode - the root stub should be first. */
 		if(ctx->env->cfg->forwards &&
-			strcmp(ctx->env->cfg->forwards->name, ".") == 0) {
+			(ctx->env->cfg->forwards->name &&
+			strcmp(ctx->env->cfg->forwards->name, ".") == 0)) {
 			s = ctx->env->cfg->forwards;
 			ctx->env->cfg->forwards = s->next;
 			s->next = NULL;
@@ -996,7 +1008,8 @@ ub_ctx_set_fwd(struct ub_ctx* ctx, const char* addr)
 	/* it parses, add root stub in front of list */
 	lock_basic_lock(&ctx->cfglock);
 	if(!ctx->env->cfg->forwards ||
-		strcmp(ctx->env->cfg->forwards->name, ".") != 0) {
+		(ctx->env->cfg->forwards->name &&
+		strcmp(ctx->env->cfg->forwards->name, ".") != 0)) {
 		s = calloc(1, sizeof(*s));
 		if(!s) {
 			lock_basic_unlock(&ctx->cfglock);
@@ -1014,6 +1027,7 @@ ub_ctx_set_fwd(struct ub_ctx* ctx, const char* addr)
 		ctx->env->cfg->forwards = s;
 	} else {
 		log_assert(ctx->env->cfg->forwards);
+		log_assert(ctx->env->cfg->forwards->name);
 		s = ctx->env->cfg->forwards;
 	}
 	dupl = strdup(addr);

libunbound/libworker.c

@@ -70,8 +70,6 @@
 #include "util/data/msgreply.h"
 #include "util/data/msgencode.h"
 #include "util/tube.h"
-#include "iterator/iter_fwd.h"
-#include "iterator/iter_hints.h"
 #include "sldns/sbuffer.h"
 #include "sldns/str2wire.h"
 #ifdef USE_DNSTAP
@@ -100,8 +98,6 @@ libworker_delete_env(struct libworker* w)
 			!w->is_bg || w->is_bg_thread);
 		sldns_buffer_free(w->env->scratch_buffer);
 		regional_destroy(w->env->scratch);
-		forwards_delete(w->env->fwds);
-		hints_delete(w->env->hints);
 		ub_randfree(w->env->rnd);
 		free(w->env);
 	}
@@ -159,30 +155,19 @@ libworker_setup(struct ub_ctx* ctx, int is_bg, struct ub_event_base* eb)
 	}
 	w->env->scratch = regional_create_custom(cfg->msg_buffer_size);
 	w->env->scratch_buffer = sldns_buffer_new(cfg->msg_buffer_size);
-	w->env->fwds = forwards_create();
-	if(w->env->fwds && !forwards_apply_cfg(w->env->fwds, cfg)) { 
-		forwards_delete(w->env->fwds);
-		w->env->fwds = NULL;
-	}
-	w->env->hints = hints_create();
-	if(w->env->hints && !hints_apply_cfg(w->env->hints, cfg)) { 
-		hints_delete(w->env->hints);
-		w->env->hints = NULL;
-	}
 #ifdef HAVE_SSL
 	w->sslctx = connect_sslctx_create(NULL, NULL,
 		cfg->tls_cert_bundle, cfg->tls_win_cert);
 	if(!w->sslctx) {
 		/* to make the setup fail after unlock */
-		hints_delete(w->env->hints);
-		w->env->hints = NULL;
+		sldns_buffer_free(w->env->scratch_buffer);
+		w->env->scratch_buffer = NULL;
 	}
 #endif
 	if(!w->is_bg || w->is_bg_thread) {
 		lock_basic_unlock(&ctx->cfglock);
 	}
-	if(!w->env->scratch || !w->env->scratch_buffer || !w->env->fwds ||
-		!w->env->hints) {
+	if(!w->env->scratch || !w->env->scratch_buffer) {
 		libworker_delete(w);
 		return NULL;
 	}
@@ -307,6 +292,7 @@ libworker_do_cmd(struct libworker* w, uint8_t* msg, uint32_t len)
 			log_err("unknown command for bg worker %d", 
 				(int)context_serial_getcmd(msg, len));
 			/* and fall through to quit */
+			ATTR_FALLTHROUGH
 			/* fallthrough */
 		case UB_LIBCMD_QUIT:
 			free(msg);
@@ -437,7 +423,7 @@ int libworker_bg(struct ub_ctx* ctx)
 static int
 fill_canon(struct ub_result* res, uint8_t* s)
 {
-	char buf[255+2];
+	char buf[LDNS_MAX_DOMAINLEN];
 	dname_str(s, buf);
 	res->canonname = strdup(buf);
 	return res->canonname != 0;
@@ -644,8 +630,9 @@ int libworker_fg(struct ub_ctx* ctx, struct ctx_query* q)
 		free(qinfo.qname);
 		return UB_NOERROR;
 	}
-	if(ctx->env->auth_zones && auth_zones_answer(ctx->env->auth_zones,
-		w->env, &qinfo, &edns, NULL, w->back->udp_buff, w->env->scratch)) {
+	if(ctx->env->auth_zones && auth_zones_downstream_answer(
+		ctx->env->auth_zones, w->env, &qinfo, &edns, NULL,
+		w->back->udp_buff, w->env->scratch)) {
 		regional_free_all(w->env->scratch);
 		libworker_fillup_fg(q, LDNS_RCODE_NOERROR, 
 			w->back->udp_buff, sec_status_insecure, NULL, 0);
@@ -723,8 +710,9 @@ int libworker_attach_mesh(struct ub_ctx* ctx, struct ctx_query* q,
 			w->back->udp_buff, sec_status_insecure, NULL, 0);
 		return UB_NOERROR;
 	}
-	if(ctx->env->auth_zones && auth_zones_answer(ctx->env->auth_zones,
-		w->env, &qinfo, &edns, NULL, w->back->udp_buff, w->env->scratch)) {
+	if(ctx->env->auth_zones && auth_zones_downstream_answer(
+		ctx->env->auth_zones, w->env, &qinfo, &edns, NULL,
+		w->back->udp_buff, w->env->scratch)) {
 		regional_free_all(w->env->scratch);
 		free(qinfo.qname);
 		libworker_event_done_cb(q, LDNS_RCODE_NOERROR,
@@ -861,8 +849,9 @@ handle_newq(struct libworker* w, uint8_t* buf, uint32_t len)
 		free(qinfo.qname);
 		return;
 	}
-	if(w->ctx->env->auth_zones && auth_zones_answer(w->ctx->env->auth_zones,
-		w->env, &qinfo, &edns, NULL, w->back->udp_buff, w->env->scratch)) {
+	if(w->ctx->env->auth_zones && auth_zones_downstream_answer(
+		w->ctx->env->auth_zones, w->env, &qinfo, &edns, NULL,
+		w->back->udp_buff, w->env->scratch)) {
 		regional_free_all(w->env->scratch);
 		q->msg_security = sec_status_insecure;
 		add_bg_result(w, q, w->back->udp_buff, UB_NOERROR, NULL, 0);
@@ -1072,3 +1061,33 @@ void dtio_mainfdcallback(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
 	log_assert(0);
 }
 #endif
+
+void fast_reload_service_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+int fast_reload_client_callback(struct comm_point* ATTR_UNUSED(c),
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(repinfo))
+{
+	log_assert(0);
+	return 0;
+}
+
+#ifdef HAVE_NGTCP2
+void doq_client_event_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+#endif
+
+#ifdef HAVE_NGTCP2
+void doq_client_timer_cb(int ATTR_UNUSED(fd), short ATTR_UNUSED(ev),
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+#endif

libunbound/python/examples/dnssec_test.py

@@ -30,6 +30,7 @@ def dnssecParse(domain, rrType=RR_TYPE_A):
 resolver = ub_ctx()
 resolver.add_ta(".   IN DS   19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5")
 resolver.add_ta(".   IN DS   20326 8 2 E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D")
+resolver.add_ta(".   IN DS   38696 8 2 683D2D0ACB8C9B712A1948B27F741219298D0A450D612C483AF444A4C0FB2B16")
 
 dnssecParse("nic.cz")
 dnssecParse("nonexistent-domain-blablabla.cz")

libunbound/unbound.h

@@ -772,6 +772,8 @@ struct ub_server_stats {
 	long long ans_bogus;
 	/** rrsets marked bogus by validator */
 	long long rrset_bogus;
+	/** number of signature validation operations performed by validator */
+	long long val_ops;
 	/** number of queries that have been ratelimited by domain recursion. */
 	long long queries_ratelimited;
 	/** unwanted traffic received on server-facing ports */
@@ -845,6 +847,16 @@ struct ub_server_stats {
 	long long qtls_resume;
 	/** RPZ action stats */
 	long long rpz_action[UB_STATS_RPZ_ACTION_NUM];
+	/** number of bytes in QUIC buffers */
+	long long mem_quic;
+	/** number of queries over (DNS over) QUIC */
+	long long qquic;
+	/** number of queries removed due to discard-timeout */
+	long long num_queries_discard_timeout;
+	/** number of queries removed due to wait-limit */
+	long long num_queries_wait_limit;
+	/** number of dns error reports generated */
+	long long num_dns_error_reports;
 };
 
 /**

makedist.sh

@@ -45,20 +45,24 @@ usage () {
 Usage $0: [-h] [-s] [-u git_url] [-b git_branch] [-w ...args...]
 Generate a distribution tar file for unbound.
 
-    -h           This usage information.
-    -s           Build a snapshot distribution file.  The current date is
-                 automatically appended to the current unbound version number.
-    -rc <nr>     Build a release candidate, the given string will be added
-                 to the version number 
-                 (which will then be unbound-<version>rc<number>)
-    -u git_url   Retrieve the source from the specified repository url.
-                 Detected from the working copy if not specified.
-    -b git_branch Retrieve the the specified branch or tag.
-                 Detected from the working copy if not specified.
+    -h                  This usage information.
+    -s                  Build a snapshot distribution file.  The current date is
+                        automatically appended to the current unbound version number.
+    -rc <nr>            Build a release candidate, the given string will be added
+                        to the version number
+                        (which will then be unbound-<version>rc<number>)
+    -u git_url          Retrieve the source from the specified repository url.
+                        Detected from the working copy if not specified.
+    -b git_branch       Retrieve the specified branch or tag.
+                        Detected from the working copy if not specified.
     -wssl openssl.xx.tar.gz Also build openssl from tarball for windows dist.
     -wxp expat.xx.tar.gz Also build expat from tarball for windows dist.
-    -w32	 32bit windows compile.
-    -w ...       Build windows binary dist. last args passed to configure.
+    -wdir directory     Build openssl and expat in a persistent directory for
+                        windows dist. If builds are already in that directory
+                        they are used right away. Useful when debugging windows
+                        builds.
+    -w32                32bit windows compile.
+    -w ...              Build windows binary dist. last args passed to configure.
 EOF
     exit 1
 }
@@ -143,6 +147,16 @@ create_temp_dir () {
     cd $temp_dir
 }
 
+activate_windebug_dir () {
+    info "Activating persistent directory for windows build."
+    if test ! -d "$WINDIR"; then
+        mkdir -p "$WINDIR"
+        info "Created $WINDIR persistent directory."
+    fi
+    cd "$WINDIR"
+    WINDIR="`pwd`"
+}
+
 # pass filename as $1 arg.
 # creates file.sha1 and file.sha256
 storehash () {
@@ -187,7 +201,9 @@ DOWIN="no"
 W64="yes"
 WINSSL=""
 WINEXPAT=""
-MINJ=""
+WINDIR=""
+# Use environment if set otherwise null
+MINJ="${MINJ:+$MINJ}"
 
 # Parse the command line arguments.
 while [ "$1" ]; do
@@ -214,6 +230,10 @@ while [ "$1" ]; do
 	    WINEXPAT="$2"
 	    shift
 	    ;;
+        "-wdir")
+	    WINDIR="$2"
+	    shift
+	    ;;
 	"-w32")
 	    W64="no"
 	    ;;
@@ -253,8 +273,27 @@ if [ "$DOWIN" = "yes" ]; then
 	shared_cross_flag=""
 
 	check_git_repo
-	create_temp_dir
 
+	if test -n "$WINDIR"; then
+		activate_windebug_dir
+	else
+		create_temp_dir
+	fi
+
+	if test -n "$WINSSL" \
+		-a -n "$WINDIR" \
+		-a -d "$WINDIR" \
+		-a -d "${WINDIR}/sslinstall" \
+		-a -d "${WINDIR}/openssl_shared" \
+		-a -d "${WINDIR}/sslsharedinstall"; then
+		info "Found already compiled openssl at $WINDIR/sslinstall; using that."
+		WINSSL=""
+		# Variables needed later on.
+		sslinstall="${WINDIR}/sslinstall"
+		cross_flag="$cross_flag --with-ssl=$sslinstall"
+		sslsharedinstall="${WINDIR}/sslsharedinstall"
+		shared_cross_flag="$shared_cross_flag --with-ssl=$sslsharedinstall"
+	fi
 	# crosscompile openssl for windows.
 	if test -n "$WINSSL"; then
 		info "Cross compile $WINSSL"
@@ -267,10 +306,10 @@ if [ "$DOWIN" = "yes" ]; then
 		# cross-compilation and it is not used anyway
 		# before 1.0.1i need --cross-compile-prefix=i686-w64-mingw32-
 		if test "$mw64" = "mingw64"; then
-			sslflags="no-asm -DOPENSSL_NO_CAPIENG mingw64"
+			sslflags="no-asm no-tests -DOPENSSL_NO_CAPIENG mingw64"
 			sspdll="/usr/x86_64-w64-mingw32/sys-root/mingw/bin/libssp-0.dll"
 		else
-			sslflags="no-asm -DOPENSSL_NO_CAPIENG mingw"
+			sslflags="no-asm no-tests -DOPENSSL_NO_CAPIENG mingw"
 			sspdll="/usr/i686-w64-mingw32/sys-root/mingw/bin/libssp-0.dll"
 		fi
 		if test -f "$sspdll"; then
@@ -282,9 +321,18 @@ if [ "$DOWIN" = "yes" ]; then
 			SSPLIB=""
 		fi
 		info "winssl: Configure no-shared $sslflags"
-		set -x # echo the configure command
-		__CNF_LDLIBS=$SSPLIB CC=${warch}-w64-mingw32-gcc AR=${warch}-w64-mingw32-ar RANLIB=${warch}-w64-mingw32-ranlib WINDRES=${warch}-w64-mingw32-windres ./Configure --prefix="$sslinstall" no-shared $sslflags || error_cleanup "OpenSSL Configure failed"
-		set +x
+		if test "$W64" = "no"; then
+			# Disable stack-protector for 32-bit windows builds.
+			# mingw passes an LDFLAGS, so there is something
+			# passed in the LDFLAGS to stop -lssp passed in it.
+			set -x # echo the configure command
+			__CNF_LDLIBS=$SSPLIB __CNF_LDFLAGS="-fno-stack-protector" CC=${warch}-w64-mingw32-gcc AR=${warch}-w64-mingw32-ar RANLIB=${warch}-w64-mingw32-ranlib WINDRES=${warch}-w64-mingw32-windres ./Configure --prefix="$sslinstall" no-shared $sslflags || error_cleanup "OpenSSL Configure failed"
+			set +x
+		else
+			set -x # echo the configure command
+			__CNF_LDLIBS=$SSPLIB CC=${warch}-w64-mingw32-gcc AR=${warch}-w64-mingw32-ar RANLIB=${warch}-w64-mingw32-ranlib WINDRES=${warch}-w64-mingw32-windres ./Configure --prefix="$sslinstall" no-shared $sslflags || error_cleanup "OpenSSL Configure failed"
+			set +x
+		fi
 		info "winssl: make"
 		make $MINJ || error_cleanup "OpenSSL crosscompile failed"
 		# only install sw not docs, which take a long time.
@@ -297,9 +345,15 @@ if [ "$DOWIN" = "yes" ]; then
 		sslsharedinstall="`pwd`/sslsharedinstall"
 		cd openssl_shared
 		info "winssl: Configure shared $sslflags"
-		set -x # echo the configure command
-		__CNF_LDLIBS=$SSPLIB CC=${warch}-w64-mingw32-gcc AR=${warch}-w64-mingw32-ar RANLIB=${warch}-w64-mingw32-ranlib WINDRES=${warch}-w64-mingw32-windres ./Configure --prefix="$sslsharedinstall" shared $sslflags || error_cleanup "OpenSSL Configure failed"
-		set +x
+		if test "$W64" = "no"; then
+			set -x # echo the configure command
+			__CNF_LDLIBS=$SSPLIB __CNF_LDFLAGS="-fno-stack-protector" CC=${warch}-w64-mingw32-gcc AR=${warch}-w64-mingw32-ar RANLIB=${warch}-w64-mingw32-ranlib WINDRES=${warch}-w64-mingw32-windres ./Configure --prefix="$sslsharedinstall" shared $sslflags || error_cleanup "OpenSSL Configure failed"
+			set +x
+		else
+			set -x # echo the configure command
+			__CNF_LDLIBS=$SSPLIB CC=${warch}-w64-mingw32-gcc AR=${warch}-w64-mingw32-ar RANLIB=${warch}-w64-mingw32-ranlib WINDRES=${warch}-w64-mingw32-windres ./Configure --prefix="$sslsharedinstall" shared $sslflags || error_cleanup "OpenSSL Configure failed"
+			set +x
+		fi
 		info "winssl: make"
 		make $MINJ || error_cleanup "OpenSSL crosscompile failed"
 		info "winssl: make install_sw"
@@ -308,6 +362,17 @@ if [ "$DOWIN" = "yes" ]; then
 		cd ..
 	fi
 
+	if test -n "$WINEXPAT" \
+		-a -n "$WINDIR" \
+		-a -d "$WINDIR" \
+		-a -d "${WINDIR}/wxpinstall"; then
+		info "Found already compiled expat at $WINDIR/wxpinstall; using that."
+		WINEXPAT=""
+		# Variables needed later on.
+		wxpinstall="${WINDIR}/wxpinstall"
+		cross_flag="$cross_flag --with-libexpat=$wxpinstall"
+		shared_cross_flag="$shared_cross_flag --with-libexpat=$wxpinstall"
+	fi
 	if test -n "$WINEXPAT"; then
 		info "Cross compile $WINEXPAT"
 		info "wxp: tar unpack"
@@ -315,7 +380,16 @@ if [ "$DOWIN" = "yes" ]; then
 		wxpinstall="`pwd`/wxpinstall"
 		cd expat-* || error_cleanup "no expat-X dir in tarball"
 		info "wxp: configure"
-		$configure --prefix="$wxpinstall" --exec-prefix="$wxpinstall" --bindir="$wxpinstall/bin" --includedir="$wxpinstall/include" --mandir="$wxpinstall/man" --libdir="$wxpinstall/lib"  || error_cleanup "libexpat configure failed"
+		if test "$W64" = "no"; then
+			# Disable stack-protector for 32-bit windows builds.
+			set -x # echo the configure command
+			$configure --prefix="$wxpinstall" --exec-prefix="$wxpinstall" --bindir="$wxpinstall/bin" --includedir="$wxpinstall/include" --mandir="$wxpinstall/man" --libdir="$wxpinstall/lib" LDFLAGS="-fno-stack-protector" || error_cleanup "libexpat configure failed"
+			set +x
+		else
+			set -x # echo the configure command
+			$configure --prefix="$wxpinstall" --exec-prefix="$wxpinstall" --bindir="$wxpinstall/bin" --includedir="$wxpinstall/include" --mandir="$wxpinstall/man" --libdir="$wxpinstall/lib"  || error_cleanup "libexpat configure failed"
+			set +x
+		fi
 		info "wxp: make"
 		make $MINJ || error_cleanup "libexpat crosscompile failed"
 		info "wxp: make install"
@@ -325,8 +399,12 @@ if [ "$DOWIN" = "yes" ]; then
 		cd ..
 	fi
 
-        info "GITREPO   is $GITREPO"
-        info "GITBRANCH is $GITBRANCH"
+	if test -n "$WINDIR"; then
+		cd "$cwd"
+		create_temp_dir
+	fi
+	info "GITREPO   is $GITREPO"
+	info "GITBRANCH is $GITBRANCH"
 	info "Exporting source from git."
 	info "git clone --depth=1 --no-tags -b $GITBRANCH $GITREPO unbound"
 	git clone --depth=1 --no-tags -b $GITBRANCH $GITREPO unbound || error_cleanup "git clone failed"
@@ -361,10 +439,14 @@ if [ "$DOWIN" = "yes" ]; then
 	autoconf -f || error_cleanup "Autoconf failed."
 	autoheader -f || error_cleanup "Autoheader failed."
     	rm -r autom4te* || echo "ignored"
+	rm -f config.h.in~ || echo "ignore absence of config.h.in~ file."
     fi
 
     if test "`uname`" = "Linux"; then 
-	    (cd ..; cp -r unbound unbound_shared)
+	    cd ..
+	    cp -r unbound unbound_shared
+	    unbound_shared="`pwd`/unbound_shared"
+	    cd unbound
     fi
 
     # procedure for making unbound installer on mingw. 
@@ -378,13 +460,15 @@ if [ "$DOWIN" = "yes" ]; then
     fi
     if test "$W64" = "no"; then
 		# Disable stack-protector for 32-bit windows builds.
-		echo "$configure"' --enable-debug --enable-static-exe --disable-flto --disable-gost '"$* $cross_flag" "$file_flag" "$file2_flag" "$file3_flag" CFLAGS='-O2 -g -fno-stack-protector'
-		$configure --enable-debug --enable-static-exe --disable-flto --disable-gost $* $cross_flag "$file_flag" "$file2_flag" "$file3_flag" CFLAGS='-O2 -g -fno-stack-protector'\
+		set -x
+		$configure --enable-debug --enable-static-exe --disable-flto --disable-gost $* $cross_flag "$file_flag" "$file2_flag" "$file3_flag" CFLAGS='-O2 -g -fno-stack-protector' LDFLAGS="-fno-stack-protector" \
 		|| error_cleanup "Could not configure"
+		set +x
     else
-		echo "$configure"' --enable-debug --enable-static-exe --disable-flto --disable-gost '"$* $cross_flag"
+		set -x
 		$configure --enable-debug --enable-static-exe --disable-flto --disable-gost $* $cross_flag \
 		|| error_cleanup "Could not configure"
+		set +x
     fi
     info "Calling make"
 	make $MINJ || error_cleanup "Could not make"
@@ -392,16 +476,18 @@ if [ "$DOWIN" = "yes" ]; then
 
     if test "`uname`" = "Linux"; then 
     info "Make DLL"
-    cd ../unbound_shared
+    cd $unbound_shared
     if test "$W64" = "no"; then
 	# Disable stack-protector for 32-bit windows builds.
-		echo "$configure"' --enable-debug --disable-flto --disable-gost '"$* $shared_cross_flag" "$file_flag" "$file2_flag" "$file3_flag" CFLAGS='-O2 -g -fno-stack-protector'
-		$configure --enable-debug --disable-flto --disable-gost $* $shared_cross_flag "$file_flag" "$file2_flag" "$file3_flag" CFLAGS='-O2 -g -fno-stack-protector'\
+		set -x
+		$configure --enable-debug --disable-flto --disable-gost $* $shared_cross_flag "$file_flag" "$file2_flag" "$file3_flag" CFLAGS='-O2 -g -fno-stack-protector' LDFLAGS="-fno-stack-protector" \
 		|| error_cleanup "Could not configure"
+		set +x
     else
-		echo "$configure"' --enable-debug --disable-flto --disable-gost '"$* $shared_cross_flag"
-        $configure --enable-debug --disable-flto --disable-gost $* $shared_cross_flag \
+		set -x
+		$configure --enable-debug --disable-flto --disable-gost $* $shared_cross_flag \
 		|| error_cleanup "Could not configure"
+		set +x
     fi
     info "Calling make for DLL"
 	make $MINJ || error_cleanup "Could not make DLL"
@@ -430,20 +516,35 @@ if [ "$DOWIN" = "yes" ]; then
     cp ../unbound.exe ../unbound-anchor.exe ../unbound-host.exe ../unbound-control.exe ../unbound-checkconf.exe ../unbound-service-install.exe ../unbound-service-remove.exe ../LICENSE ../winrc/unbound-control-setup.cmd ../winrc/unbound-website.url ../winrc/service.conf ../winrc/README.txt ../contrib/create_unbound_ad_servers.cmd ../contrib/warmup.cmd ../contrib/unbound_cache.cmd .
     mkdir libunbound
     # test to see if lib or lib64 (for openssl 3.0.0) needs to be used
-    if test -f ../../sslsharedinstall/lib/libcrypto.dll.a; then
-	cp ../../sslsharedinstall/lib/libcrypto.dll.a libunbound/.
+    if test -f $sslsharedinstall/lib/libcrypto.dll.a; then
+	cp $sslsharedinstall/lib/libcrypto.dll.a libunbound/.
     else
-	cp ../../sslsharedinstall/lib64/libcrypto.dll.a libunbound/.
+	cp $sslsharedinstall/lib64/libcrypto.dll.a libunbound/.
     fi
-    if test -f ../../sslsharedinstall/lib/libssl.dll.a; then
-	cp ../../sslsharedinstall/lib/libssl.dll.a libunbound/.
+    if test -f $sslsharedinstall/lib/libssl.dll.a; then
+	cp $sslsharedinstall/lib/libssl.dll.a libunbound/.
     else
-	cp ../../sslsharedinstall/lib64/libssl.dll.a libunbound/.
+	cp $sslsharedinstall/lib64/libssl.dll.a libunbound/.
     fi
-    cp ../../unbound_shared/unbound.h ../../unbound_shared/.libs/libunbound*.dll ../../unbound_shared/.libs/libunbound.dll.a ../../unbound_shared/.libs/libunbound.a ../../unbound_shared/.libs/libunbound*.def ../../sslsharedinstall/bin/libcrypto*.dll ../../sslsharedinstall/bin/libssl*.dll ../../wxpinstall/bin/libexpat*.dll ../../wxpinstall/lib/libexpat.dll.a libunbound/.
+    cp $unbound_shared/unbound.h               \
+	$unbound_shared/.libs/libunbound*.dll  \
+	$unbound_shared/.libs/libunbound.dll.a \
+	$unbound_shared/.libs/libunbound.a     \
+	$unbound_shared/.libs/libunbound*.def  \
+	$sslsharedinstall/bin/libcrypto*.dll   \
+	$sslsharedinstall/bin/libssl*.dll      \
+	$wxpinstall/bin/libexpat*.dll          \
+	$wxpinstall/lib/libexpat.dll.a         \
+	libunbound/.
     if test -f "$sspdll"; then
 	    cp "$sspdll" libunbound/.
     fi
+    if test "$W64" = "no"; then
+	# This could be solved with -static -static-libgcc -static-libstdc++.
+	# The dependency on c++ is probably due to libexpat. But the copy
+	# of the dll should work too. It may be needed for the libunbound.dll.
+	cp /usr/i686-w64-mingw32/sys-root/mingw/bin/libgcc_s_dw2-1.dll libunbound/.
+    fi
     # zipfile
     zip -r ../$file LICENSE README.txt unbound.exe unbound-anchor.exe unbound-host.exe unbound-control.exe unbound-checkconf.exe unbound-service-install.exe unbound-service-remove.exe unbound-control-setup.cmd example.conf service.conf root.key unbound-website.url create_unbound_ad_servers.cmd warmup.cmd unbound_cache.cmd Changelog libunbound
     info "Testing $file"
@@ -470,6 +571,11 @@ if [ "$DOWIN" = "yes" ]; then
     storehash unbound-$version.zip
     ls -lG unbound_setup_$version.exe
     ls -lG unbound-$version.zip
+
+    echo "create signed versions with:"
+    echo "  gpg --armor --detach-sign --digest-algo SHA256 unbound_setup_$version.exe"
+    echo "  gpg --armor --detach-sign --digest-algo SHA256 unbound-$version.zip"
+
     info "Done"
     exit 0
 fi
@@ -514,6 +620,7 @@ info "Building configure script (autoreconf)."
 autoreconf -f || error_cleanup "Autoconf failed."
 
 rm -r autom4te* || error_cleanup "Failed to remove autoconf cache directory."
+rm -f config.h.in~ || echo "ignore absence of config.h.in~ file."
 
 info "Building lexer and parser."
 echo "#include \"config.h\"" > util/configlexer.c || error_cleanup "Failed to create configlexer"
@@ -558,6 +665,7 @@ if [ "$RECONFIGURE" = "yes" ]; then
     info "Rebuilding configure script (autoconf) snapshot."
     autoreconf -f || error_cleanup "Autoconf failed."
     rm -r autom4te* || error_cleanup "Failed to remove autoconf cache directory."
+    rm -f config.h.in~ || echo "ignore absence of config.h.in~ file."
 fi
 
 replace_all doc/README

pythonmod/doc/modules/config.rst

@@ -176,6 +176,10 @@ config_file
    
       Harden against spoofed glue (out of zone data).
    
+   .. attribute:: harden_unverified_glue
+
+      Harden against unverified glue.
+
    .. attribute:: harden_dnssec_stripped
    
       Harden against receiving no DNSSEC data for trust anchor.

pythonmod/examples/log.py

@@ -38,12 +38,14 @@ import os
 def dataHex(data, prefix=""):
     """Converts binary string data to display representation form"""
     res = ""
-    for i in range(0, (len(data)+15)/16):
+    for i in range(0, int((len(data)+15)/16)):
         res += "%s0x%02X | " % (prefix, i*16)
-        d = map(lambda x:ord(x), data[i*16:i*16+17])
+        d = map(lambda x:x, data[i*16:i*16+17])
+        count=0
         for ch in d:
             res += "%02X " % ch
-        for i in range(0,17-len(d)):
+            count+=1
+        for i in range(0,17-count):
             res += "   "
         res += "| "
         for ch in d:
@@ -60,31 +62,31 @@ def logDnsMsg(qstate):
     r  = qstate.return_msg.rep
     q  = qstate.return_msg.qinfo
 
-    print "-"*100
+    print("-"*100)
     print("Query: %s, type: %s (%d), class: %s (%d) " % (
             qstate.qinfo.qname_str, qstate.qinfo.qtype_str, qstate.qinfo.qtype,
             qstate.qinfo.qclass_str, qstate.qinfo.qclass))
-    print "-"*100
-    print "Return    reply :: flags: %04X, QDcount: %d, Security:%d, TTL=%d" % (r.flags, r.qdcount, r.security, r.ttl)
-    print "          qinfo :: qname: %s %s, qtype: %s, qclass: %s" % (str(q.qname_list), q.qname_str, q.qtype_str, q.qclass_str)
+    print("-"*100)
+    print("Return    reply :: flags: %04X, QDcount: %d, Security:%d, TTL=%d" % (r.flags, r.qdcount, r.security, r.ttl))
+    print("          qinfo :: qname: %s %s, qtype: %s, qclass: %s" % (str(q.qname_list), q.qname_str, q.qtype_str, q.qclass_str))
 
     if (r):
-        print "Reply:"
+        print("Reply:")
         for i in range(0, r.rrset_count):
             rr = r.rrsets[i]
 
             rk = rr.rk
-            print i,":",rk.dname_list, rk.dname_str, "flags: %04X" % rk.flags,
-            print "type:",rk.type_str,"(%d)" % ntohs(rk.type), "class:",rk.rrset_class_str,"(%d)" % ntohs(rk.rrset_class)
+            print(i,":",rk.dname_list, rk.dname_str, "flags: %04X" % rk.flags,)
+            print("type:",rk.type_str,"(%d)" % ntohs(rk.type), "class:",rk.rrset_class_str,"(%d)" % ntohs(rk.rrset_class))
 
             d = rr.entry.data
             for j in range(0,d.count+d.rrsig_count):
-                print "  ",j,":","TTL=",d.rr_ttl[j],
-                if (j >= d.count): print "rrsig",
-                print 
-                print dataHex(d.rr_data[j],"       ")
+                print("  ",j,":","TTL=",d.rr_ttl[j],)
+                if (j >= d.count): print("rrsig",)
+                print()
+                print(dataHex(d.rr_data[j],"       "))
 
-    print "-"*100
+    print("-"*100)
 
 def init(id, cfg):
    log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, mod_env['script']))

pythonmod/interface.i

@@ -204,7 +204,7 @@ struct query_info {
 
 %inline %{
    PyObject* dnameAsStr(PyObject* dname) {
-       char buf[LDNS_MAX_DOMAINLEN+1];
+       char buf[LDNS_MAX_DOMAINLEN];
        buf[0] = '\0';
        dname_str((uint8_t*)PyBytes_AsString(dname), buf);
        return PyString_FromString(buf);
@@ -1009,6 +1009,7 @@ struct config_file {
    int harden_short_bufsize;
    int harden_large_queries;
    int harden_glue;
+   int harden_unverified_glue;
    int harden_dnssec_stripped;
    int harden_referral_path;
    int use_caps_bits_for_id;
@@ -1416,7 +1417,7 @@ struct delegpt* dns_cache_find_delegation(struct module_env* env,
 int iter_dp_is_useless(struct query_info* qinfo, uint16_t qflags,
         struct delegpt* dp, int supports_ipv4, int supports_ipv6, int use_nat64);
 struct iter_hints_stub* hints_lookup_stub(struct iter_hints* hints,
-        uint8_t* qname, uint16_t qclass, struct delegpt* dp);
+        uint8_t* qname, uint16_t qclass, struct delegpt* dp, int nolock);
 
 /* Custom function to perform logic similar to the one in daemon/cachedump.c */
 struct delegpt* find_delegation(struct module_qstate* qstate, char *nm, size_t nmlen);
@@ -1433,6 +1434,7 @@ struct delegpt* find_delegation(struct module_qstate* qstate, char *nm, size_t n
     struct query_info qinfo;
     struct iter_hints_stub* stub;
     uint32_t timenow = *qstate->env->now;
+    int nolock = 0;
 
     regional_free_all(region);
     qinfo.qname = (uint8_t*)nm;
@@ -1455,9 +1457,12 @@ struct delegpt* find_delegation(struct module_qstate* qstate, char *nm, size_t n
             dname_str((uint8_t*)nm, b);
             continue;
         }
-        stub = hints_lookup_stub(qstate->env->hints, qinfo.qname, qinfo.qclass, dp);
+        stub = hints_lookup_stub(qstate->env->hints, qinfo.qname,
+            qinfo.qclass, dp, nolock);
         if (stub) {
-            return stub->dp;
+            struct delegpt* stubdp = delegpt_copy(stub->dp, region);
+            lock_rw_unlock(&qstate->env->hints->lock);
+            return stubdp;
         } else {
             return dp;
         }

pythonmod/pythonmod.c

@@ -356,11 +356,11 @@ int pythonmod_init(struct module_env* env, int id)
 	return 0;
       }
 #endif
-      Py_NoSiteFlag = 1;
 #if PY_MAJOR_VERSION >= 3
       PyImport_AppendInittab(SWIG_name, (void*)SWIG_init);
 #endif
 #if PY_VERSION_HEX < 0x03080000
+      Py_NoSiteFlag = 1;
       Py_Initialize();
 #else
       PyConfig_InitPythonConfig(&config);
@@ -777,8 +777,8 @@ size_t pythonmod_get_mem(struct module_env* env, int id)
  */
 static struct module_func_block pythonmod_block = {
    "python",
-   &pythonmod_init, &pythonmod_deinit, &pythonmod_operate, &pythonmod_inform_super,
-   &pythonmod_clear, &pythonmod_get_mem
+   NULL, NULL, &pythonmod_init, &pythonmod_deinit, &pythonmod_operate,
+   &pythonmod_inform_super, &pythonmod_clear, &pythonmod_get_mem
 };
 
 struct module_func_block* pythonmod_get_funcblock(void)

pythonmod/pythonmod_utils.c

@@ -73,7 +73,7 @@ int storeQueryInCache(struct module_qstate* qstate, struct query_info* qinfo,
 
 	return dns_cache_store(qstate->env, qinfo, msgrep, is_referral,
 		qstate->prefetch_leeway, 0, NULL, qstate->query_flags,
-		qstate->qstarttime);
+		qstate->qstarttime, qstate->is_valrec);
 }
 
 /* Invalidate the message associated with query_info stored in message cache */

respip/respip.c

@@ -105,6 +105,7 @@ respip_sockaddr_find_or_create(struct respip_set* set, struct sockaddr_storage*
 		socklen_t addrlen, int net, int create, const char* ipstr)
 {
 	struct resp_addr* node;
+	log_assert(set);
 	node = (struct resp_addr*)addr_tree_find(&set->ip_tree, addr, addrlen, net);
 	if(!node && create) {
 		node = regional_alloc_zero(set->region, sizeof(*node));
@@ -128,6 +129,7 @@ void
 respip_sockaddr_delete(struct respip_set* set, struct resp_addr* node)
 {
 	struct resp_addr* prev;
+	log_assert(set);
 	prev = (struct resp_addr*)rbtree_previous((struct rbnode_type*)node);	
 	lock_rw_destroy(&node->lock);
 	(void)rbtree_delete(&set->ip_tree, node);
@@ -146,6 +148,7 @@ respip_find_or_create(struct respip_set* set, const char* ipstr, int create)
 	struct sockaddr_storage addr;
 	int net;
 	socklen_t addrlen;
+	log_assert(set);
 
 	if(!netblockstrtoaddr(ipstr, 0, &addr, &addrlen, &net)) {
 		log_err("cannot parse netblock: '%s'", ipstr);
@@ -160,6 +163,7 @@ respip_tag_cfg(struct respip_set* set, const char* ipstr,
 	const uint8_t* taglist, size_t taglen)
 {
 	struct resp_addr* node;
+	log_assert(set);
 
 	if(!(node=respip_find_or_create(set, ipstr, 1)))
 		return 0;
@@ -183,6 +187,7 @@ respip_action_cfg(struct respip_set* set, const char* ipstr,
 {
 	struct resp_addr* node;
 	enum respip_action action;
+	log_assert(set);
 
 	if(!(node=respip_find_or_create(set, ipstr, 1)))
 		return 0;
@@ -325,6 +330,7 @@ static int
 respip_data_cfg(struct respip_set* set, const char* ipstr, const char* rrstr)
 {
 	struct resp_addr* node;
+	log_assert(set);
 
 	node=respip_find_or_create(set, ipstr, 0);
 	if(!node || node->action == respip_none) {
@@ -344,6 +350,7 @@ respip_set_apply_cfg(struct respip_set* set, char* const* tagname, int num_tags,
 	struct config_strbytelist* p;
 	struct config_str2list* pa;
 	struct config_str2list* pd;
+	log_assert(set);
 
 	set->tagname = tagname;
 	set->num_tags = num_tags;
@@ -609,6 +616,7 @@ respip_addr_lookup(const struct reply_info *rep, struct respip_set* rs,
 	struct resp_addr* ra;
 	struct sockaddr_storage ss;
 	socklen_t addrlen;
+	log_assert(rs);
 
 	lock_rw_rdlock(&rs->lock);
 	for(i=0; i<rep->an_numrrsets; i++) {
@@ -867,7 +875,8 @@ respip_rewrite_reply(const struct query_info* qinfo,
 	const struct respip_client_info* cinfo, const struct reply_info* rep,
 	struct reply_info** new_repp, struct respip_action_info* actinfo,
 	struct ub_packed_rrset_key** alias_rrset, int search_only,
-	struct regional* region, struct auth_zones* az, int* rpz_passthru)
+	struct regional* region, struct auth_zones* az, int* rpz_passthru,
+	struct views* views, struct respip_set* ipset)
 {
 	const uint8_t* ctaglist;
 	size_t ctaglen;
@@ -876,7 +885,6 @@ respip_rewrite_reply(const struct query_info* qinfo,
 	struct config_strlist** tag_datas;
 	size_t tag_datas_size;
 	struct view* view = NULL;
-	struct respip_set* ipset = NULL;
 	size_t rrset_id = 0, rr_id = 0;
 	enum respip_action action = respip_none;
 	int tag = -1;
@@ -899,8 +907,20 @@ respip_rewrite_reply(const struct query_info* qinfo,
 	tag_actions_size = cinfo->tag_actions_size;
 	tag_datas = cinfo->tag_datas;
 	tag_datas_size = cinfo->tag_datas_size;
-	view = cinfo->view;
-	ipset = cinfo->respip_set;
+	if(cinfo->view) {
+		view = cinfo->view;
+		lock_rw_rdlock(&view->lock);
+	} else if(cinfo->view_name) {
+		view = views_find_view(views, cinfo->view_name, 0);
+		if(!view) {
+			/* If the view no longer exists, the rewrite can not
+			 * be processed further. */
+			verbose(VERB_ALGO, "respip: failed because view %s no "
+				"longer exists", cinfo->view_name);
+			return 0;
+		}
+		/* The view is rdlocked by views_find_view. */
+	}
 
 	log_assert(ipset);
 
@@ -915,7 +935,6 @@ respip_rewrite_reply(const struct query_info* qinfo,
 	  * Note also that we assume 'view' is valid in this function, which
 	  * should be safe (see unbound bug #1191) */
 	if(view) {
-		lock_rw_rdlock(&view->lock);
 		if(view->respip_set) {
 			if((raddr = respip_addr_lookup(rep,
 				view->respip_set, &rrset_id, &rr_id))) {
@@ -961,7 +980,7 @@ respip_rewrite_reply(const struct query_info* qinfo,
 						struct sockaddr_storage ss;
 						socklen_t ss_len = 0;
 						char nm[256], ip[256];
-						char qn[255+1];
+						char qn[LDNS_MAX_DOMAINLEN];
 						if(!rdata2sockaddr(rep->rrsets[rrset_id]->entry.data, ntohs(rep->rrsets[rrset_id]->rk.type), rr_id, &ss, &ss_len))
 							snprintf(ip, sizeof(ip), "invalidRRdata");
 						else
@@ -1101,7 +1120,8 @@ respip_operate(struct module_qstate* qstate, enum module_ev event, int id,
 				qstate->client_info, qstate->return_msg->rep,
 				&new_rep, &actinfo, &alias_rrset, 0,
 				qstate->region, qstate->env->auth_zones,
-				&qstate->rpz_passthru)) {
+				&qstate->rpz_passthru, qstate->env->views,
+				qstate->env->respip_set)) {
 				goto servfail;
 			}
 			if(actinfo.action != respip_none) {
@@ -1149,7 +1169,8 @@ respip_merge_cname(struct reply_info* base_rep,
 	const struct query_info* qinfo, const struct reply_info* tgt_rep,
 	const struct respip_client_info* cinfo, int must_validate,
 	struct reply_info** new_repp, struct regional* region,
-	struct auth_zones* az)
+	struct auth_zones* az, struct views* views,
+	struct respip_set* respip_set)
 {
 	struct reply_info* new_rep;
 	struct reply_info* tmp_rep = NULL; /* just a placeholder */
@@ -1176,7 +1197,7 @@ respip_merge_cname(struct reply_info* base_rep,
 
 	/* see if the target reply would be subject to a response-ip action. */
 	if(!respip_rewrite_reply(qinfo, cinfo, tgt_rep, &tmp_rep, &actinfo,
-		&alias_rrset, 1, region, az, NULL))
+		&alias_rrset, 1, region, az, NULL, views, respip_set))
 		return 0;
 	if(actinfo.action != respip_none) {
 		log_info("CNAME target of redirect response-ip action would "
@@ -1229,7 +1250,8 @@ respip_inform_super(struct module_qstate* qstate, int id,
 	if(!respip_merge_cname(super->return_msg->rep, &qstate->qinfo,
 		qstate->return_msg->rep, super->client_info,
 		super->env->need_to_validate, &new_rep, super->region,
-		qstate->env->auth_zones))
+		qstate->env->auth_zones, qstate->env->views,
+		qstate->env->respip_set))
 		goto fail;
 	super->return_msg->rep = new_rep;
 	return;
@@ -1259,8 +1281,8 @@ respip_get_mem(struct module_env* env, int id)
  */
 static struct module_func_block respip_block = {
 	"respip",
-	&respip_init, &respip_deinit, &respip_operate, &respip_inform_super,
-	&respip_clear, &respip_get_mem
+	NULL, NULL, &respip_init, &respip_deinit, &respip_operate,
+	&respip_inform_super, &respip_clear, &respip_get_mem
 };
 
 struct module_func_block*
@@ -1326,3 +1348,35 @@ respip_inform_print(struct respip_action_info* respip_actinfo, uint8_t* qname,
 		(actionstr) ? actionstr : "inform", srcip, port);
 	log_nametypeclass(NO_VERBOSE, txt, qname, qtype, qclass);
 }
+
+size_t respip_set_get_mem(struct respip_set* set)
+{
+	size_t m;
+	if(!set) return 0;
+	m = sizeof(*set);
+	lock_rw_rdlock(&set->lock);
+	m += regional_get_mem(set->region);
+	lock_rw_unlock(&set->lock);
+	return m;
+}
+
+void
+respip_set_swap_tree(struct respip_set* respip_set,
+	struct respip_set* data)
+{
+	rbnode_type* oldroot = respip_set->ip_tree.root;
+	size_t oldcount = respip_set->ip_tree.count;
+	struct regional* oldregion = respip_set->region;
+	char* const* oldtagname = respip_set->tagname;
+	int oldnum_tags = respip_set->num_tags;
+	respip_set->ip_tree.root = data->ip_tree.root;
+	respip_set->ip_tree.count = data->ip_tree.count;
+	respip_set->region = data->region;
+	respip_set->tagname = data->tagname;
+	respip_set->num_tags = data->num_tags;
+	data->ip_tree.root = oldroot;
+	data->ip_tree.count = oldcount;
+	data->region = oldregion;
+	data->tagname = oldtagname;
+	data->num_tags = oldnum_tags;
+}

respip/respip.h

@@ -23,7 +23,8 @@
 struct respip_set {
 	struct regional* region;
 	struct rbtree_type ip_tree;
-	lock_rw_type lock;	/* lock on the respip tree */
+	lock_rw_type lock;	/* lock on the respip tree. It is ordered
+		after views and before hints, stubs and local zones. */
 	char* const* tagname;	/* shallow copy of tag names, for logging */
 	int num_tags;		/* number of tagname entries */
 };
@@ -59,7 +60,6 @@ struct respip_addr_info;
  * This is essentially a subset of acl_addr (except for respip_set) but
  * defined as a separate structure to avoid dependency on the daemon-specific
  * structure.
- * respip_set is supposed to refer to the response-ip set for the global view.
  */
 struct respip_client_info {
 	uint8_t* taglist;
@@ -68,8 +68,12 @@ struct respip_client_info {
 	size_t tag_actions_size;
 	struct config_strlist** tag_datas;
 	size_t tag_datas_size;
+	/** The view for the action, during cache callback that is by
+	 * pointer. */
 	struct view* view;
-	struct respip_set* respip_set;
+	/** If from module query state, the view pointer is NULL, but the
+	 * name is stored in reference to the view. */
+	char* view_name;
 };
 
 /**
@@ -149,13 +153,16 @@ int respip_views_apply_cfg(struct views* vs, struct config_file* cfg,
  *   on error.
  * @param region: allocator to build *new_repp.
  * @param az: auth zones containing RPZ information.
+ * @param views: views tree to lookup view used.
+ * @param respip_set: the respip set for the global view.
  * @return 1 on success, 0 on error.
  */
 int respip_merge_cname(struct reply_info* base_rep,
 	const struct query_info* qinfo, const struct reply_info* tgt_rep,
 	const struct respip_client_info* cinfo, int must_validate,
 	struct reply_info** new_repp, struct regional* region,
-	struct auth_zones* az);
+	struct auth_zones* az, struct views* views,
+	struct respip_set* respip_set);
 
 /**
  * See if any IP-based action should apply to any IP address of AAAA/A answer
@@ -178,6 +185,8 @@ int respip_merge_cname(struct reply_info* base_rep,
  * @param region: allocator to build *new_repp.
  * @param rpz_passthru: keeps track of query state can have passthru that
  *   stops further rpz processing. Or NULL for cached answer processing.
+ * @param views: views tree to lookup view used.
+ * @param ipset: the respip set for the global view.
  * @return 1 on success, 0 on error.
  */
 int respip_rewrite_reply(const struct query_info* qinfo,
@@ -186,7 +195,7 @@ int respip_rewrite_reply(const struct query_info* qinfo,
 	struct respip_action_info* actinfo,
 	struct ub_packed_rrset_key** alias_rrset,
 	int search_only, struct regional* region, struct auth_zones* az,
-	int* rpz_passthru);
+	int* rpz_passthru, struct views* views, struct respip_set* ipset);
 
 /**
  * Get the response-ip function block.
@@ -267,7 +276,7 @@ void respip_inform_print(struct respip_action_info* respip_actinfo,
  * @param addrlen: length of addr.
  * @param net: netblock to lookup.
  * @param create: create node if it does not exist when 1.
- * @param ipstr: human redable ip string, for logging.
+ * @param ipstr: human readable ip string, for logging.
  * @return newly created of found node, not holding lock.
  */
 struct resp_addr*
@@ -302,4 +311,18 @@ respip_sockaddr_delete(struct respip_set* set, struct resp_addr* node);
 
 struct ub_packed_rrset_key*
 respip_copy_rrset(const struct ub_packed_rrset_key* key, struct regional* region);
+
+/** Get memory usage of respip set tree. The routine locks and unlocks the
+ * set for reading. */
+size_t respip_set_get_mem(struct respip_set* set);
+
+/**
+ * Swap internal tree with preallocated entries. Caller should manage
+ * the locks.
+ * @param respip_set: response ip tree
+ * @param data: preallocated information.
+ */
+void respip_set_swap_tree(struct respip_set* respip_set,
+	struct respip_set* data);
+
 #endif	/* RESPIP_RESPIP_H */

services/authzone.h

@@ -70,7 +70,8 @@ struct auth_chunk;
  * Authoritative zones, shared.
  */
 struct auth_zones {
-	/** lock on the authzone trees */
+	/** lock on the authzone trees. It is locked after views, respip,
+	 * local_zones and before fwds and stubs. */
 	lock_rw_type lock;
 	/** rbtree of struct auth_zone */
 	rbtree_type ztree;
@@ -78,10 +79,6 @@ struct auth_zones {
 	rbtree_type xtree;
 	/** do we have downstream enabled */
 	int have_downstream;
-	/** number of queries upstream */
-	size_t num_query_up;
-	/** number of queries downstream */
-	size_t num_query_down;
 	/** first auth zone containing rpz item in linked list */
 	struct auth_zone* rpz_first;
 	/** rw lock for rpz linked list, needed when iterating or editing linked
@@ -121,6 +118,8 @@ struct auth_zone {
 	char* zonefile;
 	/** fallback to the internet on failure or ttl-expiry of auth zone */
 	int fallback_enabled;
+	/** the time when zone was transferred from upstream */
+	time_t soa_zone_acquired;
 	/** the zone has expired (enabled by the xfer worker), fallback
 	 * happens if that option is enabled. */
 	int zone_expired;
@@ -211,7 +210,9 @@ struct auth_xfer {
 	 * one of the tasks. 
 	 * Once it has the task assigned to it, the worker can access the
 	 * other elements of the task structure without a lock, because that
-	 * is necessary for the eventloop and callbacks from that. */
+	 * is necessary for the eventloop and callbacks from that.
+	 * The auth_zone->lock is locked before this lock.
+	 */
 	lock_basic_type lock;
 
 	/** zone name, in uncompressed wireformat */
@@ -262,6 +263,8 @@ struct auth_xfer {
 	int zone_expired;
 	/** do we have a zone (if 0, no zone data at all) */
 	int have_zone;
+	/** the time when zone was transferred from upstream */
+	time_t soa_zone_acquired;
 
 	/** current serial (from SOA), if we have no zone, 0 */
 	uint32_t serial;
@@ -551,9 +554,10 @@ int auth_zones_lookup(struct auth_zones* az, struct query_info* qinfo,
  * @param temp: temporary storage region.
  * @return false if not answered
  */
-int auth_zones_answer(struct auth_zones* az, struct module_env* env,
+int auth_zones_downstream_answer(struct auth_zones* az, struct module_env* env,
 	struct query_info* qinfo, struct edns_data* edns,
-	struct comm_reply* repinfo, struct sldns_buffer* buf, struct regional* temp);
+	struct comm_reply* repinfo, struct sldns_buffer* buf,
+	struct regional* temp);
 
 /** 
  * Find the auth zone that is above the given qname.
@@ -787,4 +791,41 @@ void auth_zonemd_dnskey_lookup_callback(void* arg, int rcode,
 void auth_zones_pickup_zonemd_verify(struct auth_zones* az,
 	struct module_env* env);
 
+/** Get memory usage for auth zones. The routine locks and unlocks
+ * for reading. */
+size_t auth_zones_get_mem(struct auth_zones* zones);
+
+/**
+ * Initial pick up of the auth zone nextprobe timeout and that turns
+ * into further zone transfer work, if any. Also sets the lease time.
+ * @param x: xfer structure, locked by caller.
+ * @param env: environment of the worker that picks up the task.
+ */
+void auth_xfer_pickup_initial_zone(struct auth_xfer* x,
+	struct module_env* env);
+
+/**
+ * Initial pick up of the auth zone, it sets the acquired time.
+ * @param z: the zone, write locked by caller.
+ * @param env: environment of the worker, with current time.
+ */
+void auth_zone_pickup_initial_zone(struct auth_zone* z,
+	struct module_env* env);
+
+/**
+ * Delete auth xfer structure
+ * @param xfr: delete this xfer and its tasks.
+ */
+void auth_xfer_delete(struct auth_xfer* xfr);
+
+/**
+ * Disown tasks from the xfr that belong to this worker.
+ * Only tasks for the worker in question, the comm point and timer
+ * delete functions need to run in the thread of that worker to be
+ * able to delete the callback from the event base.
+ * @param xfr: xfr structure
+ * @param worker: the worker for which to stop tasks.
+ */
+void xfr_disown_tasks(struct auth_xfer* xfr, struct worker* worker);
+
 #endif /* SERVICES_AUTHZONE_H */

services/cache/dns.c

@@ -80,6 +80,7 @@ store_rrsets(struct module_env* env, struct reply_info* rep, time_t now,
 	struct regional* region, time_t qstarttime)
 {
 	size_t i;
+	time_t ttl, min_ttl = rep->ttl;
 	/* see if rrset already exists in cache, if not insert it. */
 	for(i=0; i<rep->rrset_count; i++) {
 		rep->ref[i].key = rep->rrsets[i];
@@ -87,7 +88,7 @@ store_rrsets(struct module_env* env, struct reply_info* rep, time_t now,
 		/* update ref if it was in the cache */
 		switch(rrset_cache_update(env->rrset_cache, &rep->ref[i],
 				env->alloc, ((ntohs(rep->ref[i].key->rk.type)==
-				LDNS_RR_TYPE_NS && !pside)?qstarttime:now + leeway))) {
+				LDNS_RR_TYPE_NS && !pside)?qstarttime:now) + leeway)) {
 		case 0: /* ref unchanged, item inserted */
 			break;
 		case 2: /* ref updated, cache is superior */
@@ -95,7 +96,8 @@ store_rrsets(struct module_env* env, struct reply_info* rep, time_t now,
 				struct ub_packed_rrset_key* ck;
 				lock_rw_rdlock(&rep->ref[i].key->entry.lock);
 				/* if deleted rrset, do not copy it */
-				if(rep->ref[i].key->id == 0)
+				if(rep->ref[i].key->id == 0 ||
+					rep->ref[i].id != rep->ref[i].key->id)
 					ck = NULL;
 				else 	ck = packed_rrset_copy_region(
 					rep->ref[i].key, region, now);
@@ -108,11 +110,28 @@ store_rrsets(struct module_env* env, struct reply_info* rep, time_t now,
 			/* no break: also copy key item */
 			/* the line below is matched by gcc regex and silences
 			 * the fallthrough warning */
+			ATTR_FALLTHROUGH
 			/* fallthrough */
 		case 1: /* ref updated, item inserted */
 			rep->rrsets[i] = rep->ref[i].key;
+			/* ref was updated; make sure the message ttl is
+			 * updated to the minimum of the current rrsets. */
+			lock_rw_rdlock(&rep->ref[i].key->entry.lock);
+			/* if deleted, skip ttl update. */
+			if(rep->ref[i].key->id != 0 &&
+				rep->ref[i].id == rep->ref[i].key->id) {
+				ttl = ((struct packed_rrset_data*)
+				    rep->rrsets[i]->entry.data)->ttl;
+				if(ttl < min_ttl) min_ttl = ttl;
+			}
+			lock_rw_unlock(&rep->ref[i].key->entry.lock);
 		}
 	}
+	if(min_ttl < rep->ttl) {
+		rep->ttl = min_ttl;
+		rep->prefetch_ttl = PREFETCH_TTL_CALC(rep->ttl);
+		rep->serve_expired_ttl = rep->ttl + SERVE_EXPIRED_TTL;
+	}
 }
 
 /** delete message from message cache */
@@ -143,7 +162,7 @@ dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
 	size_t i;
 
 	/* store RRsets */
-        for(i=0; i<rep->rrset_count; i++) {
+	for(i=0; i<rep->rrset_count; i++) {
 		rep->ref[i].key = rep->rrsets[i];
 		rep->ref[i].id = rep->rrsets[i]->id;
 	}
@@ -178,51 +197,12 @@ dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
 	reply_info_sortref(rep);
 	if(!(e = query_info_entrysetup(qinfo, rep, hash))) {
 		log_err("store_msg: malloc failed");
+		reply_info_delete(rep, NULL);
 		return;
 	}
 	slabhash_insert(env->msg_cache, hash, &e->entry, rep, env->alloc);
 }
 
-/** see if an rrset is expired above the qname, return upper qname. */
-static int
-rrset_expired_above(struct module_env* env, uint8_t** qname, size_t* qnamelen,
-	uint16_t searchtype, uint16_t qclass, time_t now, uint8_t* expiretop,
-	size_t expiretoplen)
-{
-	struct ub_packed_rrset_key *rrset;
-	uint8_t lablen;
-
-	while(*qnamelen > 0) {
-		/* look one label higher */
-		lablen = **qname;
-		*qname += lablen + 1;
-		*qnamelen -= lablen + 1;
-		if(*qnamelen <= 0)
-			break;
-
-		/* looks up with a time of 0, to see expired entries */
-		if((rrset = rrset_cache_lookup(env->rrset_cache, *qname,
-			*qnamelen, searchtype, qclass, 0, 0, 0))) {
-			struct packed_rrset_data* data =
-				(struct packed_rrset_data*)rrset->entry.data;
-			if(now > data->ttl) {
-				/* it is expired, this is not wanted */
-				lock_rw_unlock(&rrset->entry.lock);
-				log_nametypeclass(VERB_ALGO, "this rrset is expired", *qname, searchtype, qclass);
-				return 1;
-			}
-			/* it is not expired, continue looking */
-			lock_rw_unlock(&rrset->entry.lock);
-		}
-
-		/* do not look above the expiretop. */
-		if(expiretop && *qnamelen == expiretoplen &&
-			query_dname_compare(*qname, expiretop)==0)
-			break;
-	}
-	return 0;
-}
-
 /** find closest NS or DNAME and returns the rrset (locked) */
 static struct ub_packed_rrset_key*
 find_closest_of_type(struct module_env* env, uint8_t* qname, size_t qnamelen, 
@@ -256,12 +236,12 @@ find_closest_of_type(struct module_env* env, uint8_t* qname, size_t qnamelen,
 			/* check for expiry, but we have to let go of the rrset
 			 * for the lock ordering */
 			lock_rw_unlock(&rrset->entry.lock);
-			/* the expired_above function always takes off one
-			 * label (if qnamelen>0) and returns the final qname
-			 * where it searched, so we can continue from there
-			 * turning the O N*N search into O N. */
-			if(!rrset_expired_above(env, &qname, &qnamelen,
-				searchtype, qclass, now, expiretop,
+			/* the rrset_cache_expired_above function always takes
+			 * off one label (if qnamelen>0) and returns the final
+			 * qname where it searched, so we can continue from
+			 * there turning the O N*N search into O N. */
+			if(!rrset_cache_expired_above(env->rrset_cache, &qname,
+				&qnamelen, searchtype, qclass, now, expiretop,
 				expiretoplen)) {
 				/* we want to return rrset, but it may be
 				 * gone from cache, if so, just loop like
@@ -367,6 +347,13 @@ find_add_addrs(struct module_env* env, uint16_t qclass,
 			 * not use dns64 translation */
 			neg = msg_cache_lookup(env, ns->name, ns->namelen,
 				LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+			/* Because recursion for lookup uses BIT_CD, check
+			 * for that so it stops the recursion lookup, if a
+			 * negative answer is cached. Because the cache uses
+			 * the CD flag for type AAAA. */
+			if(!neg)
+				neg = msg_cache_lookup(env, ns->name, ns->namelen,
+					LDNS_RR_TYPE_AAAA, qclass, BIT_CD, now, 0);
 			if(neg) {
 				delegpt_add_neg_msg(dp, neg);
 				lock_rw_unlock(&neg->entry.lock);
@@ -379,7 +366,7 @@ find_add_addrs(struct module_env* env, uint16_t qclass,
 /** find and add A and AAAA records for missing nameservers in delegpt */
 int
 cache_fill_missing(struct module_env* env, uint16_t qclass, 
-	struct regional* region, struct delegpt* dp)
+	struct regional* region, struct delegpt* dp, uint32_t flags)
 {
 	struct delegpt_ns* ns;
 	struct msgreply_entry* neg;
@@ -390,7 +377,7 @@ cache_fill_missing(struct module_env* env, uint16_t qclass,
 			continue;
 		ns->cache_lookup_count++;
 		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
-			ns->namelen, LDNS_RR_TYPE_A, qclass, 0, now, 0);
+			ns->namelen, LDNS_RR_TYPE_A, qclass, flags, now, 0);
 		if(akey) {
 			if(!delegpt_add_rrset_A(dp, region, akey, ns->lame,
 				NULL)) {
@@ -411,7 +398,7 @@ cache_fill_missing(struct module_env* env, uint16_t qclass,
 			}
 		}
 		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
-			ns->namelen, LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+			ns->namelen, LDNS_RR_TYPE_AAAA, qclass, flags, now, 0);
 		if(akey) {
 			if(!delegpt_add_rrset_AAAA(dp, region, akey, ns->lame,
 				NULL)) {
@@ -426,6 +413,13 @@ cache_fill_missing(struct module_env* env, uint16_t qclass,
 			 * not use dns64 translation */
 			neg = msg_cache_lookup(env, ns->name, ns->namelen,
 				LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+			/* Because recursion for lookup uses BIT_CD, check
+			 * for that so it stops the recursion lookup, if a
+			 * negative answer is cached. Because the cache uses
+			 * the CD flag for type AAAA. */
+			if(!neg)
+				neg = msg_cache_lookup(env, ns->name, ns->namelen,
+					LDNS_RR_TYPE_AAAA, qclass, BIT_CD, now, 0);
 			if(neg) {
 				delegpt_add_neg_msg(dp, neg);
 				lock_rw_unlock(&neg->entry.lock);
@@ -614,22 +608,8 @@ tomsg(struct module_env* env, struct query_info* q, struct reply_info* r,
 	time_t now_control = now;
 	if(now > r->ttl) {
 		/* Check if we are allowed to serve expired */
-		if(allow_expired) {
-			if(env->cfg->serve_expired_ttl &&
-				r->serve_expired_ttl < now) {
-				return NULL;
-			}
-			/* Ignore expired failure answers */
-			if(FLAGS_GET_RCODE(r->flags) !=
-				LDNS_RCODE_NOERROR &&
-				FLAGS_GET_RCODE(r->flags) !=
-				LDNS_RCODE_NXDOMAIN &&
-				FLAGS_GET_RCODE(r->flags) !=
-				LDNS_RCODE_YXDOMAIN)
-				return 0;
-		} else {
+		if(!allow_expired || !reply_info_can_answer_expired(r, now))
 			return NULL;
-		}
 		/* Change the current time so we can pass the below TTL checks when
 		 * serving expired data. */
 		now_control = r->ttl - env->cfg->serve_expired_reply_ttl;
@@ -648,6 +628,7 @@ tomsg(struct module_env* env, struct query_info* q, struct reply_info* r,
 	else
 		msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 	msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
+	msg->rep->serve_expired_norec_ttl = 0;
 	msg->rep->security = r->security;
 	msg->rep->an_numrrsets = r->an_numrrsets;
 	msg->rep->ns_numrrsets = r->ns_numrrsets;
@@ -731,6 +712,7 @@ rrset_msg(struct ub_packed_rrset_key* rrset, struct regional* region,
 	msg->rep->ttl = d->ttl - now;
 	msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 	msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
+	msg->rep->serve_expired_norec_ttl = 0;
 	msg->rep->security = sec_status_unchecked;
 	msg->rep->an_numrrsets = 1;
 	msg->rep->ns_numrrsets = 0;
@@ -770,6 +752,7 @@ synth_dname_msg(struct ub_packed_rrset_key* rrset, struct regional* region,
 	msg->rep->ttl = d->ttl - now;
 	msg->rep->prefetch_ttl = PREFETCH_TTL_CALC(msg->rep->ttl);
 	msg->rep->serve_expired_ttl = msg->rep->ttl + SERVE_EXPIRED_TTL;
+	msg->rep->serve_expired_norec_ttl = 0;
 	msg->rep->security = sec_status_unchecked;
 	msg->rep->an_numrrsets = 1;
 	msg->rep->ns_numrrsets = 0;
@@ -818,7 +801,7 @@ synth_dname_msg(struct ub_packed_rrset_key* rrset, struct regional* region,
 	if(!newd)
 		return NULL;
 	ck->entry.data = newd;
-	newd->ttl = 0; /* 0 for synthesized CNAME TTL */
+	newd->ttl = d->ttl - now; /* RFC6672: synth CNAME TTL == DNAME TTL */
 	newd->count = 1;
 	newd->rrsig_count = 0;
 	newd->trust = rrset_trust_ans_noAA;
@@ -1073,10 +1056,76 @@ dns_cache_lookup(struct module_env* env,
 
 int
 dns_cache_store(struct module_env* env, struct query_info* msgqinf,
-        struct reply_info* msgrep, int is_referral, time_t leeway, int pside,
-	struct regional* region, uint32_t flags, time_t qstarttime)
+	struct reply_info* msgrep, int is_referral, time_t leeway, int pside,
+	struct regional* region, uint32_t flags, time_t qstarttime,
+	int is_valrec)
 {
 	struct reply_info* rep = NULL;
+	if(SERVE_EXPIRED) {
+		/* We are serving expired records. Before caching, check if a
+		 * useful expired record exists. */
+		struct msgreply_entry* e = msg_cache_lookup(env,
+			msgqinf->qname, msgqinf->qname_len, msgqinf->qtype,
+			msgqinf->qclass, flags, 0, 1);
+		if(e) {
+			struct reply_info* cached = e->entry.data;
+			if(cached->ttl < *env->now
+				&& reply_info_could_use_expired(cached, *env->now)
+				/* If we are validating make sure only
+				 * validating modules can update such messages.
+				 * In that case don't cache it and let a
+				 * subsequent module handle the caching. For
+				 * example, the iterator should not replace an
+				 * expired secure answer with a fresh unchecked
+				 * one and let the validator manage caching. */
+				&& cached->security != sec_status_bogus
+				&& (env->need_to_validate &&
+				msgrep->security == sec_status_unchecked)
+				/* Exceptions to that rule are:
+				 * o recursions that don't need validation but
+				 *   need to update the cache for coherence
+				 *   (delegation information while iterating,
+				 *   DNSKEY and DS lookups from validator)
+				 * o explicit RRSIG queries that are not
+				 *   validated. */
+				&& !is_valrec
+				&& msgqinf->qtype != LDNS_RR_TYPE_RRSIG) {
+				if((int)FLAGS_GET_RCODE(msgrep->flags) !=
+					LDNS_RCODE_NOERROR &&
+					(int)FLAGS_GET_RCODE(msgrep->flags) !=
+					LDNS_RCODE_NXDOMAIN) {
+					/* The current response has an
+					 * erroneous rcode. Adjust norec time
+					 * so that additional lookups are not
+					 * performed for some time. */
+					verbose(VERB_ALGO, "set "
+						"serve-expired-norec-ttl for "
+						"response in cache");
+					cached->serve_expired_norec_ttl =
+						NORR_TTL + *env->now;
+					if(env->cfg->serve_expired_ttl_reset &&
+					    cached->serve_expired_ttl
+					    < *env->now +
+					    env->cfg->serve_expired_ttl) {
+						/* Reset serve-expired-ttl for
+						 * valid response in cache. */
+						verbose(VERB_ALGO, "reset "
+							"serve-expired-ttl "
+							"for response in cache");
+						cached->serve_expired_ttl =
+						    *env->now +
+						    env->cfg->serve_expired_ttl;
+					}
+				}
+				verbose(VERB_ALGO, "a validated expired entry "
+					"could be overwritten, skip caching "
+					"the new message at this stage");
+				lock_rw_unlock(&e->entry.lock);
+				return 1;
+			}
+			lock_rw_unlock(&e->entry.lock);
+		}
+	}
 	/* alloc, malloc properly (not in region, like msg is) */
 	rep = reply_info_copy(msgrep, env->alloc, NULL);
 	if(!rep)

services/cache/dns.h

@@ -90,11 +90,14 @@ struct dns_msg {
  *   (See DNSCACHE_STORE_xxx flags).
  * @param qstarttime: time when the query was started, and thus when the
  * 	delegations were looked up.
+ * @param is_valrec: if the query is validation recursion and does not get
+ *	dnssec validation itself.
  * @return 0 on alloc error (out of memory).
  */
 int dns_cache_store(struct module_env* env, struct query_info* qinf,
         struct reply_info* rep, int is_referral, time_t leeway, int pside,
-	struct regional* region, uint32_t flags, time_t qstarttime);
+	struct regional* region, uint32_t flags, time_t qstarttime,
+	int is_valrec);
 
 /**
  * Store message in the cache. Stores in message cache and rrset cache.
@@ -202,10 +205,11 @@ struct dns_msg* dns_cache_lookup(struct module_env* env,
  * @param qclass: which class to look in.
  * @param region: where to store new dp info.
  * @param dp: delegation point to fill missing entries.
+ * @param flags: rrset flags, or 0.
  * @return false on alloc failure.
  */
 int cache_fill_missing(struct module_env* env, uint16_t qclass, 
-	struct regional* region, struct delegpt* dp);
+	struct regional* region, struct delegpt* dp, uint32_t flags);
 
 /**
  * Utility, create new, unpacked data structure for cache response.