tag for release 1.9.0rc1

git-svn-id: file:///svn/unbound/tags/release-1.9.0rc1@5085 be551aaa-1e26-0410-a405-d3ace91eadb9

.gitignore

@@ -36,4 +36,7 @@
 /streamtcp
 /testbound
 /unittest
+/contrib/libunbound.pc
+/contrib/unbound.service
+/contrib/unbound.socket
 

aclocal.m4

@@ -1,6 +1,6 @@
-# generated automatically by aclocal 1.15.1 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.1 -*- Autoconf -*-
 
-# Copyright (C) 1996-2017 Free Software Foundation, Inc.
+# Copyright (C) 1996-2018 Free Software Foundation, Inc.
 
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -9390,7 +9390,7 @@ AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
 
 # AM_CONDITIONAL                                            -*- Autoconf -*-
 
-# Copyright (C) 1997-2017 Free Software Foundation, Inc.
+# Copyright (C) 1997-2018 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -9421,7 +9421,7 @@ AC_CONFIG_COMMANDS_PRE(
 Usually this means the macro was only invoked conditionally.]])
 fi])])
 
-# Copyright (C) 2006-2017 Free Software Foundation, Inc.
+# Copyright (C) 2006-2018 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,

compat/arc4random.c

@@ -140,6 +140,7 @@ nodevrandom:
 static inline void
 _rs_init(u_char *buf, size_t n)
 {
+	assert(buf);
 	if (n < KEYSZ + IVSZ)
 		return;
 

config.guess

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/usr/bin/sh
 # Attempt to guess a canonical system name.
 #   Copyright 1992-2016 Free Software Foundation, Inc.
 

config.h.in

@@ -1,5 +1,11 @@
 /* config.h.in.  Generated from configure.ac by autoheader.  */
 
+/* apply the noreturn attribute to a function that exits the program */
+#undef ATTR_NORETURN
+
+/* apply the weak attribute to a symbol */
+#undef ATTR_WEAK
+
 /* Directory to chroot to */
 #undef CHROOT_DIR
 
@@ -45,6 +51,9 @@
 /* Whether the C compiler accepts the "format" attribute */
 #undef HAVE_ATTR_FORMAT
 
+/* Whether the C compiler accepts the "noreturn" attribute */
+#undef HAVE_ATTR_NORETURN
+
 /* Whether the C compiler accepts the "unused" attribute */
 #undef HAVE_ATTR_UNUSED
 
@@ -169,6 +178,9 @@
 /* Define to 1 if you have the <event.h> header file. */
 #undef HAVE_EVENT_H
 
+/* Define to 1 if you have the `EVP_aes_256_cbc' function. */
+#undef HAVE_EVP_AES_256_CBC
+
 /* Define to 1 if you have the `EVP_cleanup' function. */
 #undef HAVE_EVP_CLEANUP
 
@@ -178,6 +190,9 @@
 /* Define to 1 if you have the `EVP_dss1' function. */
 #undef HAVE_EVP_DSS1
 
+/* Define to 1 if you have the `EVP_EncryptInit_ex' function. */
+#undef HAVE_EVP_ENCRYPTINIT_EX
+
 /* Define to 1 if you have the `EVP_MD_CTX_new' function. */
 #undef HAVE_EVP_MD_CTX_NEW
 
@@ -199,6 +214,9 @@
 /* Define to 1 if you have the <expat.h> header file. */
 #undef HAVE_EXPAT_H
 
+/* Define to 1 if you have the `explicit_bzero' function. */
+#undef HAVE_EXPLICIT_BZERO
+
 /* Define to 1 if you have the `fcntl' function. */
 #undef HAVE_FCNTL
 
@@ -247,6 +265,9 @@
 /* Define to 1 if you have the <hiredis/hiredis.h> header file. */
 #undef HAVE_HIREDIS_HIREDIS_H
 
+/* Define to 1 if you have the `HMAC_Init_ex' function. */
+#undef HAVE_HMAC_INIT_EX
+
 /* If you have HMAC_Update */
 #undef HAVE_HMAC_UPDATE
 
@@ -439,9 +460,15 @@
 /* Define if you have the SSL libraries installed. */
 #undef HAVE_SSL
 
+/* Define to 1 if you have the `SSL_CTX_set_ciphersuites' function. */
+#undef HAVE_SSL_CTX_SET_CIPHERSUITES
+
 /* Define to 1 if you have the `SSL_CTX_set_security_level' function. */
 #undef HAVE_SSL_CTX_SET_SECURITY_LEVEL
 
+/* Define to 1 if you have the `SSL_CTX_set_tlsext_ticket_key_cb' function. */
+#undef HAVE_SSL_CTX_SET_TLSEXT_TICKET_KEY_CB
+
 /* Define to 1 if you have the `SSL_get0_peername' function. */
 #undef HAVE_SSL_GET0_PEERNAME
 
@@ -574,6 +601,9 @@
 /* Define to 1 if you have the <ws2tcpip.h> header file. */
 #undef HAVE_WS2TCPIP_H
 
+/* Define to 1 if you have the `X509_VERIFY_PARAM_set1_host' function. */
+#undef HAVE_X509_VERIFY_PARAM_SET1_HOST
+
 /* Define to 1 if you have the `_beginthreadex' function. */
 #undef HAVE__BEGINTHREADEX
 
@@ -651,6 +681,9 @@
 /* Define as the return type of signal handlers (`int' or `void'). */
 #undef RETSIGTYPE
 
+/* if REUSEPORT is enabled by default */
+#undef REUSEPORT_DEFAULT
+
 /* default rootkey location */
 #undef ROOT_ANCHOR_FILE
 
@@ -1144,6 +1177,11 @@ char *strsep(char **stringp, const char *delim);
 int isblank(int c);
 #endif
 
+#ifndef HAVE_EXPLICIT_BZERO
+#define explicit_bzero unbound_explicit_bzero
+void explicit_bzero(void* buf, size_t len);
+#endif
+
 #if defined(HAVE_INET_NTOP) && !HAVE_DECL_INET_NTOP
 const char *inet_ntop(int af, const void *src, char *dst, size_t size);
 #endif
@@ -1176,7 +1214,6 @@ void *reallocarray(void *ptr, size_t nmemb, size_t size);
 #  endif
 #endif /* HAVE_LIBRESSL */
 #ifndef HAVE_ARC4RANDOM
-void explicit_bzero(void* buf, size_t len);
 int getentropy(void* buf, size_t len);
 uint32_t arc4random(void);
 void arc4random_buf(void* buf, size_t n);

config.sub

@@ -1,4 +1,4 @@
-#! /bin/sh
+#!/usr/bin/sh
 # Configuration validation subroutine script.
 #   Copyright 1992-2016 Free Software Foundation, Inc.
 

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for unbound 1.8.0.
+# Generated by GNU Autoconf 2.69 for unbound 1.9.0.
 #
 # Report bugs to <unbound-bugs@nlnetlabs.nl>.
 #
@@ -590,8 +590,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='unbound'
 PACKAGE_TARNAME='unbound'
-PACKAGE_VERSION='1.8.0'
-PACKAGE_STRING='unbound 1.8.0'
+PACKAGE_VERSION='1.9.0'
+PACKAGE_STRING='unbound 1.9.0'
 PACKAGE_BUGREPORT='unbound-bugs@nlnetlabs.nl'
 PACKAGE_URL=''
 
@@ -1440,7 +1440,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures unbound 1.8.0 to adapt to many kinds of systems.
+\`configure' configures unbound 1.9.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1505,7 +1505,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of unbound 1.8.0:";;
+     short | recursive ) echo "Configuration of unbound 1.9.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1722,7 +1722,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-unbound configure 1.8.0
+unbound configure 1.9.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2431,7 +2431,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by unbound $as_me 1.8.0, which was
+It was created by unbound $as_me 1.9.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2781,14 +2781,14 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 UNBOUND_VERSION_MAJOR=1
 
-UNBOUND_VERSION_MINOR=8
+UNBOUND_VERSION_MINOR=9
 
 UNBOUND_VERSION_MICRO=0
 
 
-LIBUNBOUND_CURRENT=8
+LIBUNBOUND_CURRENT=9
 LIBUNBOUND_REVISION=0
-LIBUNBOUND_AGE=0
+LIBUNBOUND_AGE=1
 # 1.0.0 had 0:12:0
 # 1.0.1 had 0:13:0
 # 1.0.2 had 0:14:0
@@ -2850,7 +2850,11 @@ LIBUNBOUND_AGE=0
 # 1.7.1 had 7:9:5
 # 1.7.2 had 7:10:5
 # 1.7.3 had 7:11:5
-# 1.7.4 had 8:0:0 # changes the event callback function signature
+# 1.8.0 had 8:0:0 # changes the event callback function signature
+# 1.8.1 had 8:1:0
+# 1.8.2 had 8:2:0
+# 1.8.3 had 8:3:0
+# 1.8.4 had 9:0:1 # add ub_ctx_set_tls
 
 #   Current  -- the number of the binary API that we're implementing
 #   Revision -- which iteration of the implementation of the binary
@@ -6265,6 +6269,54 @@ if test $ac_cv_c_weak_attribute = yes; then
 
 $as_echo "#define HAVE_ATTR_WEAK 1" >>confdefs.h
 
+
+$as_echo "#define ATTR_WEAK __attribute__((weak))" >>confdefs.h
+
+fi
+
+
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler (${CC-cc}) accepts the \"noreturn\" attribute" >&5
+$as_echo_n "checking whether the C compiler (${CC-cc}) accepts the \"noreturn\" attribute... " >&6; }
+if ${ac_cv_c_noreturn_attribute+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_cv_c_noreturn_attribute=no
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+ #include <stdio.h>
+__attribute__((noreturn)) void f(int x) { printf("%d", x); }
+
+int
+main ()
+{
+
+   f(1);
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  ac_cv_c_noreturn_attribute="yes"
+else
+  ac_cv_c_noreturn_attribute="no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_noreturn_attribute" >&5
+$as_echo "$ac_cv_c_noreturn_attribute" >&6; }
+if test $ac_cv_c_noreturn_attribute = yes; then
+
+$as_echo "#define HAVE_ATTR_NORETURN 1" >>confdefs.h
+
+
+$as_echo "#define ATTR_NORETURN __attribute__((__noreturn__))" >>confdefs.h
+
 fi
 
 
@@ -15917,6 +15969,17 @@ fi
 done
 
 
+# check if we can use SO_REUSEPORT
+if echo "$host" | grep -i -e linux -e dragonfly >/dev/null; then
+
+$as_echo "#define REUSEPORT_DEFAULT 1" >>confdefs.h
+
+else
+
+$as_echo "#define REUSEPORT_DEFAULT 0" >>confdefs.h
+
+fi
+
 # set memory allocation checking if requested
 # Check whether --enable-alloc-checks was given.
 if test "${enable_alloc_checks+set}" = set; then :
@@ -17930,7 +17993,7 @@ fi
 
 done
 
-for ac_func in OPENSSL_config EVP_sha1 EVP_sha256 EVP_sha512 FIPS_mode EVP_MD_CTX_new OpenSSL_add_all_digests OPENSSL_init_crypto EVP_cleanup ERR_load_crypto_strings CRYPTO_cleanup_all_ex_data ERR_free_strings RAND_cleanup DSA_SIG_set0 EVP_dss1 EVP_DigestVerify
+for ac_func in OPENSSL_config EVP_sha1 EVP_sha256 EVP_sha512 FIPS_mode EVP_MD_CTX_new OpenSSL_add_all_digests OPENSSL_init_crypto EVP_cleanup ERR_load_crypto_strings CRYPTO_cleanup_all_ex_data ERR_free_strings RAND_cleanup DSA_SIG_set0 EVP_dss1 EVP_DigestVerify SSL_CTX_set_tlsext_ticket_key_cb EVP_aes_256_cbc EVP_EncryptInit_ex HMAC_Init_ex
 do :
   as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
@@ -17946,7 +18009,7 @@ done
 # these check_funcs need -lssl
 BAKLIBS="$LIBS"
 LIBS="-lssl $LIBS"
-for ac_func in OPENSSL_init_ssl SSL_CTX_set_security_level SSL_set1_host SSL_get0_peername
+for ac_func in OPENSSL_init_ssl SSL_CTX_set_security_level SSL_set1_host SSL_get0_peername X509_VERIFY_PARAM_set1_host SSL_CTX_set_ciphersuites
 do :
   as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
@@ -20033,6 +20096,20 @@ esac
 fi
 
 
+ac_fn_c_check_func "$LINENO" "explicit_bzero" "ac_cv_func_explicit_bzero"
+if test "x$ac_cv_func_explicit_bzero" = xyes; then :
+  $as_echo "#define HAVE_EXPLICIT_BZERO 1" >>confdefs.h
+
+else
+  case " $LIBOBJS " in
+  *" explicit_bzero.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS explicit_bzero.$ac_objext"
+ ;;
+esac
+
+fi
+
+
 LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS"
 
 ac_fn_c_check_func "$LINENO" "reallocarray" "ac_cv_func_reallocarray"
@@ -20079,12 +20156,6 @@ fi
 
 
 	if test "$ac_cv_func_arc4random" = "no"; then
-		case " $LIBOBJS " in
-  *" explicit_bzero.$ac_objext "* ) ;;
-  *) LIBOBJS="$LIBOBJS explicit_bzero.$ac_objext"
- ;;
-esac
-
 		case " $LIBOBJS " in
   *" arc4_lock.$ac_objext "* ) ;;
   *) LIBOBJS="$LIBOBJS arc4_lock.$ac_objext"
@@ -21077,7 +21148,7 @@ _ACEOF
 
 
 
-version=1.8.0
+version=1.9.0
 
 date=`date +'%b %e, %Y'`
 
@@ -21596,7 +21667,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by unbound $as_me 1.8.0, which was
+This file was extended by unbound $as_me 1.9.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -21662,7 +21733,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-unbound config.status 1.8.0
+unbound config.status 1.9.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -10,16 +10,16 @@ sinclude(dnscrypt/dnscrypt.m4)
 
 # must be numbers. ac_defun because of later processing
 m4_define([VERSION_MAJOR],[1])
-m4_define([VERSION_MINOR],[8])
+m4_define([VERSION_MINOR],[9])
 m4_define([VERSION_MICRO],[0])
 AC_INIT(unbound, m4_defn([VERSION_MAJOR]).m4_defn([VERSION_MINOR]).m4_defn([VERSION_MICRO]), unbound-bugs@nlnetlabs.nl, unbound)
 AC_SUBST(UNBOUND_VERSION_MAJOR, [VERSION_MAJOR])
 AC_SUBST(UNBOUND_VERSION_MINOR, [VERSION_MINOR])
 AC_SUBST(UNBOUND_VERSION_MICRO, [VERSION_MICRO])
 
-LIBUNBOUND_CURRENT=8
+LIBUNBOUND_CURRENT=9
 LIBUNBOUND_REVISION=0
-LIBUNBOUND_AGE=0
+LIBUNBOUND_AGE=1
 # 1.0.0 had 0:12:0
 # 1.0.1 had 0:13:0
 # 1.0.2 had 0:14:0
@@ -81,7 +81,11 @@ LIBUNBOUND_AGE=0
 # 1.7.1 had 7:9:5
 # 1.7.2 had 7:10:5
 # 1.7.3 had 7:11:5
-# 1.7.4 had 8:0:0 # changes the event callback function signature
+# 1.8.0 had 8:0:0 # changes the event callback function signature
+# 1.8.1 had 8:1:0
+# 1.8.2 had 8:2:0
+# 1.8.3 had 8:3:0
+# 1.8.4 had 9:0:1 # add ub_ctx_set_tls
 
 #   Current  -- the number of the binary API that we're implementing
 #   Revision -- which iteration of the implementation of the binary
@@ -310,11 +314,36 @@ __attribute__((weak)) void f(int x) { printf("%d", x); }
 AC_MSG_RESULT($ac_cv_c_weak_attribute)
 if test $ac_cv_c_weak_attribute = yes; then
   AC_DEFINE(HAVE_ATTR_WEAK, 1, [Whether the C compiler accepts the "weak" attribute])
+  AC_DEFINE(ATTR_WEAK, [__attribute__((weak))], [apply the weak attribute to a symbol])
 fi
 ])dnl End of CHECK_WEAK_ATTRIBUTE
 
 CHECK_WEAK_ATTRIBUTE
 
+AC_DEFUN([CHECK_NORETURN_ATTRIBUTE],
+[AC_REQUIRE([AC_PROG_CC])
+AC_MSG_CHECKING(whether the C compiler (${CC-cc}) accepts the "noreturn" attribute)
+AC_CACHE_VAL(ac_cv_c_noreturn_attribute,
+[ac_cv_c_noreturn_attribute=no
+AC_TRY_COMPILE(
+[ #include <stdio.h>
+__attribute__((noreturn)) void f(int x) { printf("%d", x); }
+], [
+   f(1);
+],
+[ac_cv_c_noreturn_attribute="yes"],
+[ac_cv_c_noreturn_attribute="no"])
+])
+
+AC_MSG_RESULT($ac_cv_c_noreturn_attribute)
+if test $ac_cv_c_noreturn_attribute = yes; then
+  AC_DEFINE(HAVE_ATTR_NORETURN, 1, [Whether the C compiler accepts the "noreturn" attribute])
+  AC_DEFINE(ATTR_NORETURN, [__attribute__((__noreturn__))], [apply the noreturn attribute to a function that exits the program])
+fi
+])dnl End of CHECK_NORETURN_ATTRIBUTE
+
+CHECK_NORETURN_ATTRIBUTE
+
 if test "$srcdir" != "."; then
 	CPPFLAGS="$CPPFLAGS -I$srcdir"
 fi
@@ -435,6 +464,13 @@ ACX_CHECK_NONBLOCKING_BROKEN
 ACX_MKDIR_ONE_ARG
 AC_CHECK_FUNCS([strptime],[AC_CHECK_STRPTIME_WORKS],[AC_LIBOBJ([strptime])])
 
+# check if we can use SO_REUSEPORT
+if echo "$host" | grep -i -e linux -e dragonfly >/dev/null; then
+	AC_DEFINE(REUSEPORT_DEFAULT, 1, [if REUSEPORT is enabled by default])
+else
+	AC_DEFINE(REUSEPORT_DEFAULT, 0, [if REUSEPORT is enabled by default])
+fi
+
 # set memory allocation checking if requested
 AC_ARG_ENABLE(alloc-checks, AC_HELP_STRING([--enable-alloc-checks],
 	[ enable to memory allocation statistics, for debug purposes ]), 
@@ -745,12 +781,12 @@ else
 	AC_MSG_RESULT([no])
 fi
 AC_CHECK_HEADERS([openssl/conf.h openssl/engine.h openssl/bn.h openssl/dh.h openssl/dsa.h openssl/rsa.h],,, [AC_INCLUDES_DEFAULT])
-AC_CHECK_FUNCS([OPENSSL_config EVP_sha1 EVP_sha256 EVP_sha512 FIPS_mode EVP_MD_CTX_new OpenSSL_add_all_digests OPENSSL_init_crypto EVP_cleanup ERR_load_crypto_strings CRYPTO_cleanup_all_ex_data ERR_free_strings RAND_cleanup DSA_SIG_set0 EVP_dss1 EVP_DigestVerify])
+AC_CHECK_FUNCS([OPENSSL_config EVP_sha1 EVP_sha256 EVP_sha512 FIPS_mode EVP_MD_CTX_new OpenSSL_add_all_digests OPENSSL_init_crypto EVP_cleanup ERR_load_crypto_strings CRYPTO_cleanup_all_ex_data ERR_free_strings RAND_cleanup DSA_SIG_set0 EVP_dss1 EVP_DigestVerify SSL_CTX_set_tlsext_ticket_key_cb EVP_aes_256_cbc EVP_EncryptInit_ex HMAC_Init_ex])
 
 # these check_funcs need -lssl
 BAKLIBS="$LIBS"
 LIBS="-lssl $LIBS"
-AC_CHECK_FUNCS([OPENSSL_init_ssl SSL_CTX_set_security_level SSL_set1_host SSL_get0_peername])
+AC_CHECK_FUNCS([OPENSSL_init_ssl SSL_CTX_set_security_level SSL_set1_host SSL_get0_peername X509_VERIFY_PARAM_set1_host SSL_CTX_set_ciphersuites])
 LIBS="$BAKLIBS"
 
 AC_CHECK_DECLS([SSL_COMP_get_compression_methods,sk_SSL_COMP_pop_free,SSL_CTX_set_ecdh_auto], [], [], [
@@ -1396,6 +1432,7 @@ AC_REPLACE_FUNCS(strlcpy)
 AC_REPLACE_FUNCS(memmove)
 AC_REPLACE_FUNCS(gmtime_r)
 AC_REPLACE_FUNCS(isblank)
+AC_REPLACE_FUNCS(explicit_bzero)
 dnl without CTIME, ARC4-functions and without reallocarray.
 LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS"
 AC_SUBST(LIBOBJ_WITHOUT_CTIMEARC4)
@@ -1404,7 +1441,6 @@ if test "$USE_NSS" = "no"; then
 	AC_REPLACE_FUNCS(arc4random)
 	AC_REPLACE_FUNCS(arc4random_uniform)
 	if test "$ac_cv_func_arc4random" = "no"; then
-		AC_LIBOBJ(explicit_bzero)
 		AC_LIBOBJ(arc4_lock)
 		AC_CHECK_FUNCS([getentropy],,[
 		    if test "$USE_WINSOCK" = 1; then
@@ -1729,6 +1765,11 @@ char *strsep(char **stringp, const char *delim);
 int isblank(int c);
 #endif
 
+#ifndef HAVE_EXPLICIT_BZERO
+#define explicit_bzero unbound_explicit_bzero
+void explicit_bzero(void* buf, size_t len);
+#endif
+
 #if defined(HAVE_INET_NTOP) && !HAVE_DECL_INET_NTOP
 const char *inet_ntop(int af, const void *src, char *dst, size_t size);
 #endif
@@ -1761,7 +1802,6 @@ void *reallocarray(void *ptr, size_t nmemb, size_t size);
 #  endif
 #endif /* HAVE_LIBRESSL */
 #ifndef HAVE_ARC4RANDOM
-void explicit_bzero(void* buf, size_t len);
 int getentropy(void* buf, size_t len);
 uint32_t arc4random(void);
 void arc4random_buf(void* buf, size_t n);

contrib/README

@@ -38,3 +38,5 @@ distribution but may be helpful.
 * unbound-querycachedb.py: utility to show data stored in cachedb backend
   for a particular query name and type.  It requires dnspython and (for
   redis backend) redis Python modules.
+* unbound-fuzzme.patch: adds unbound-fuzzme program that parses a packet from
+  stdin.  Used with fuzzers, patch from Jacob Hoffman-Andrews.

contrib/fastrpz.patch

@@ -1,15 +1,11 @@
 Description: based on the included patch contrib/fastrpz.patch
 Author: fastrpz@farsightsecurity.com
 ---
-This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+Index: unboundfastrpz/Makefile.in
 ===================================================================
-RCS file: ./RCS/Makefile.in,v
-retrieving revision 1.1
-Index: unbound-1.7.0~rc1/Makefile.in
-===================================================================
---- unbound-1.7.0~rc1.orig/Makefile.in
-+++ unbound-1.7.0~rc1/Makefile.in
-@@ -23,6 +23,8 @@ CHECKLOCK_SRC=testcode/checklocks.c
+--- unboundfastrpz/Makefile.in	(revision 5073)
++++ unboundfastrpz/Makefile.in	(working copy)
+@@ -23,6 +23,8 @@
  CHECKLOCK_OBJ=@CHECKLOCK_OBJ@
  DNSTAP_SRC=@DNSTAP_SRC@
  DNSTAP_OBJ=@DNSTAP_OBJ@
@@ -18,7 +14,7 @@ Index: unbound-1.7.0~rc1/Makefile.in
  DNSCRYPT_SRC=@DNSCRYPT_SRC@
  DNSCRYPT_OBJ=@DNSCRYPT_OBJ@
  WITH_PYTHONMODULE=@WITH_PYTHONMODULE@
-@@ -125,7 +127,7 @@ validator/val_sigcrypt.c validator/val_u
+@@ -126,7 +128,7 @@
  edns-subnet/edns-subnet.c edns-subnet/subnetmod.c \
  edns-subnet/addrtree.c edns-subnet/subnet-whitelist.c \
  cachedb/cachedb.c cachedb/redis.c respip/respip.c $(CHECKLOCK_SRC) \
@@ -27,16 +23,16 @@ Index: unbound-1.7.0~rc1/Makefile.in
  COMMON_OBJ_WITHOUT_NETCALL=dns.lo infra.lo rrset.lo dname.lo msgencode.lo \
  as112.lo msgparse.lo msgreply.lo packed_rrset.lo iterator.lo iter_delegpt.lo \
  iter_donotq.lo iter_fwd.lo iter_hints.lo iter_priv.lo iter_resptype.lo \
-@@ -137,7 +139,7 @@ slabhash.lo timehist.lo tube.lo winsock_
+@@ -139,7 +141,7 @@
  validator.lo val_kcache.lo val_kentry.lo val_neg.lo val_nsec3.lo val_nsec.lo \
- val_secalgo.lo val_sigcrypt.lo val_utils.lo dns64.lo cachedb.lo authzone.lo\
+ val_secalgo.lo val_sigcrypt.lo val_utils.lo dns64.lo cachedb.lo redis.lo authzone.lo \
  $(SUBNET_OBJ) $(PYTHONMOD_OBJ) $(CHECKLOCK_OBJ) $(DNSTAP_OBJ) $(DNSCRYPT_OBJ) \
 -$(IPSECMOD_OBJ) respip.lo
 +$(FASTRPZ_OBJ) $(IPSECMOD_OBJ) respip.lo
  COMMON_OBJ_WITHOUT_UB_EVENT=$(COMMON_OBJ_WITHOUT_NETCALL) netevent.lo listen_dnsport.lo \
  outside_network.lo
  COMMON_OBJ=$(COMMON_OBJ_WITHOUT_UB_EVENT) ub_event.lo
-@@ -400,6 +402,11 @@ dnscrypt.lo dnscrypt.o: $(srcdir)/dnscry
+@@ -405,6 +407,11 @@
  	$(srcdir)/util/config_file.h $(srcdir)/util/log.h \
  	$(srcdir)/util/netevent.h
  
@@ -48,11 +44,11 @@ Index: unbound-1.7.0~rc1/Makefile.in
  # Python Module
  pythonmod.lo pythonmod.o: $(srcdir)/pythonmod/pythonmod.c config.h \
  	pythonmod/interface.h \
-Index: unbound-1.7.0~rc1/config.h.in
+Index: unboundfastrpz/config.h.in
 ===================================================================
---- unbound-1.7.0~rc1.orig/config.h.in
-+++ unbound-1.7.0~rc1/config.h.in
-@@ -1228,4 +1228,11 @@ void *unbound_stat_realloc_log(void *ptr
+--- unboundfastrpz/config.h.in	(revision 5073)
++++ unboundfastrpz/config.h.in	(working copy)
+@@ -1293,4 +1293,11 @@
  /** the version of unbound-control that this software implements */
  #define UNBOUND_CONTROL_VERSION 1
  
@@ -65,11 +61,11 @@ Index: unbound-1.7.0~rc1/config.h.in
 +#undef FASTRPZ_LIB_OPEN
 +/** turn on fastrpz response policy zones */
 +#undef ENABLE_FASTRPZ
-Index: unbound-1.7.0~rc1/configure.ac
+Index: unboundfastrpz/configure.ac
 ===================================================================
---- unbound-1.7.0~rc1.orig/configure.ac
-+++ unbound-1.7.0~rc1/configure.ac
-@@ -6,6 +6,7 @@ sinclude(ax_pthread.m4)
+--- unboundfastrpz/configure.ac	(revision 5073)
++++ unboundfastrpz/configure.ac	(working copy)
+@@ -6,6 +6,7 @@
  sinclude(acx_python.m4)
  sinclude(ac_pkg_swig.m4)
  sinclude(dnstap/dnstap.m4)
@@ -77,7 +73,7 @@ Index: unbound-1.7.0~rc1/configure.ac
  sinclude(dnscrypt/dnscrypt.m4)
  
  # must be numbers. ac_defun because of later processing
-@@ -1453,6 +1454,9 @@ case "$enable_ipsecmod" in
+@@ -1575,6 +1576,9 @@
  		;;
  esac
  
@@ -87,11 +83,11 @@ Index: unbound-1.7.0~rc1/configure.ac
  AC_MSG_CHECKING([if ${MAKE:-make} supports $< with implicit rule in scope])
  # on openBSD, the implicit rule make $< work.
  # on Solaris, it does not work ($? is changed sources, $^ lists dependencies).
-Index: unbound-1.7.0~rc1/daemon/daemon.c
+Index: unboundfastrpz/daemon/daemon.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/daemon/daemon.c
-+++ unbound-1.7.0~rc1/daemon/daemon.c
-@@ -90,6 +90,9 @@
+--- unboundfastrpz/daemon/daemon.c	(revision 5073)
++++ unboundfastrpz/daemon/daemon.c	(working copy)
+@@ -91,6 +91,9 @@
  #include "sldns/keyraw.h"
  #include "respip/respip.h"
  #include <signal.h>
@@ -101,7 +97,7 @@ Index: unbound-1.7.0~rc1/daemon/daemon.c
  
  #ifdef HAVE_SYSTEMD
  #include <systemd/sd-daemon.h>
-@@ -461,6 +464,14 @@ daemon_create_workers(struct daemon* dae
+@@ -462,6 +465,14 @@
  		fatal_exit("dnstap enabled in config but not built with dnstap support");
  #endif
  	}
@@ -116,9 +112,9 @@ Index: unbound-1.7.0~rc1/daemon/daemon.c
  	for(i=0; i<daemon->num; i++) {
  		if(!(daemon->workers[i] = worker_create(daemon, i,
  			shufport+numport*i/daemon->num, 
-@@ -710,6 +721,9 @@ daemon_cleanup(struct daemon* daemon)
- #ifdef USE_DNSCRYPT
+@@ -719,6 +730,9 @@
  	dnsc_delete(daemon->dnscenv);
+ 	daemon->dnscenv = NULL;
  #endif
 +#ifdef ENABLE_FASTRPZ
 +	rpz_delete(&daemon->rpz_clist, &daemon->rpz_client);
@@ -126,11 +122,11 @@ Index: unbound-1.7.0~rc1/daemon/daemon.c
  	daemon->cfg = NULL;
  }
  
-Index: unbound-1.7.0~rc1/daemon/daemon.h
+Index: unboundfastrpz/daemon/daemon.h
 ===================================================================
---- unbound-1.7.0~rc1.orig/daemon/daemon.h
-+++ unbound-1.7.0~rc1/daemon/daemon.h
-@@ -134,6 +134,11 @@ struct daemon {
+--- unboundfastrpz/daemon/daemon.h	(revision 5073)
++++ unboundfastrpz/daemon/daemon.h	(working copy)
+@@ -136,6 +136,11 @@
  	/** the dnscrypt environment */
  	struct dnsc_env* dnscenv;
  #endif
@@ -142,11 +138,11 @@ Index: unbound-1.7.0~rc1/daemon/daemon.h
  };
  
  /**
-Index: unbound-1.7.0~rc1/daemon/worker.c
+Index: unboundfastrpz/daemon/worker.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/daemon/worker.c
-+++ unbound-1.7.0~rc1/daemon/worker.c
-@@ -74,6 +74,9 @@
+--- unboundfastrpz/daemon/worker.c	(revision 5073)
++++ unboundfastrpz/daemon/worker.c	(working copy)
+@@ -75,6 +75,9 @@
  #include "libunbound/context.h"
  #include "libunbound/libworker.h"
  #include "sldns/sbuffer.h"
@@ -156,7 +152,7 @@ Index: unbound-1.7.0~rc1/daemon/worker.c
  #include "sldns/wire2str.h"
  #include "util/shm_side/shm_main.h"
  #include "dnscrypt/dnscrypt.h"
-@@ -527,8 +530,27 @@ answer_norec_from_cache(struct worker* w
+@@ -533,8 +536,27 @@
  			/* not secure */
  			secure = 0;
  			break;
@@ -182,9 +178,9 @@ Index: unbound-1.7.0~rc1/daemon/worker.c
 +	}
 +#endif
  	/* return this delegation from the cache */
+ 	edns_bak = *edns;
  	edns->edns_version = EDNS_ADVERTISED_VERSION;
- 	edns->udp_size = EDNS_ADVERTISED_SIZE;
-@@ -689,6 +711,23 @@ answer_from_cache(struct worker* worker,
+@@ -702,6 +724,23 @@
  			secure = 0;
  		}
  	} else	secure = 0;
@@ -206,9 +202,9 @@ Index: unbound-1.7.0~rc1/daemon/worker.c
 +        }
 +#endif
  
+ 	edns_bak = *edns;
  	edns->edns_version = EDNS_ADVERTISED_VERSION;
- 	edns->udp_size = EDNS_ADVERTISED_SIZE;
-@@ -1291,6 +1330,15 @@ worker_handle_request(struct comm_point*
+@@ -1407,6 +1446,15 @@
  		log_addr(VERB_ALGO, "refused nonrec (cache snoop) query from",
  			&repinfo->addr, repinfo->addrlen);
  		goto send_reply;
@@ -224,7 +220,7 @@ Index: unbound-1.7.0~rc1/daemon/worker.c
  	}
  
  	/* If we've found a local alias, replace the qname with the alias
-@@ -1339,12 +1387,21 @@ lookup_cache:
+@@ -1455,12 +1503,21 @@
  		h = query_info_hash(lookup_qinfo, sldns_buffer_read_u16_at(c->buffer, 2));
  		if((e=slabhash_lookup(worker->env.msg_cache, h, lookup_qinfo, 0))) {
  			/* answer from cache - we have acquired a readlock on it */
@@ -248,7 +244,7 @@ Index: unbound-1.7.0~rc1/daemon/worker.c
  				/* prefetch it if the prefetch TTL expired.
  				 * Note that if there is more than one pass
  				 * its qname must be that used for cache
-@@ -1398,11 +1455,19 @@ lookup_cache:
+@@ -1514,11 +1571,19 @@
  			lock_rw_unlock(&e->lock);
  		}
  		if(!LDNS_RD_WIRE(sldns_buffer_begin(c->buffer))) {
@@ -270,11 +266,11 @@ Index: unbound-1.7.0~rc1/daemon/worker.c
  				goto send_reply;
  			}
  			verbose(VERB_ALGO, "answer norec from cache -- "
-Index: unbound-1.7.0~rc1/doc/unbound.conf.5.in
+Index: unboundfastrpz/doc/unbound.conf.5.in
 ===================================================================
---- unbound-1.7.0~rc1.orig/doc/unbound.conf.5.in
-+++ unbound-1.7.0~rc1/doc/unbound.conf.5.in
-@@ -1705,6 +1705,81 @@ It must be /96 or shorter.  The default
+--- unboundfastrpz/doc/unbound.conf.5.in	(revision 5073)
++++ unboundfastrpz/doc/unbound.conf.5.in	(working copy)
+@@ -1781,6 +1781,81 @@
  used by dns64 processing instead.  Can be entered multiple times, list a
  new domain for which it applies, one per line.  Applies also to names
  underneath the name given.
@@ -356,10 +352,10 @@ Index: unbound-1.7.0~rc1/doc/unbound.conf.5.in
  .SS "DNSCrypt Options"
  .LP
  The
-Index: unbound-1.7.0~rc1/fastrpz/librpz.h
+Index: unboundfastrpz/fastrpz/librpz.h
 ===================================================================
---- /dev/null
-+++ unbound-1.7.0~rc1/fastrpz/librpz.h
+--- unboundfastrpz/fastrpz/librpz.h	(nonexistent)
++++ unboundfastrpz/fastrpz/librpz.h	(working copy)
 @@ -0,0 +1,957 @@
 +/*
 + * Define the interface from a DNS resolver to the Response Policy Zone
@@ -1318,11 +1314,11 @@ Index: unbound-1.7.0~rc1/fastrpz/librpz.h
 +#endif /* LIBRPZ_LIB_OPEN */
 +
 +#endif /* LIBRPZ_H */
-Index: unbound-1.7.0~rc1/fastrpz/rpz.c
+Index: unboundfastrpz/fastrpz/rpz.c
 ===================================================================
---- /dev/null
-+++ unbound-1.7.0~rc1/fastrpz/rpz.c
-@@ -0,0 +1,1357 @@
+--- unboundfastrpz/fastrpz/rpz.c	(nonexistent)
++++ unboundfastrpz/fastrpz/rpz.c	(working copy)
+@@ -0,0 +1,1352 @@
 +/*
 + * fastrpz/rpz.c - interface to the fastrpz response policy zone library
 + *
@@ -1438,8 +1434,6 @@ Index: unbound-1.7.0~rc1/fastrpz/rpz.c
 +static void
 +log_fnc(librpz_log_level_t level, void* ATTR_UNUSED(ctx), const char* buf)
 +{
-+	char label_buf[sizeof("rpz ")+8];
-+
 +	/* Setting librpz_log_level overrides the unbound "verbose" level. */
 +	if(level > LIBRPZ_LOG_TRACE1 &&
 +	   level <= librpz->log_level_val(LIBRPZ_LOG_INVALID))
@@ -1949,12 +1943,9 @@ Index: unbound-1.7.0~rc1/fastrpz/rpz.c
 +	case st_ck_ns:
 +		/* An NSDNAME or NSIP check failed for lack of cached data. */
 +		return false;
-+#pragma clang diagnostic push
-+#pragma clang diagnostic ignored "-Wunreachable-code"
 +	default:
 +		fatal_exit("impossible RPZ state %d in rpz_worker_cache()",
 +			   rpz->st);
-+#pragma clang diagnostic pop
 +	}
 +
 +	/* Wait for a trigger. */
@@ -2680,10 +2671,10 @@ Index: unbound-1.7.0~rc1/fastrpz/rpz.c
 +}
 +
 +#endif /* ENABLE_FASTRPZ */
-Index: unbound-1.7.0~rc1/fastrpz/rpz.h
+Index: unboundfastrpz/fastrpz/rpz.h
 ===================================================================
---- /dev/null
-+++ unbound-1.7.0~rc1/fastrpz/rpz.h
+--- unboundfastrpz/fastrpz/rpz.h	(nonexistent)
++++ unboundfastrpz/fastrpz/rpz.h	(working copy)
 @@ -0,0 +1,138 @@
 +/*
 + * fastrpz/rpz.h - interface to the fastrpz response policy zone library
@@ -2823,10 +2814,10 @@ Index: unbound-1.7.0~rc1/fastrpz/rpz.h
 +
 +#endif /* ENABLE_FASTRPZ */
 +#endif /* UNBOUND_FASTRPZ_RPZ_H */
-Index: unbound-1.7.0~rc1/fastrpz/rpz.m4
+Index: unboundfastrpz/fastrpz/rpz.m4
 ===================================================================
---- /dev/null
-+++ unbound-1.7.0~rc1/fastrpz/rpz.m4
+--- unboundfastrpz/fastrpz/rpz.m4	(nonexistent)
++++ unboundfastrpz/fastrpz/rpz.m4	(working copy)
 @@ -0,0 +1,64 @@
 +# fastrpz/rpz.m4
 +
@@ -2892,10 +2883,10 @@ Index: unbound-1.7.0~rc1/fastrpz/rpz.m4
 +    AC_MSG_WARN([[dlopen and librpz.so needed for fastrpz]])
 +  fi
 +])
-Index: unbound-1.7.0~rc1/iterator/iterator.c
+Index: unboundfastrpz/iterator/iterator.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/iterator/iterator.c
-+++ unbound-1.7.0~rc1/iterator/iterator.c
+--- unboundfastrpz/iterator/iterator.c	(revision 5073)
++++ unboundfastrpz/iterator/iterator.c	(working copy)
 @@ -68,6 +68,9 @@
  #include "sldns/str2wire.h"
  #include "sldns/parseutil.h"
@@ -2904,9 +2895,9 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
 +#include "fastrpz/rpz.h"
 +#endif
  
- int 
- iter_init(struct module_env* env, int id)
-@@ -511,6 +514,23 @@ handle_cname_response(struct module_qsta
+ /* in msec */
+ int UNKNOWN_SERVER_NICENESS = 376;
+@@ -551,6 +554,23 @@
  		if(ntohs(r->rk.type) == LDNS_RR_TYPE_CNAME &&
  			query_dname_compare(*mname, r->rk.dname) == 0 &&
  			!iter_find_rrset_in_prepend_answer(iq, r)) {
@@ -2930,7 +2921,7 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
  			/* Add this relevant CNAME rrset to the prepend list.*/
  			if(!iter_add_prepend_answer(qstate, iq, r))
  				return 0;
-@@ -519,6 +539,9 @@ handle_cname_response(struct module_qsta
+@@ -559,6 +579,9 @@
  
  		/* Other rrsets in the section are ignored. */
  	}
@@ -2940,7 +2931,7 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
  	/* add authority rrsets to authority prepend, for wildcarded CNAMEs */
  	for(i=msg->rep->an_numrrsets; i<msg->rep->an_numrrsets +
  		msg->rep->ns_numrrsets; i++) {
-@@ -1148,6 +1171,7 @@ processInitRequest(struct module_qstate*
+@@ -1195,6 +1218,7 @@
  	uint8_t* delname;
  	size_t delnamelen;
  	struct dns_msg* msg = NULL;
@@ -2948,7 +2939,7 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
  
  	log_query_info(VERB_DETAIL, "resolving", &qstate->qinfo);
  	/* check effort */
-@@ -1223,8 +1247,7 @@ processInitRequest(struct module_qstate*
+@@ -1281,8 +1305,7 @@
  	}
  	if(msg) {
  		/* handle positive cache response */
@@ -2958,7 +2949,7 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
  		if(verbosity >= VERB_ALGO) {
  			log_dns_msg("msg from cache lookup", &msg->qinfo, 
  				msg->rep);
-@@ -1232,7 +1255,22 @@ processInitRequest(struct module_qstate*
+@@ -1290,7 +1313,22 @@
  				(int)msg->rep->ttl, 
  				(int)msg->rep->prefetch_ttl);
  		}
@@ -2981,7 +2972,7 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
  		if(type == RESPONSE_TYPE_CNAME) {
  			uint8_t* sname = 0;
  			size_t slen = 0;
-@@ -2552,6 +2590,62 @@ processQueryResponse(struct module_qstat
+@@ -2694,6 +2732,62 @@
  			sock_list_insert(&qstate->reply_origin, 
  				&qstate->reply->addr, qstate->reply->addrlen, 
  				qstate->region);
@@ -3041,10 +3032,10 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
 +			}
 +		}
 +#endif
- 		if(iq->minimisation_state != DONOT_MINIMISE_STATE) {
+ 		if(iq->minimisation_state != DONOT_MINIMISE_STATE
+ 			&& !(iq->chase_flags & BIT_RD)) {
  			if(FLAGS_GET_RCODE(iq->response->rep->flags) != 
- 				LDNS_RCODE_NOERROR) {
-@@ -3273,12 +3367,44 @@ processFinished(struct module_qstate* qs
+@@ -3440,6 +3534,10 @@
  		 * but only if we did recursion. The nonrecursion referral
  		 * from cache does not need to be stored in the msg cache. */
  		if(!qstate->no_cache_store && qstate->query_flags&BIT_RD) {
@@ -3055,6 +3046,7 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
  			iter_dns_store(qstate->env, &qstate->qinfo, 
  				iq->response->rep, 0, qstate->prefetch_leeway,
  				iq->dp&&iq->dp->has_parent_side_NS,
+@@ -3446,6 +3544,34 @@
  				qstate->region, qstate->query_flags);
  		}
  	}
@@ -3089,11 +3081,11 @@ Index: unbound-1.7.0~rc1/iterator/iterator.c
  	qstate->return_rcode = LDNS_RCODE_NOERROR;
  	qstate->return_msg = iq->response;
  	return 0;
-Index: unbound-1.7.0~rc1/iterator/iterator.h
+Index: unboundfastrpz/iterator/iterator.h
 ===================================================================
---- unbound-1.7.0~rc1.orig/iterator/iterator.h
-+++ unbound-1.7.0~rc1/iterator/iterator.h
-@@ -383,6 +383,16 @@ struct iter_qstate {
+--- unboundfastrpz/iterator/iterator.h	(revision 5073)
++++ unboundfastrpz/iterator/iterator.h	(working copy)
+@@ -386,6 +386,16 @@
  	 */
  	int minimise_count;
  
@@ -3110,11 +3102,11 @@ Index: unbound-1.7.0~rc1/iterator/iterator.h
  	/**
  	 * Count number of time-outs. Used to prevent resolving failures when
  	 * the QNAME minimisation QTYPE is blocked. */
-Index: unbound-1.7.0~rc1/services/cache/dns.c
+Index: unboundfastrpz/services/cache/dns.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/services/cache/dns.c
-+++ unbound-1.7.0~rc1/services/cache/dns.c
-@@ -876,6 +876,14 @@ dns_cache_store(struct module_env* env,
+--- unboundfastrpz/services/cache/dns.c	(revision 5073)
++++ unboundfastrpz/services/cache/dns.c	(working copy)
+@@ -939,6 +939,14 @@
  	struct regional* region, uint32_t flags)
  {
  	struct reply_info* rep = NULL;
@@ -3129,11 +3121,11 @@ Index: unbound-1.7.0~rc1/services/cache/dns.c
  	/* alloc, malloc properly (not in region, like msg is) */
  	rep = reply_info_copy(msgrep, env->alloc, NULL);
  	if(!rep)
-Index: unbound-1.7.0~rc1/services/mesh.c
+Index: unboundfastrpz/services/mesh.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/services/mesh.c
-+++ unbound-1.7.0~rc1/services/mesh.c
-@@ -59,6 +59,9 @@
+--- unboundfastrpz/services/mesh.c	(revision 5073)
++++ unboundfastrpz/services/mesh.c	(working copy)
+@@ -60,6 +60,9 @@
  #include "sldns/wire2str.h"
  #include "services/localzone.h"
  #include "util/data/dname.h"
@@ -3141,9 +3133,9 @@ Index: unbound-1.7.0~rc1/services/mesh.c
 +#include "fastrpz/rpz.h"
 +#endif
  #include "respip/respip.h"
+ #include "services/listen_dnsport.h"
  
- /** subtract timers and the values do not overflow or become negative */
-@@ -1050,6 +1053,13 @@ mesh_send_reply(struct mesh_state* m, in
+@@ -1072,6 +1075,13 @@
  	else	secure = 0;
  	if(!rep && rcode == LDNS_RCODE_NOERROR)
  		rcode = LDNS_RCODE_SERVFAIL;
@@ -3157,7 +3149,7 @@ Index: unbound-1.7.0~rc1/services/mesh.c
  	/* send the reply */
  	/* We don't reuse the encoded answer if either the previous or current
  	 * response has a local alias.  We could compare the alias records
-@@ -1199,6 +1209,7 @@ struct mesh_state* mesh_area_find(struct
+@@ -1247,6 +1257,7 @@
  	key.s.is_valrec = valrec;
  	key.s.qinfo = *qinfo;
  	key.s.query_flags = qflags;
@@ -3165,7 +3157,7 @@ Index: unbound-1.7.0~rc1/services/mesh.c
  	/* We are searching for a similar mesh state when we DO want to
  	 * aggregate the state. Thus unique is set to NULL. (default when we
  	 * desire aggregation).*/
-@@ -1245,6 +1256,10 @@ int mesh_state_add_reply(struct mesh_sta
+@@ -1293,6 +1304,10 @@
  	if(!r)
  		return 0;
  	r->query_reply = *rep;
@@ -3176,11 +3168,11 @@ Index: unbound-1.7.0~rc1/services/mesh.c
  	r->edns = *edns;
  	if(edns->opt_list) {
  		r->edns.opt_list = edns_opt_copy_region(edns->opt_list,
-Index: unbound-1.7.0~rc1/util/config_file.c
+Index: unboundfastrpz/util/config_file.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/config_file.c
-+++ unbound-1.7.0~rc1/util/config_file.c
-@@ -1323,6 +1323,8 @@ config_delete(struct config_file* cfg)
+--- unboundfastrpz/util/config_file.c	(revision 5073)
++++ unboundfastrpz/util/config_file.c	(working copy)
+@@ -1418,6 +1418,8 @@
  	free(cfg->dnstap_socket_path);
  	free(cfg->dnstap_identity);
  	free(cfg->dnstap_version);
@@ -3189,11 +3181,11 @@ Index: unbound-1.7.0~rc1/util/config_file.c
  	config_deldblstrlist(cfg->ratelimit_for_domain);
  	config_deldblstrlist(cfg->ratelimit_below_domain);
  #ifdef USE_IPSECMOD
-Index: unbound-1.7.0~rc1/util/config_file.h
+Index: unboundfastrpz/util/config_file.h
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/config_file.h
-+++ unbound-1.7.0~rc1/util/config_file.h
-@@ -431,6 +431,11 @@ struct config_file {
+--- unboundfastrpz/util/config_file.h	(revision 5073)
++++ unboundfastrpz/util/config_file.h	(working copy)
+@@ -490,6 +490,11 @@
  	/** true to disable DNSSEC lameness check in iterator */
  	int disable_dnssec_lame_check;
  
@@ -3205,11 +3197,11 @@ Index: unbound-1.7.0~rc1/util/config_file.h
  	/** ratelimit for ip addresses. 0 is off, otherwise qps (unless overridden) */
  	int ip_ratelimit;
  	/** number of slabs for ip_ratelimit cache */
-Index: unbound-1.7.0~rc1/util/configlexer.lex
+Index: unboundfastrpz/util/configlexer.lex
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/configlexer.lex
-+++ unbound-1.7.0~rc1/util/configlexer.lex
-@@ -412,6 +412,10 @@ dnstap-log-forwarder-query-messages{COLO
+--- unboundfastrpz/util/configlexer.lex	(revision 5073)
++++ unboundfastrpz/util/configlexer.lex	(working copy)
+@@ -439,6 +439,10 @@
  		YDVAR(1, VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES) }
  dnstap-log-forwarder-response-messages{COLON}	{
  		YDVAR(1, VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES) }
@@ -3220,11 +3212,11 @@ Index: unbound-1.7.0~rc1/util/configlexer.lex
  disable-dnssec-lame-check{COLON} { YDVAR(1, VAR_DISABLE_DNSSEC_LAME_CHECK) }
  ip-ratelimit{COLON}		{ YDVAR(1, VAR_IP_RATELIMIT) }
  ratelimit{COLON}		{ YDVAR(1, VAR_RATELIMIT) }
-Index: unbound-1.7.0~rc1/util/configparser.y
+Index: unboundfastrpz/util/configparser.y
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/configparser.y
-+++ unbound-1.7.0~rc1/util/configparser.y
-@@ -124,6 +124,7 @@ extern struct config_parser_state* cfg_p
+--- unboundfastrpz/util/configparser.y	(revision 5073)
++++ unboundfastrpz/util/configparser.y	(working copy)
+@@ -125,6 +125,7 @@
  %token VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES
  %token VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES
  %token VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES
@@ -3232,7 +3224,7 @@ Index: unbound-1.7.0~rc1/util/configparser.y
  %token VAR_RESPONSE_IP_TAG VAR_RESPONSE_IP VAR_RESPONSE_IP_DATA
  %token VAR_HARDEN_ALGO_DOWNGRADE VAR_IP_TRANSPARENT
  %token VAR_DISABLE_DNSSEC_LAME_CHECK
-@@ -158,7 +159,7 @@ extern struct config_parser_state* cfg_p
+@@ -170,7 +171,7 @@
  
  %%
  toplevelvars: /* empty */ | toplevelvars toplevelvar ;
@@ -3241,8 +3233,8 @@ Index: unbound-1.7.0~rc1/util/configparser.y
  	forwardstart contents_forward | pythonstart contents_py | 
  	rcstart contents_rc | dtstart contents_dt | viewstart contents_view |
  	dnscstart contents_dnsc | cachedbstart contents_cachedb |
-@@ -2384,6 +2385,50 @@ dt_dnstap_log_forwarder_response_message
- 			(strcmp($2, "yes")==0);
+@@ -2708,6 +2709,50 @@
+ 		free($2);
  	}
  	;
 +rpzstart: VAR_RPZ
@@ -3269,7 +3261,7 @@ Index: unbound-1.7.0~rc1/util/configparser.y
 +
 +		OUTYY(("P(rpz_zone:%s)\n", $2));
 +		old_cstr = cfg_parser->cfg->rpz_cstr;
-+		asprintf(&new_cstr, "%s\nzone %s", old_cstr?old_cstr:"", $2);
++		(void)asprintf(&new_cstr, "%s\nzone %s", old_cstr?old_cstr:"", $2);
 +		if(!new_cstr)
 +			yyerror("out of memory");
 +		free(old_cstr);
@@ -3282,7 +3274,7 @@ Index: unbound-1.7.0~rc1/util/configparser.y
 +
 +		OUTYY(("P(rpz_option:%s)\n", $2));
 +		old_cstr = cfg_parser->cfg->rpz_cstr;
-+		asprintf(&new_cstr, "%s\n%s", old_cstr ? old_cstr : "", $2);
++		(void)asprintf(&new_cstr, "%s\n%s", old_cstr ? old_cstr : "", $2);
 +		if(!new_cstr)
 +			yyerror("out of memory");
 +		free(old_cstr);
@@ -3292,11 +3284,11 @@ Index: unbound-1.7.0~rc1/util/configparser.y
  pythonstart: VAR_PYTHON
  	{ 
  		OUTYY(("\nP(python:)\n")); 
-Index: unbound-1.7.0~rc1/util/data/msgencode.c
+Index: unboundfastrpz/util/data/msgencode.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/data/msgencode.c
-+++ unbound-1.7.0~rc1/util/data/msgencode.c
-@@ -585,6 +585,35 @@ insert_section(struct reply_info* rep, s
+--- unboundfastrpz/util/data/msgencode.c	(revision 5073)
++++ unboundfastrpz/util/data/msgencode.c	(working copy)
+@@ -590,6 +590,35 @@
  	return RETVAL_OK;
  }
  
@@ -3332,7 +3324,7 @@ Index: unbound-1.7.0~rc1/util/data/msgencode.c
  /** store query section in wireformat buffer, return RETVAL */
  static int
  insert_query(struct query_info* qinfo, struct compress_tree_node** tree, 
-@@ -750,6 +779,19 @@ reply_info_encode(struct query_info* qin
+@@ -753,6 +782,19 @@
  			return 0;
  		}
  		sldns_buffer_write_u16_at(buffer, 10, arcount);
@@ -3352,13 +3344,13 @@ Index: unbound-1.7.0~rc1/util/data/msgencode.c
  	}
  	sldns_buffer_flip(buffer);
  	return 1;
-Index: unbound-1.7.0~rc1/util/data/packed_rrset.c
+Index: unboundfastrpz/util/data/packed_rrset.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/data/packed_rrset.c
-+++ unbound-1.7.0~rc1/util/data/packed_rrset.c
-@@ -254,6 +254,10 @@ sec_status_to_string(enum sec_status s)
- 	case sec_status_indeterminate: 	return "sec_status_indeterminate";
+--- unboundfastrpz/util/data/packed_rrset.c	(revision 5073)
++++ unboundfastrpz/util/data/packed_rrset.c	(working copy)
+@@ -255,6 +255,10 @@
  	case sec_status_insecure: 	return "sec_status_insecure";
+ 	case sec_status_secure_sentinel_fail: 	return "sec_status_secure_sentinel_fail";
  	case sec_status_secure: 	return "sec_status_secure";
 +#ifdef ENABLE_FASTRPZ
 +	case sec_status_rpz_rewritten: 	return "sec_status_rpz_rewritten";
@@ -3367,12 +3359,12 @@ Index: unbound-1.7.0~rc1/util/data/packed_rrset.c
  	}
  	return "unknown_sec_status_value";
  }
-Index: unbound-1.7.0~rc1/util/data/packed_rrset.h
+Index: unboundfastrpz/util/data/packed_rrset.h
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/data/packed_rrset.h
-+++ unbound-1.7.0~rc1/util/data/packed_rrset.h
-@@ -189,7 +189,15 @@ enum sec_status {
- 	sec_status_insecure,
+--- unboundfastrpz/util/data/packed_rrset.h	(revision 5073)
++++ unboundfastrpz/util/data/packed_rrset.h	(working copy)
+@@ -193,7 +193,15 @@
+ 	sec_status_secure_sentinel_fail,
  	/** SECURE means that the object (RRset or message) validated 
  	 * according to local policy. */
 -	sec_status_secure
@@ -3388,11 +3380,11 @@ Index: unbound-1.7.0~rc1/util/data/packed_rrset.h
  };
  
  /**
-Index: unbound-1.7.0~rc1/util/netevent.c
+Index: unboundfastrpz/util/netevent.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/netevent.c
-+++ unbound-1.7.0~rc1/util/netevent.c
-@@ -54,6 +54,9 @@
+--- unboundfastrpz/util/netevent.c	(revision 5073)
++++ unboundfastrpz/util/netevent.c	(working copy)
+@@ -57,6 +57,9 @@
  #ifdef HAVE_OPENSSL_ERR_H
  #include <openssl/err.h>
  #endif
@@ -3402,7 +3394,7 @@ Index: unbound-1.7.0~rc1/util/netevent.c
  
  /* -------- Start of local definitions -------- */
  /** if CMSG_ALIGN is not defined on this platform, a workaround */
-@@ -585,6 +588,9 @@ comm_point_udp_ancil_callback(int fd, sh
+@@ -590,6 +593,9 @@
  	struct cmsghdr* cmsg;
  #endif /* S_SPLINT_S */
  
@@ -3412,7 +3404,7 @@ Index: unbound-1.7.0~rc1/util/netevent.c
  	rep.c = (struct comm_point*)arg;
  	log_assert(rep.c->type == comm_udp);
  
-@@ -674,6 +680,9 @@ comm_point_udp_callback(int fd, short ev
+@@ -679,6 +685,9 @@
  	int i;
  	struct sldns_buffer *buffer;
  
@@ -3422,7 +3414,7 @@ Index: unbound-1.7.0~rc1/util/netevent.c
  	rep.c = (struct comm_point*)arg;
  	log_assert(rep.c->type == comm_udp);
  
-@@ -717,6 +726,9 @@ comm_point_udp_callback(int fd, short ev
+@@ -722,6 +731,9 @@
  			(void)comm_point_send_udp_msg(rep.c, buffer,
  				(struct sockaddr*)&rep.addr, rep.addrlen);
  		}
@@ -3432,9 +3424,9 @@ Index: unbound-1.7.0~rc1/util/netevent.c
  		if(!rep.c || rep.c->fd != fd) /* commpoint closed to -1 or reused for
  		another UDP port. Note rep.c cannot be reused with TCP fd. */
  			break;
-@@ -2956,6 +2968,9 @@ comm_point_send_reply(struct comm_reply
- 		comm_point_start_listening(repinfo->c, -1,
- 			repinfo->c->tcp_timeout_msec);
+@@ -3108,6 +3120,9 @@
+ 				repinfo->c->tcp_timeout_msec);
+ 		}
  	}
 +#ifdef ENABLE_FASTRPZ
 +	rpz_end(repinfo);
@@ -3442,7 +3434,7 @@ Index: unbound-1.7.0~rc1/util/netevent.c
  }
  
  void 
-@@ -2965,6 +2980,9 @@ comm_point_drop_reply(struct comm_reply*
+@@ -3117,6 +3132,9 @@
  		return;
  	log_assert(repinfo && repinfo->c);
  	log_assert(repinfo->c->type != comm_tcp_accept);
@@ -3451,8 +3443,8 @@ Index: unbound-1.7.0~rc1/util/netevent.c
 +#endif
  	if(repinfo->c->type == comm_udp)
  		return;
- 	reclaim_tcp_handler(repinfo->c);
-@@ -2984,6 +3002,9 @@ comm_point_start_listening(struct comm_p
+ 	if(repinfo->c->tcp_req_info)
+@@ -3138,6 +3156,9 @@
  {
  	verbose(VERB_ALGO, "comm point start listening %d", 
  		c->fd==-1?newfd:c->fd);
@@ -3462,11 +3454,11 @@ Index: unbound-1.7.0~rc1/util/netevent.c
  	if(c->type == comm_tcp_accept && !c->tcp_free) {
  		/* no use to start listening no free slots. */
  		return;
-Index: unbound-1.7.0~rc1/util/netevent.h
+Index: unboundfastrpz/util/netevent.h
 ===================================================================
---- unbound-1.7.0~rc1.orig/util/netevent.h
-+++ unbound-1.7.0~rc1/util/netevent.h
-@@ -119,6 +119,10 @@ struct comm_reply {
+--- unboundfastrpz/util/netevent.h	(revision 5073)
++++ unboundfastrpz/util/netevent.h	(working copy)
+@@ -120,6 +120,10 @@
  	/** return type 0 (none), 4(IP4), 6(IP6) */
  	int srctype;
  	/* DnsCrypt context */
@@ -3477,11 +3469,11 @@ Index: unbound-1.7.0~rc1/util/netevent.h
  #ifdef USE_DNSCRYPT
  	uint8_t client_nonce[crypto_box_HALF_NONCEBYTES];
  	uint8_t nmkey[crypto_box_BEFORENMBYTES];
-Index: unbound-1.7.0~rc1/validator/validator.c
+Index: unboundfastrpz/validator/validator.c
 ===================================================================
---- unbound-1.7.0~rc1.orig/validator/validator.c
-+++ unbound-1.7.0~rc1/validator/validator.c
-@@ -2688,6 +2688,12 @@ ds_response_to_ke(struct module_qstate*
+--- unboundfastrpz/validator/validator.c	(revision 5073)
++++ unboundfastrpz/validator/validator.c	(working copy)
+@@ -2755,6 +2755,12 @@
  			default:
  				/* NSEC proof did not work, try next */
  				break;
@@ -3494,7 +3486,7 @@ Index: unbound-1.7.0~rc1/validator/validator.c
  		}
  
  		sec = nsec3_prove_nods(qstate->env, ve, 
-@@ -2721,6 +2727,12 @@ ds_response_to_ke(struct module_qstate*
+@@ -2788,6 +2794,12 @@
  			default:
  				/* NSEC3 proof did not work */
  				break;
@@ -3507,4 +3499,3 @@ Index: unbound-1.7.0~rc1/validator/validator.c
  		}
  
  		/* Apparently, no available NSEC/NSEC3 proved NODATA, so 
-

contrib/unbound-fuzzme.patch

@@ -0,0 +1,148 @@
+>From cc9b927f8f29d989ddb8415fe6508a538546abca Mon Sep 17 00:00:00 2001
+From: Jacob Hoffman-Andrews <github@hoffman-andrews.com>
+Date: Wed, 2 Jan 2019 22:52:51 -0800
+Subject: [PATCH] Add unbound-fuzzme.
+
+This is a small program that simply parses a packet provided on stdout,
+for the purposes of fuzzing.
+---
+ .gitignore                |  1 +
+ Makefile.in               | 22 ++++++++++++++++++++--
+ smallapp/unbound-fuzzme.c | 38 ++++++++++++++++++++++++++++++++++++++
+ 3 files changed, 59 insertions(+), 2 deletions(-)
+ create mode 100644 smallapp/unbound-fuzzme.c
+
+diff --git a/.gitignore b/.gitignore
+index f4527fd8..6163f905 100644
+--- a/.gitignore
++++ b/.gitignore
+@@ -24,6 +24,7 @@
+ /unbound-checkconf
+ /unbound-control
+ /unbound-control-setup
++/unbound-fuzzme
+ /unbound-host
+ /unbound.h
+ /asynclook
+diff --git a/Makefile.in b/Makefile.in
+index af5b10f6..dacf1ab5 100644
+--- a/Makefile.in
++++ b/Makefile.in
+@@ -177,6 +177,10 @@ shm_main.lo remote.lo stats.lo unbound.lo \
+ worker.lo @WIN_DAEMON_OBJ@
+ DAEMON_OBJ_LINK=$(DAEMON_OBJ) $(COMMON_OBJ_ALL_SYMBOLS) $(SLDNS_OBJ) \
+ $(COMPAT_OBJ) @WIN_DAEMON_OBJ_LINK@
++FUZZME_SRC=smallapp/unbound-fuzzme.c
++FUZZME_OBJ=unbound-fuzzme.lo
++FUZZME_OBJ_LINK=$(FUZZME_OBJ) worker_cb.lo $(COMMON_OBJ_ALL_SYMBOLS) $(SLDNS_OBJ) \
++$(COMPAT_OBJ)
+ CHECKCONF_SRC=smallapp/unbound-checkconf.c smallapp/worker_cb.c
+ CHECKCONF_OBJ=unbound-checkconf.lo worker_cb.lo
+ CHECKCONF_OBJ_LINK=$(CHECKCONF_OBJ) $(COMMON_OBJ_ALL_SYMBOLS) $(SLDNS_OBJ) \
+@@ -252,6 +256,7 @@ RSRC_OBJ=rsrc_svcinst.o rsrc_svcuninst.o rsrc_anchorupd.o rsrc_unbound.o \
+ 	rsrc_unbound_checkconf.o
+ 
+ ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) \
++	$(FUZZME_SRC) \
+ 	$(TESTBOUND_SRC) $(LOCKVERIFY_SRC) $(PKTVIEW_SRC) \
+ 	$(MEMSTATS_SRC) $(CHECKCONF_SRC) $(LIBUNBOUND_SRC) $(HOST_SRC) \
+ 	$(ASYNCLOOK_SRC) $(STREAMTCP_SRC) $(PERF_SRC) $(DELAYER_SRC) \
+@@ -259,6 +264,7 @@ ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) \
+ 	$(PYTHONMOD_SRC) $(PYUNBOUND_SRC) $(WIN_DAEMON_THE_SRC)\
+ 	$(SVCINST_SRC) $(SVCUNINST_SRC) $(ANCHORUPD_SRC) $(SLDNS_SRC)
+ ALL_OBJ=$(COMMON_OBJ) $(UNITTEST_OBJ) $(DAEMON_OBJ) \
++	$(FUZZME_OBJ) \
+ 	$(TESTBOUND_OBJ) $(LOCKVERIFY_OBJ) $(PKTVIEW_OBJ) \
+ 	$(MEMSTATS_OBJ) $(CHECKCONF_OBJ) $(LIBUNBOUND_OBJ) $(HOST_OBJ) \
+ 	$(ASYNCLOOK_OBJ) $(STREAMTCP_OBJ) $(PERF_OBJ) $(DELAYER_OBJ) \
+@@ -274,7 +280,7 @@ LINK_LIB=$(LIBTOOL) --tag=CC --mode=link $(CC) $(RUNTIME_PATH) $(CPPFLAGS) $(CFL
+ 
+ all:	$(COMMON_OBJ) $(ALLTARGET)
+ 
+-alltargets:	unbound$(EXEEXT) unbound-checkconf$(EXEEXT) lib unbound-host$(EXEEXT) unbound-control$(EXEEXT) unbound-anchor$(EXEEXT) unbound-control-setup $(WINAPPS) $(PYUNBOUND_TARGET)
++alltargets:	unbound$(EXEEXT) unbound-checkconf$(EXEEXT) lib unbound-host$(EXEEXT) unbound-control$(EXEEXT) unbound-anchor$(EXEEXT) unbound-control-setup unbound-fuzzme$(EXEEXT) $(WINAPPS) $(PYUNBOUND_TARGET)
+ 
+ # compat with BSD make, register suffix, and an implicit rule to actualise it.
+ .SUFFIXES: .lo
+@@ -325,6 +331,9 @@ libunbound.la:	$(LIBUNBOUND_OBJ_LINK)
+ unbound$(EXEEXT):	$(DAEMON_OBJ_LINK) libunbound.la
+ 	$(LINK) -o $@ $(DAEMON_OBJ_LINK) $(EXTRALINK) $(SSLLIB) $(LIBS)
+ 
++unbound-fuzzme$(EXEEXT):	$(FUZZME_OBJ_LINK) libunbound.la
++	$(LINK) -o $@ $(FUZZME_OBJ_LINK) $(EXTRALINK) $(SSLLIB) $(LIBS)
++
+ unbound-checkconf$(EXEEXT):	$(CHECKCONF_OBJ_LINK) libunbound.la
+ 	$(LINK) -o $@ $(CHECKCONF_OBJ_LINK) $(EXTRALINK) $(SSLLIB) $(LIBS)
+ 
+@@ -447,7 +456,7 @@ util/configparser.c util/configparser.h:  $(srcdir)/util/configparser.y
+ 
+ clean:
+ 	rm -f *.o *.d *.lo *~ tags
+-	rm -f unbound$(EXEEXT) unbound-checkconf$(EXEEXT) unbound-host$(EXEEXT) unbound-control$(EXEEXT) unbound-anchor$(EXEEXT) unbound-control-setup libunbound.la unbound.h
++	rm -f unbound$(EXEEXT) unbound-checkconf$(EXEEXT) unbound-fuzzme$(EXEEXT) unbound-host$(EXEEXT) unbound-control$(EXEEXT) unbound-anchor$(EXEEXT) unbound-control-setup libunbound.la unbound.h
+ 	rm -f $(ALL_SRC:.c=.lint)
+ 	rm -f _unbound.la libunbound/python/libunbound_wrap.c libunbound/python/unbound.py pythonmod/interface.h pythonmod/unboundmodule.py
+ 	rm -rf autom4te.cache .libs build doc/html doc/xml
+@@ -1183,6 +1192,15 @@ stats.lo stats.o: $(srcdir)/daemon/stats.c config.h $(srcdir)/daemon/stats.h $(s
+  $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/infra.h $(srcdir)/util/storage/dnstree.h \
+  $(srcdir)/util/rtt.h $(srcdir)/services/authzone.h $(srcdir)/validator/val_kcache.h \
+  $(srcdir)/validator/val_neg.h
++unbound-fuzzme.lo unbound-fuzzme.o: $(srcdir)/smallapp/unbound-fuzzme.c \
++ $(srcdir)/util/locks.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h  \
++  $(srcdir)/daemon/remote.h $(srcdir)/util/config_file.h \
++ $(srcdir)/util/storage/slabhash.h $(srcdir)/util/storage/lruhash.h $(srcdir)/services/listen_dnsport.h \
++ $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h $(srcdir)/services/cache/rrset.h \
++ $(srcdir)/util/data/packed_rrset.h $(srcdir)/services/cache/infra.h $(srcdir)/util/storage/dnstree.h \
++ $(srcdir)/util/rbtree.h $(srcdir)/util/rtt.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/fptr_wlist.h \
++ $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \
++ $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/util/net_help.h $(srcdir)/util/ub_event.h
+ unbound.lo unbound.o: $(srcdir)/daemon/unbound.c config.h $(srcdir)/util/log.h $(srcdir)/daemon/daemon.h \
+  $(srcdir)/util/locks.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h  \
+   $(srcdir)/daemon/remote.h \
+diff --git a/smallapp/unbound-fuzzme.c b/smallapp/unbound-fuzzme.c
+new file mode 100644
+index 00000000..74ae5204
+--- /dev/null
++++ b/smallapp/unbound-fuzzme.c
+@@ -0,0 +1,38 @@
++/*
++ * unbound-fuzzme.c - parse a packet provided on stdin (for fuzzing).
++ *
++ */
++#include "config.h"
++#include "util/regional.h"
++#include "util/fptr_wlist.h"
++#include "sldns/sbuffer.h"
++
++#define SZ 10000
++
++int main() {
++	char buffer[SZ];
++	size_t n_read = fread(buffer, 1, SZ, stdin);
++	if (n_read == SZ) {
++		printf("input too big\n");
++		return 1;
++	}
++	sldns_buffer *pkt = sldns_buffer_new(n_read);
++	sldns_buffer_init_frm_data(pkt, buffer, n_read);
++
++	struct regional *region = regional_create();
++
++	struct msg_parse* prs;
++	struct edns_data edns;
++	prs = (struct msg_parse*)malloc(sizeof(struct msg_parse));
++	if(!prs) {
++		printf("out of memory on incoming message\n");
++		return 1;
++	}
++	memset(prs, 0, sizeof(*prs));
++	memset(&edns, 0, sizeof(edns));
++	sldns_buffer_set_position(pkt, 0);
++	if(parse_packet(pkt, prs, region) != LDNS_RCODE_NOERROR) {
++		printf("parse error\n");
++		return 1;
++	}
++}
+-- 
+2.17.1
+

daemon/daemon.c

@@ -660,18 +660,15 @@ daemon_fork(struct daemon* daemon)
 
 	/* Start resolver service on main thread. */
 #ifdef HAVE_SYSTEMD
-	if(daemon->cfg->use_systemd)
-		sd_notify(0, "READY=1");
+	sd_notify(0, "READY=1");
 #endif
 	log_info("start of service (%s).", PACKAGE_STRING);
 	worker_work(daemon->workers[0]);
 #ifdef HAVE_SYSTEMD
-	if(daemon->cfg->use_systemd) {
-		if (daemon->workers[0]->need_to_exit)
-			sd_notify(0, "STOPPING=1");
-		else
-			sd_notify(0, "RELOADING=1");
-	}
+	if (daemon->workers[0]->need_to_exit)
+		sd_notify(0, "STOPPING=1");
+	else
+		sd_notify(0, "RELOADING=1");
 #endif
 	log_info("service stopped (%s).", PACKAGE_STRING);
 
@@ -752,6 +749,7 @@ daemon_delete(struct daemon* daemon)
 	free(daemon->pidfile);
 	free(daemon->env);
 #ifdef HAVE_SSL
+	listen_sslctx_delete_ticket_keys();
 	SSL_CTX_free((SSL_CTX*)daemon->listen_sslctx);
 	SSL_CTX_free((SSL_CTX*)daemon->connect_sslctx);
 #endif

daemon/remote.c

@@ -275,12 +275,13 @@ add_open(const char* ip, int nr, struct listen_port** list, int noproto_is_err,
 	struct addrinfo hints;
 	struct addrinfo* res;
 	struct listen_port* n;
-	int noproto;
+	int noproto = 0;
 	int fd, r;
 	char port[15];
 	snprintf(port, sizeof(port), "%d", nr);
 	port[sizeof(port)-1]=0;
 	memset(&hints, 0, sizeof(hints));
+	log_assert(ip);
 
 	if(ip[0] == '/') {
 		/* This looks like a local socket */
@@ -788,7 +789,8 @@ print_longnum(RES* ssl, const char* desc, size_t x)
 
 /** print mem stats */
 static int
-print_mem(RES* ssl, struct worker* worker, struct daemon* daemon)
+print_mem(RES* ssl, struct worker* worker, struct daemon* daemon,
+	struct ub_stats_info* s)
 {
 	size_t msg, rrset, val, iter, respip;
 #ifdef CLIENT_SUBNET
@@ -846,6 +848,9 @@ print_mem(RES* ssl, struct worker* worker, struct daemon* daemon)
 			dnscrypt_nonce))
 		return 0;
 #endif /* USE_DNSCRYPT */
+	if(!print_longnum(ssl, "mem.streamwait"SQ,
+		(size_t)s->svr.mem_stream_wait))
+		return 0;
 	return 1;
 }
 
@@ -968,6 +973,8 @@ print_ext(RES* ssl, struct ub_stats_info* s)
 		(unsigned long)s->svr.qtcp_outgoing)) return 0;
 	if(!ssl_printf(ssl, "num.query.tls"SQ"%lu\n", 
 		(unsigned long)s->svr.qtls)) return 0;
+	if(!ssl_printf(ssl, "num.query.tls.resume"SQ"%lu\n", 
+		(unsigned long)s->svr.qtls_resume)) return 0;
 	if(!ssl_printf(ssl, "num.query.ipv6"SQ"%lu\n", 
 		(unsigned long)s->svr.qipv6)) return 0;
 	/* flags */
@@ -1069,6 +1076,7 @@ do_stats(RES* ssl, struct daemon_remote* rc, int reset)
 	struct ub_stats_info total;
 	struct ub_stats_info s;
 	int i;
+	memset(&total, 0, sizeof(total));
 	log_assert(daemon->num > 0);
 	/* gather all thread statistics in one place */
 	for(i=0; i<daemon->num; i++) {
@@ -1086,7 +1094,7 @@ do_stats(RES* ssl, struct daemon_remote* rc, int reset)
 	if(!print_uptime(ssl, rc->worker, reset))
 		return;
 	if(daemon->cfg->stat_extended) {
-		if(!print_mem(ssl, rc->worker, daemon)) 
+		if(!print_mem(ssl, rc->worker, daemon, &total)) 
 			return;
 		if(!print_hist(ssl, &total))
 			return;
@@ -1426,6 +1434,28 @@ do_view_data_add(RES* ssl, struct worker* worker, char* arg)
 	lock_rw_unlock(&v->lock);
 }
 
+/** Add new RR data from stdin to view */
+static void
+do_view_datas_add(RES* ssl, struct worker* worker, char* arg)
+{
+	struct view* v;
+	v = views_find_view(worker->daemon->views,
+		arg, 1 /* get write lock*/);
+	if(!v) {
+		ssl_printf(ssl,"no view with name: %s\n", arg);
+		return;
+	}
+	if(!v->local_zones) {
+		if(!(v->local_zones = local_zones_create())){
+			lock_rw_unlock(&v->lock);
+			ssl_printf(ssl,"error out of memory\n");
+			return;
+		}
+	}
+	do_datas_add(ssl, v->local_zones);
+	lock_rw_unlock(&v->lock);
+}
+
 /** Remove RR data from view */
 static void
 do_view_data_remove(RES* ssl, struct worker* worker, char* arg)
@@ -2454,7 +2484,7 @@ do_auth_zone_reload(RES* ssl, struct worker* worker, char* arg)
 		(void)ssl_printf(ssl, "error no auth-zone %s\n", arg);
 		return;
 	}
-	if(!auth_zone_read_zonefile(z)) {
+	if(!auth_zone_read_zonefile(z, worker->env.cfg)) {
 		lock_rw_unlock(&z->lock);
 		(void)ssl_printf(ssl, "error failed to read %s\n", arg);
 		return;
@@ -2961,6 +2991,8 @@ execute_cmd(struct daemon_remote* rc, RES* ssl, char* cmd,
 		do_view_data_remove(ssl, worker, skipwhite(p+22));
 	} else if(cmdcmp(p, "view_local_data", 15)) {
 		do_view_data_add(ssl, worker, skipwhite(p+15));
+	} else if(cmdcmp(p, "view_local_datas", 16)) {
+		do_view_datas_add(ssl, worker, skipwhite(p+16));
 	} else if(cmdcmp(p, "flush_zone", 10)) {
 		do_flush_zone(ssl, worker, skipwhite(p+10));
 	} else if(cmdcmp(p, "flush_type", 10)) {

daemon/stats.c

@@ -66,6 +66,9 @@
 #ifdef CLIENT_SUBNET
 #include "edns-subnet/subnetmod.h"
 #endif
+#ifdef HAVE_SSL
+#include <openssl/ssl.h>
+#endif
 
 /** add timers and the values do not overflow or become negative */
 static void
@@ -328,6 +331,8 @@ server_stats_compile(struct worker* worker, struct ub_stats_info* s, int reset)
 		}
 		lock_rw_unlock(&worker->env.auth_zones->lock);
 	}
+	s->svr.mem_stream_wait =
+		(long long)tcp_req_info_get_stream_buffer_size();
 
 	/* Set neg cache usage numbers */
 	set_neg_cache_stats(worker, &s->svr, reset);
@@ -412,6 +417,7 @@ void server_stats_add(struct ub_stats_info* total, struct ub_stats_info* a)
 		total->svr.qtcp += a->svr.qtcp;
 		total->svr.qtcp_outgoing += a->svr.qtcp_outgoing;
 		total->svr.qtls += a->svr.qtls;
+		total->svr.qtls_resume += a->svr.qtls_resume;
 		total->svr.qipv6 += a->svr.qipv6;
 		total->svr.qbit_QR += a->svr.qbit_QR;
 		total->svr.qbit_AA += a->svr.qbit_AA;
@@ -468,8 +474,13 @@ void server_stats_insquery(struct ub_server_stats* stats, struct comm_point* c,
 	stats->qopcode[ LDNS_OPCODE_WIRE(sldns_buffer_begin(c->buffer)) ]++;
 	if(c->type != comm_udp) {
 		stats->qtcp++;
-		if(c->ssl != NULL)
+		if(c->ssl != NULL) {
 			stats->qtls++;
+#ifdef HAVE_SSL
+			if(SSL_session_reused(c->ssl)) 
+				stats->qtls_resume++;
+#endif
+		}
 	}
 	if(repinfo && addr_is_ip6(&repinfo->addr, repinfo->addrlen))
 		stats->qipv6++;

daemon/unbound.c

@@ -67,6 +67,7 @@
 #ifdef HAVE_GRP_H
 #include <grp.h>
 #endif
+#include <openssl/ssl.h>
 
 #ifndef S_SPLINT_S
 /* splint chokes on this system header file */
@@ -430,6 +431,23 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode,
 		if(!(daemon->listen_sslctx = listen_sslctx_create(
 			cfg->ssl_service_key, cfg->ssl_service_pem, NULL)))
 			fatal_exit("could not set up listen SSL_CTX");
+		if(cfg->tls_ciphers && cfg->tls_ciphers[0]) {
+			if (!SSL_CTX_set_cipher_list(daemon->listen_sslctx, cfg->tls_ciphers)) {
+				fatal_exit("failed to set tls-cipher %s", cfg->tls_ciphers);
+			}
+		}
+#ifdef HAVE_SSL_CTX_SET_CIPHERSUITES
+		if(cfg->tls_ciphersuites && cfg->tls_ciphersuites[0]) {
+			if (!SSL_CTX_set_ciphersuites(daemon->listen_sslctx, cfg->tls_ciphersuites)) {
+				fatal_exit("failed to set tls-ciphersuites %s", cfg->tls_ciphersuites);
+			}
+		}
+#endif
+		if(cfg->tls_session_ticket_keys.first) {
+			if(!listen_sslctx_setup_ticket_keys(daemon->listen_sslctx, cfg->tls_session_ticket_keys.first)) {
+				fatal_exit("could not set session ticket SSL_CTX");
+			}
+		}
 	}
 	if(!(daemon->connect_sslctx = connect_sslctx_create(NULL, NULL,
 		cfg->tls_cert_bundle, cfg->tls_win_cert)))
@@ -730,7 +748,7 @@ main(int argc, char* argv[])
 		}
 	}
 	argc -= optind;
-	argv += optind;
+	/* argv += optind; not using further arguments */
 
 	if(winopt) {
 #ifdef UB_ON_WINDOWS

daemon/worker.c

@@ -1088,7 +1088,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 	struct ub_packed_rrset_key* alias_rrset = NULL;
 	struct reply_info* partial_rep = NULL;
 	struct query_info* lookup_qinfo = &qinfo;
-	struct query_info qinfo_tmp; /* placeholdoer for lookup_qinfo */
+	struct query_info qinfo_tmp; /* placeholder for lookup_qinfo */
 	struct respip_client_info* cinfo = NULL, cinfo_tmp;
 	memset(&qinfo, 0, sizeof(qinfo));
 
@@ -1175,12 +1175,12 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		/* See if we are passed through with slip factor */
 		if(worker->env.cfg->ip_ratelimit_factor != 0 &&
 			ub_random_max(worker->env.rnd,
-						  worker->env.cfg->ip_ratelimit_factor) == 1) {
+						  worker->env.cfg->ip_ratelimit_factor) == 0) {
 
 			char addrbuf[128];
 			addr_to_str(&repinfo->addr, repinfo->addrlen,
 						addrbuf, sizeof(addrbuf));
-		  verbose(VERB_OPS, "ip_ratelimit allowed through for ip address %s ",
+		  verbose(VERB_QUERY, "ip_ratelimit allowed through for ip address %s because of slip in ip_ratelimit_factor",
 				  addrbuf);
 		} else {
 			worker->stats.num_queries_ip_ratelimited++;
@@ -1208,7 +1208,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 	if(worker->env.cfg->log_queries) {
 		char ip[128];
 		addr_to_str(&repinfo->addr, repinfo->addrlen, ip, sizeof(ip));
-		log_nametypeclass(0, ip, qinfo.qname, qinfo.qtype, qinfo.qclass);
+		log_query_in(ip, qinfo.qname, qinfo.qtype, qinfo.qclass);
 	}
 	if(qinfo.qtype == LDNS_RR_TYPE_AXFR || 
 		qinfo.qtype == LDNS_RR_TYPE_IXFR) {
@@ -1671,14 +1671,14 @@ worker_create(struct daemon* daemon, int id, int* ports, int n)
 		(((unsigned int)worker->thread_num)<<17);
 		/* shift thread_num so it does not match out pid bits */
 	if(!(worker->rndstate = ub_initstate(seed, daemon->rand))) {
-		seed = 0;
+		explicit_bzero(&seed, sizeof(seed));
 		log_err("could not init random numbers.");
 		tube_delete(worker->cmd);
 		free(worker->ports);
 		free(worker);
 		return NULL;
 	}
-	seed = 0;
+	explicit_bzero(&seed, sizeof(seed));
 #ifdef USE_DNSTAP
 	if(daemon->cfg->dnstap) {
 		log_assert(daemon->dtenv != NULL);
@@ -1802,8 +1802,6 @@ worker_init(struct worker* worker, struct config_file *cfg,
 	alloc_set_id_cleanup(&worker->alloc, &worker_alloc_cleanup, worker);
 	worker->env = *worker->daemon->env;
 	comm_base_timept(worker->base, &worker->env.now, &worker->env.now_tv);
-	if(worker->thread_num == 0)
-		log_set_time(worker->env.now);
 	worker->env.worker = worker;
 	worker->env.worker_base = worker->base;
 	worker->env.send_query = &worker_send_query;
@@ -1909,7 +1907,6 @@ worker_delete(struct worker* worker)
 	comm_timer_delete(worker->env.probe_timer);
 	free(worker->ports);
 	if(worker->thread_num == 0) {
-		log_set_time(NULL);
 #ifdef UB_ON_WINDOWS
 		wsvc_desetup_worker(worker);
 #endif /* UB_ON_WINDOWS */

dns64/dns64.c

@@ -70,12 +70,9 @@ static const char DEFAULT_DNS64_PREFIX[] = "64:ff9b::/96";
 #define MAX_PTR_QNAME_IPV4 30
 
 /**
- * Per-query module-specific state. This is usually a dynamically-allocated
- * structure, but in our case we only need to store one variable describing the
- * state the query is in. So we repurpose the minfo pointer by storing an
- * integer in there.
+ * State of DNS64 processing for a query.
  */
-enum dns64_qstate {
+enum dns64_state {
     DNS64_INTERNAL_QUERY,    /**< Internally-generated query, no DNS64
                                   processing. */
     DNS64_NEW_QUERY,         /**< Query for which we're the first module in
@@ -84,6 +81,19 @@ enum dns64_qstate {
                                   for which this sub-query is finished. */
 };
 
+/**
+ * Per-query module-specific state.  For the DNS64 module.
+ */
+struct dns64_qstate {
+	/** State of the DNS64 module. */
+	enum dns64_state state;
+	/** If the dns64 module started with no_cache bool set in the qstate,
+	 * a message to tell it to not modify the cache contents, then this
+	 * is true.  The dns64 module is then free to modify that flag for
+	 * its own purposes.
+	 * Otherwise, it is false, the dns64 module was not told to no_cache */
+	int started_no_cache_store;
+};
 
 /******************************************************************************
  *                                                                            *
@@ -470,7 +480,7 @@ handle_ipv6_ptr(struct module_qstate* qstate, int id)
     if (subq) {
         subq->curmod = id;
         subq->ext_state[id] = module_state_initial;
-        subq->minfo[id] = NULL;
+	subq->minfo[id] = NULL;
     }
 
     return module_wait_subquery;
@@ -540,7 +550,8 @@ dns64_always_synth_for_qname(struct module_qstate* qstate, int id)
 static enum module_ext_state
 handle_event_pass(struct module_qstate* qstate, int id)
 {
-	if ((uintptr_t)qstate->minfo[id] == DNS64_NEW_QUERY
+	struct dns64_qstate* iq = (struct dns64_qstate*)qstate->minfo[id];
+	if (iq && iq->state == DNS64_NEW_QUERY
             && qstate->qinfo.qtype == LDNS_RR_TYPE_PTR
             && qstate->qinfo.qname_len == 74
             && !strcmp((char*)&qstate->qinfo.qname[64], "\03ip6\04arpa"))
@@ -548,12 +559,12 @@ handle_event_pass(struct module_qstate* qstate, int id)
         return handle_ipv6_ptr(qstate, id);
 
 	if (qstate->env->cfg->dns64_synthall &&
-	    (uintptr_t)qstate->minfo[id] == DNS64_NEW_QUERY
+	    iq && iq->state == DNS64_NEW_QUERY
 	    && qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA)
 		return generate_type_A_query(qstate, id);
 
 	if(dns64_always_synth_for_qname(qstate, id) &&
-	    (uintptr_t)qstate->minfo[id] == DNS64_NEW_QUERY
+	    iq && iq->state == DNS64_NEW_QUERY
 	    && !(qstate->query_flags & BIT_CD)
 	    && qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA) {
 		verbose(VERB_ALGO, "dns64: ignore-aaaa and synthesize anyway");
@@ -561,7 +572,7 @@ handle_event_pass(struct module_qstate* qstate, int id)
 	}
 
 	/* We are finished when our sub-query is finished. */
-	if ((uintptr_t)qstate->minfo[id] == DNS64_SUBQUERY_FINISHED)
+	if (iq && iq->state == DNS64_SUBQUERY_FINISHED)
 		return module_finished;
 
 	/* Otherwise, pass request to next module. */
@@ -582,6 +593,7 @@ handle_event_pass(struct module_qstate* qstate, int id)
 static enum module_ext_state
 handle_event_moddone(struct module_qstate* qstate, int id)
 {
+	struct dns64_qstate* iq = (struct dns64_qstate*)qstate->minfo[id];
     /*
      * In many cases we have nothing special to do. From most to least common:
      *
@@ -593,7 +605,7 @@ handle_event_moddone(struct module_qstate* qstate, int id)
      *        synthesize in (sec 5.1.2 of RFC6147).
      *   - A successful AAAA query with an answer.
      */
-	if((enum dns64_qstate)qstate->minfo[id] != DNS64_INTERNAL_QUERY
+	if((!iq || iq->state != DNS64_INTERNAL_QUERY)
             && qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA
 	    && !(qstate->query_flags & BIT_CD)
 	    && !(qstate->return_msg &&
@@ -604,7 +616,7 @@ handle_event_moddone(struct module_qstate* qstate, int id)
 		 * So, this is a AAAA noerror/nodata answer */
 		return generate_type_A_query(qstate, id);
 
-	if((enum dns64_qstate)qstate->minfo[id] != DNS64_INTERNAL_QUERY
+	if((!iq || iq->state != DNS64_INTERNAL_QUERY)
 	    && qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA
 	    && !(qstate->query_flags & BIT_CD)
 	    && dns64_always_synth_for_qname(qstate, id)) {
@@ -614,6 +626,13 @@ handle_event_moddone(struct module_qstate* qstate, int id)
 		return generate_type_A_query(qstate, id);
 	}
 
+	/* Store the response in cache. */
+	if ( (!iq || !iq->started_no_cache_store) &&
+		qstate->return_msg && qstate->return_msg->rep &&
+		!dns_cache_store(qstate->env, &qstate->qinfo, qstate->return_msg->rep,
+		0, 0, 0, NULL, qstate->query_flags))
+		log_err("out of memory");
+
 	/* do nothing */
 	return module_finished;
 }
@@ -634,6 +653,7 @@ void
 dns64_operate(struct module_qstate* qstate, enum module_ev event, int id,
 		struct outbound_entry* outbound)
 {
+	struct dns64_qstate* iq;
 	(void)outbound;
 	verbose(VERB_QUERY, "dns64[module %d] operate: extstate:%s event:%s",
 			id, strextstate(qstate->ext_state[id]),
@@ -643,7 +663,12 @@ dns64_operate(struct module_qstate* qstate, enum module_ev event, int id,
 	switch(event) {
 		case module_event_new:
 			/* Tag this query as being new and fall through. */
-			qstate->minfo[id] = (void*)DNS64_NEW_QUERY;
+			iq = (struct dns64_qstate*)regional_alloc(
+				qstate->region, sizeof(*iq));
+			qstate->minfo[id] = iq;
+			iq->state = DNS64_NEW_QUERY;
+			iq->started_no_cache_store = qstate->no_cache_store;
+			qstate->no_cache_store = 1;
   			/* fallthrough */
 		case module_event_pass:
 			qstate->ext_state[id] = handle_event_pass(qstate, id);
@@ -655,6 +680,11 @@ dns64_operate(struct module_qstate* qstate, enum module_ev event, int id,
 			qstate->ext_state[id] = module_finished;
 			break;
 	}
+	if(qstate->ext_state[id] == module_finished) {
+		iq = (struct dns64_qstate*)qstate->minfo[id];
+		if(iq && iq->state != DNS64_INTERNAL_QUERY)
+			qstate->no_cache_store = iq->started_no_cache_store;
+	}
 }
 
 static void
@@ -867,9 +897,10 @@ dns64_adjust_ptr(struct module_qstate* qstate, struct module_qstate* super)
      * initial query's domain name.
      */
     answer = reply_find_answer_rrset(&qstate->qinfo, super->return_msg->rep);
-    log_assert(answer);
-    answer->rk.dname = super->qinfo.qname;
-    answer->rk.dname_len = super->qinfo.qname_len;
+    if(answer) {
+	    answer->rk.dname = super->qinfo.qname;
+	    answer->rk.dname_len = super->qinfo.qname_len;
+    }
 }
 
 /**
@@ -885,6 +916,7 @@ void
 dns64_inform_super(struct module_qstate* qstate, int id,
 		struct module_qstate* super)
 {
+	struct dns64_qstate* super_dq = (struct dns64_qstate*)super->minfo[id];
 	log_query_info(VERB_ALGO, "dns64: inform_super, sub is",
 		       &qstate->qinfo);
 	log_query_info(VERB_ALGO, "super is", &super->qinfo);
@@ -893,15 +925,21 @@ dns64_inform_super(struct module_qstate* qstate, int id,
 	 * Signal that the sub-query is finished, no matter whether we are
 	 * successful or not. This lets the state machine terminate.
 	 */
-	super->minfo[id] = (void*)DNS64_SUBQUERY_FINISHED;
+	if(!super_dq) {
+		super_dq = (struct dns64_qstate*)regional_alloc(super->region,
+			sizeof(*super_dq));
+		super->minfo[id] = super_dq;
+		memset(super_dq, 0, sizeof(*super_dq));
+		super_dq->started_no_cache_store = super->no_cache_store;
+	}
+	super_dq->state = DNS64_SUBQUERY_FINISHED;
 
 	/* If there is no successful answer, we're done. */
 	if (qstate->return_rcode != LDNS_RCODE_NOERROR
 	    || !qstate->return_msg
-	    || !qstate->return_msg->rep
-	    || !reply_find_answer_rrset(&qstate->qinfo,
-					qstate->return_msg->rep))
+	    || !qstate->return_msg->rep) {
 		return;
+	}
 
 	/* Use return code from A query in response to client. */
 	if (super->return_rcode != LDNS_RCODE_NOERROR)
@@ -916,7 +954,7 @@ dns64_inform_super(struct module_qstate* qstate, int id,
 	}
 
 	/* Store the generated response in cache. */
-	if (!super->no_cache_store &&
+	if ( (!super_dq || !super_dq->started_no_cache_store) &&
 		!dns_cache_store(super->env, &super->qinfo, super->return_msg->rep,
 		0, 0, 0, NULL, super->query_flags))
 		log_err("out of memory");

dnscrypt/dnscrypt.c

@@ -772,12 +772,13 @@ key_get_es_version(uint8_t version[2])
         const char *name;
     };
 
+    const int num_versions = 2;
     struct es_version es_versions[] = {
         {{0x00, 0x01}, "X25519-XSalsa20Poly1305"},
         {{0x00, 0x02}, "X25519-XChacha20Poly1305"},
     };
     int i;
-    for(i=0; i < (int)sizeof(es_versions); i++){
+    for(i=0; i < num_versions; i++){
         if(es_versions[i].es_version[0] == version[0] &&
            es_versions[i].es_version[1] == version[1]){
             return es_versions[i].name;

dnstap/dnstap.c

@@ -39,6 +39,10 @@
 #include "config.h"
 #include <string.h>
 #include <sys/time.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#include <errno.h>
 #include "sldns/sbuffer.h"
 #include "util/config_file.h"
 #include "util/net_help.h"
@@ -118,6 +122,18 @@ dt_msg_init(const struct dt_env *env,
 	}
 }
 
+/* check that the socket file can be opened and exists, print error if not */
+static void
+check_socket_file(const char* socket_path)
+{
+	struct stat statbuf;
+	memset(&statbuf, 0, sizeof(statbuf));
+	if(stat(socket_path, &statbuf) < 0) {
+		log_warn("could not open dnstap-socket-path: %s, %s",
+			socket_path, strerror(errno));
+	}
+}
+
 struct dt_env *
 dt_create(const char *socket_path, unsigned num_workers)
 {
@@ -134,6 +150,7 @@ dt_create(const char *socket_path, unsigned num_workers)
 		socket_path);
 	log_assert(socket_path != NULL);
 	log_assert(num_workers > 0);
+	check_socket_file(socket_path);
 
 	env = (struct dt_env *) calloc(1, sizeof(struct dt_env));
 	if (!env)

doc/Changelog

@@ -1,5 +1,349 @@
+28 January 2019: Wouter
+	- ub_ctx_set_tls call for libunbound that enables DoT for the machines
+	  set with ub_ctx_set_fwd.  Patch from Florian Obser.
+	- Set build system for added call in the libunbound API.
+	- List example config for root zone copy locally hosted with auth-zone
+	  as suggested from draft-ietf-dnsop-7706-bis-02.  But with updated
+	  B root address.
+	- set version to 1.9.0 for release.
+
+25 January 2019: Wouter
+	- Fix that tcp for auth zone and outgoing does not remove and
+	  then gets the ssl read again applied to the deleted commpoint.
+	- updated contrib/fastrpz.patch to cleanly diff.
+	- no lock when threads disabled in tcp request buffer count.
+	- remove compile warnings from libnettle compile.
+	- output of newer lex 2.6.1 and bison 3.0.5.
+
+24 January 2019: Wouter
+	- Newer aclocal and libtoolize used for generating configure scripts,
+	  aclocal 1.16.1 and libtoolize 2.4.6.
+	- Fix unit test for python 3.7 new keyword 'async'.
+	- clang analysis fixes, assert arc4random buffer in init,
+	  no check for already checked delegation pointer in iterator,
+	  in testcode check for NULL packet matches, in perf do not copy
+	  from NULL start list when growing capacity.  Adjust host and file
+	  only when present in test header read to please checker.  In
+	  testcode for unknown macro operand give zero result. Initialise the
+	  passed argv array in test code.  In test code add EDNS data
+	  segment copy only when nonempty.
+	- Patch from Florian Obser fixes some compiler warnings:
+	  include mini_event.h to have a prototype for mini_ev_cmp
+	  include edns.h to have a prototype for apply_edns_options
+	  sldns_wire2str_edns_keepalive_print is only called in the wire2str,
+	  module declare it static to get rid of compiler warning:
+	  no previous prototype for function
+	  infra_find_ip_ratedata() is only called in the infra module,
+	  declare it static to get rid of compiler warning:
+	  no previous prototype for function
+	  do not shadow local variable buf in authzone
+	  auth_chunks_delete and az_nsec3_findnode are only called in the
+	  authzone module, declare them static to get rid of compiler warning:
+	  no previous prototype for function...
+	  copy_rrset() is only called in the respip module, declare it
+	  static to get rid of compiler warning:
+	  no previous prototype for function 'copy_rrset'
+	  no need for another variable "r"; gets rid of compiler warning:
+	  declaration shadows a local variable in libunbound.c
+	  no need for another variable "ns"; gets rid of compiler warning:
+	  declaration shadows a local variable in iterator.c
+	- Moved includes and make depend.
+
+23 January 2019: Wouter
+	- Patch from Manabu Sonoda with tls-ciphers and tls-ciphersuites
+	  options for unbound.conf.
+	- Fixes for the patch, and man page entry.
+	- Fix configure to detect SSL_CTX_set_ciphersuites, for better
+	  library compatibility when compiling.
+	- Patch for TLS session resumption from Manabu Sonoda,
+	  enable with tls-session-ticket-keys in unbound.conf.
+	- Fixes for patch (includes, declarations, warnings).  Free at end
+	  and keep config options in order read from file to keep the first
+	  one as the first one.
+	- Fix for IXFR fallback to reset counter when IXFR does not timeout.
+
+22 January 2019: Wouter
+	- Fix space calculation for tcp req buffer size.
+	- Doc for stream-wait-size and unit test.
+	- unbound-control stats has mem.streamwait that counts TCP and TLS
+	  waiting result buffers.
+	- Fix for #4219: secondaries not updated after serial change, unbound
+	  falls back to AXFR after IXFR gives several timeout failures.
+	- Fix that auth zone after IXFR fallback tries the same master.
+
+21 January 2019: Wouter
+	- Fix tcp idle timeout test, for difference in the tcp reply code.
+	- Unit test for tcp request reorder and timeouts.
+	- Unit tests for ssl out of order processing.
+	- Fix that multiple dns fragments can be carried in one TLS frame.
+	- Add stream-wait-size: 4m config option to limit the maximum
+	  memory used by waiting tcp and tls stream replies.  This avoids
+	  a denial of service where these replies use up all of the memory.
+
+17 January 2019: Wouter
+	- For caps-for-id fallback, use the whitelist to avoid timeout
+	  starting a fallback sequence for it.
+	- increase mesh max activation count for capsforid long fetches.
+
+16 January 2019: Ralph
+	- Get ready for the DNS flag day: remove EDNS lame procedure, do not
+	  re-query without EDNS after timeout.
+
+15 January 2019: Wouter
+	- In the out of order processing, reset byte count for (potential)
+	  partial read.
+	- Review fixes in out of order processing.
+
+14 January 2019: Wouter
+	- streamtcp option -a send queries consecutively and prints answers
+	  as they arrive.
+	- Fix for out of order processing administration quit cleanup.
+	- unit test for tcp out of order processing.
+
+11 January 2019: Wouter
+	- Initial commit for out-of-order processing for TCP and TLS.
+
+9 January 2019: Wouter
+	- Log query name for looping module errors.
+
+8 January 2019: Wouter
+	- Fix syntax in comment of local alias processing.
+	- Fix NSEC3 record that is returned in wildcard replies from
+	  auth-zone zones with NSEC3 and wildcards.
+
+7 January 2019: Wouter
+	- On FreeBSD warn if systcl settings do not allow server TCP FASTOPEN,
+	  and server tcp fastopen is enabled at compile time.
+	- Document interaction between the tls-upstream option in the server
+	  section and forward-tls-upstream option in the forward-zone sections.
+	- Add contrib/unbound-fuzzme.patch from Jacob Hoffman-Andrews,
+	  the patch adds a program used for fuzzing.
+
+12 December 2018: Wouter
+	- Fix for crash in dns64 module if response is null.
+
+10 December 2018: Wouter
+	- Fix config parser memory leaks.
+	- ip-ratelimit-factor of 1 allows all traffic through, instead of the
+	  previous blocking everything.
+	- Fix for FreeBSD port make with dnscrypt and dnstap enabled.
+	- Fix #4206: support openssl 1.0.2 for TLS hostname verification,
+	  alongside the 1.1.0 and later support that is already there.
+	- Fixup openssl 1.0.2 compile
+
+6 December 2018: Wouter
+	- Fix dns64 allocation in wrong region for returned internal queries.
+
+3 December 2018: Wouter
+	- Fix icon, no ragged edges and nicer resolutions available, for eg.
+	  Win 7 and Windows 10 display.
+	- cache-max-ttl also defines upperbound of initial TTL in response.
+
+30 November 2018: Wouter
+	- Patch for typo in unbound.conf man page.
+	- log-tag-queryreply: yes in unbound.conf tags the log-queries and
+	  log-replies in the log file for easier log filter maintenance.
+
+29 November 2018: Wouter
+	- iana portlist updated.
+	- Fix chroot auth-zone fix to remove chroot prefix.
+	- tag for 1.8.2rc1, which became 1.8.2 on 4 dec 2018, with icon
+	  updated.  Trunk contains 1.8.3 in development.
+	  Which became 1.8.3 on 11 december with only the dns64 fix of 6 dec.
+	  Trunk then became 1.8.4 in development.
+	- Fix that unbound-checkconf does not complains if the config file
+	  is not placed inside the chroot.
+	- Refuse to start with no ports.
+	- Remove clang analysis warnings.
+
+28 November 2018: Wouter
+	- Fix leak in chroot fix for auth-zone.
+	- Fix clang analysis for outside directory build test.
+
+27 November 2018: Wouter
+	- Fix DNS64 to not store intermediate results in cache, this avoids
+	  other threads from picking up the wrong data.  The module restores
+	  the previous no_cache_store setting when the the module is finished.
+	- Fix #4208: 'stub-no-cache' and 'forward-no-cache' not work.
+	- New and better fix for Fix #4193: Fix that prefetch failure does
+	  not overwrite valid cache entry with SERVFAIL.
+	- auth-zone give SERVFAIL when expired, fallback activates when
+	  expired, and this is documented in the man page.
+	- stat count SERVFAIL downstream auth-zone queries for expired zones.
+	- Put new logos into windows installer.
+	- Fix windows compile for new rrset roundrobin fix.
+	- Update contrib fastrpz patch for latest release.
+
+26 November 2018: Wouter
+	- Fix to not set GLOB_NOSORT so the unbound.conf include: files are
+	  sorted and in a predictable order.
+	- Fix #4193: Fix that prefetch failure does not overwrite valid cache
+	  entry with SERVFAIL.
+	- Add unbound-control view_local_datas command, like local_datas.
+	- Fix that unbound-control can send file for view_local_datas.
+
+22 November 2018: Wouter
+	- With ./configure --with-pyunbound --with-pythonmodule
+	  PYTHON_VERSION=3.6 or with 2.7 unbound can compile and unit tests
+	  succeed for the python module.
+	- pythonmod logs the python error and traceback on failure.
+	- ignore debug python module for test in doxygen output.
+	- review fixes for python module.
+	- Fix #4209: Crash in libunbound when called from getdns.
+	- auth zone zonefiles can be in a chroot, the chroot directory
+	  components are removed before use.
+	- Fix that empty zonefile means the zonefile is not set and not used.
+	- make depend.
+
+21 November 2018: Wouter
+	- Scrub NS records from NODATA responses as well.
+
+20 November 2018: Wouter
+	- Scrub NS records from NXDOMAIN responses to stop fragmentation
+	  poisoning of the cache.
+	- Add patch from Jan Vcelak for pythonmod,
+	  add sockaddr_storage getters, add support for query callbacks,
+	  allow raw address access via comm_reply and update API documentation.
+	- Removed compile warnings in pythonmod sockaddr routines.
+
+19 November 2018: Wouter
+	- Support SO_REUSEPORT_LB in FreeBSD 12 with the so-reuseport: yes
+	  option in unbound.conf.
+
+6 November 2018: Ralph
+	- Bugfix min-client-subnet-ipv6
+
+25 October 2018: Ralph
+	- Add min-client-subnet-ipv6 and min-client-subnet-ipv4 options.
+
+25 October 2018: Wouter
+	- Fix #4191: NXDOMAIN vs SERVFAIL during dns64 PTR query.
+	- Fix #4190: Please create a "ANY" deny option, adds the option
+	  deny-any: yes in unbound.conf.  This responds with an empty message
+	  to queries of type ANY.
+	- Fix #4141: More randomness to rrset-roundrobin.
+	- Fix #4132: Openness/closeness of RANGE intervals in rpl files.
+	- Fix #4126: RTT_band too low on VSAT links with 600+ms latency,
+	  adds the option unknown-server-time-limit to unbound.conf that
+	  can be increased to avoid the problem.
+	- remade makefile dependencies.
+	- Fix #4152: Logs shows wrong time when using log-time-ascii: yes.
+
+24 October 2018: Ralph
+	- Add markdel function to ECS slabhash.
+	- Limit ECS scope returned to client to the scope used for caching.
+	- Make lint like previous #4154 fix.
+
+22 October 2018: Wouter
+	- Fix #4192: unbound-control-setup generates keys not readable by
+	  group.
+	- check that the dnstap socket file can be opened and exists, print
+	  error if not.
+	- Fix #4154: make ECS_MAX_TREESIZE configurable, with
+	  the max-ecs-tree-size-ipv4 and max-ecs-tree-size-ipv6 options.
+
+22 October 2018: Ralph
+	- Change fast-server-num default to 3.
+
+8 October 2018: Ralph
+	- Add fast-server-permil and fast-server-num options.
+	- Deprecate low-rtt and low-rtt-permil options.
+
+8 October 2018: Wouter
+	- Squelch log of failed to tcp initiate after TCP Fastopen failure.
+
+5 October 2018: Wouter
+	- Squelch EADDRNOTAVAIL errors when the interface goes away,
+	  this omits 'can't assign requested address' errors unless
+	  verbosity is set to a high value.
+	- Set default for so-reuseport to no for FreeBSD.  It is enabled
+	  by default for Linux and DragonFlyBSD.  The setting can 
+	  be configured in unbound.conf to override the default.
+	- iana port update.
+
+2 October 2018: Wouter
+	- updated contrib/fastrpz.patch to apply for this version
+	- dnscrypt.c removed sizeof to get array bounds.
+	- Fix testlock code to set noreturn on error routine.
+	- Remove unused variable from contrib fastrpz/rpz.c and
+	  remove unused diagnostic pragmas that themselves generate warnings
+	- clang analyze test is used only when assertions are enabled.
+
+1 October 2018: Wouter
+	- tag for release 1.8.1rc1.  Became release 1.8.1 on 8 oct, with
+	  fastrpz.patch fix included.  Trunk has 1.8.2 in development.
+
+27 September 2018: Wouter
+	- Fix #4188: IPv6 forwarders without ipv6 result in SERVFAIL, fixes
+	  qname minimisation with a forwarder when connectivity has issues
+	  from rejecting responses.
+
+25 September 2018: Wouter
+	- Perform TLS SNI indication of the host that is being contacted
+	  for DNS over TLS service.  It sets the configured tls auth name.
+	  This is useful for hosts that apart from the DNS over TLS services
+	  also provide other (web) services.
+	- Fix #4149: Add SSL cleanup for tcp timeout.
+
+17 September 2018: Wouter
+	- Fix compile on Mac for unbound, provide explicit_bzero when libc
+	  does not have it.
+	- Fix unbound for openssl in FIPS mode, it uses the digests with
+	  the EVP call contexts.
+	- Fix that with harden-below-nxdomain and qname minisation enabled
+	  some iterator states for nonresponsive domains can get into a
+	  state where they waited for an empty list.
+	- Stop UDP to TCP failover after timeouts that causes the ping count
+	  to be reset by the TCP time measurement (that exists for TLS),
+	  because that causes the UDP part to not be measured as timeout.
+	- Fix #4156: Fix systemd service manager state change notification.
+
+13 September 2018: Wouter
+	- Fix seed for random backup code to use explicit zero when wiped.
+	- exit log routine is annotated as noreturn function.
+	- free memory leaks in config strlist and str2list insert functions.
+	- do not move unused argv variable after getopt.
+	- Remove unused if clause in testcode.
+	- in testcode, free async ids, initialise array, and check for null
+	  pointer during test of the test.  And use exit for return to note
+	  irregular program stop.
+	- Free memory leak in config strlist append.
+	- make sure nsec3 comparison salt is initialized.
+	- unit test has clang analysis.
+	- remove unused variable assignment from iterator scrub routine.
+	- check for null in delegation point during iterator refetch
+	  in forward zone.
+	- neater pointer cast in libunbound context quit routine.
+	- initialize statistics totals for printout.
+	- in authzone check that node exists before adding rrset.
+	- in unbound-anchor, use readwrite memory BIO.
+	- assertion in autotrust that packed rrset is formed correctly.
+	- Fix memory leak when message parse fails partway through copy.
+	- remove unused udpsize assignment in message encode.
+	- nicer bio free code in unbound-anchor.
+	- annotate exit functions with noreturn in unbound-control.
+
+11 September 2018: Wouter
+	- Fixed unused return value warnings in contrib/fastrpz.patch for
+	  asprintf.
+	- Fix to squelch respip warning in unit test, it is printed at
+	  higher verbosity settings.
+	- Fix spelling errors.
+	- Fix initialisation in remote.c
+
+10 September 2018: Wouter
+	- 1.8.1 in svn trunk. (changes from 4,5,.. sep apply).
+	- iana port update.
+
+5 September 2018: Wouter
+	- Fix spelling error in header, from getdns commit by Andreas Gelmini.
+
+4 September 2018: Ralph
+	- More explicitly mention the type of ratelimit when applying
+	  ip-ratelimit.
+
 4 September 2018: Wouter
-	- Tag for 1.8.0rc1 release.
+	- Tag for 1.8.0rc1 release, became 1.8.0 release on 10 Sep 2018.
 
 31 August 2018: Wouter
 	- Disable minimal-responses in subnet unit tests.
@@ -1856,7 +2200,7 @@
 	  compatibility with cisco dns guard.  This lowers false positives.
 
 18 April 2016: Wouter
-	- Fix some malformed reponses to edns queries get fallback to nonedns.
+	- Fix some malformed responses to edns queries get fallback to nonedns.
 
 15 April 2016: Wouter
 	- cachedb module event handling design.

doc/example.conf.in

@@ -123,6 +123,9 @@ server:
 	# Suggested values are 512 to 4096. Default is 4096. 65536 disables it.
 	# max-udp-size: 4096
 
+	# max memory to use for stream(tcp and tls) waiting result buffers.
+	# stream-wait-size: 4m
+
 	# buffer size for handling DNS data. No messages larger than this
 	# size can be sent or received, by UDP or TCP. In bytes.
 	# msg-buffer-size: 65552
@@ -145,6 +148,10 @@ server:
 	# msec to wait before close of port on timeout UDP. 0 disables.
 	# delay-close: 0
 
+	# msec for waiting for an unknown server to reply.  Increase if you
+	# are behind a slow satellite link, to eg. 1128.
+	# unknown-server-time-limit: 376
+
 	# the amount of memory to use for the RRset cache.
 	# plain value in bytes or you can append k, m or G. default is "4Mb".
 	# rrset-cache-size: 4m
@@ -318,6 +325,10 @@ server:
 	# timetoresolve, fromcache and responsesize.
 	# log-replies: no
 
+	# log with tag 'query' and 'reply' instead of 'info' for
+	# filtering log-queries and log-replies from the log.
+	# log-tag-queryreply: no
+
 	# log the local-zone actions, like local-zone type inform is enabled
 	# also for the other local zone types.
 	# log-local-actions: no
@@ -449,6 +460,9 @@ server:
 	# if yes, perform key lookups adjacent to normal lookups.
 	# prefetch-key: no
 
+	# deny queries of type ANY with an empty response.
+	# deny-any: no
+
 	# if yes, Unbound rotates RRSet order in response.
 	# rrset-roundrobin: no
 
@@ -701,6 +715,19 @@ server:
 	# tls-service-pem: "path/to/publiccertfile.pem"
 	# tls-port: 853
 
+	# cipher setting for TLSv1.2
+	# tls-chiphers: "DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256"
+	# cipher setting for TLSv1.3 
+	# tls-ciphersuites: "TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_8_SHA256:TLS_AES_128_CCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256"
+
+	# Add the secret file for TLS Session Ticket.
+	# Secret file must be 80 bytes of random data.
+	# First key use to encrypt and decrypt TLS session tickets.
+	# Other keys use to decrypt only.
+	# requires restart to take effect.
+	# tls-session-ticket-keys: "path/to/secret_file1"
+	# tls-session-ticket-keys: "path/to/secret_file2"
+
 	# request upstream over TLS (with plain DNS inside the TLS stream).
 	# Default is no.  Can be turned on and off with unbound-control.
 	# tls-upstream: no
@@ -757,12 +784,12 @@ server:
 	# Limit the number of connections simultaneous from a netblock
 	# tcp-connection-limit: 192.0.2.0/24 12
 
-	# what is considered a low rtt (ping time for upstream server), in msec
-	# low-rtt: 45
-	# select low rtt this many times out of 1000. 0 means the fast server
-	# select is disabled.  prefetches are not sped up.
-	# low-rtt-permil: 0
-
+	# select from the fastest servers this many times out of 1000. 0 means
+	# the fast server select is disabled. prefetches are not sped up.
+	# fast-server-permil: 0
+	# the number of servers that will be used in the fast server selection.
+	# fast-server-num: 3
+	
 	# Specific options for ipsecmod. unbound needs to be configured with
 	# --enable-ipsecmod for these to take effect.
 	#
@@ -879,15 +906,25 @@ remote-control:
 # notifies.
 # auth-zone:
 #	name: "."
-#	for-downstream: no
+#	master: 199.9.14.201         # b.root-servers.net	
+#	master: 192.33.4.12          # c.root-servers.net	
+#	master: 199.7.91.13          # d.root-servers.net	
+#	master: 192.5.5.241          # f.root-servers.net	
+#	master: 192.112.36.4         # g.root-servers.net	
+#	master: 193.0.14.129         # k.root-servers.net	
+#	master: 192.0.47.132         # xfr.cjr.dns.icann.org	
+#	master: 192.0.32.132         # xfr.lax.dns.icann.org	
+#	master: 2001:500:200::b      # b.root-servers.net	
+#	master: 2001:500:2::c        # c.root-servers.net	
+#	master: 2001:500:2d::d       # d.root-servers.net	
+#	master: 2001:500:2f::f       # f.root-servers.net	
+#	master: 2001:500:12::d0d     # g.root-servers.net	
+#	master: 2001:7fd::1          # k.root-servers.net	
+#	master: 2620:0:2830:202::132 # xfr.cjr.dns.icann.org	
+#	master: 2620:0:2d0:202::132  # xfr.lax.dns.icann.org	
+#	fallback-enabled: yes	
+#	for-downstream: no	
 #	for-upstream: yes
-#	fallback-enabled: yes
-#	master: b.root-servers.net
-#	master: c.root-servers.net
-#	master: e.root-servers.net
-#	master: f.root-servers.net
-#	master: g.root-servers.net
-#	master: k.root-servers.net
 # auth-zone:
 #	name: "example.org"
 #	for-downstream: yes

doc/libunbound.3.in

@@ -20,6 +20,7 @@
 .B ub_ctx_config,
 .B ub_ctx_set_fwd,
 .B ub_ctx_set_stub,
+.B ub_ctx_set_tls,
 .B ub_ctx_resolvconf,
 .B ub_ctx_hosts,
 .B ub_ctx_add_ta,
@@ -72,6 +73,9 @@
 		\fIint\fR isprime);
 .LP
 \fIint\fR
+\fBub_ctx_set_tls\fR(\fIstruct ub_ctx*\fR ctx, \fIint\fR tls);
+.LP
+\fIint\fR
 \fBub_ctx_resolvconf\fR(\fIstruct ub_ctx*\fR ctx, \fIchar*\fR fname);
 .LP
 \fIint\fR
@@ -227,6 +231,12 @@ for different zones, or to add multiple addresses for a particular zone.
 At this time it is only possible to set configuration before the
 first resolve is done.
 .TP
+.B ub_ctx_set_tls
+Enable DNS over TLS (DoT) for machines set with 
+.B ub_ctx_set_fwd.
+At this time it is only possible to set configuration before the
+first resolve is done.
+.TP
 .B ub_ctx_resolvconf
 By default the root servers are queried and full resolver mode is used, but
 you can use this call to read the list of nameservers to use from the

doc/unbound-control.8.in

@@ -301,7 +301,7 @@ you set unbound to validate with for-upstream yes and that can be cleared
 with \fBflush_zone\fR \fIzone\fR.
 .TP
 .B auth_zone_transfer \fIzone\fR
-Tranfer the auth zone from master.  The auth zone probe sequence is started,
+Transfer the auth zone from master.  The auth zone probe sequence is started,
 where the masters are probed to see if they have an updated zone (with the SOA
 serial check).  And then the zone is transferred for a newer zone version.
 .TP
@@ -322,6 +322,9 @@ serial check).  And then the zone is transferred for a newer zone version.
 .TP
 .B view_local_data_remove \fIview\fR \fIname
 \fIlocal_data_remove\fR for given view.
+.TP
+.B view_local_datas \fIview\fR
+Add a list of \fIlocal_data\fR for given view from stdin.  Like local_datas.
 .SH "EXIT CODE"
 The unbound\-control program exits with status code 1 on error, 0 on success.
 .SH "SET UP"
@@ -496,6 +499,10 @@ Memory in bytes in use by the iterator module.
 Memory in bytes in use by the validator module. Includes the key cache and
 negative cache.
 .TP
+.I mem.streamwait
+Memory in bytes in used by the TCP and TLS stream wait buffers.  These are
+answers waiting to be written back to the clients.
+.TP
 .I histogram.<sec>.<usec>.to.<sec>.<usec>
 Shows a histogram, summed over all threads. Every element counts the
 recursive queries whose reply time fit between the lower and upper bound.
@@ -531,6 +538,10 @@ other servers.
 Number of queries that were made using TLS towards the unbound server.
 These are also counted in num.query.tcp, because TLS uses TCP.
 .TP
+.I num.query.tls.resume
+Number of TLS session resumptions, these are queries over TLS towards
+the unbound server where the client negotiated a TLS session resumption key.
+.TP
 .I num.query.ipv6
 Number of queries that were made using IPv6 towards the unbound server.
 .TP

doc/unbound.conf.5.in

@@ -207,6 +207,16 @@ Maximum UDP response size (not applied to TCP response).  65536 disables the
 udp response size maximum, and uses the choice from the client, always.
 Suggested values are 512 to 4096. Default is 4096.
 .TP
+.B stream\-wait\-size: \fI<number>
+Number of bytes size maximum to use for waiting stream buffers.  Default is
+4 megabytes.  A plain number is in bytes, append 'k', 'm' or 'g' for kilobytes,
+megabytes or gigabytes (1024*1024 bytes in a megabyte).  As TCP and TLS streams
+queue up multiple results, the amount of memory used for these buffers does
+not exceed this number, otherwise the responses are dropped.  This manages
+the total memory usage of the server (under heavy use), the number of requests
+that can be queued up per connection is also limited, with further requests
+waiting in TCP buffers.
+.TP
 .B msg\-buffer\-size: \fI<number>
 Number of bytes size of the message buffers. Default is 65552 bytes, enough
 for 64 Kb packets, the maximum DNS message size. No message larger than this
@@ -253,6 +263,12 @@ eg. 1500 msec.  When timeouts happen you need extra sockets, it checks
 the ID and remote IP of packets, and unwanted packets are added to the
 unwanted packet counter.
 .TP
+.B unknown\-server\-time\-limit: \fI<msec>
+The wait time in msec for waiting for an unknown server to reply.
+Increase this if you are behind a slow satellite link, to eg. 1128.
+That would then avoid re\-querying every initial query because it times out.
+Default is 376 msec.
+.TP
 .B so\-rcvbuf: \fI<number>
 If not 0, then set the SO_RCVBUF socket option to get more buffer
 space on UDP port 53 incoming queries.  So that short spikes on busy
@@ -314,11 +330,9 @@ Must be set to a power of 2.
 .TP
 .B cache\-max\-ttl: \fI<seconds>
 Time to live maximum for RRsets and messages in the cache. Default is
-86400 seconds (1 day). If the maximum kicks in, responses to clients
-still get decrementing TTLs based on the original (larger) values.
-When the internal TTL expires, the cache item has expired.
+86400 seconds (1 day).  When the TTL expires, the cache item has expired.
 Can be set lower to force the resolver to query for data often, and not
-trust (very large) TTL values.
+trust (very large) TTL values.  Downstream clients also see the lower TTL.
 .TP
 .B cache\-min\-ttl: \fI<seconds>
 Time to live minimum for RRsets and messages in the cache. Default is 0.
@@ -436,13 +450,15 @@ TCP wireformat.  The other server must support this (see
 \fBtls\-service\-key\fR).
 If you enable this, also configure a tls\-cert\-bundle or use tls\-win\-cert to
 load CA certs, otherwise the connections cannot be authenticated.
+This option enables TLS for all of them, but if you do not set this you can
+configure TLS specifically for some forward zones with forward\-tls\-upstream.  And also with stub\-tls\-upstream.
 .TP
 .B ssl\-upstream: \fI<yes or no>
 Alternate syntax for \fBtls\-upstream\fR.  If both are present in the config
 file the last is used.
 .TP
 .B tls\-service\-key: \fI<file>
-If enabled, the server provider TLS service on its TCP sockets.  The clients
+If enabled, the server provides TLS service on its TCP sockets.  The clients
 have to use tls\-upstream: yes.  The file is the private key for the TLS
 session.  The public certificate is in the tls\-service\-pem file.  Default
 is "", turned off.  Requires a restart (a reload is not enough) if changed,
@@ -488,6 +504,27 @@ List portnumbers as tls\-additional\-port, and when interfaces are defined,
 eg. with the @port suffix, as this port number, they provide dns over TLS
 service.  Can list multiple, each on a new statement.
 .TP
+.B tls-session-ticket-keys: \fI<file>
+If not "", lists files with 80 bytes of random contents that are used to
+perform TLS session resumption for clients using the unbound server.
+These files contain the secret key for the TLS session tickets.
+First key use to encrypt and decrypt TLS session tickets.
+Other keys use to decrypt only.  With this you can roll over to new keys,
+by generating a new first file and allowing decrypt of the old file by
+listing it after the first file for some time, after the wait clients are not
+using the old key any more and the old key can be removed.
+One way to create the file is dd if=/dev/random bs=1 count=80 of=ticket.dat
+The first 16 bytes should be different from the old one if you create a second key, that is the name used to identify the key.  Then there is 32 bytes random
+data for an AES key and then 32 bytes random data for the HMAC key.
+.TP
+.B tls\-ciphers: \fI<string with cipher list>
+Set the list of ciphers to allow when serving TLS.  Use "" for defaults,
+and that is the default.
+.TP
+.B tls\-ciphersuites: \fI<string with ciphersuites list>
+Set the list of ciphersuites to allow when serving TLS.  This is for newer
+TLS 1.3 connections.  Use "" for defaults, and that is the default.
+.TP
 .B use\-systemd: \fI<yes or no>
 Enable or disable systemd socket activation.
 Default is no.
@@ -498,7 +535,7 @@ a daemon.  Set the value to \fIno\fR when unbound runs as systemd service.
 Default is yes.
 .TP
 .B tcp\-connection\-limit: \fI<IP netblock> <limit>
-Allow up to \fIlimit\R simultaneous TCP connections from the given netblock.
+Allow up to \fIlimit\fR simultaneous TCP connections from the given netblock.
 When at the limit, further connections are accepted but closed immediately.
 This option is experimental at this time.
 .TP
@@ -655,6 +692,11 @@ Default is no.  Note that it takes time to print these
 lines which makes the server (significantly) slower.  Odd (nonprintable)
 characters in names are printed as '?'.
 .TP
+.B log\-tag\-queryreply: \fI<yes or no>
+Prints the word 'query' and 'reply' with log\-queries and log\-replies.
+This makes filtering logs easier.  The default is off (for backwards
+compatibility).
+.TP
 .B log\-local\-actions: \fI<yes or no>
 Print log lines to inform about local zone actions.  These lines are like the
 local\-zone type inform prints out, but they are also printed for the other
@@ -848,12 +890,18 @@ keep the cache up to date.  Default is no.  Turning it on gives about
 10 percent more traffic and load on the machine, but popular items do
 not expire from the cache.
 .TP
-.B prefetch-key: \fI<yes or no>
+.B prefetch\-key: \fI<yes or no>
 If yes, fetch the DNSKEYs earlier in the validation process, when a DS
 record is encountered.  This lowers the latency of requests.  It does use
 a little more CPU.  Also if the cache is set to 0, it is no use. Default is no.
 .TP
-.B rrset-roundrobin: \fI<yes or no>
+.B deny\-any: \fI<yes or no>
+If yes, deny queries of type ANY with an empty response.  Default is no.
+If disabled, unbound responds with a short list of resource records if some
+can be found in the cache and makes the upstream type ANY query if there
+are none.
+.TP
+.B rrset\-roundrobin: \fI<yes or no>
 If yes, Unbound rotates RRSet order in response (the random number is taken
 from the query ID, for speed and thread safety).  Default is no.
 .TP
@@ -1391,22 +1439,20 @@ This can make ordinary queries complete (if repeatedly queried for),
 and enter the cache, whilst also mitigating the traffic flow by the
 factor given.
 .TP 5
-.B low\-rtt: \fI<msec time>
-Set the time in millisecond that is considere a low ping time for fast
-server selection with the low\-rtt\-permil option, that turns this on or off.
-The default is 45 msec, a number from IPv6 quick response documents.
+.B fast\-server\-permil: \fI<number>
+Specify how many times out of 1000 to pick from the set of fastest servers.
+0 turns the feature off.  A value of 900 would pick from the fastest
+servers 90 percent of the time, and would perform normal exploration of random
+servers for the remaining time. When prefetch is enabled (or serve\-expired),
+such prefetches are not sped up, because there is no one waiting for it, and it
+presents a good moment to perform server exploration. The
+\fBfast\-server\-num\fR option can be used to specify the size of the fastest
+servers set. The default for fast\-server\-permil is 0.
 .TP 5
-.B low\-rtt\-permil: \fI<number>
-Specify how many times out of 1000 to pick the fast server from the low
-rtt band.  0 turns the feature off.  A value of 900 would pick the fast
-server when such fast servers are available 90 percent of the time, and
-the remaining time perform normal exploration of random servers.
-When prefetch is enabled (or serve\-expired), such prefetches are not
-sped up, because there is no one waiting for it, and it presents a good
-moment to perform server exploration.  The low\-rtt option can be used
-to specify which servers are picked for fast server selection, servers
-with a ping roundtrip time below that value are considered.
-The default for low\-rtt\-permil is 0.
+.B fast\-server\-num: \fI<number>
+Set the number of servers that should be used for fast server selection. Only
+use the fastest specified number of servers with the fast\-server\-permil
+option, that turns this on or off. The default is to use the fastest 3 servers.
 .SS "Remote Control Options"
 In the
 .B remote\-control:
@@ -1604,6 +1650,13 @@ lookups of that data.
 Authority zones can be read from zonefile.  And can be kept updated via
 AXFR and IXFR.  After update the zonefile is rewritten.  The update mechanism
 uses the SOA timer values and performs SOA UDP queries to detect zone changes.
+.LP
+If the update fetch fails, the timers in the SOA record are used to time
+another fetch attempt.  Until the SOA expiry timer is reached.  Then the
+zone is expired.  When a zone is expired, queries are SERVFAIL, and
+any new serial number is accepted from the master (even if older), and if
+fallback is enabled, the fallback activates to fetch from the upstream instead
+of the SERVFAIL.
 .TP
 .B name: \fI<zone name>
 Name of the authority zone.
@@ -1843,6 +1896,24 @@ to expose to third parties for IPv6.  Defaults to 56.
 .B max\-client\-subnet\-ipv4: \fI<number>\fR
 Specifies the maximum prefix length of the client source address we are willing
 to expose to third parties for IPv4. Defaults to 24.
+.TP
+.B min\-client\-subnet\-ipv6: \fI<number>\fR
+Specifies the minimum prefix length of the IPv6 source mask we are willing to
+accept in queries. Shorter source masks result in REFUSED answers. Source mask
+of 0 is always accepted. Default is 0.
+.TP
+.B min\-client\-subnet\-ipv4: \fI<number>\fR
+Specifies the minimum prefix length of the IPv4 source mask we are willing to
+accept in queries. Shorter source masks result in REFUSED answers. Source mask 
+of 0 is always accepted. Default is 0.
+.TP
+.B max\-ecs\-tree\-size\-ipv4: \fI<number>\fR
+Specifies the maximum number of subnets ECS answers kept in the ECS radix tree.
+This number applies for each qname/qclass/qtype tuple. Defaults to 100.
+.TP
+.B max\-ecs\-tree\-size\-ipv6: \fI<number>\fR
+Specifies the maximum number of subnets ECS answers kept in the ECS radix tree.
+This number applies for each qname/qclass/qtype tuple. Defaults to 100.
 .SS "Opportunistic IPsec Support Module Options"
 .LP
 The IPsec module must be configured in the \fBmodule\-config:\fR "ipsecmod

doc/unbound.doxygen

@@ -612,17 +612,22 @@ RECURSIVE              = YES
 
 EXCLUDE                = ./build \
                          ./compat \
+			 ./contrib \
                          util/configparser.c \
                          util/configparser.h \
                          util/configlexer.c \
                          util/locks.h \
+                         pythonmod/doc \
+                         pythonmod/examples \
                          pythonmod/unboundmodule.py \
                          pythonmod/interface.h \
-                         pythonmod/examples/resgen.py \
-                         pythonmod/examples/resmod.py \
-                         pythonmod/examples/resip.py \
+                         pythonmod/ubmodule-msg.py \
+                         pythonmod/ubmodule-tst.py \
+			 unboundmodule.py \
                          libunbound/python/unbound.py \
                          libunbound/python/libunbound_wrap.c \
+                         libunbound/python/doc \
+                         libunbound/python/examples \
                          ./ldns-src \
 			 doc/control_proto_spec.txt \
 			 doc/requirements.txt

edns-subnet/addrtree.c

@@ -119,7 +119,7 @@ node_size(const struct addrtree *tree, const struct addrnode *n)
 
 struct addrtree * 
 addrtree_create(addrlen_t max_depth, void (*delfunc)(void *, void *), 
-	size_t (*sizefunc)(void *), void *env, unsigned int max_node_count)
+	size_t (*sizefunc)(void *), void *env, uint32_t max_node_count)
 {
 	struct addrtree *tree;
 	log_assert(delfunc != NULL);

edns-subnet/addrtree.h

@@ -66,10 +66,10 @@ struct addrtree {
 	struct addrnode *root;
 	/** Number of elements in the tree (not always equal to number of 
 	 * nodes) */
-	unsigned int node_count;
+	uint32_t node_count;
 	/** Maximum number of allowed nodes, will be enforced by LRU list.
 	 * Excluding the root node, 0 for unlimited */
-	unsigned int max_node_count;
+	uint32_t max_node_count;
 	/** Size of tree in bytes */
 	size_t size_bytes;
 	/** Maximum prefix length we are willing to cache. */
@@ -137,7 +137,7 @@ size_t addrtree_size(const struct addrtree *tree);
  */
 struct addrtree * 
 addrtree_create(addrlen_t max_depth, void (*delfunc)(void *, void *), 
-	size_t (*sizefunc)(void *), void *env, unsigned int max_node_count);
+	size_t (*sizefunc)(void *), void *env, uint32_t max_node_count);
 
 /** 
  * Free tree and all nodes below.

edns-subnet/subnetmod.c

@@ -55,8 +55,7 @@
 #include "util/config_file.h"
 #include "util/data/msgreply.h"
 #include "sldns/sbuffer.h"
-
-#define ECS_MAX_TREESIZE 100
+#include "iterator/iter_utils.h"
 
 /** externally called */
 void 
@@ -93,6 +92,7 @@ subnet_new_qstate(struct module_qstate *qstate, int id)
 		return 0;
 	qstate->minfo[id] = sq;
 	memset(sq, 0, sizeof(*sq));
+	sq->started_no_cache_store = qstate->no_cache_store;
 	return 1;
 }
 
@@ -150,7 +150,9 @@ int ecs_whitelist_check(struct query_info* qinfo,
 
 	/* Cache by default, might be disabled after parsing EDNS option
 	 * received from nameserver. */
-	qstate->no_cache_store = 0;
+	if(!iter_stub_fwd_no_cache(qstate, &qstate->qinfo)) {
+		qstate->no_cache_store = 0;
+	}
 
 	if(sq->ecs_server_out.subnet_validdata && ((sq->subnet_downstream &&
 		qstate->env->cfg->client_subnet_always_forward) ||
@@ -177,6 +179,14 @@ int ecs_whitelist_check(struct query_info* qinfo,
 }
 
 
+void
+subnet_markdel(void* key)
+{
+	struct msgreply_entry *e = (struct msgreply_entry*)key;
+	e->key.qtype = 0;
+	e->key.qclass = 0;
+}
+
 int
 subnetmod_init(struct module_env *env, int id)
 {
@@ -193,6 +203,7 @@ subnetmod_init(struct module_env *env, int id)
 		HASH_DEFAULT_STARTARRAY, env->cfg->msg_cache_size,
 		msg_cache_sizefunc, query_info_compare, query_entry_delete,
 		subnet_data_delete, NULL);
+	slabhash_setmarkdel(sn_env->subnet_msg_cache, &subnet_markdel);
 	if(!sn_env->subnet_msg_cache) {
 		log_err("subnet: could not create cache");
 		free(sn_env);
@@ -291,13 +302,13 @@ get_tree(struct subnet_msg_cache_data *data, struct ecs_data *edns,
 		if (!data->tree4)
 			data->tree4 = addrtree_create(
 				cfg->max_client_subnet_ipv4, &delfunc,
-				&sizefunc, env, ECS_MAX_TREESIZE);
+				&sizefunc, env, cfg->max_ecs_tree_size_ipv4);
 		tree = data->tree4;
 	} else {
 		if (!data->tree6)
 			data->tree6 = addrtree_create(
 				cfg->max_client_subnet_ipv6, &delfunc,
-				&sizefunc, env, ECS_MAX_TREESIZE);
+				&sizefunc, env, cfg->max_ecs_tree_size_ipv6);
 		tree = data->tree6;
 	}
 	return tree;
@@ -487,9 +498,11 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 		 * is still usefull to put it in the edns subnet cache for
 		 * when a client explicitly asks for subnet specific answer. */
 		verbose(VERB_QUERY, "subnet: Authority indicates no support");
-		lock_rw_wrlock(&sne->biglock);
-		update_cache(qstate, id);
-		lock_rw_unlock(&sne->biglock);
+		if(!sq->started_no_cache_store) {
+			lock_rw_wrlock(&sne->biglock);
+			update_cache(qstate, id);
+			lock_rw_unlock(&sne->biglock);
+		}
 		if (sq->subnet_downstream)
 			cp_edns_bad_response(c_out, c_in);
 		return module_finished;
@@ -515,7 +528,9 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 	}
 
 	lock_rw_wrlock(&sne->biglock);
-	update_cache(qstate, id);
+	if(!sq->started_no_cache_store) {
+		update_cache(qstate, id);
+	}
 	sne->num_msg_nocache++;
 	lock_rw_unlock(&sne->biglock);
 	
@@ -526,6 +541,19 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 		c_out->subnet_source_mask = c_in->subnet_source_mask;
 		memcpy(&c_out->subnet_addr, &c_in->subnet_addr, INET6_SIZE);
 		c_out->subnet_scope_mask = s_in->subnet_scope_mask;
+		/* Limit scope returned to client to scope used for caching. */
+		if(c_out->subnet_addr_fam == EDNSSUBNET_ADDRFAM_IP4) {
+			if(c_out->subnet_scope_mask >
+				qstate->env->cfg->max_client_subnet_ipv4) {
+				c_out->subnet_scope_mask =
+					qstate->env->cfg->max_client_subnet_ipv4;
+			}
+		}
+		else if(c_out->subnet_scope_mask >
+				qstate->env->cfg->max_client_subnet_ipv6) {
+				c_out->subnet_scope_mask =
+					qstate->env->cfg->max_client_subnet_ipv6;
+		}
 		c_out->subnet_validdata = 1;
 	}
 	return module_finished;
@@ -697,6 +725,17 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 			return;
 		}
 
+		/* Limit to minimum allowed source mask */
+		if(sq->ecs_client_in.subnet_source_mask != 0 && (
+			(sq->ecs_client_in.subnet_addr_fam == EDNSSUBNET_ADDRFAM_IP4 &&
+			 sq->ecs_client_in.subnet_source_mask < qstate->env->cfg->min_client_subnet_ipv4) ||
+			(sq->ecs_client_in.subnet_addr_fam == EDNSSUBNET_ADDRFAM_IP6 &&
+			 sq->ecs_client_in.subnet_source_mask < qstate->env->cfg->min_client_subnet_ipv6))) {
+				qstate->return_rcode = LDNS_RCODE_REFUSED;
+				qstate->ext_state[id] = module_finished;
+				return;
+		}
+
 		lock_rw_wrlock(&sne->biglock);
 		if (lookup_and_reply(qstate, id, sq)) {
 			sne->num_msg_cache++;
@@ -753,6 +792,7 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 			ecs_opt_list_append(&sq->ecs_client_out,
 				&qstate->edns_opts_front_out, qstate);
 		}
+		qstate->no_cache_store = sq->started_no_cache_store;
 		return;
 	}
 	if(sq && outbound) {

edns-subnet/subnetmod.h

@@ -83,6 +83,8 @@ struct subnet_qstate {
 	struct ecs_data	ecs_server_out;
 	int subnet_downstream;
 	int subnet_sent;
+	/** has the subnet module been started with no_cache_store? */
+	int started_no_cache_store;
 };
 
 void subnet_data_delete(void* d, void* ATTR_UNUSED(arg));
@@ -131,4 +133,7 @@ int ecs_edns_back_parsed(struct module_qstate* qstate, int id, void* cbargs);
 int ecs_query_response(struct module_qstate* qstate, struct dns_msg* response,
 	int id, void* cbargs);
 
+/** mark subnet msg to be deleted */
+void subnet_markdel(void* key);
+
 #endif /* SUBNETMOD_H */

install-sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/sh
 # install - install a program, script, or datafile
 
 scriptversion=2013-12-25.23; # UTC

iterator/iter_scrub.c

@@ -316,6 +316,18 @@ sub_of_pkt(sldns_buffer* pkt, uint8_t* zone, uint8_t* comprname)
 	return dname_subdomain_c(zone, buf);
 }
 
+/** Check if there are SOA records in the authority section (negative) */
+static int
+soa_in_auth(struct msg_parse* msg)
+{
+	struct rrset_parse* rrset;
+	for(rrset = msg->rrset_first; rrset; rrset = rrset->rrset_all_next)
+		if(rrset->type == LDNS_RR_TYPE_SOA &&
+			rrset->section == LDNS_SECTION_AUTHORITY) 
+			return 1;
+	return 0;
+}
+
 /**
  * This routine normalizes a response. This includes removing "irrelevant"
  * records from the answer and additional sections and (re)synthesizing
@@ -437,7 +449,9 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 					rrset->rrset_all_next =
 						nx->rrset_all_next;
 					nx->rrset_all_next = rrset;
-					prev = nx;
+					/* prev = nx; unused, enable if there
+					 * is other rrset removal code after
+					 * this */
 				}
 			}
 
@@ -495,6 +509,19 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 					"RRset:", pkt, msg, prev, &rrset);
 				continue;
 			}
+			/* we don't want NS sets for NXDOMAIN answers,
+			 * because they could contain poisonous contents,
+			 * from. eg. fragmentation attacks, inserted after
+			 * long RRSIGs in the packet get to the packet
+			 * border and such */
+			/* also for NODATA answers */
+			if(FLAGS_GET_RCODE(msg->flags) == LDNS_RCODE_NXDOMAIN ||
+			   (FLAGS_GET_RCODE(msg->flags) == LDNS_RCODE_NOERROR
+			    && soa_in_auth(msg) && msg->an_rrsets == 0)) {
+				remove_rrset("normalize: removing irrelevant "
+					"RRset:", pkt, msg, prev, &rrset);
+				continue;
+			}
 			if(nsset == NULL) {
 				nsset = rrset;
 			} else {
@@ -593,18 +620,6 @@ store_rrset(sldns_buffer* pkt, struct msg_parse* msg, struct module_env* env,
 	(void)rrset_cache_update(env->rrset_cache, &ref, env->alloc, now);
 }
 
-/** Check if there are SOA records in the authority section (negative) */
-static int
-soa_in_auth(struct msg_parse* msg)
-{
-	struct rrset_parse* rrset;
-	for(rrset = msg->rrset_first; rrset; rrset = rrset->rrset_all_next)
-		if(rrset->type == LDNS_RR_TYPE_SOA &&
-			rrset->section == LDNS_SECTION_AUTHORITY) 
-			return 1;
-	return 0;
-}
- 
 /**
  * Check if right hand name in NSEC is within zone
  * @param rrset: the NSEC rrset

iterator/iter_utils.c

@@ -282,10 +282,13 @@ iter_filter_unsuitable(struct iter_env* iter_env, struct module_env* env,
 static int
 iter_fill_rtt(struct iter_env* iter_env, struct module_env* env,
 	uint8_t* name, size_t namelen, uint16_t qtype, time_t now, 
-	struct delegpt* dp, int* best_rtt, struct sock_list* blacklist)
+	struct delegpt* dp, int* best_rtt, struct sock_list* blacklist,
+	size_t* num_suitable_results)
 {
 	int got_it = 0;
 	struct delegpt_addr* a;
+	*num_suitable_results = 0;
+
 	if(dp->bogus)
 		return 0; /* NS bogus, all bogus, nothing found */
 	for(a=dp->result_list; a; a = a->next_result) {
@@ -301,11 +304,58 @@ iter_fill_rtt(struct iter_env* iter_env, struct module_env* env,
 			} else if(a->sel_rtt < *best_rtt) {
 				*best_rtt = a->sel_rtt;
 			}
+			(*num_suitable_results)++;
 		}
 	}
 	return got_it;
 }
 
+/** compare two rtts, return -1, 0 or 1 */
+static int
+rtt_compare(const void* x, const void* y)
+{
+	if(*(int*)x == *(int*)y)
+		return 0;
+	if(*(int*)x > *(int*)y)
+		return 1;
+	return -1;
+}
+
+/** get RTT for the Nth fastest server */
+static int
+nth_rtt(struct delegpt_addr* result_list, size_t num_results, size_t n)
+{
+	int rtt_band;
+	size_t i;
+	int* rtt_list, *rtt_index;
+	
+	if(num_results < 1 || n >= num_results) {
+		return -1;
+	}
+
+	rtt_list = calloc(num_results, sizeof(int));
+	if(!rtt_list) {
+		log_err("malloc failure: allocating rtt_list");
+		return -1;
+	}
+	rtt_index = rtt_list;
+
+	for(i=0; i<num_results && result_list; i++) {
+		if(result_list->sel_rtt != -1) {
+			*rtt_index = result_list->sel_rtt;
+			rtt_index++;
+		}
+		result_list=result_list->next_result;
+	}
+	qsort(rtt_list, num_results, sizeof(*rtt_list), rtt_compare);
+
+	log_assert(n > 0);
+	rtt_band = rtt_list[n-1];
+	free(rtt_list);
+
+	return rtt_band;
+}
+
 /** filter the address list, putting best targets at front,
  * returns number of best targets (or 0, no suitable targets) */
 static int
@@ -314,12 +364,13 @@ iter_filter_order(struct iter_env* iter_env, struct module_env* env,
 	struct delegpt* dp, int* selected_rtt, int open_target, 
 	struct sock_list* blacklist, time_t prefetch)
 {
-	int got_num = 0, low_rtt = 0, swap_to_front, rtt_band = RTT_BAND;
+	int got_num = 0, low_rtt = 0, swap_to_front, rtt_band = RTT_BAND, nth;
+	size_t num_results;
 	struct delegpt_addr* a, *n, *prev=NULL;
 
 	/* fillup sel_rtt and find best rtt in the bunch */
 	got_num = iter_fill_rtt(iter_env, env, name, namelen, qtype, now, dp, 
-		&low_rtt, blacklist);
+		&low_rtt, blacklist, &num_results);
 	if(got_num == 0) 
 		return 0;
 	if(low_rtt >= USEFUL_SERVER_TOP_TIMEOUT &&
@@ -329,14 +380,19 @@ iter_filter_order(struct iter_env* iter_env, struct module_env* env,
 			     return 0 to force the caller to fetch more */
 	}
 
-	if(env->cfg->low_rtt_permil != 0 && prefetch == 0 &&
-		low_rtt < env->cfg->low_rtt &&
-		ub_random_max(env->rnd, 1000) < env->cfg->low_rtt_permil) {
+	if(env->cfg->fast_server_permil != 0 && prefetch == 0 &&
+		num_results > env->cfg->fast_server_num &&
+		ub_random_max(env->rnd, 1000) < env->cfg->fast_server_permil) {
 		/* the query is not prefetch, but for a downstream client,
-		 * there is a low_rtt (fast) server.  We choose that x% of the
-		 * time */
-		/* pick rtt numbers from 0..LOWBAND_RTT */
-		rtt_band = env->cfg->low_rtt - low_rtt;
+		 * there are more servers available then the fastest N we want
+		 * to choose from. Limit our choice to the fastest servers. */
+		nth = nth_rtt(dp->result_list, num_results,
+			env->cfg->fast_server_num);
+		if(nth > 0) {
+			rtt_band = nth - low_rtt;
+			if(rtt_band > RTT_BAND)
+				rtt_band = RTT_BAND;
+		}
 	}
 
 	got_num = 0;
@@ -1210,3 +1266,50 @@ int iter_dp_cangodown(struct query_info* qinfo, struct delegpt* dp)
 		return 0;
 	return 1;
 }
+
+int
+iter_stub_fwd_no_cache(struct module_qstate *qstate, struct query_info *qinf)
+{
+	struct iter_hints_stub *stub;
+	struct delegpt *dp;
+
+	/* Check for stub. */
+	stub = hints_lookup_stub(qstate->env->hints, qinf->qname,
+	    qinf->qclass, NULL);
+	dp = forwards_lookup(qstate->env->fwds, qinf->qname, qinf->qclass);
+
+	/* see if forward or stub is more pertinent */
+	if(stub && stub->dp && dp) {
+		if(dname_strict_subdomain(dp->name, dp->namelabs,
+			stub->dp->name, stub->dp->namelabs)) {
+			stub = NULL; /* ignore stub, forward is lower */
+		} else {
+			dp = NULL; /* ignore forward, stub is lower */
+		}
+	}
+
+	/* check stub */
+	if (stub != NULL && stub->dp != NULL) {
+		if(stub->dp->no_cache) {
+			char qname[255+1];
+			char dpname[255+1];
+			dname_str(qinf->qname, qname);
+			dname_str(stub->dp->name, dpname);
+			verbose(VERB_ALGO, "stub for %s %s has no_cache", qname, dpname);
+		}
+		return (stub->dp->no_cache);
+	}
+
+	/* Check for forward. */
+	if (dp) {
+		if(dp->no_cache) {
+			char qname[255+1];
+			char dpname[255+1];
+			dname_str(qinf->qname, qname);
+			dname_str(dp->name, dpname);
+			verbose(VERB_ALGO, "forward for %s %s has no_cache", qname, dpname);
+		}
+		return (dp->no_cache);
+	}
+	return 0;
+}

iterator/iter_utils.h

@@ -369,4 +369,13 @@ int iter_ds_toolow(struct dns_msg* msg, struct delegpt* dp);
  */
 int iter_dp_cangodown(struct query_info* qinfo, struct delegpt* dp);
 
+/** 
+ * Lookup if no_cache is set in stub or fwd.
+ * @param qstate: query state with env with hints and fwds.
+ * @param qinf: query name to lookup for.
+ * @return true if no_cache is set in stub or fwd.
+ */
+int iter_stub_fwd_no_cache(struct module_qstate *qstate,
+	struct query_info *qinf);
+
 #endif /* ITERATOR_ITER_UTILS_H */

iterator/iterator.c

@@ -69,6 +69,9 @@
 #include "sldns/parseutil.h"
 #include "sldns/sbuffer.h"
 
+/* in msec */
+int UNKNOWN_SERVER_NICENESS = 376;
+
 int 
 iter_init(struct module_env* env, int id)
 {
@@ -324,6 +327,29 @@ error_response_cache(struct module_qstate* qstate, int id, int rcode)
 			/* serving expired contents, but nothing is cached
 			 * at all, so the servfail cache entry is useful
 			 * (stops waste of time on this servfail NORR_TTL) */
+		} else {
+			/* don't overwrite existing (non-expired) data in
+			 * cache with a servfail */
+			struct msgreply_entry* msg;
+			if((msg=msg_cache_lookup(qstate->env,
+				qstate->qinfo.qname, qstate->qinfo.qname_len,
+				qstate->qinfo.qtype, qstate->qinfo.qclass,
+				qstate->query_flags, *qstate->env->now, 0))
+				!= NULL) {
+				struct reply_info* rep = (struct reply_info*)
+					msg->entry.data;
+				if(FLAGS_GET_RCODE(rep->flags) ==
+					LDNS_RCODE_NOERROR ||
+					FLAGS_GET_RCODE(rep->flags) ==
+					LDNS_RCODE_NXDOMAIN) {
+					/* we have a good entry,
+					 * don't overwrite */
+					lock_rw_unlock(&msg->entry.lock);
+					return error_response(qstate, id, rcode);
+				}
+				lock_rw_unlock(&msg->entry.lock);
+			}
+			
 		}
 		memset(&err, 0, sizeof(err));
 		err.flags = (uint16_t)(BIT_QR | BIT_RA);
@@ -1125,7 +1151,7 @@ forward_request(struct module_qstate* qstate, struct iter_qstate* iq)
 	struct delegpt* dp;
 	uint8_t* delname = iq->qchase.qname;
 	size_t delnamelen = iq->qchase.qname_len;
-	if(iq->refetch_glue) {
+	if(iq->refetch_glue && iq->dp) {
 		delname = iq->dp->name;
 		delnamelen = iq->dp->namelen;
 	}
@@ -1144,53 +1170,6 @@ forward_request(struct module_qstate* qstate, struct iter_qstate* iq)
 	return 1;
 }
 
-static int
-iter_stub_fwd_no_cache(struct module_qstate *qstate, struct iter_qstate *iq)
-{
-	struct iter_hints_stub *stub;
-	struct delegpt *dp;
-
-	/* Check for stub. */
-	stub = hints_lookup_stub(qstate->env->hints, iq->qchase.qname,
-	    iq->qchase.qclass, iq->dp);
-	dp = forwards_lookup(qstate->env->fwds, iq->qchase.qname, iq->qchase.qclass);
-
-	/* see if forward or stub is more pertinent */
-	if(stub && stub->dp && dp) {
-		if(dname_strict_subdomain(dp->name, dp->namelabs,
-			stub->dp->name, stub->dp->namelabs)) {
-			stub = NULL; /* ignore stub, forward is lower */
-		} else {
-			dp = NULL; /* ignore forward, stub is lower */
-		}
-	}
-
-	/* check stub */
-	if (stub != NULL && stub->dp != NULL) {
-		if(stub->dp->no_cache) {
-			char qname[255+1];
-			char dpname[255+1];
-			dname_str(iq->qchase.qname, qname);
-			dname_str(stub->dp->name, dpname);
-			verbose(VERB_ALGO, "stub for %s %s has no_cache", qname, dpname);
-		}
-		return (stub->dp->no_cache);
-	}
-
-	/* Check for forward. */
-	if (dp) {
-		if(dp->no_cache) {
-			char qname[255+1];
-			char dpname[255+1];
-			dname_str(iq->qchase.qname, qname);
-			dname_str(dp->name, dpname);
-			verbose(VERB_ALGO, "forward for %s %s has no_cache", qname, dpname);
-		}
-		return (dp->no_cache);
-	}
-	return 0;
-}
-
 /** 
  * Process the initial part of the request handling. This state roughly
  * corresponds to resolver algorithms steps 1 (find answer in cache) and 2
@@ -1268,7 +1247,7 @@ processInitRequest(struct module_qstate* qstate, struct iter_qstate* iq,
 	/* This either results in a query restart (CNAME cache response), a
 	 * terminating response (ANSWER), or a cache miss (null). */
 	
-	if (iter_stub_fwd_no_cache(qstate, iq)) {
+	if (iter_stub_fwd_no_cache(qstate, &iq->qchase)) {
 		/* Asked to not query cache. */
 		verbose(VERB_ALGO, "no-cache set, going to the network");
 		qstate->no_cache_lookup = 1;
@@ -1903,7 +1882,6 @@ processLastResort(struct module_qstate* qstate, struct iter_qstate* iq,
 		struct delegpt* p = hints_lookup_root(qstate->env->hints,
 			iq->qchase.qclass);
 		if(p) {
-			struct delegpt_ns* ns;
 			struct delegpt_addr* a;
 			iq->chase_flags &= ~BIT_RD; /* go to authorities */
 			for(ns = p->nslist; ns; ns=ns->next) {
@@ -2174,7 +2152,8 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq,
 		return 0;
 	}
 
-	if(iq->minimisation_state == INIT_MINIMISE_STATE) {
+	if(iq->minimisation_state == INIT_MINIMISE_STATE
+		&& !(iq->chase_flags & BIT_RD)) {
 		/* (Re)set qinfo_out to (new) delegation point, except when
 		 * qinfo_out is already a subdomain of dp. This happens when
 		 * increasing by more than one label at once (QNAMEs with more
@@ -2319,7 +2298,7 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq,
 		errinf(qstate, "auth zone lookup failed, fallback is off");
 		return error_response(qstate, id, LDNS_RCODE_SERVFAIL);
 	}
-	if(iq->dp && iq->dp->auth_dp) {
+	if(iq->dp->auth_dp) {
 		/* we wanted to fallback, but had no delegpt, only the
 		 * auth zone generated delegpt, create an actual one */
 		iq->auth_zone_avoid = 1;
@@ -2715,7 +2694,8 @@ processQueryResponse(struct module_qstate* qstate, struct iter_qstate* iq,
 			sock_list_insert(&qstate->reply_origin, 
 				&qstate->reply->addr, qstate->reply->addrlen, 
 				qstate->region);
-		if(iq->minimisation_state != DONOT_MINIMISE_STATE) {
+		if(iq->minimisation_state != DONOT_MINIMISE_STATE
+			&& !(iq->chase_flags & BIT_RD)) {
 			if(FLAGS_GET_RCODE(iq->response->rep->flags) != 
 				LDNS_RCODE_NOERROR) {
 				if(qstate->env->cfg->qname_minimisation_strict)
@@ -2752,6 +2732,12 @@ processQueryResponse(struct module_qstate* qstate, struct iter_qstate* iq,
 						verbose(VERB_ALGO,
 						"could not validate NXDOMAIN "
 						"response");
+					outbound_list_clear(&iq->outlist);
+					iq->num_current_queries = 0;
+					fptr_ok(fptr_whitelist_modenv_detach_subs(
+						qstate->env->detach_subs));
+					(*qstate->env->detach_subs)(qstate);
+					iq->num_target_queries = 0;
 				}
 			}
 			return next_state(iq, QUERYTARGETS_STATE);
@@ -2993,7 +2979,7 @@ processQueryResponse(struct module_qstate* qstate, struct iter_qstate* iq,
 			iq->dp->name, iq->dp->namelen, qstate->qinfo.qclass)) {
 			verbose(VERB_ALGO, "auth zone response bad, and no"
 				" fallback possible, servfail");
-			errinf_dname(qstate, "reponse is bad, no fallback, "
+			errinf_dname(qstate, "response is bad, no fallback, "
 				"for auth zone", iq->dp->name);
 			return error_response(qstate, id, LDNS_RCODE_SERVFAIL);
 		}
@@ -3584,7 +3570,7 @@ process_response(struct module_qstate* qstate, struct iter_qstate* iq,
 	if(event == module_event_noreply || event == module_event_error) {
 		if(event == module_event_noreply && iq->sent_count >= 3 &&
 			qstate->env->cfg->use_caps_bits_for_id &&
-			!iq->caps_fallback) {
+			!iq->caps_fallback && !is_caps_whitelisted(ie, iq)) {
 			/* start fallback */
 			iq->caps_fallback = 1;
 			iq->caps_server = 0;

iterator/iterator.h

@@ -83,7 +83,7 @@ struct rbtree_type;
 /** how nice is a server without further information, in msec 
  * Equals rtt initial timeout value.
  */
-#define UNKNOWN_SERVER_NICENESS 376
+extern int UNKNOWN_SERVER_NICENESS;
 /** maximum timeout before a host is deemed unsuitable, in msec. 
  * After host_ttl this will be timed out and the host will be tried again. 
  * Equals RTT_MAX_TIMEOUT

libunbound/context.c

@@ -392,12 +392,12 @@ struct ctx_query* context_deserialize_cancel(struct ub_ctx* ctx,
 uint8_t* 
 context_serialize_quit(uint32_t* len)
 {
-	uint8_t* p = (uint8_t*)malloc(sizeof(uint32_t));
+	uint32_t* p = (uint32_t*)malloc(sizeof(uint32_t));
 	if(!p)
 		return NULL;
 	*len = sizeof(uint32_t);
 	sldns_write_uint32(p, UB_LIBCMD_QUIT);
-	return p;
+	return (uint8_t*)p;
 }
 
 enum ub_ctx_cmd context_serial_getcmd(uint8_t* p, uint32_t len)

libunbound/libunbound.c

@@ -109,13 +109,13 @@ static struct ub_ctx* ub_ctx_create_nopipe(void)
 	alloc_init(&ctx->superalloc, NULL, 0);
 	seed = (unsigned int)time(NULL) ^ (unsigned int)getpid();
 	if(!(ctx->seed_rnd = ub_initstate(seed, NULL))) {
-		seed = 0;
+		explicit_bzero(&seed, sizeof(seed));
 		ub_randfree(ctx->seed_rnd);
 		free(ctx);
 		errno = ENOMEM;
 		return NULL;
 	}
-	seed = 0;
+	explicit_bzero(&seed, sizeof(seed));
 	lock_basic_init(&ctx->qqpipe_lock);
 	lock_basic_init(&ctx->rrpipe_lock);
 	lock_basic_init(&ctx->cfglock);
@@ -392,7 +392,6 @@ ub_ctx_add_ta(struct ub_ctx* ctx, const char* ta)
 	}
 	if(!cfg_strlist_insert(&ctx->env->cfg->trust_anchor_list, dup)) {
 		lock_basic_unlock(&ctx->cfglock);
-		free(dup);
 		return UB_NOMEM;
 	}
 	lock_basic_unlock(&ctx->cfglock);
@@ -412,7 +411,6 @@ ub_ctx_add_ta_file(struct ub_ctx* ctx, const char* fname)
 	}
 	if(!cfg_strlist_insert(&ctx->env->cfg->trust_anchor_file_list, dup)) {
 		lock_basic_unlock(&ctx->cfglock);
-		free(dup);
 		return UB_NOMEM;
 	}
 	lock_basic_unlock(&ctx->cfglock);
@@ -432,7 +430,6 @@ int ub_ctx_add_ta_autr(struct ub_ctx* ctx, const char* fname)
 	if(!cfg_strlist_insert(&ctx->env->cfg->auto_trust_anchor_file_list,
 		dup)) {
 		lock_basic_unlock(&ctx->cfglock);
-		free(dup);
 		return UB_NOMEM;
 	}
 	lock_basic_unlock(&ctx->cfglock);
@@ -452,7 +449,6 @@ ub_ctx_trustedkeys(struct ub_ctx* ctx, const char* fname)
 	}
 	if(!cfg_strlist_insert(&ctx->env->cfg->trusted_keys_file_list, dup)) {
 		lock_basic_unlock(&ctx->cfglock);
-		free(dup);
 		return UB_NOMEM;
 	}
 	lock_basic_unlock(&ctx->cfglock);
@@ -728,7 +724,7 @@ ub_resolve_event(struct ub_ctx* ctx, const char* name, int rrtype,
 		*async_id = 0;
 	lock_basic_lock(&ctx->cfglock);
 	if(!ctx->finalized) {
-		int r = context_finalize(ctx);
+		r = context_finalize(ctx);
 		if(r) {
 			lock_basic_unlock(&ctx->cfglock);
 			return r;
@@ -962,7 +958,6 @@ ub_ctx_set_fwd(struct ub_ctx* ctx, const char* addr)
 		return UB_NOMEM;
 	}
 	if(!cfg_strlist_insert(&s->addrs, dupl)) {
-		free(dupl);
 		lock_basic_unlock(&ctx->cfglock);
 		errno=ENOMEM;
 		return UB_NOMEM;
@@ -971,6 +966,19 @@ ub_ctx_set_fwd(struct ub_ctx* ctx, const char* addr)
 	return UB_NOERROR;
 }
 
+int ub_ctx_set_tls(struct ub_ctx* ctx, int tls)
+{
+	lock_basic_lock(&ctx->cfglock);
+	if(ctx->finalized) {
+		lock_basic_unlock(&ctx->cfglock);
+		errno=EINVAL;
+		return UB_AFTERFINAL;
+	}
+	ctx->env->cfg->ssl_upstream = tls;
+	lock_basic_unlock(&ctx->cfglock);
+	return UB_NOERROR;
+}
+
 int ub_ctx_set_stub(struct ub_ctx* ctx, const char* zone, const char* addr,
 	int isprime)
 {
@@ -1045,7 +1053,6 @@ int ub_ctx_set_stub(struct ub_ctx* ctx, const char* zone, const char* addr,
 	}
 	if(!cfg_strlist_insert(&elem->addrs, a)) {
 		lock_basic_unlock(&ctx->cfglock);
-		free(a);
 		errno = ENOMEM;
 		return UB_NOMEM;
 	}
@@ -1233,7 +1240,6 @@ ub_ctx_hosts(struct ub_ctx* ctx, const char* fname)
 				ins)) {
 				lock_basic_unlock(&ctx->cfglock);
 				fclose(in);
-				free(ins);
 				errno=ENOMEM;
 				return UB_NOMEM;
 			}

libunbound/libworker.c

@@ -187,7 +187,7 @@ libworker_setup(struct ub_ctx* ctx, int is_bg, struct ub_event_base* eb)
 		if(!w->is_bg || w->is_bg_thread) {
 			lock_basic_unlock(&ctx->cfglock);
 		}
-		seed = 0;
+		explicit_bzero(&seed, sizeof(seed));
 		libworker_delete(w);
 		return NULL;
 	}
@@ -207,7 +207,7 @@ libworker_setup(struct ub_ctx* ctx, int is_bg, struct ub_event_base* eb)
 			hash_set_raninit((uint32_t)ub_random(w->env->rnd));
 		}
 	}
-	seed = 0;
+	explicit_bzero(&seed, sizeof(seed));
 
 	if(eb)
 		w->base = comm_base_create_event(eb);
@@ -657,8 +657,8 @@ libworker_event_done_cb(void* arg, int rcode, sldns_buffer* buf,
 			sec = 1;
 		else if(s == sec_status_secure)
 			sec = 2;
-		(*cb)(cb_arg, rcode, (void*)sldns_buffer_begin(buf),
-			(int)sldns_buffer_limit(buf), sec, why_bogus, was_ratelimited);
+		(*cb)(cb_arg, rcode, (buf?(void*)sldns_buffer_begin(buf):NULL),
+			(buf?(int)sldns_buffer_limit(buf):0), sec, why_bogus, was_ratelimited);
 	}
 }
 

libunbound/ubsyms.def

@@ -20,6 +20,7 @@ ub_ctx_set_event
 ub_ctx_set_fwd
 ub_ctx_set_option
 ub_ctx_set_stub
+ub_ctx_set_tls
 ub_ctx_trustedkeys
 ub_ctx_zone_add
 ub_ctx_zone_remove

libunbound/unbound.h

@@ -309,6 +309,17 @@ int ub_ctx_config(struct ub_ctx* ctx, const char* fname);
  */
 int ub_ctx_set_fwd(struct ub_ctx* ctx, const char* addr);
 
+/**
+ * Use DNS over TLS to send queries to machines set with ub_ctx_set_fwd().
+ *
+ * @param ctx: context.
+ *	At this time it is only possible to set configuration before the
+ *	first resolve is done.
+ * @param tls: enable or disable DNS over TLS
+ * @return 0 if OK, else error.
+ */
+int ub_ctx_set_tls(struct ub_ctx* ctx, int tls);
+
 /**
  * Add a stub zone, with given address to send to.  This is for custom
  * root hints or pointing to a local authoritative dns server.
@@ -770,6 +781,10 @@ struct ub_server_stats {
 	/** number of queries answered from edns-subnet specific data, and
 	 * the answer was from the edns-subnet cache. */
 	long long num_query_subnet_cache;
+	/** number of bytes in the stream wait buffers */
+	long long mem_stream_wait;
+	/** number of TLS connection resume */
+	long long qtls_resume;
 };
 
 /** 

ltmain.sh

@@ -2124,7 +2124,7 @@ fi
 # a configuration failure hint, and exit.
 func_fatal_configuration ()
 {
-    func__fatal_error ${1+"$@"} \
+    func_fatal_error ${1+"$@"} \
       "See the $PACKAGE documentation for more information." \
       "Fatal configuration error."
 }
@@ -7272,10 +7272,12 @@ func_mode_link ()
       # -tp=*                Portland pgcc target processor selection
       # --sysroot=*          for sysroot support
       # -O*, -g*, -flto*, -fwhopr*, -fuse-linker-plugin GCC link-time optimization
+      # -specs=*             GCC specs files
       # -stdlib=*            select c++ std lib with clang
       -64|-mips[0-9]|-r[0-9][0-9]*|-xarch=*|-xtarget=*|+DA*|+DD*|-q*|-m*| \
       -t[45]*|-txscale*|-p|-pg|--coverage|-fprofile-*|-F*|@*|-tp=*|--sysroot=*| \
-      -O*|-g*|-flto*|-fwhopr*|-fuse-linker-plugin|-fstack-protector*|-stdlib=*)
+      -O*|-g*|-flto*|-fwhopr*|-fuse-linker-plugin|-fstack-protector*|-stdlib=*| \
+      -specs=*)
         func_quote_for_eval "$arg"
 	arg=$func_quote_for_eval_result
         func_append compile_command " $arg"

pythonmod/doc/modules/functions.rst

@@ -103,42 +103,67 @@ Inplace callbacks
     :param opt_list_out: :class:`edns_option`. EDNS option list to append options to.
     :param region: :class:`regional`
 
-.. function:: register_inplace_cb_reply(py_cb, env)
+.. function:: inplace_cb_query(qinfo, flags, qstate, addr, zone, region)
+
+    Function prototype for callback functions used in
+    `register_inplace_cb_query`_.
+
+    :param qinfo: :class:`query_info`
+    :param flags: query flags (integer)
+    :param qstate: :class:`module_qstate`
+    :param addr: :class:`sockaddr_storage`
+    :param zone: zone name in wire format (bytes)
+    :param region: :class:`regional`
+
+.. function:: register_inplace_cb_reply(py_cb, env, id)
 
     Register py_cb as an inplace reply callback function.
 
     :param py_cb: Python function that follows `inplace_cb_reply`_'s prototype. **Must** be callable.
     :param env: :class:`module_env`
+    :param id: Module ID.
     :return: True on success, False otherwise
     :rtype: boolean
 
-.. function:: register_inplace_cb_reply_cache(py_cb, env)
+.. function:: register_inplace_cb_reply_cache(py_cb, env, id)
 
     Register py_cb as an inplace reply_cache callback function.
 
     :param py_cb: Python function that follows `inplace_cb_reply`_'s prototype. **Must** be callable.
     :param env: :class:`module_env`
+    :param id: Module ID.
     :return: True on success, False otherwise
     :rtype: boolean
 
-.. function:: register_inplace_cb_reply_local(py_cb, env)
+.. function:: register_inplace_cb_reply_local(py_cb, env, id)
 
     Register py_cb as an inplace reply_local callback function.
 
     :param py_cb: Python function that follows `inplace_cb_reply`_'s prototype. **Must** be callable.
     :param env: :class:`module_env`
+    :param id: Module ID.
     :return: True on success, False otherwise
     :rtype: boolean
 
-.. function:: register_inplace_cb_reply_servfail(py_cb, env)
+.. function:: register_inplace_cb_reply_servfail(py_cb, env, id)
 
     Register py_cb as an inplace reply_servfail callback function.
 
     :param py_cb: Python function that follows `inplace_cb_reply`_'s prototype. **Must** be callable.
     :param env: :class:`module_env`
+    :param id: Module ID.
     :return: True on success, False otherwise
     :rtype: boolean
 
+.. function:: register_inplace_cb_query(py_cb, env, id)
+
+    Register py_cb as an inplace query callback function.
+
+    :param py_cb: Python function that follows `inplace_cb_query`_'s prototype. **Must** be callable.
+    :param env: :class:`module_env`
+    :param id: Module ID.
+    :return: True on success, False otherwise
+    :rtype: boolean
 
 Logging
 -------

pythonmod/doc/modules/struct.rst

@@ -514,3 +514,33 @@ pythonmod_qstate
 	
 	   Here you can keep your own private data (each thread has own data object).
 
+sockaddr_storage
+-------------------------
+
+.. class:: sockaddr_storage
+
+   The :class:`sockaddr_storage` provides these data attributes:
+
+   .. attribute:: family
+
+      Address family name as a string. Possible values are `ip4`, `ip6`, and `unix`.
+
+   .. attribute:: addr
+
+      Address in presentation format.
+
+   .. attribute:: raw_addr
+
+      Address in network wire format.
+
+   .. attribute:: port
+
+      Port number. Invalid for Unix address.
+
+   .. attribute:: flowinfo
+
+      Flow info value. Valid only for IPv6 address.
+
+   .. attribute:: scope_id
+
+      Scope ID value. Valid only for IPv6 address.

pythonmod/examples/inplace_callbacks.py

@@ -247,6 +247,25 @@ def inplace_servfail_callback(qinfo, qstate, rep, rcode, edns, opt_list_out,
     return True
 
 
+def inplace_query_callback(qinfo, flags, qstate, addr, zone, region, **kwargs):
+    """
+    Function that will be registered as an inplace callback function.
+    It will be called before sending a query to a backend server.
+
+    :param qinfo: query_info struct;
+    :param flags: flags of the query;
+    :param qstate: module qstate. opt_lists are available here;
+    :param addr: struct sockaddr_storage. Address of the backend server;
+    :param zone: zone name in binary;
+    :param region: region to allocate temporary data. Needs to be used when we
+                   want to append a new option to opt_lists.
+    :param **kwargs: Dictionary that may contain parameters added in a future
+                     release.
+    """
+    log_info("python: outgoing query to {}@{}".format(addr.addr, addr.port))
+    return True
+
+
 def init_standard(id, env):
     """
     New version of the init function.
@@ -281,6 +300,11 @@ def init_standard(id, env):
     if not register_inplace_cb_reply_servfail(inplace_servfail_callback, env, id):
         return False
 
+    # Register the inplace_query_callback function as an inplace callback
+    # before sending a query to a backend server.
+    if not register_inplace_cb_query(inplace_query_callback, env, id):
+        return False
+
     return True
 
 

pythonmod/interface.i

@@ -12,6 +12,8 @@
    #include <sys/socket.h>
    #include <netinet/in.h>
    #include <arpa/inet.h>
+   #include <netdb.h>
+   #include <sys/un.h>
    #include <stdarg.h>
    #include "config.h"
    #include "util/log.h"
@@ -43,15 +45,21 @@
 
      i = 0;
      while (i < len) {
-        i += name[i] + 1;
+        i += ((unsigned int)name[i]) + 1;
         cnt++;
      }
 
      list = PyList_New(cnt);
      i = 0; cnt = 0;
      while (i < len) {
-        PyList_SetItem(list, cnt, PyBytes_FromStringAndSize(name + i + 1, name[i]));
-        i += name[i] + 1;
+        char buf[LDNS_MAX_LABELLEN+1];
+        if(((unsigned int)name[i])+1 <= (unsigned int)sizeof(buf) &&
+                i+(int)((unsigned int)name[i]) < len) {
+                memmove(buf, name + i + 1, (unsigned int)name[i]);
+                buf[(unsigned int)name[i]] = 0;
+                PyList_SetItem(list, cnt, PyString_FromString(buf));
+        }
+        i += ((unsigned int)name[i]) + 1;
         cnt++;
      }
      return list;
@@ -159,11 +167,11 @@ struct query_info {
 %}
 
 %inline %{
-   PyObject* dnameAsStr(const char* dname) {
+   PyObject* dnameAsStr(PyObject* dname) {
        char buf[LDNS_MAX_DOMAINLEN+1];
        buf[0] = '\0';
-       dname_str((uint8_t*)dname, buf);
-       return PyBytes_FromString(buf);
+       dname_str((uint8_t*)PyBytes_AsString(dname), buf);
+       return PyString_FromString(buf);
    }
 %}
 
@@ -426,6 +434,164 @@ struct dns_msg {
    %}
 }
 
+/* ************************************************************************************ *
+   Structure sockaddr_storage
+ * ************************************************************************************ */
+
+struct sockaddr_storage {};
+
+%inline %{
+    static size_t _sockaddr_storage_len(const struct sockaddr_storage *ss) {
+        if (ss == NULL) {
+            return 0;
+        }
+
+        switch (ss->ss_family) {
+        case AF_INET:  return sizeof(struct sockaddr_in);
+        case AF_INET6: return sizeof(struct sockaddr_in6);
+        case AF_UNIX:  return sizeof(struct sockaddr_un);
+        default:
+            return 0;
+        }
+    }
+
+    PyObject *_sockaddr_storage_family(const struct sockaddr_storage *ss) {
+        if (ss == NULL) {
+            return Py_None;
+        }
+
+        switch (ss->ss_family) {
+        case AF_INET:  return PyUnicode_FromString("ip4");
+        case AF_INET6: return PyUnicode_FromString("ip6");
+        case AF_UNIX:  return PyUnicode_FromString("unix");
+        default:
+            return Py_None;
+        }
+    }
+
+    PyObject *_sockaddr_storage_addr(const struct sockaddr_storage *ss) {
+        const struct sockaddr *sa;
+        size_t sa_len;
+        char name[NI_MAXHOST] = {0};
+
+        if (ss == NULL) {
+            return Py_None;
+        }
+
+        sa = (struct sockaddr *)ss;
+        sa_len = _sockaddr_storage_len(ss);
+        if (sa_len == 0) {
+            return Py_None;
+        }
+
+        if (getnameinfo(sa, sa_len, name, sizeof(name), NULL, 0, NI_NUMERICHOST) != 0) {
+            return Py_None;
+        }
+
+        return PyUnicode_FromString(name);
+    }
+
+    PyObject *_sockaddr_storage_raw_addr(const struct sockaddr_storage *ss) {
+        size_t sa_len;
+
+        if (ss == NULL) {
+            return Py_None;
+        }
+
+        sa_len = _sockaddr_storage_len(ss);
+        if (sa_len == 0) {
+            return Py_None;
+        }
+
+        if (ss->ss_family == AF_INET) {
+            const struct sockaddr_in *sa = (struct sockaddr_in *)ss;
+            const struct in_addr *raw = (struct in_addr *)&sa->sin_addr;
+            return PyBytes_FromStringAndSize((const char *)raw, sizeof(*raw));
+        }
+
+        if (ss->ss_family == AF_INET6) {
+            const struct sockaddr_in6 *sa = (struct sockaddr_in6 *)ss;
+            const struct in6_addr *raw = (struct in6_addr *)&sa->sin6_addr;
+            return PyBytes_FromStringAndSize((const char *)raw, sizeof(*raw));
+        }
+
+        if (ss->ss_family == AF_UNIX) {
+            const struct sockaddr_un *sa = (struct sockaddr_un *)ss;
+            return PyBytes_FromString(sa->sun_path);
+        }
+
+        return Py_None;
+    }
+
+    PyObject *_sockaddr_storage_port(const struct sockaddr_storage *ss) {
+        if (ss == NULL) {
+            return Py_None;
+        }
+
+        if (ss->ss_family == AF_INET) {
+            const struct sockaddr_in *sa4 = (struct sockaddr_in *)ss;
+            return PyInt_FromLong(ntohs(sa4->sin_port));
+        }
+
+        if (ss->ss_family == AF_INET6) {
+            const struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *)ss;
+            return PyInt_FromLong(ntohs(sa6->sin6_port));
+        }
+
+        return Py_None;
+    }
+
+    PyObject *_sockaddr_storage_flowinfo(const struct sockaddr_storage *ss) {
+        const struct sockaddr_in6 *sa6;
+
+        if (ss == NULL || ss->ss_family != AF_INET6) {
+            return Py_None;
+        }
+
+        sa6 = (struct sockaddr_in6 *)ss;
+        return PyInt_FromLong(ntohl(sa6->sin6_flowinfo));
+    }
+
+    PyObject *_sockaddr_storage_scope_id(const struct sockaddr_storage *ss) {
+        const struct sockaddr_in6 *sa6;
+
+        if (ss == NULL || ss->ss_family != AF_INET6) {
+            return Py_None;
+        }
+
+        sa6 = (struct sockaddr_in6 *)ss;
+        return PyInt_FromLong(ntohl(sa6->sin6_scope_id));
+    }
+%}
+
+%extend sockaddr_storage {
+   %pythoncode %{
+        def _family_get(self): return _sockaddr_storage_family(self)
+        __swig_getmethods__["family"] = _family_get
+        if _newclass: family = _swig_property(_family_get)
+
+        def _addr_get(self): return _sockaddr_storage_addr(self)
+        __swig_getmethods__["addr"] = _addr_get
+        if _newclass: addr = _swig_property(_addr_get)
+
+        def _raw_addr_get(self): return _sockaddr_storage_raw_addr(self)
+        __swig_getmethods__["raw_addr"] = _raw_addr_get
+        if _newclass: raw_addr = _swig_property(_raw_addr_get)
+
+        def _port_get(self): return _sockaddr_storage_port(self)
+        __swig_getmethods__["port"] = _port_get
+        if _newclass: port = _swig_property(_port_get)
+
+        def _flowinfo_get(self): return _sockaddr_storage_flowinfo(self)
+        __swig_getmethods__["flowinfo"] = _flowinfo_get
+        if _newclass: flowinfo = _swig_property(_flowinfo_get)
+
+        def _scope_id_get(self): return _sockaddr_storage_scope_id(self)
+        __swig_getmethods__["scope_id"] = _scope_id_get
+        if _newclass: scope_id = _swig_property(_scope_id_get)
+   %}
+}
+
 /* ************************************************************************************ *
    Structure mesh_state
  * ************************************************************************************ */
@@ -438,52 +604,22 @@ struct mesh_reply {
    struct comm_reply query_reply;
 };
 
+%rename(_addr) comm_reply::addr;
 struct comm_reply {
-
+   struct sockaddr_storage addr;
 };
 
-%inline %{
-
-  PyObject* _comm_reply_addr_get(struct comm_reply* reply) {
-     char dest[64];
-     reply_addr2str(reply, dest, 64);
-     if (dest[0] == 0)
-        return Py_None;
-     return PyBytes_FromString(dest);
-  }
-
-  PyObject* _comm_reply_family_get(struct comm_reply* reply) {
-
-        int af = (int)((struct sockaddr_in*) &(reply->addr))->sin_family;
-
-        switch(af) {
-           case AF_INET: return PyBytes_FromString("ip4");
-           case AF_INET6: return PyBytes_FromString("ip6");
-           case AF_UNIX: return PyBytes_FromString("unix");
-        }
-
-        return Py_None;
-  }
-
-  PyObject* _comm_reply_port_get(struct comm_reply* reply) {
-     uint16_t port;
-     port = ntohs(((struct sockaddr_in*)&(reply->addr))->sin_port);
-     return PyInt_FromLong(port);
-  }
-
-%}
-
 %extend comm_reply {
    %pythoncode %{
-        def _addr_get(self): return _comm_reply_addr_get(self)
+        def _addr_get(self): return _sockaddr_storage_addr(self._addr)
         __swig_getmethods__["addr"] = _addr_get
         if _newclass:addr = _swig_property(_addr_get)
 
-        def _port_get(self): return _comm_reply_port_get(self)
+        def _port_get(self): return _sockaddr_storage_port(self._addr)
         __swig_getmethods__["port"] = _port_get
         if _newclass:port = _swig_property(_port_get)
 
-        def _family_get(self): return _comm_reply_family_get(self)
+        def _family_get(self): return _sockaddr_storage_family(self._addr)
         __swig_getmethods__["family"] = _family_get
         if _newclass:family = _swig_property(_family_get)
    %}
@@ -1081,7 +1217,7 @@ int checkList(PyObject *l)
        for (i=0; i < PyList_Size(l); i++)
        {
            item = PyList_GetItem(l, i);
-           if (!PyBytes_Check(item))
+           if (!PyBytes_Check(item) && !PyUnicode_Check(item))
               return 0;
        }
        return 1;
@@ -1096,23 +1232,40 @@ int pushRRList(sldns_buffer* qb, PyObject *l, uint32_t default_ttl, int qsec,
     PyObject* item;
     int i;
     size_t len;
+    char* s;
+    PyObject* ascstr;
 
     for (i=0; i < PyList_Size(l); i++)
     {
+        ascstr = NULL;
         item = PyList_GetItem(l, i);
+        if(PyObject_TypeCheck(item, &PyBytes_Type)) {
+                s = PyBytes_AsString(item);
+        } else {
+                ascstr = PyUnicode_AsASCIIString(item);
+                s = PyBytes_AsString(ascstr);
+        }
 
         len = sldns_buffer_remaining(qb);
         if(qsec) {
-                if(sldns_str2wire_rr_question_buf(PyBytes_AsString(item),
+                if(sldns_str2wire_rr_question_buf(s,
                         sldns_buffer_current(qb), &len, NULL, NULL, 0, NULL, 0)
-                        != 0)
+                        != 0) {
+                        if(ascstr)
+                            Py_DECREF(ascstr);
                         return 0;
+                }
         } else {
-                if(sldns_str2wire_rr_buf(PyBytes_AsString(item),
+                if(sldns_str2wire_rr_buf(s,
                         sldns_buffer_current(qb), &len, NULL, default_ttl,
-                        NULL, 0, NULL, 0) != 0)
+                        NULL, 0, NULL, 0) != 0) {
+                        if(ascstr)
+                            Py_DECREF(ascstr);
                         return 0;
+                }
         }
+        if(ascstr)
+            Py_DECREF(ascstr);
         sldns_buffer_skip(qb, len);
 
         sldns_buffer_write_u16_at(qb, count_offset,
@@ -1437,6 +1590,54 @@ int edns_opt_list_append(struct edns_option** list, uint16_t code, size_t len,
         return python_inplace_cb_register(inplace_cb_reply_servfail,
                 py_cb, env, id);
     }
+
+    int python_inplace_cb_query_generic(
+        struct query_info* qinfo, uint16_t flags, struct module_qstate* qstate,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        uint8_t* zone, size_t zonelen, struct regional* region, int id,
+        void* python_callback)
+    {
+        int res = 0;
+        PyObject *func = python_callback;
+
+        PyGILState_STATE gstate = PyGILState_Ensure();
+
+        PyObject *py_qinfo = SWIG_NewPointerObj((void*) qinfo, SWIGTYPE_p_query_info, 0);
+        PyObject *py_qstate = SWIG_NewPointerObj((void*) qstate, SWIGTYPE_p_module_qstate, 0);
+        PyObject *py_addr = SWIG_NewPointerObj((void *) addr, SWIGTYPE_p_sockaddr_storage, 0);
+        PyObject *py_zone = PyBytes_FromStringAndSize((const char *)zone, zonelen);
+        PyObject *py_region = SWIG_NewPointerObj((void*) region, SWIGTYPE_p_regional, 0);
+
+        PyObject *py_args = Py_BuildValue("(OiOOOO)", py_qinfo, flags, py_qstate, py_addr, py_zone, py_region);
+        PyObject *py_kwargs = Py_BuildValue("{}");
+        PyObject *result = PyObject_Call(func, py_args, py_kwargs);
+        if (result) {
+            res = PyInt_AsLong(result);
+        }
+
+        Py_XDECREF(py_qinfo);
+        Py_XDECREF(py_qstate);
+        Py_XDECREF(py_addr);
+        Py_XDECREF(py_zone);
+        Py_XDECREF(py_region);
+
+        Py_XDECREF(py_args);
+        Py_XDECREF(py_kwargs);
+        Py_XDECREF(result);
+
+        PyGILState_Release(gstate);
+
+        return res;
+    }
+
+    static int register_inplace_cb_query(PyObject* py_cb,
+        struct module_env* env, int id)
+    {
+        int ret = inplace_cb_register(python_inplace_cb_query_generic,
+                inplace_cb_query, (void*) py_cb, env, id);
+        if (ret) Py_INCREF(py_cb);
+        return ret;
+    }
 %}
 /* C declarations */
 int inplace_cb_register(void* cb, enum inplace_cb_list_type type, void* cbarg,
@@ -1451,3 +1652,5 @@ static int register_inplace_cb_reply_local(PyObject* py_cb,
     struct module_env* env, int id);
 static int register_inplace_cb_reply_servfail(PyObject* py_cb,
     struct module_env* env, int id);
+static int register_inplace_cb_query(PyObject *py_cb,
+    struct module_env* env, int id);

pythonmod/pythonmod.c

@@ -110,6 +110,136 @@ struct pythonmod_qstate {
 #include "pythonmod/interface.h"
 #endif
 
+/** log python error */
+static void
+log_py_err(void)
+{
+	char *result = NULL;
+	const char* iomod = "cStringIO";
+	PyObject *modStringIO = NULL;
+	PyObject *modTB = NULL;
+	PyObject *obFuncStringIO = NULL;
+	PyObject *obStringIO = NULL;
+	PyObject *obFuncTB = NULL;
+	PyObject *argsTB = NULL;
+	PyObject *obResult = NULL;
+	PyObject *ascstr = NULL;
+	PyObject *exc_typ, *exc_val, *exc_tb;
+
+	/* Fetch the error state now before we cruch it */
+	/* exc val contains the error message
+	 * exc tb contains stack traceback and other info. */
+	PyErr_Fetch(&exc_typ, &exc_val, &exc_tb);
+	PyErr_NormalizeException(&exc_typ, &exc_val, &exc_tb);
+
+	/* Import the modules we need - cStringIO and traceback */
+	modStringIO = PyImport_ImportModule("cStringIO");
+	if (modStringIO==NULL) {
+		/* python 1.4 and before */
+		modStringIO = PyImport_ImportModule("StringIO");
+		iomod = "StringIO";
+	}
+	if (modStringIO==NULL) {
+		/* python 3 */
+		modStringIO = PyImport_ImportModule("io");
+		iomod = "io";
+	}
+	if (modStringIO==NULL) {
+		log_err("pythonmod: cannot print exception, "
+			"cannot ImportModule cStringIO or StringIO or io");
+		goto cleanup;
+	}
+	modTB = PyImport_ImportModule("traceback");
+	if (modTB==NULL) {
+		log_err("pythonmod: cannot print exception, "
+			"cannot ImportModule traceback");
+		goto cleanup;
+	}
+
+	/* Construct a cStringIO object */
+	obFuncStringIO = PyObject_GetAttrString(modStringIO, "StringIO");
+	if (obFuncStringIO==NULL) {
+		log_err("pythonmod: cannot print exception, "
+			"cannot GetAttrString %s.StringIO", iomod);
+		goto cleanup;
+	}
+	obStringIO = PyObject_CallObject(obFuncStringIO, NULL);
+	if (obStringIO==NULL) {
+		log_err("pythonmod: cannot print exception, "
+			"cannot call %s.StringIO()", iomod);
+		goto cleanup;
+	}
+
+	/* Get the traceback.print_exception function, and call it. */
+	obFuncTB = PyObject_GetAttrString(modTB, "print_exception");
+	if (obFuncTB==NULL) {
+		log_err("pythonmod: cannot print exception, "
+			"cannot GetAttrString traceback.print_exception");
+		goto cleanup;
+	}
+	argsTB = Py_BuildValue("OOOOO", (exc_typ ? exc_typ : Py_None),
+		(exc_val ? exc_val : Py_None), (exc_tb  ? exc_tb  : Py_None),
+		Py_None, obStringIO);
+	if (argsTB==NULL) {
+		log_err("pythonmod: cannot print exception, "
+			"cannot BuildValue for print_exception");
+		goto cleanup;
+	}
+
+	obResult = PyObject_CallObject(obFuncTB, argsTB);
+	if (obResult==NULL) {
+		PyErr_Print();
+		log_err("pythonmod: cannot print exception, "
+			"call traceback.print_exception() failed");
+		goto cleanup;
+	}
+
+	/* Now call the getvalue() method in the StringIO instance */
+	Py_DECREF(obFuncStringIO);
+	obFuncStringIO = PyObject_GetAttrString(obStringIO, "getvalue");
+	if (obFuncStringIO==NULL) {
+		log_err("pythonmod: cannot print exception, "
+			"cannot GetAttrString StringIO.getvalue");
+		goto cleanup;
+	}
+	Py_DECREF(obResult);
+	obResult = PyObject_CallObject(obFuncStringIO, NULL);
+	if (obResult==NULL) {
+		log_err("pythonmod: cannot print exception, "
+			"call StringIO.getvalue() failed");
+		goto cleanup;
+	}
+
+	/* And it should be a string all ready to go - duplicate it. */
+	if (!PyString_Check(obResult) && !PyUnicode_Check(obResult)) {
+		log_err("pythonmod: cannot print exception, "
+			"StringIO.getvalue() result did not String_Check"
+			" or Unicode_Check");
+		goto cleanup;
+	}
+	if(PyString_Check(obResult)) {
+		result = PyString_AsString(obResult);
+	} else {
+		ascstr = PyUnicode_AsASCIIString(obResult);
+		result = PyBytes_AsString(ascstr);
+	}
+	log_err("pythonmod: python error: %s", result);
+
+cleanup:
+	Py_XDECREF(modStringIO);
+	Py_XDECREF(modTB);
+	Py_XDECREF(obFuncStringIO);
+	Py_XDECREF(obStringIO);
+	Py_XDECREF(obFuncTB);
+	Py_XDECREF(argsTB);
+	Py_XDECREF(obResult);
+	Py_XDECREF(ascstr);
+
+	/* clear the exception, by not restoring it */
+	/* Restore the exception state */
+	/* PyErr_Restore(exc_typ, exc_val, exc_tb); */
+}
+
 int pythonmod_init(struct module_env* env, int id)
 {
    /* Initialize module */
@@ -193,13 +323,26 @@ int pythonmod_init(struct module_env* env, int id)
 
    /* TODO: deallocation of pe->... if an error occurs */
 
-   if (PyRun_SimpleFile(script_py, pe->fname) < 0)
-   {
+   if (PyRun_SimpleFile(script_py, pe->fname) < 0) {
       log_err("pythonmod: can't parse Python script %s", pe->fname);
+      /* print the error to logs too, run it again */
+      fseek(script_py, 0, SEEK_SET);
+      /* we don't run the file, like this, because then side-effects
+       *    s = PyRun_File(script_py, pe->fname, Py_file_input, 
+       *        PyModule_GetDict(PyImport_AddModule("__main__")), pe->dict);
+       * could happen (again). Instead we parse the file again to get
+       * the error string in the logs, for when the daemon has stderr
+       * removed.  SimpleFile run already printed to stderr, for then
+       * this is called from unbound-checkconf or unbound -dd the user
+       * has a nice formatted error.
+      */
+      /* ignore the NULL return of _node, it is NULL due to the parse failure
+       * that we are expecting */
+      (void)PyParser_SimpleParseFile(script_py, pe->fname, Py_file_input);
+      log_py_err();
       PyGILState_Release(gil);
       return 0;
    }
-
    fclose(script_py);
 
    if ((pe->func_init = PyDict_GetItemString(pe->dict, "init_standard")) == NULL)
@@ -244,7 +387,7 @@ int pythonmod_init(struct module_env* env, int id)
    if (PyErr_Occurred())
    {
       log_err("pythonmod: Exception occurred in function init");
-      PyErr_Print();
+      log_py_err();
       Py_XDECREF(res);
       Py_XDECREF(py_init_arg);
       PyGILState_Release(gil);
@@ -274,7 +417,7 @@ void pythonmod_deinit(struct module_env* env, int id)
       res = PyObject_CallFunction(pe->func_deinit, "i", id);
       if (PyErr_Occurred()) {
          log_err("pythonmod: Exception occurred in function deinit");
-         PyErr_Print();
+         log_py_err();
       }
       /* Free result if any */
       Py_XDECREF(res);
@@ -312,7 +455,7 @@ void pythonmod_inform_super(struct module_qstate* qstate, int id, struct module_
    if (PyErr_Occurred())
    {
       log_err("pythonmod: Exception occurred in function inform_super");
-      PyErr_Print();
+      log_py_err();
       qstate->ext_state[id] = module_error;
    }
    else if ((res == NULL)  || (!PyObject_IsTrue(res)))
@@ -353,7 +496,7 @@ void pythonmod_operate(struct module_qstate* qstate, enum module_ev event,
    if (PyErr_Occurred())
    {
       log_err("pythonmod: Exception occurred in function operate, event: %s", strmodulevent(event));
-      PyErr_Print();
+      log_py_err();
       qstate->ext_state[id] = module_error;
    }
    else if ((res == NULL)  || (!PyObject_IsTrue(res)))

pythonmod/pythonmod.h

@@ -74,4 +74,12 @@ int python_inplace_cb_reply_generic(struct query_info* qinfo,
 	struct edns_data* edns, struct edns_option** opt_list_out,
 	struct comm_reply* repinfo, struct regional* region, int id,
 	void* python_callback);
+
+/** Declared here for fptr_wlist access. The definition is in interface.i. */
+int python_inplace_cb_query_generic(
+        struct query_info* qinfo, uint16_t flags, struct module_qstate* qstate,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        uint8_t* zone, size_t zonelen, struct regional* region, int id,
+        void* python_callback);
+
 #endif /* PYTHONMOD_H */

respip/respip.c

@@ -172,7 +172,7 @@ respip_action_cfg(struct respip_set* set, const char* ipstr,
 	if(!(node=respip_find_or_create(set, ipstr, 1)))
 		return 0;
 	if(node->action != respip_none) {
-		log_warn("duplicate response-ip action for '%s', overridden.",
+		verbose(VERB_QUERY, "duplicate response-ip action for '%s', overridden.",
 			ipstr);
 	}
         if(strcmp(actnstr, "deny") == 0)
@@ -451,7 +451,7 @@ respip_views_apply_cfg(struct views* vs, struct config_file* cfg,
  * This function returns the copied rrset key on success, and NULL on memory
  * allocation failure.
  */
-struct ub_packed_rrset_key*
+static struct ub_packed_rrset_key*
 copy_rrset(const struct ub_packed_rrset_key* key, struct regional* region)
 {
 	struct ub_packed_rrset_key* ck = regional_alloc(region,

services/authzone.c

@@ -88,6 +88,9 @@
 #define AUTH_HTTPS_PORT 443
 /* max depth for nested $INCLUDEs */
 #define MAX_INCLUDE_DEPTH 10
+/** number of timeouts before we fallback from IXFR to AXFR,
+ * because some versions of servers (eg. dnsmasq) drop IXFR packets. */
+#define NUM_TIMEOUTS_FALLBACK_IXFR 3
 
 /** pick up nextprobe task to start waiting to perform transfer actions */
 static void xfr_set_timeout(struct auth_xfer* xfr, struct module_env* env,
@@ -218,7 +221,7 @@ msg_add_rrset_an(struct auth_zone* z, struct regional* region,
 {
 	log_assert(msg->rep->ns_numrrsets == 0);
 	log_assert(msg->rep->ar_numrrsets == 0);
-	if(!rrset)
+	if(!rrset || !node)
 		return 1;
 	if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 		z->dclass))
@@ -242,7 +245,7 @@ msg_add_rrset_ns(struct auth_zone* z, struct regional* region,
 	struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 {
 	log_assert(msg->rep->ar_numrrsets == 0);
-	if(!rrset)
+	if(!rrset || !node)
 		return 1;
 	if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 		z->dclass))
@@ -265,7 +268,7 @@ static int
 msg_add_rrset_ar(struct auth_zone* z, struct regional* region,
 	struct dns_msg* msg, struct auth_data* node, struct auth_rrset* rrset)
 {
-	if(!rrset)
+	if(!rrset || !node)
 		return 1;
 	if(msg_rrset_duplicate(msg, node->name, node->namelen, rrset->type,
 		z->dclass))
@@ -1450,11 +1453,13 @@ az_remove_rr_decompress(struct auth_zone* z, uint8_t* pkt, size_t pktlen,
  *	The lineno is set at 1 and then increased by the function.
  * @param fname: file name.
  * @param depth: recursion depth for includes
+ * @param cfg: config for chroot.
  * returns false on failure, has printed an error message
  */
 static int
 az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
-	struct sldns_file_parse_state* state, char* fname, int depth)
+	struct sldns_file_parse_state* state, char* fname, int depth,
+	struct config_file* cfg)
 {
 	size_t rr_len, dname_len;
 	int status;
@@ -1480,6 +1485,11 @@ az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
 				/* skip spaces */
 				while(*incfile == ' ' || *incfile == '\t')
 					incfile++;
+				/* adjust for chroot on include file */
+				if(cfg->chrootdir && cfg->chrootdir[0] &&
+					strncmp(incfile, cfg->chrootdir,
+						strlen(cfg->chrootdir)) == 0)
+					incfile += strlen(cfg->chrootdir);
 				incfile = strdup(incfile);
 				if(!incfile) {
 					log_err("malloc failure");
@@ -1490,7 +1500,7 @@ az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
 				inc = fopen(incfile, "r");
 				if(!inc) {
 					log_err("%s:%d cannot open include "
-						"file %s: %s", z->zonefile,
+						"file %s: %s", fname,
 						lineno_orig, incfile,
 						strerror(errno));
 					free(incfile);
@@ -1498,7 +1508,7 @@ az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
 				}
 				/* recurse read that file now */
 				if(!az_parse_file(z, inc, rr, rrbuflen,
-					state, incfile, depth+1)) {
+					state, incfile, depth+1, cfg)) {
 					log_err("%s:%d cannot parse include "
 						"file %s", fname,
 						lineno_orig, incfile);
@@ -1538,30 +1548,36 @@ az_parse_file(struct auth_zone* z, FILE* in, uint8_t* rr, size_t rrbuflen,
 }
 
 int
-auth_zone_read_zonefile(struct auth_zone* z)
+auth_zone_read_zonefile(struct auth_zone* z, struct config_file* cfg)
 {
 	uint8_t rr[LDNS_RR_BUF_SIZE];
 	struct sldns_file_parse_state state;
+	char* zfilename;
 	FILE* in;
 	if(!z || !z->zonefile || z->zonefile[0]==0)
 		return 1; /* no file, or "", nothing to read */
+	
+	zfilename = z->zonefile;
+	if(cfg->chrootdir && cfg->chrootdir[0] && strncmp(zfilename,
+		cfg->chrootdir, strlen(cfg->chrootdir)) == 0)
+		zfilename += strlen(cfg->chrootdir);
 	if(verbosity >= VERB_ALGO) {
 		char nm[255+1];
 		dname_str(z->name, nm);
-		verbose(VERB_ALGO, "read zonefile %s for %s", z->zonefile, nm);
+		verbose(VERB_ALGO, "read zonefile %s for %s", zfilename, nm);
 	}
-	in = fopen(z->zonefile, "r");
+	in = fopen(zfilename, "r");
 	if(!in) {
 		char* n = sldns_wire2str_dname(z->name, z->namelen);
 		if(z->zone_is_slave && errno == ENOENT) {
 			/* we fetch the zone contents later, no file yet */
 			verbose(VERB_ALGO, "no zonefile %s for %s",
-				z->zonefile, n?n:"error");
+				zfilename, n?n:"error");
 			free(n);
 			return 1;
 		}
 		log_err("cannot open zonefile %s for %s: %s",
-			z->zonefile, n?n:"error", strerror(errno));
+			zfilename, n?n:"error", strerror(errno));
 		free(n);
 		return 0;
 	}
@@ -1579,10 +1595,10 @@ auth_zone_read_zonefile(struct auth_zone* z)
 		state.origin_len = z->namelen;
 	}
 	/* parse the (toplevel) file */
-	if(!az_parse_file(z, in, rr, sizeof(rr), &state, z->zonefile, 0)) {
+	if(!az_parse_file(z, in, rr, sizeof(rr), &state, zfilename, 0, cfg)) {
 		char* n = sldns_wire2str_dname(z->name, z->namelen);
 		log_err("error parsing zonefile %s for %s",
-			z->zonefile, n?n:"error");
+			zfilename, n?n:"error");
 		free(n);
 		fclose(in);
 		return 0;
@@ -1710,13 +1726,13 @@ int auth_zone_write_file(struct auth_zone* z, const char* fname)
 
 /** read all auth zones from file (if they have) */
 static int
-auth_zones_read_zones(struct auth_zones* az)
+auth_zones_read_zones(struct auth_zones* az, struct config_file* cfg)
 {
 	struct auth_zone* z;
 	lock_rw_wrlock(&az->lock);
 	RBTREE_FOR(z, struct auth_zone*, &az->ztree) {
 		lock_rw_wrlock(&z->lock);
-		if(!auth_zone_read_zonefile(z)) {
+		if(!auth_zone_read_zonefile(z, cfg)) {
 			lock_rw_unlock(&z->lock);
 			lock_rw_unlock(&az->lock);
 			return 0;
@@ -1953,7 +1969,7 @@ int auth_zones_apply_cfg(struct auth_zones* az, struct config_file* cfg,
 		}
 	}
 	az_delete_deleted_zones(az);
-	if(!auth_zones_read_zones(az))
+	if(!auth_zones_read_zones(az, cfg))
 		return 0;
 	if(setup) {
 		if(!auth_zones_setup_zones(az))
@@ -1966,7 +1982,7 @@ int auth_zones_apply_cfg(struct auth_zones* az, struct config_file* cfg,
  * @param at: transfer structure with chunks list.  The chunks and their
  * 	data are freed.
  */
-void
+static void
 auth_chunks_delete(struct auth_transfer* at)
 {
 	if(at->chunks_first) {
@@ -2605,7 +2621,7 @@ az_nsec3_hashname(struct auth_zone* z, uint8_t* hashname, size_t* hashnmlen,
 }
 
 /** Find the datanode that covers the nsec3hash-name */
-struct auth_data*
+static struct auth_data*
 az_nsec3_findnode(struct auth_zone* z, uint8_t* hashnm, size_t hashnmlen)
 {
 	struct query_info qinfo;
@@ -2730,13 +2746,14 @@ az_nsec3_insert(struct auth_zone* z, struct regional* region,
  * 	that is an exact match that should exist for it.
  * 	If that does not exist, a higher exact match + nxproof is enabled
  * 	(for some sort of opt-out empty nonterminal cases).
+ * ceproof: include ce proof NSEC3 (omitted for wildcard replies).
  * nxproof: include denial of the qname.
  * wcproof: include denial of wildcard (wildcard.ce).
  */
 static int
 az_add_nsec3_proof(struct auth_zone* z, struct regional* region,
 	struct dns_msg* msg, uint8_t* cenm, size_t cenmlen, uint8_t* qname,
-	size_t qname_len, int nxproof, int wcproof)
+	size_t qname_len, int ceproof, int nxproof, int wcproof)
 {
 	int algo;
 	size_t iter, saltlen;
@@ -2748,11 +2765,13 @@ az_add_nsec3_proof(struct auth_zone* z, struct regional* region,
 	if(!az_nsec3_param(z, &algo, &iter, &salt, &saltlen))
 		return 1; /* no nsec3 */
 	/* find ce that has an NSEC3 */
-	node = az_nsec3_find_ce(z, &cenm, &cenmlen, &no_exact_ce,
-		algo, iter, salt, saltlen);
-	if(no_exact_ce) nxproof = 1;
-	if(!az_nsec3_insert(z, region, msg, node))
-		return 0;
+	if(ceproof) {
+		node = az_nsec3_find_ce(z, &cenm, &cenmlen, &no_exact_ce,
+			algo, iter, salt, saltlen);
+		if(no_exact_ce) nxproof = 1;
+		if(!az_nsec3_insert(z, region, msg, node))
+			return 0;
+	}
 
 	if(nxproof) {
 		uint8_t* nx;
@@ -2828,7 +2847,7 @@ az_generate_any_answer(struct auth_zone* z, struct regional* region,
 		if(!msg_add_rrset_an(z, region, msg, node, rrset)) return 0;
 		added++;
 	}
-	if(added == 0 && node->rrsets) {
+	if(added == 0 && node && node->rrsets) {
 		if(!msg_add_rrset_an(z, region, msg, node,
 			node->rrsets)) return 0;
 	}
@@ -2897,7 +2916,7 @@ az_generate_notype_answer(struct auth_zone* z, struct regional* region,
 		/* DNSSEC denial NSEC3 */
 		if(!az_add_nsec3_proof(z, region, msg, node->name,
 			node->namelen, msg->qinfo.qname,
-			msg->qinfo.qname_len, 0, 0))
+			msg->qinfo.qname_len, 1, 0, 0))
 			return 0;
 	}
 	return 1;
@@ -2924,7 +2943,7 @@ az_generate_referral_answer(struct auth_zone* z, struct regional* region,
 		} else {
 			if(!az_add_nsec3_proof(z, region, msg, ce->name,
 				ce->namelen, msg->qinfo.qname,
-				msg->qinfo.qname_len, 0, 0))
+				msg->qinfo.qname_len, 1, 0, 0))
 				return 0;
 		}
 	}
@@ -2995,9 +3014,12 @@ az_generate_wildcard_answer(struct auth_zone* z, struct query_info* qinfo,
 	if((nsec=az_find_nsec_cover(z, &node)) != NULL) {
 		if(!msg_add_rrset_ns(z, region, msg, node, nsec)) return 0;
 	} else if(ce) {
-		if(!az_add_nsec3_proof(z, region, msg, ce->name,
-			ce->namelen, msg->qinfo.qname,
-			msg->qinfo.qname_len, 1, 0))
+		uint8_t* wildup = wildcard->name;
+		size_t wilduplen= wildcard->namelen;
+		dname_remove_label(&wildup, &wilduplen);
+		if(!az_add_nsec3_proof(z, region, msg, wildup,
+			wilduplen, msg->qinfo.qname,
+			msg->qinfo.qname_len, 0, 1, 0))
 			return 0;
 	}
 
@@ -3023,7 +3045,7 @@ az_generate_nxdomain_answer(struct auth_zone* z, struct regional* region,
 	} else if(ce) {
 		if(!az_add_nsec3_proof(z, region, msg, ce->name,
 			ce->namelen, msg->qinfo.qname,
-			msg->qinfo.qname_len, 1, 1))
+			msg->qinfo.qname_len, 1, 1, 1))
 			return 0;
 	}
 	return 1;
@@ -3163,6 +3185,11 @@ int auth_zones_lookup(struct auth_zones* az, struct query_info* qinfo,
 		*fallback = 1;
 		return 0;
 	}
+	if(z->zone_expired) {
+		*fallback = z->fallback_enabled;
+		lock_rw_unlock(&z->lock);
+		return 0;
+	}
 	/* see what answer that zone would generate */
 	r = auth_zone_generate_answer(z, qinfo, region, msg, fallback);
 	lock_rw_unlock(&z->lock);
@@ -3250,6 +3277,19 @@ int auth_zones_answer(struct auth_zones* az, struct module_env* env,
 		lock_rw_unlock(&z->lock);
 		return 0;
 	}
+	if(z->zone_expired) {
+		if(z->fallback_enabled) {
+			lock_rw_unlock(&z->lock);
+			return 0;
+		}
+		lock_rw_unlock(&z->lock);
+		lock_rw_wrlock(&az->lock);
+		az->num_query_down++;
+		lock_rw_unlock(&az->lock);
+		auth_error_encode(qinfo, env, edns, repinfo, buf, temp,
+			LDNS_RCODE_SERVFAIL);
+		return 1;
+	}
 
 	/* answer it from zone z */
 	r = auth_zone_generate_answer(z, qinfo, temp, &msg, &fallback);
@@ -4773,8 +4813,10 @@ auth_zone_write_chunks(struct auth_xfer* xfr, const char* fname)
 static void
 xfr_write_after_update(struct auth_xfer* xfr, struct module_env* env)
 {
+	struct config_file* cfg = env->cfg;
 	struct auth_zone* z;
 	char tmpfile[1024];
+	char* zfilename;
 	lock_basic_unlock(&xfr->lock);
 
 	/* get lock again, so it is a readlock and concurrently queries
@@ -4792,20 +4834,24 @@ xfr_write_after_update(struct auth_xfer* xfr, struct module_env* env)
 	lock_basic_lock(&xfr->lock);
 	lock_rw_unlock(&env->auth_zones->lock);
 
-	if(z->zonefile == NULL) {
+	if(z->zonefile == NULL || z->zonefile[0] == 0) {
 		lock_rw_unlock(&z->lock);
 		/* no write needed, no zonefile set */
 		return;
 	}
+	zfilename = z->zonefile;
+	if(cfg->chrootdir && cfg->chrootdir[0] && strncmp(zfilename,
+		cfg->chrootdir, strlen(cfg->chrootdir)) == 0)
+		zfilename += strlen(cfg->chrootdir);
 
 	/* write to tempfile first */
-	if((size_t)strlen(z->zonefile) + 16 > sizeof(tmpfile)) {
+	if((size_t)strlen(zfilename) + 16 > sizeof(tmpfile)) {
 		verbose(VERB_ALGO, "tmpfilename too long, cannot update "
-			" zonefile %s", z->zonefile);
+			" zonefile %s", zfilename);
 		lock_rw_unlock(&z->lock);
 		return;
 	}
-	snprintf(tmpfile, sizeof(tmpfile), "%s.tmp%u", z->zonefile,
+	snprintf(tmpfile, sizeof(tmpfile), "%s.tmp%u", zfilename,
 		(unsigned)getpid());
 	if(xfr->task_transfer->master->http) {
 		/* use the stored chunk list to write them */
@@ -4818,8 +4864,8 @@ xfr_write_after_update(struct auth_xfer* xfr, struct module_env* env)
 		lock_rw_unlock(&z->lock);
 		return;
 	}
-	if(rename(tmpfile, z->zonefile) < 0) {
-		log_err("could not rename(%s, %s): %s", tmpfile, z->zonefile,
+	if(rename(tmpfile, zfilename) < 0) {
+		log_err("could not rename(%s, %s): %s", tmpfile, zfilename,
 			strerror(errno));
 		unlink(tmpfile);
 		lock_rw_unlock(&z->lock);
@@ -4951,12 +4997,12 @@ xfr_transfer_lookup_host(struct auth_xfer* xfr, struct module_env* env)
 		qinfo.qtype = LDNS_RR_TYPE_AAAA;
 	qinfo.local_alias = NULL;
 	if(verbosity >= VERB_ALGO) {
-		char buf[512];
+		char buf1[512];
 		char buf2[LDNS_MAX_DOMAINLEN+1];
 		dname_str(xfr->name, buf2);
-		snprintf(buf, sizeof(buf), "auth zone %s: master lookup"
+		snprintf(buf1, sizeof(buf1), "auth zone %s: master lookup"
 			" for task_transfer", buf2);
-		log_query_info(VERB_ALGO, buf, &qinfo);
+		log_query_info(VERB_ALGO, buf1, &qinfo);
 	}
 	edns.edns_present = 1;
 	edns.ext_rcode = 0;
@@ -5593,15 +5639,33 @@ auth_xfer_transfer_tcp_callback(struct comm_point* c, void* arg, int err,
 		 * and continue task_transfer*/
 		verbose(VERB_ALGO, "xfr stopped, connection lost to %s",
 			xfr->task_transfer->master->host);
+
+		/* see if IXFR caused the failure, if so, try AXFR */
+		if(xfr->task_transfer->on_ixfr) {
+			xfr->task_transfer->ixfr_possible_timeout_count++;
+			if(xfr->task_transfer->ixfr_possible_timeout_count >=
+				NUM_TIMEOUTS_FALLBACK_IXFR) {
+				verbose(VERB_ALGO, "xfr to %s, fallback "
+					"from IXFR to AXFR (because of timeouts)",
+					xfr->task_transfer->master->host);
+				xfr->task_transfer->ixfr_fail = 1;
+				gonextonfail = 0;
+			}
+		}
+
 	failed:
 		/* delete transferred data from list */
 		auth_chunks_delete(xfr->task_transfer);
 		comm_point_delete(xfr->task_transfer->cp);
 		xfr->task_transfer->cp = NULL;
-		xfr_transfer_nextmaster(xfr);
+		if(gonextonfail)
+			xfr_transfer_nextmaster(xfr);
 		xfr_transfer_nexttarget_or_end(xfr, env);
 		return 0;
 	}
+	/* note that IXFR worked without timeout */
+	if(xfr->task_transfer->on_ixfr)
+		xfr->task_transfer->ixfr_possible_timeout_count = 0;
 
 	/* handle returned packet */
 	/* if it fails, cleanup and end this transfer */
@@ -5973,12 +6037,12 @@ xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
 		qinfo.qtype = LDNS_RR_TYPE_AAAA;
 	qinfo.local_alias = NULL;
 	if(verbosity >= VERB_ALGO) {
-		char buf[512];
+		char buf1[512];
 		char buf2[LDNS_MAX_DOMAINLEN+1];
 		dname_str(xfr->name, buf2);
-		snprintf(buf, sizeof(buf), "auth zone %s: master lookup"
+		snprintf(buf1, sizeof(buf1), "auth zone %s: master lookup"
 			" for task_probe", buf2);
-		log_query_info(VERB_ALGO, buf, &qinfo);
+		log_query_info(VERB_ALGO, buf1, &qinfo);
 	}
 	edns.edns_present = 1;
 	edns.ext_rcode = 0;

services/authzone.h

@@ -378,6 +378,8 @@ struct auth_transfer {
 	 * data or add of duplicate data).  Flag is cleared once the retry
 	 * with axfr is done. */
 	int ixfr_fail;
+	/** we saw an ixfr-indicating timeout, count of them */
+	int ixfr_possible_timeout_count;
 	/** we are doing IXFR right now */
 	int on_ixfr;
 	/** did we detect the current AXFR/IXFR serial number yet, 0 not yet,
@@ -599,7 +601,7 @@ int auth_zones_startprobesequence(struct auth_zones* az,
 	struct module_env* env, uint8_t* nm, size_t nmlen, uint16_t dclass);
 
 /** read auth zone from zonefile. caller must lock zone. false on failure */
-int auth_zone_read_zonefile(struct auth_zone* z);
+int auth_zone_read_zonefile(struct auth_zone* z, struct config_file* cfg);
 
 /** find serial number of zone or false if none (no SOA record) */
 int auth_zone_get_serial(struct auth_zone* z, uint32_t* serial);

services/cache/dns.c

@@ -721,6 +721,17 @@ fill_any(struct module_env* env,
 	int i, num=6; /* number of RR types to look up */
 	log_assert(lookup[num] == 0);
 
+	if(env->cfg->deny_any) {
+		/* return empty message */
+		msg = dns_msg_create(qname, qnamelen, qtype, qclass,
+			region, 0);
+		if(!msg) {
+			return NULL;
+		}
+		msg->rep->security = sec_status_indeterminate;
+		return msg;
+	}
+
 	for(i=0; i<num; i++) {
 		/* look up this RR for inclusion in type ANY response */
 		struct ub_packed_rrset_key* rrset = rrset_cache_lookup(

services/cache/infra.c

@@ -808,7 +808,7 @@ static struct lruhash_entry* infra_find_ratedata(struct infra_cache* infra,
 }
 
 /** find data item in array for ip addresses */
-struct lruhash_entry* infra_find_ip_ratedata(struct infra_cache* infra,
+static struct lruhash_entry* infra_find_ip_ratedata(struct infra_cache* infra,
 	struct comm_reply* repinfo, int wr)
 {
 	struct ip_rate_key key;
@@ -1013,8 +1013,8 @@ int infra_ip_ratelimit_inc(struct infra_cache* infra,
 			char client_ip[128];
 			addr_to_str((struct sockaddr_storage *)&repinfo->addr,
 				repinfo->addrlen, client_ip, sizeof(client_ip));
-			verbose(VERB_OPS, "ratelimit exceeded %s %d", client_ip,
-				infra_ip_ratelimit);
+			verbose(VERB_OPS, "ip_ratelimit exceeded %s %d",
+				client_ip, infra_ip_ratelimit);
 		}
 		return (max <= infra_ip_ratelimit);
 	}

services/listen_dnsport.c

@@ -53,6 +53,9 @@
 #include "util/config_file.h"
 #include "util/net_help.h"
 #include "sldns/sbuffer.h"
+#include "services/mesh.h"
+#include "util/fptr_wlist.h"
+#include "util/locks.h"
 
 #ifdef HAVE_NETDB_H
 #include <netdb.h>
@@ -70,6 +73,18 @@
 /** number of queued TCP connections for listen() */
 #define TCP_BACKLOG 256 
 
+/** number of simultaneous requests a client can have */
+#define TCP_MAX_REQ_SIMULTANEOUS 32
+
+#ifndef THREADS_DISABLED
+/** lock on the counter of stream buffer memory */
+static lock_basic_type stream_wait_count_lock;
+#endif
+/** size (in bytes) of stream wait buffers */
+static size_t stream_wait_count = 0;
+/** is the lock initialised for stream wait buffers */
+static int stream_wait_lock_inited = 0;
+
 /**
  * Debug print of the getaddrinfo returned address.
  * @param addr: the address returned.
@@ -247,6 +262,26 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		}
 #endif /* SO_REUSEADDR */
 #ifdef SO_REUSEPORT
+#  ifdef SO_REUSEPORT_LB
+		/* on FreeBSD 12 we have SO_REUSEPORT_LB that does loadbalance
+		 * like SO_REUSEPORT on Linux.  This is what the users want
+		 * with the config option in unbound.conf; if we actually
+		 * need local address and port reuse they'll also need to
+		 * have SO_REUSEPORT set for them, assume it was _LB they want.
+		 */
+		if (reuseport && *reuseport &&
+		    setsockopt(s, SOL_SOCKET, SO_REUSEPORT_LB, (void*)&on,
+			(socklen_t)sizeof(on)) < 0) {
+#ifdef ENOPROTOOPT
+			if(errno != ENOPROTOOPT || verbosity >= 3)
+				log_warn("setsockopt(.. SO_REUSEPORT_LB ..) failed: %s",
+					strerror(errno));
+#endif
+			/* this option is not essential, we can continue */
+			*reuseport = 0;
+		}
+#  else /* no SO_REUSEPORT_LB */
+
 		/* try to set SO_REUSEPORT so that incoming
 		 * queries are distributed evenly among the receiving threads.
 		 * Each thread must have its own socket bound to the same port,
@@ -263,6 +298,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 			/* this option is not essential, we can continue */
 			*reuseport = 0;
 		}
+#  endif /* SO_REUSEPORT_LB */
 #else
 		(void)reuseport;
 #endif /* defined(SO_REUSEPORT) */
@@ -565,7 +601,11 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		if(family==AF_INET6 && errno==EINVAL)
 			*noproto = 1;
 		else if(errno != EADDRINUSE &&
-			!(errno == EACCES && verbosity < 4 && !listen)) {
+			!(errno == EACCES && verbosity < 4 && !listen)
+#ifdef EADDRNOTAVAIL
+			&& !(errno == EADDRNOTAVAIL && verbosity < 4 && !listen)
+#endif
+			) {
 			log_err_addr("can't bind socket", strerror(errno),
 				(struct sockaddr_storage*)addr, addrlen);
 		}
@@ -813,7 +853,11 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
 		   disabled, except when verbosity enabled for debugging */
 		if(errno != ENOPROTOOPT || verbosity >= 3)
 #endif
-		  log_err("Setting TCP Fast Open as server failed: %s", strerror(errno));
+		  if(errno == EPERM) {
+		  	log_warn("Setting TCP Fast Open as server failed: %s ; this could likely be because sysctl net.inet.tcp.fastopen.enabled, net.inet.tcp.fastopen.server_enable, or net.ipv4.tcp_fastopen is disabled", strerror(errno));
+		  } else {
+		  	log_err("Setting TCP Fast Open as server failed: %s", strerror(errno));
+		  }
 	}
 #endif
 	return s;
@@ -1235,6 +1279,10 @@ listen_create(struct comm_base* base, struct listen_port* ports,
 		free(front);
 		return NULL;
 	}
+	if(!stream_wait_lock_inited) {
+		lock_basic_init(&stream_wait_count_lock);
+		stream_wait_lock_inited = 1;
+	}
 
 	/* create comm points as needed */
 	while(ports) {
@@ -1247,11 +1295,13 @@ listen_create(struct comm_base* base, struct listen_port* ports,
 				ports->ftype == listen_type_tcp_dnscrypt)
 			cp = comm_point_create_tcp(base, ports->fd, 
 				tcp_accept_count, tcp_idle_timeout,
-				tcp_conn_limit, bufsize, cb, cb_arg);
+				tcp_conn_limit, bufsize, front->udp_buff,
+				cb, cb_arg);
 		else if(ports->ftype == listen_type_ssl) {
 			cp = comm_point_create_tcp(base, ports->fd, 
 				tcp_accept_count, tcp_idle_timeout,
-				tcp_conn_limit, bufsize, cb, cb_arg);
+				tcp_conn_limit, bufsize, front->udp_buff,
+				cb, cb_arg);
 			cp->ssl = sslctx;
 		} else if(ports->ftype == listen_type_udpancil ||
 				  ports->ftype == listen_type_udpancil_dnscrypt)
@@ -1322,6 +1372,10 @@ listen_delete(struct listen_dnsport* front)
 #endif
 	sldns_buffer_free(front->udp_buff);
 	free(front);
+	if(stream_wait_lock_inited) {
+		stream_wait_lock_inited = 0;
+		lock_basic_destroy(&stream_wait_count_lock);
+	}
 }
 
 struct listen_port* 
@@ -1479,3 +1533,367 @@ void listen_start_accept(struct listen_dnsport* listen)
 	}
 }
 
+struct tcp_req_info*
+tcp_req_info_create(struct sldns_buffer* spoolbuf)
+{
+	struct tcp_req_info* req = (struct tcp_req_info*)malloc(sizeof(*req));
+	if(!req) {
+		log_err("malloc failure for new stream outoforder processing structure");
+		return NULL;
+	}
+	memset(req, 0, sizeof(*req));
+	req->spool_buffer = spoolbuf;
+	return req;
+}
+
+void
+tcp_req_info_delete(struct tcp_req_info* req)
+{
+	if(!req) return;
+	tcp_req_info_clear(req);
+	/* cp is pointer back to commpoint that owns this struct and
+	 * called delete on us */
+	/* spool_buffer is shared udp buffer, not deleted here */
+	free(req);
+}
+
+void tcp_req_info_clear(struct tcp_req_info* req)
+{
+	struct tcp_req_open_item* open, *nopen;
+	struct tcp_req_done_item* item, *nitem;
+	if(!req) return;
+
+	/* free outstanding request mesh reply entries */
+	open = req->open_req_list;
+	while(open) {
+		nopen = open->next;
+		mesh_state_remove_reply(open->mesh, open->mesh_state, req->cp);
+		free(open);
+		open = nopen;
+	}
+	req->open_req_list = NULL;
+	req->num_open_req = 0;
+	
+	/* free pending writable result packets */
+	item = req->done_req_list;
+	while(item) {
+		nitem = item->next;
+		lock_basic_lock(&stream_wait_count_lock);
+		stream_wait_count -= (sizeof(struct tcp_req_done_item)
+			+item->len);
+		lock_basic_unlock(&stream_wait_count_lock);
+		free(item->buf);
+		free(item);
+		item = nitem;
+	}
+	req->done_req_list = NULL;
+	req->num_done_req = 0;
+	req->read_is_closed = 0;
+}
+
+void
+tcp_req_info_remove_mesh_state(struct tcp_req_info* req, struct mesh_state* m)
+{
+	struct tcp_req_open_item* open, *prev = NULL;
+	if(!req || !m) return;
+	open = req->open_req_list;
+	while(open) {
+		if(open->mesh_state == m) {
+			struct tcp_req_open_item* next;
+			if(prev) prev->next = open->next;
+			else req->open_req_list = open->next;
+			/* caller has to manage the mesh state reply entry */
+			next = open->next;
+			free(open);
+			req->num_open_req --;
+
+			/* prev = prev; */
+			open = next;
+			continue;
+		}
+		prev = open;
+		open = open->next;
+	}
+}
+
+/** setup listening for read or write */
+static void
+tcp_req_info_setup_listen(struct tcp_req_info* req)
+{
+	int wr = 0;
+	int rd = 0;
+
+	if(req->cp->tcp_byte_count != 0) {
+		/* cannot change, halfway through */
+		return;
+	}
+
+	if(!req->cp->tcp_is_reading)
+		wr = 1;
+	if(req->num_open_req + req->num_done_req < TCP_MAX_REQ_SIMULTANEOUS &&
+		!req->read_is_closed)
+		rd = 1;
+	
+	if(wr) {
+		req->cp->tcp_is_reading = 0;
+		comm_point_start_listening(req->cp, -1,
+			req->cp->tcp_timeout_msec);
+	} else if(rd) {
+		req->cp->tcp_is_reading = 1;
+		comm_point_start_listening(req->cp, -1,
+			req->cp->tcp_timeout_msec);
+		/* and also read it (from SSL stack buffers), so
+		 * no event read event is expected since the remainder of
+		 * the TLS frame is sitting in the buffers. */
+		req->read_again = 1;
+	} else {
+		comm_point_start_listening(req->cp, -1,
+			req->cp->tcp_timeout_msec);
+		comm_point_listen_for_rw(req->cp, 0, 0);
+	}
+}
+
+/** remove first item from list of pending results */
+static struct tcp_req_done_item*
+tcp_req_info_pop_done(struct tcp_req_info* req)
+{
+	struct tcp_req_done_item* item;
+	log_assert(req->num_done_req > 0 && req->done_req_list);
+	item = req->done_req_list;
+	lock_basic_lock(&stream_wait_count_lock);
+	stream_wait_count -= (sizeof(struct tcp_req_done_item)+item->len);
+	lock_basic_unlock(&stream_wait_count_lock);
+	req->done_req_list = req->done_req_list->next;
+	req->num_done_req --;
+	return item;
+}
+
+/** Send given buffer and setup to write */
+static void
+tcp_req_info_start_write_buf(struct tcp_req_info* req, uint8_t* buf,
+	size_t len)
+{
+	sldns_buffer_clear(req->cp->buffer);
+	sldns_buffer_write(req->cp->buffer, buf, len);
+	sldns_buffer_flip(req->cp->buffer);
+
+	req->cp->tcp_is_reading = 0; /* we are now writing */
+}
+
+/** pick up the next result and start writing it to the channel */
+static void
+tcp_req_pickup_next_result(struct tcp_req_info* req)
+{
+	if(req->num_done_req > 0) {
+		/* unlist the done item from the list of pending results */
+		struct tcp_req_done_item* item = tcp_req_info_pop_done(req);
+		tcp_req_info_start_write_buf(req, item->buf, item->len);
+		free(item->buf);
+		free(item);
+	}
+}
+
+/** the read channel has closed */
+int
+tcp_req_info_handle_read_close(struct tcp_req_info* req)
+{
+	verbose(VERB_ALGO, "tcp channel read side closed %d", req->cp->fd);
+	/* reset byte count for (potential) partial read */
+	req->cp->tcp_byte_count = 0;
+	/* if we still have results to write, pick up next and write it */
+	if(req->num_done_req != 0) {
+		tcp_req_pickup_next_result(req);
+		tcp_req_info_setup_listen(req);
+		return 1;
+	}
+	/* if nothing to do, this closes the connection */
+	if(req->num_open_req == 0 && req->num_done_req == 0)
+		return 0;
+	/* otherwise, we must be waiting for dns resolve, wait with timeout */
+	req->read_is_closed = 1;
+	tcp_req_info_setup_listen(req);
+	return 1;
+}
+
+void
+tcp_req_info_handle_writedone(struct tcp_req_info* req)
+{
+	/* back to reading state, we finished this write event */
+	sldns_buffer_clear(req->cp->buffer);
+	if(req->num_done_req == 0 && req->read_is_closed) {
+		/* no more to write and nothing to read, close it */
+		comm_point_drop_reply(&req->cp->repinfo);
+		return;
+	}
+	req->cp->tcp_is_reading = 1;
+	/* see if another result needs writing */
+	tcp_req_pickup_next_result(req);
+
+	/* see if there is more to write, if not stop_listening for writing */
+	/* see if new requests are allowed, if so, start_listening
+	 * for reading */
+	tcp_req_info_setup_listen(req);
+}
+
+void
+tcp_req_info_handle_readdone(struct tcp_req_info* req)
+{
+	struct comm_point* c = req->cp;
+
+	/* we want to read up several requests, unless there are
+	 * pending answers */
+
+	req->is_drop = 0;
+	req->is_reply = 0;
+	req->in_worker_handle = 1;
+	/* handle the current request */
+	/* this calls the worker handle request routine that could give
+	 * a cache response, or localdata response, or drop the reply,
+	 * or schedule a mesh entry for later */
+	fptr_ok(fptr_whitelist_comm_point(c->callback));
+	if( (*c->callback)(c, c->cb_arg, NETEVENT_NOERROR, &c->repinfo) ) {
+		req->in_worker_handle = 0;
+		/* there is an answer, put it up.  It is already in the
+		 * c->buffer, just send it. */
+		/* since we were just reading a query, the channel is
+		 * clear to write to */
+	send_it:
+		c->tcp_is_reading = 0;
+		comm_point_start_listening(c, -1, c->tcp_timeout_msec);
+		return;
+	}
+	req->in_worker_handle = 0;
+	/* it should be waiting in the mesh for recursion.
+	 * If mesh failed to add a new entry and called commpoint_drop_reply. 
+	 * Then the mesh state has been cleared. */
+	if(req->is_drop) {
+		/* we can now call drop_reply without recursing into ourselves
+		 * whilst in the callback */
+		/* we have to close the stream because there is no reply,
+		 * no servfail to send, but the query needs an action, for
+		 * a stream that is close the connection */
+		sldns_buffer_clear(c->buffer);
+		comm_point_drop_reply(&c->repinfo);
+		return;
+	}
+	/* If mesh failed(mallocfail) and called commpoint_send_reply with
+	 * something like servfail then we pick up that reply below. */
+	if(req->is_reply) {
+		goto send_it;
+	}
+
+	sldns_buffer_clear(c->buffer);
+	/* if pending answers, pick up an answer and start sending it */
+	tcp_req_pickup_next_result(req);
+
+	/* if answers pending, start sending answers */
+	/* read more requests if we can have more requests */
+	tcp_req_info_setup_listen(req);
+}
+
+int
+tcp_req_info_add_meshstate(struct tcp_req_info* req,
+	struct mesh_area* mesh, struct mesh_state* m)
+{
+	struct tcp_req_open_item* item;
+	log_assert(req && mesh && m);
+	item = (struct tcp_req_open_item*)malloc(sizeof(*item));
+	if(!item) return 0;
+	item->next = req->open_req_list;
+	item->mesh = mesh;
+	item->mesh_state = m;
+	req->open_req_list = item;
+	req->num_open_req++;
+	return 1;
+}
+
+/** Add a result to the result list.  At the end. */
+static int
+tcp_req_info_add_result(struct tcp_req_info* req, uint8_t* buf, size_t len)
+{
+	struct tcp_req_done_item* last = NULL;
+	struct tcp_req_done_item* item;
+	size_t space;
+
+	/* see if we have space */
+	space = sizeof(struct tcp_req_done_item) + len;
+	lock_basic_lock(&stream_wait_count_lock);
+	if(stream_wait_count + space > stream_wait_max) {
+		lock_basic_unlock(&stream_wait_count_lock);
+		verbose(VERB_ALGO, "drop stream reply, no space left, in stream-wait-size");
+		return 0;
+	}
+	stream_wait_count += space;
+	lock_basic_unlock(&stream_wait_count_lock);
+
+	/* find last element */
+	last = req->done_req_list;
+	while(last && last->next)
+		last = last->next;
+	
+	/* create new element */
+	item = (struct tcp_req_done_item*)malloc(sizeof(*item));
+	if(!item) {
+		log_err("malloc failure, for stream result list");
+		return 0;
+	}
+	item->next = NULL;
+	item->len = len;
+	item->buf = memdup(buf, len);
+	if(!item->buf) {
+		free(item);
+		log_err("malloc failure, adding reply to stream result list");
+		return 0;
+	}
+
+	/* link in */
+	if(last) last->next = item;
+	else req->done_req_list = item;
+	req->num_done_req++;
+	return 1;
+}
+
+void
+tcp_req_info_send_reply(struct tcp_req_info* req)
+{
+	if(req->in_worker_handle) {
+		/* It is in the right buffer to answer straight away */
+		req->is_reply = 1;
+		return;
+	}
+	/* now that the query has been handled, that mesh_reply entry
+	 * should be removed, from the tcp_req_info list,
+	 * the mesh state cleanup removes then with region_cleanup and
+	 * replies_sent true. */
+	/* see if we can send it straight away (we are not doing
+	 * anything else).  If so, copy to buffer and start */
+	if(req->cp->tcp_is_reading && req->cp->tcp_byte_count == 0) {
+		/* buffer is free, and was ready to read new query into,
+		 * but we are now going to use it to send this answer */
+		tcp_req_info_start_write_buf(req,
+			sldns_buffer_begin(req->spool_buffer),
+			sldns_buffer_limit(req->spool_buffer));
+		/* switch to listen to write events */
+		comm_point_stop_listening(req->cp);
+		comm_point_start_listening(req->cp, -1,
+			req->cp->tcp_timeout_msec);
+		return;
+	}
+	/* queue up the answer behind the others already pending */
+	if(!tcp_req_info_add_result(req, sldns_buffer_begin(req->spool_buffer),
+		sldns_buffer_limit(req->spool_buffer))) {
+		/* drop the connection, we are out of resources */
+		comm_point_drop_reply(&req->cp->repinfo);
+	}
+}
+
+size_t tcp_req_info_get_stream_buffer_size(void)
+{
+	size_t s;
+	if(!stream_wait_lock_inited)
+		return stream_wait_count;
+	lock_basic_lock(&stream_wait_count_lock);
+	s = stream_wait_count;
+	lock_basic_unlock(&stream_wait_count_lock);
+	return s;
+}

services/listen_dnsport.h

@@ -237,4 +237,134 @@ int create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
  */
 int create_local_accept_sock(const char* path, int* noproto, int use_systemd);
 
+/**
+ * TCP request info.  List of requests outstanding on the channel, that
+ * are asked for but not yet answered back.
+ */
+struct tcp_req_info {
+	/** the TCP comm point for this.  Its buffer is used for read/write */
+	struct comm_point* cp;
+	/** the buffer to use to spool reply from mesh into,
+	 * it can then be copied to the result list and written.
+	 * it is a pointer to the shared udp buffer. */
+	struct sldns_buffer* spool_buffer;
+	/** are we in worker_handle function call (for recursion callback)*/
+	int in_worker_handle;
+	/** is the comm point dropped (by worker handle).
+	 * That means we have to disconnect the channel. */
+	int is_drop;
+	/** is the comm point set to send_reply (by mesh new client in worker
+	 * handle), if so answer is available in c.buffer */
+	int is_reply;
+	/** read channel has closed, just write pending results */
+	int read_is_closed;
+	/** read again */
+	int read_again;
+	/** number of outstanding requests */
+	int num_open_req;
+	/** list of outstanding requests */
+	struct tcp_req_open_item* open_req_list;
+	/** number of pending writeable results */
+	int num_done_req;
+	/** list of pending writable result packets, malloced one at a time */
+	struct tcp_req_done_item* done_req_list;
+};
+
+/**
+ * List of open items in TCP channel
+ */
+struct tcp_req_open_item {
+	/** next in list */
+	struct tcp_req_open_item* next;
+	/** the mesh area of the mesh_state */
+	struct mesh_area* mesh;
+	/** the mesh state */
+	struct mesh_state* mesh_state;
+};
+
+/**
+ * List of done items in TCP channel
+ */
+struct tcp_req_done_item {
+	/** next in list */
+	struct tcp_req_done_item* next;
+	/** the buffer with packet contents */
+	uint8_t* buf;
+	/** length of the buffer */
+	size_t len;
+};
+
+/**
+ * Create tcp request info structure that keeps track of open
+ * requests on the TCP channel that are resolved at the same time,
+ * and the pending results that have to get written back to that client.
+ * @param spoolbuf: shared buffer
+ * @return new structure or NULL on alloc failure.
+ */
+struct tcp_req_info* tcp_req_info_create(struct sldns_buffer* spoolbuf);
+
+/**
+ * Delete tcp request structure.  Called by owning commpoint.
+ * Removes mesh entry references and stored results from the lists.
+ * @param req: the tcp request info
+ */
+void tcp_req_info_delete(struct tcp_req_info* req);
+
+/**
+ * Clear tcp request structure.  Removes list entries, sets it up ready
+ * for the next connection.
+ * @param req: tcp request info structure.
+ */
+void tcp_req_info_clear(struct tcp_req_info* req);
+
+/**
+ * Remove mesh state entry from list in tcp_req_info.
+ * caller has to manage the mesh state reply entry in the mesh state.
+ * @param req: the tcp req info that has the entry removed from the list.
+ * @param m: the state removed from the list.
+ */
+void tcp_req_info_remove_mesh_state(struct tcp_req_info* req,
+	struct mesh_state* m);
+
+/**
+ * Handle write done of the last result packet
+ * @param req: the tcp req info.
+ */
+void tcp_req_info_handle_writedone(struct tcp_req_info* req);
+
+/**
+ * Handle read done of a new request from the client
+ * @param req: the tcp req info.
+ */
+void tcp_req_info_handle_readdone(struct tcp_req_info* req);
+
+/**
+ * Add mesh state to the tcp req list of open requests.
+ * So the comm_reply can be removed off the mesh reply list when
+ * the tcp channel has to be closed (for other reasons then that that
+ * request was done, eg. channel closed by client or some format error).
+ * @param req: tcp req info structure.  It keeps track of the simultaneous
+ * 	requests and results on a tcp (or TLS) channel.
+ * @param mesh: mesh area for the state.
+ * @param m: mesh state to add.
+ * @return 0 on failure (malloc failure).
+ */
+int tcp_req_info_add_meshstate(struct tcp_req_info* req,
+	struct mesh_area* mesh, struct mesh_state* m);
+
+/**
+ * Send reply on tcp simultaneous answer channel.  May queue it up.
+ * @param req: request info structure.
+ */
+void tcp_req_info_send_reply(struct tcp_req_info* req);
+
+/** the read channel has closed
+ * @param req: request. remaining queries are looked up and answered. 
+ * @return zero if nothing to do, just close the tcp.
+ */
+int tcp_req_info_handle_read_close(struct tcp_req_info* req);
+
+/** get the size of currently used tcp stream wait buffers (in bytes) */
+size_t tcp_req_info_get_stream_buffer_size(void);
+
 #endif /* LISTEN_DNSPORT_H */

services/mesh.c

@@ -61,6 +61,7 @@
 #include "services/localzone.h"
 #include "util/data/dname.h"
 #include "respip/respip.h"
+#include "services/listen_dnsport.h"
 
 /** subtract timers and the values do not overflow or become negative */
 static void
@@ -429,6 +430,7 @@ void mesh_new_client(struct mesh_area* mesh, struct query_info* qinfo,
 	/* add reply to s */
 	if(!mesh_state_add_reply(s, edns, rep, qid, qflags, qinfo)) {
 			log_err("mesh_new_client: out of memory; SERVFAIL");
+		servfail_mem:
 			if(!inplace_cb_reply_servfail_call(mesh->env, qinfo, &s->s,
 				NULL, LDNS_RCODE_SERVFAIL, edns, rep, mesh->env->scratch))
 					edns->opt_list = NULL;
@@ -439,6 +441,12 @@ void mesh_new_client(struct mesh_area* mesh, struct query_info* qinfo,
 				mesh_state_delete(&s->s);
 			return;
 	}
+	if(rep->c->tcp_req_info) {
+		if(!tcp_req_info_add_meshstate(rep->c->tcp_req_info, mesh, s)) {
+			log_err("mesh_new_client: out of memory add tcpreqinfo");
+			goto servfail_mem;
+		}
+	}
 	/* update statistics */
 	if(was_detached) {
 		log_assert(mesh->num_detached_states > 0);
@@ -732,9 +740,13 @@ mesh_state_cleanup(struct mesh_state* mstate)
 	mesh = mstate->s.env->mesh;
 	/* drop unsent replies */
 	if(!mstate->replies_sent) {
-		struct mesh_reply* rep;
+		struct mesh_reply* rep = mstate->reply_list;
 		struct mesh_cb* cb;
-		for(rep=mstate->reply_list; rep; rep=rep->next) {
+		/* in tcp_req_info, the mstates linked are removed, but
+		 * the reply_list is now NULL, so the remove-from-empty-list
+		 * takes no time and also it does not do the mesh accounting */
+		mstate->reply_list = NULL;
+		for(; rep; rep=rep->next) {
 			comm_point_drop_reply(&rep->query_reply);
 			mesh->num_reply_addrs--;
 		}
@@ -1031,11 +1043,14 @@ mesh_do_callback(struct mesh_state* m, int rcode, struct reply_info* rep,
  * @param rcode: if not 0, error code.
  * @param rep: reply to send (or NULL if rcode is set).
  * @param r: reply entry
+ * @param r_buffer: buffer to use for reply entry.
  * @param prev: previous reply, already has its answer encoded in buffer.
+ * @param prev_buffer: buffer for previous reply.
  */
 static void
 mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
-	struct mesh_reply* r, struct mesh_reply* prev)
+	struct mesh_reply* r, struct sldns_buffer* r_buffer,
+	struct mesh_reply* prev, struct sldns_buffer* prev_buffer)
 {
 	struct timeval end_time;
 	struct timeval duration;
@@ -1063,7 +1078,7 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 	 * and still reuse the previous answer if they are the same, but that
 	 * would be complicated and error prone for the relatively minor case.
 	 * So we err on the side of safety. */
-	if(prev && prev->qflags == r->qflags && 
+	if(prev && prev_buffer && prev->qflags == r->qflags && 
 		!prev->local_alias && !r->local_alias &&
 		prev->edns.edns_present == r->edns.edns_present && 
 		prev->edns.bits == r->edns.bits && 
@@ -1071,13 +1086,11 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 		edns_opt_list_compare(prev->edns.opt_list, r->edns.opt_list)
 		== 0) {
 		/* if the previous reply is identical to this one, fix ID */
-		if(prev->query_reply.c->buffer != r->query_reply.c->buffer)
-			sldns_buffer_copy(r->query_reply.c->buffer, 
-				prev->query_reply.c->buffer);
-		sldns_buffer_write_at(r->query_reply.c->buffer, 0, 
-			&r->qid, sizeof(uint16_t));
-		sldns_buffer_write_at(r->query_reply.c->buffer, 12, 
-			r->qname, m->s.qinfo.qname_len);
+		if(prev_buffer != r_buffer)
+			sldns_buffer_copy(r_buffer, prev_buffer);
+		sldns_buffer_write_at(r_buffer, 0, &r->qid, sizeof(uint16_t));
+		sldns_buffer_write_at(r_buffer, 12, r->qname,
+			m->s.qinfo.qname_len);
 		comm_point_send_reply(&r->query_reply);
 	} else if(rcode) {
 		m->s.qinfo.qname = r->qname;
@@ -1091,8 +1104,8 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 				&r->edns, NULL, m->s.region))
 					r->edns.opt_list = NULL;
 		}
-		error_encode(r->query_reply.c->buffer, rcode, &m->s.qinfo,
-			r->qid, r->qflags, &r->edns);
+		error_encode(r_buffer, rcode, &m->s.qinfo, r->qid,
+			r->qflags, &r->edns);
 		comm_point_send_reply(&r->query_reply);
 	} else {
 		size_t udp_size = r->edns.udp_size;
@@ -1108,16 +1121,15 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 				m->s.env->cfg, r->query_reply.c,
 				m->s.region) ||
 			!reply_info_answer_encode(&m->s.qinfo, rep, r->qid, 
-			r->qflags, r->query_reply.c->buffer, 0, 1, 
-			m->s.env->scratch, udp_size, &r->edns, 
-			(int)(r->edns.bits & EDNS_DO), secure)) 
+			r->qflags, r_buffer, 0, 1, m->s.env->scratch,
+			udp_size, &r->edns, (int)(r->edns.bits & EDNS_DO),
+			secure)) 
 		{
 			if(!inplace_cb_reply_servfail_call(m->s.env, &m->s.qinfo, &m->s,
 			rep, LDNS_RCODE_SERVFAIL, &r->edns, NULL, m->s.region))
 				r->edns.opt_list = NULL;
-			error_encode(r->query_reply.c->buffer, 
-				LDNS_RCODE_SERVFAIL, &m->s.qinfo, r->qid, 
-				r->qflags, &r->edns);
+			error_encode(r_buffer, LDNS_RCODE_SERVFAIL,
+				&m->s.qinfo, r->qid, r->qflags, &r->edns);
 		}
 		r->edns = edns_bak;
 		comm_point_send_reply(&r->query_reply);
@@ -1132,19 +1144,17 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 	timeval_add(&m->s.env->mesh->replies_sum_wait, &duration);
 	timehist_insert(m->s.env->mesh->histogram, &duration);
 	if(m->s.env->cfg->stat_extended) {
-		uint16_t rc = FLAGS_GET_RCODE(sldns_buffer_read_u16_at(r->
-			query_reply.c->buffer, 2));
+		uint16_t rc = FLAGS_GET_RCODE(sldns_buffer_read_u16_at(
+			r_buffer, 2));
 		if(secure) m->s.env->mesh->ans_secure++;
 		m->s.env->mesh->ans_rcode[ rc ] ++;
-		if(rc == 0 && LDNS_ANCOUNT(sldns_buffer_begin(r->
-			query_reply.c->buffer)) == 0)
+		if(rc == 0 && LDNS_ANCOUNT(sldns_buffer_begin(r_buffer)) == 0)
 			m->s.env->mesh->ans_nodata++;
 	}
 	/* Log reply sent */
 	if(m->s.env->cfg->log_replies) {
 		log_reply_info(0, &m->s.qinfo, &r->query_reply.addr,
-			r->query_reply.addrlen, duration, 0,
-			r->query_reply.c->buffer);
+			r->query_reply.addrlen, duration, 0, r_buffer);
 	}
 }
 
@@ -1152,6 +1162,7 @@ void mesh_query_done(struct mesh_state* mstate)
 {
 	struct mesh_reply* r;
 	struct mesh_reply* prev = NULL;
+	struct sldns_buffer* prev_buffer = NULL;
 	struct mesh_cb* c;
 	struct reply_info* rep = (mstate->s.return_msg?
 		mstate->s.return_msg->rep:NULL);
@@ -1180,9 +1191,15 @@ void mesh_query_done(struct mesh_state* mstate)
 		if(mstate->s.is_drop)
 			comm_point_drop_reply(&r->query_reply);
 		else {
+			struct sldns_buffer* r_buffer = r->query_reply.c->buffer;
+			if(r->query_reply.c->tcp_req_info)
+				r_buffer = r->query_reply.c->tcp_req_info->spool_buffer;
 			mesh_send_reply(mstate, mstate->s.return_rcode, rep,
-				r, prev);
+				r, r_buffer, prev, prev_buffer);
+			if(r->query_reply.c->tcp_req_info)
+				tcp_req_info_remove_mesh_state(r->query_reply.c->tcp_req_info, mstate);
 			prev = r;
+			prev_buffer = r_buffer;
 		}
 	}
 	mstate->replies_sent = 1;
@@ -1392,7 +1409,7 @@ mesh_continue(struct mesh_area* mesh, struct mesh_state* mstate,
 		/* module is looping. Stop it. */
 		log_err("internal error: looping module (%s) stopped",
 			mesh->mods.mod[mstate->s.curmod]->name);
-		log_query_info(VERB_QUERY, "pass error for qstate",
+		log_query_info(0, "pass error for qstate",
 			&mstate->s.qinfo);
 		s = module_error;
 	}
@@ -1613,3 +1630,38 @@ void mesh_list_remove(struct mesh_state* m, struct mesh_state** fp,
 		m->prev->next = m->next;
 	else	*fp = m->next;
 }
+
+void mesh_state_remove_reply(struct mesh_area* mesh, struct mesh_state* m,
+	struct comm_point* cp)
+{
+	struct mesh_reply* n, *prev = NULL;
+	n = m->reply_list;
+	/* when in mesh_cleanup, it sets the reply_list to NULL, so that
+	 * there is no accounting twice */
+	if(!n) return; /* nothing to remove, also no accounting needed */
+	while(n) {
+		if(n->query_reply.c == cp) {
+			/* unlink it */
+			if(prev) prev->next = n->next;
+			else m->reply_list = n->next;
+			/* delete it, but allocated in m region */
+			mesh->num_reply_addrs--;
+
+			/* prev = prev; */
+			n = n->next;
+			continue;
+		}
+		prev = n;
+		n = n->next;
+	}
+	/* it was not detached (because it had a reply list), could be now */
+	if(!m->reply_list && !m->cb_list
+		&& m->super_set.count == 0) {
+		mesh->num_detached_states++;
+	}
+	/* if not replies any more in mstate, it is no longer a reply_state */
+	if(!m->reply_list && !m->cb_list) {
+		log_assert(mesh->num_reply_states > 0);
+		mesh->num_reply_states--;
+	}
+}

services/mesh.h

@@ -65,7 +65,7 @@ struct respip_client_info;
  * Maximum number of mesh state activations. Any more is likely an
  * infinite loop in the module. It is then terminated.
  */
-#define MESH_MAX_ACTIVATION 3000
+#define MESH_MAX_ACTIVATION 10000
 
 /**
  * Max number of references-to-references-to-references.. search size.
@@ -633,4 +633,14 @@ void mesh_list_insert(struct mesh_state* m, struct mesh_state** fp,
 void mesh_list_remove(struct mesh_state* m, struct mesh_state** fp,
 	struct mesh_state** lp);
 
+/**
+ * Remove mesh reply entry from the reply entry list.  Searches for
+ * the comm_point pointer.
+ * @param mesh: to update the counters.
+ * @param m: the mesh state.
+ * @param cp: the comm_point to remove from the list.
+ */
+void mesh_state_remove_reply(struct mesh_area* mesh, struct mesh_state* m,
+	struct comm_point* cp);
+
 #endif /* SERVICES_MESH_H */

services/outside_network.c

@@ -63,6 +63,9 @@
 #ifdef HAVE_OPENSSL_SSL_H
 #include <openssl/ssl.h>
 #endif
+#ifdef HAVE_X509_VERIFY_PARAM_SET1_HOST
+#include <openssl/x509v3.h>
+#endif
 
 #ifdef HAVE_NETDB_H
 #include <netdb.h>
@@ -365,6 +368,11 @@ outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
 		comm_point_tcp_win_bio_cb(pend->c, pend->c->ssl);
 #endif
 		pend->c->ssl_shake_state = comm_ssl_shake_write;
+		if(w->tls_auth_name) {
+#ifdef HAVE_SSL
+			(void)SSL_set_tlsext_host_name(pend->c->ssl, w->tls_auth_name);
+#endif
+		}
 #ifdef HAVE_SSL_SET1_HOST
 		if(w->tls_auth_name) {
 			SSL_set_verify(pend->c->ssl, SSL_VERIFY_PEER, NULL);
@@ -374,10 +382,28 @@ outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
                         if(!SSL_set1_host(pend->c->ssl, w->tls_auth_name)) {
                                 log_err("SSL_set1_host failed");
 				pend->c->fd = s;
+				SSL_free(pend->c->ssl);
+				pend->c->ssl = NULL;
 				comm_point_close(pend->c);
 				return 0;
 			}
 		}
+#elif defined(HAVE_X509_VERIFY_PARAM_SET1_HOST)
+		/* openssl 1.0.2 has this function that can be used for
+		 * set1_host like verification */
+		if(w->tls_auth_name) {
+			X509_VERIFY_PARAM* param = SSL_get0_param(pend->c->ssl);
+			X509_VERIFY_PARAM_set_hostflags(param, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS);
+			if(!X509_VERIFY_PARAM_set1_host(param, w->tls_auth_name, strlen(w->tls_auth_name))) {
+				log_err("X509_VERIFY_PARAM_set1_host failed");
+				pend->c->fd = s;
+				SSL_free(pend->c->ssl);
+				pend->c->ssl = NULL;
+				comm_point_close(pend->c);
+				return 0;
+			}
+			SSL_set_verify(pend->c->ssl, SSL_VERIFY_PEER, NULL);
+		}
 #endif /* HAVE_SSL_SET1_HOST */
 	}
 	w->pkt = NULL;
@@ -752,7 +778,7 @@ outside_network_create(struct comm_base *base, size_t bufsize,
 		outnet->delay_tv.tv_usec = (delayclose%1000)*1000;
 	}
 #endif
-	if(numavailports == 0) {
+	if(numavailports == 0 || num_ports == 0) {
 		log_err("no outgoing ports available");
 		outside_network_delete(outnet);
 		return NULL;
@@ -1261,6 +1287,13 @@ outnet_tcptimer(void* arg)
 	} else {
 		/* it was in use */
 		struct pending_tcp* pend=(struct pending_tcp*)w->next_waiting;
+		if(pend->c->ssl) {
+#ifdef HAVE_SSL
+			SSL_shutdown(pend->c->ssl);
+			SSL_free(pend->c->ssl);
+			pend->c->ssl = NULL;
+#endif
+		}
 		comm_point_close(pend->c);
 		pend->query = NULL;
 		pend->next_free = outnet->tcp_free;
@@ -1473,7 +1506,6 @@ serviced_delete(struct serviced_query* sq)
 		/* clear up the pending query */
 		if(sq->status == serviced_query_UDP_EDNS ||
 			sq->status == serviced_query_UDP ||
-			sq->status == serviced_query_PROBE_EDNS ||
 			sq->status == serviced_query_UDP_EDNS_FRAG ||
 			sq->status == serviced_query_UDP_EDNS_fallback) {
 			struct pending* p = (struct pending*)sq->pending;
@@ -1600,15 +1632,7 @@ serviced_udp_send(struct serviced_query* sq, sldns_buffer* buff)
 	sq->last_rtt = rtt;
 	verbose(VERB_ALGO, "EDNS lookup known=%d vs=%d", edns_lame_known, vs);
 	if(sq->status == serviced_initial) {
-		if(edns_lame_known == 0 && rtt > 5000 && rtt < 10001) {
-			/* perform EDNS lame probe - check if server is
-			 * EDNS lame (EDNS queries to it are dropped) */
-			verbose(VERB_ALGO, "serviced query: send probe to see "
-				" if use of EDNS causes timeouts");
-			/* even 700 msec may be too small */
-			rtt = 1000;
-			sq->status = serviced_query_PROBE_EDNS;
-		} else if(vs != -1) {
+		if(vs != -1) {
 			sq->status = serviced_query_UDP_EDNS;
 		} else { 	
 			sq->status = serviced_query_UDP; 
@@ -1862,7 +1886,7 @@ serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff)
 	if(!sq->pending) {
 		/* delete from tree so that a retry by above layer does not
 		 * clash with this entry */
-		log_err("serviced_tcp_initiate: failed to send tcp query");
+		verbose(VERB_ALGO, "serviced_tcp_initiate: failed to send tcp query");
 		serviced_callbacks(sq, NETEVENT_CLOSED, NULL, NULL);
 	}
 }
@@ -1945,12 +1969,6 @@ serviced_udp_callback(struct comm_point* c, void* arg, int error,
 	sq->pending = NULL; /* removed after callback */
 	if(error == NETEVENT_TIMEOUT) {
 		int rto = 0;
-		if(sq->status == serviced_query_PROBE_EDNS) {
-			/* non-EDNS probe failed; we do not know its status,
-			 * keep trying with EDNS, timeout may not be caused
-			 * by EDNS. */
-			sq->status = serviced_query_UDP_EDNS;
-		}
 		if(sq->status == serviced_query_UDP_EDNS && sq->last_rtt < 5000) {
 			/* fallback to 1480/1280 */
 			sq->status = serviced_query_UDP_EDNS_FRAG;
@@ -1979,7 +1997,7 @@ serviced_udp_callback(struct comm_point* c, void* arg, int error,
 			return 0;
 		}
 		if(rto >= RTT_MAX_TIMEOUT) {
-			fallback_tcp = 1;
+			/* fallback_tcp = 1; */
 			/* UDP does not work, fallback to TCP below */
 		} else {
 			serviced_callbacks(sq, NETEVENT_TIMEOUT, c, rep);
@@ -2014,18 +2032,6 @@ serviced_udp_callback(struct comm_point* c, void* arg, int error,
 			serviced_callbacks(sq, NETEVENT_CLOSED, c, rep);
 		}
 		return 0;
-	    } else if(sq->status == serviced_query_PROBE_EDNS) {
-		/* probe without EDNS succeeds, so we conclude that this
-		 * host likely has EDNS packets dropped */
-		log_addr(VERB_DETAIL, "timeouts, concluded that connection to "
-			"host drops EDNS packets", &sq->addr, sq->addrlen);
-		/* only store noEDNS in cache if domain is noDNSSEC */
-		if(!sq->want_dnssec)
-		  if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
-			sq->zone, sq->zonelen, -1, (time_t)now.tv_sec)) {
-			log_err("Out of memory caching no edns for host");
-		  }
-		sq->status = serviced_query_UDP;
 	    } else if(sq->status == serviced_query_UDP_EDNS && 
 		!sq->edns_lame_known) {
 		/* now we know that edns queries received answers store that */
@@ -2389,6 +2395,18 @@ outnet_comm_point_for_http(struct outside_network* outnet,
 				return NULL;
 			}
 		}
+#elif defined(HAVE_X509_VERIFY_PARAM_SET1_HOST)
+		/* openssl 1.0.2 has this function that can be used for
+		 * set1_host like verification */
+		if((SSL_CTX_get_verify_mode(outnet->sslctx)&SSL_VERIFY_PEER)) {
+			X509_VERIFY_PARAM* param = SSL_get0_param(cp->ssl);
+			X509_VERIFY_PARAM_set_hostflags(param, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS);
+			if(!X509_VERIFY_PARAM_set1_host(param, host, strlen(host))) {
+				log_err("X509_VERIFY_PARAM_set1_host failed");
+				comm_point_delete(cp);
+				return NULL;
+			}
+		}
 #endif /* HAVE_SSL_SET1_HOST */
 	}
 
@@ -2494,7 +2512,6 @@ serviced_get_mem(struct serviced_query* sq)
 		s += sizeof(*sb);
 	if(sq->status == serviced_query_UDP_EDNS ||
 		sq->status == serviced_query_UDP ||
-		sq->status == serviced_query_PROBE_EDNS ||
 		sq->status == serviced_query_UDP_EDNS_FRAG ||
 		sq->status == serviced_query_UDP_EDNS_fallback) {
 		s += sizeof(struct pending);

services/outside_network.h

@@ -359,8 +359,6 @@ struct serviced_query {
 		serviced_query_TCP_EDNS,
 		/** TCP without EDNS sent */
 		serviced_query_TCP,
-		/** probe to test EDNS lameness (EDNS is dropped) */
-		serviced_query_PROBE_EDNS,
 		/** probe to test noEDNS0 (EDNS gives FORMERRorNOTIMP) */
 		serviced_query_UDP_EDNS_fallback,
 		/** probe to test TCP noEDNS0 (EDNS gives FORMERRorNOTIMP) */

sldns/sbuffer.h

@@ -130,7 +130,7 @@ struct sldns_buffer
 	/** If the buffer is fixed it cannot be resized */
 	unsigned _fixed : 1;
 
-	/** If the buffer is vfixed, no more than capacity bytes willl be
+	/** If the buffer is vfixed, no more than capacity bytes will be
 	 * written to _data, however the _position counter will be updated
 	 * with the amount that would have been written in consecutive
 	 * writes.  This allows for a modus operandi in which a sequence is
@@ -160,7 +160,7 @@ sldns_buffer_invariant(sldns_buffer *buffer)
 	assert(buffer != NULL);
 	assert(buffer->_position <= buffer->_limit || buffer->_vfixed);
 	assert(buffer->_limit <= buffer->_capacity);
-	assert(buffer->_data != NULL || (buffer->_vfixed && buffer->_capacity == 0));
+	assert(buffer->_data != NULL || (buffer->_vfixed && buffer->_capacity == 0 && buffer->_limit == 0));
 }
 #endif
 

sldns/wire2str.c

@@ -1908,8 +1908,8 @@ int sldns_wire2str_edns_subnet_print(char** s, size_t* sl, uint8_t* data,
 	return w;
 }
 
-int sldns_wire2str_edns_keepalive_print(char** s, size_t* sl, uint8_t* data,
-	size_t len)
+static int sldns_wire2str_edns_keepalive_print(char** s, size_t* sl,
+	uint8_t* data, size_t len)
 {
 	int w = 0;
 	uint16_t timeout;

smallapp/unbound-anchor.c

@@ -1097,11 +1097,14 @@ read_http_result(SSL* ssl)
 	}
 	if(!data) return NULL;
 	if(verb >= 4) print_data("read data", data, (int)len);
-	m = BIO_new_mem_buf(data, (int)len);
+	m = BIO_new(BIO_s_mem());
 	if(!m) {
 		if(verb) printf("out of memory\n");
+		free(data);
 		exit(0);
 	}
+	BIO_write(m, data, (int)len);
+	free(data);
 	return m;
 }
 
@@ -1168,17 +1171,6 @@ https(struct ip_list* ip_list, const char* pathname, const char* urlname)
 	return bio;
 }
 
-/** free up a downloaded file BIO */
-static void
-free_file_bio(BIO* bio)
-{
-	char* pp = NULL;
-	(void)BIO_reset(bio);
-	(void)BIO_get_mem_data(bio, &pp);
-	free(pp);
-	BIO_free(bio);
-}
-
 /** XML parse private data during the parse */
 struct xml_data {
 	/** the parser, reference */
@@ -1603,7 +1595,7 @@ xml_parse(BIO* xml, time_t now)
 	xml_parse_setup(parser, &data, now);
 
 	/* parse it */
-	(void)BIO_reset(xml);
+	(void)BIO_seek(xml, 0);
 	len = (int)BIO_get_mem_data(xml, &pp);
 	if(!len || !pp) {
 		if(verb) printf("out of memory\n");
@@ -1777,8 +1769,8 @@ verify_p7sig(BIO* data, BIO* p7s, STACK_OF(X509)* trust, const char* p7signer)
 	X509_VERIFY_PARAM_free(param);
 #endif
 
-	(void)BIO_reset(p7s);
-	(void)BIO_reset(data);
+	(void)BIO_seek(p7s, 0);
+	(void)BIO_seek(data, 0);
 
 	/* convert p7s to p7 (the signature) */
 	p7 = d2i_PKCS7_bio(p7s, NULL);
@@ -1956,8 +1948,8 @@ do_certupdate(const char* root_anchor_file, const char* root_cert_file,
 	if(verb) printf("success: the anchor has been updated "
 			"using the cert\n");
 
-	free_file_bio(xml);
-	free_file_bio(p7s);
+	BIO_free(xml);
+	BIO_free(p7s);
 #ifndef S_SPLINT_S
 	sk_X509_pop_free(cert, X509_free);
 #endif
@@ -2349,7 +2341,7 @@ int main(int argc, char* argv[])
 		}
 	}
 	argc -= optind;
-	argv += optind;
+	/* argv += optind; not using further arguments */
 	if(argc != 0)
 		usage();
 

smallapp/unbound-checkconf.c

@@ -431,7 +431,7 @@ check_modules_exist(const char* module_conf)
 
 /** check configuration for errors */
 static void
-morechecks(struct config_file* cfg, const char* fname)
+morechecks(struct config_file* cfg)
 {
 	warn_hosts("stub-host", cfg->stubs);
 	warn_hosts("forward-host", cfg->forwards);
@@ -463,19 +463,6 @@ morechecks(struct config_file* cfg, const char* fname)
 		!is_dir(cfg->chrootdir)) {
 		fatal_exit("bad chroot directory");
 	}
-	if(cfg->chrootdir && cfg->chrootdir[0]) {
-		char buf[10240];
-		buf[0] = 0;
-		if(fname[0] != '/') {
-			if(getcwd(buf, sizeof(buf)) == NULL)
-				fatal_exit("getcwd: %s", strerror(errno));
-			(void)strlcat(buf, "/", sizeof(buf));
-		}
-		(void)strlcat(buf, fname, sizeof(buf));
-		if(strncmp(buf, cfg->chrootdir, strlen(cfg->chrootdir)) != 0)
-			fatal_exit("config file %s is not inside chroot %s",
-				buf, cfg->chrootdir);
-	}
 	if(cfg->directory && cfg->directory[0]) {
 		char* ad = fname_after_chroot(cfg->directory, cfg, 0);
 		if(!ad) fatal_exit("out of memory");
@@ -680,7 +667,7 @@ checkconf(const char* cfgfile, const char* opt, int final)
 		config_delete(cfg);
 		return;
 	}
-	morechecks(cfg, cfgfile);
+	morechecks(cfg);
 	check_mod(cfg, iter_get_funcblock());
 	check_mod(cfg, val_get_funcblock());
 #ifdef WITH_PYTHONMODULE

smallapp/unbound-control-setup.sh.in

@@ -148,8 +148,8 @@ test -f $CTL_BASE.pem || error "could not create $CTL_BASE.pem"
 # echo "empty password is used, simply click OK on the password dialog box."
 # openssl pkcs12 -export -in $CTL_BASE"_trust.pem" -inkey $CTL_BASE.key -name "unbound remote control client cert" -out $CTL_BASE"_browser.pfx" -password "pass:" || error "could not create browser certificate"
 
-# remove unused permissions
-chmod o-rw $SVR_BASE.pem $SVR_BASE.key $CTL_BASE.pem $CTL_BASE.key
+# set desired permissions
+chmod 0640 $SVR_BASE.pem $SVR_BASE.key $CTL_BASE.pem $CTL_BASE.key
 
 # remove crap
 rm -f request.cfg

smallapp/unbound-control.c

@@ -73,6 +73,10 @@
 #include <sys/un.h>
 #endif
 
+static void usage(void) ATTR_NORETURN;
+static void ssl_err(const char* s) ATTR_NORETURN;
+static void ssl_path_err(const char* s, const char *path) ATTR_NORETURN;
+
 /** Give unbound-control usage, and exit (1). */
 static void
 usage(void)
@@ -150,6 +154,8 @@ usage(void)
 	printf("  view_local_zone view name type  	add local-zone in view\n");
 	printf("  view_local_zone_remove view name  	remove local-zone in view\n");
 	printf("  view_local_data view RR...		add local-data in view\n");
+	printf("  view_local_datas view 		add list of local-data to view\n");
+	printf("  					one entry per line read from stdin\n");
 	printf("  view_local_data_remove view name  	remove local-data in view\n");
 	printf("Version %s\n", PACKAGE_VERSION);
 	printf("BSD licensed, see LICENSE in source package for details.\n");
@@ -241,7 +247,8 @@ static void print_uptime(struct ub_shm_stat_info* shm_stat)
 }
 
 /** print memory usage */
-static void print_mem(struct ub_shm_stat_info* shm_stat)
+static void print_mem(struct ub_shm_stat_info* shm_stat,
+	struct ub_stats_info* s)
 {
 	PR_LL("mem.cache.rrset", shm_stat->mem.rrset);
 	PR_LL("mem.cache.message", shm_stat->mem.msg);
@@ -260,6 +267,7 @@ static void print_mem(struct ub_shm_stat_info* shm_stat)
 	PR_LL("mem.cache.dnscrypt_nonce",
 		shm_stat->mem.dnscrypt_nonce);
 #endif
+	PR_LL("mem.streamwait", s->svr.mem_stream_wait);
 }
 
 /** print histogram */
@@ -322,6 +330,7 @@ static void print_extended(struct ub_stats_info* s)
 	PR_UL("num.query.tcp", s->svr.qtcp);
 	PR_UL("num.query.tcpout", s->svr.qtcp_outgoing);
 	PR_UL("num.query.tls", s->svr.qtls);
+	PR_UL("num.query.tls_resume", s->svr.qtls_resume);
 	PR_UL("num.query.ipv6", s->svr.qipv6);
 
 	/* flags */
@@ -393,7 +402,7 @@ static void do_stats_shm(struct config_file* cfg, struct ub_stats_info* stats,
 	pr_stats("total", &stats[0]);
 	print_uptime(shm_stat);
 	if(cfg->stat_extended) {
-		print_mem(shm_stat);
+		print_mem(shm_stat, &stats[0]);
 		print_hist(stats);
 		print_extended(stats);
 	}
@@ -713,9 +722,10 @@ go_cmd(SSL* ssl, int fd, int quiet, int argc, char* argv[])
 	if(argc == 1 && strcmp(argv[0], "load_cache") == 0) {
 		send_file(ssl, fd, stdin, buf, sizeof(buf));
 	}
-	else if(argc == 1 && (strcmp(argv[0], "local_zones") == 0 ||
+	else if(argc >= 1 && (strcmp(argv[0], "local_zones") == 0 ||
 		strcmp(argv[0], "local_zones_remove") == 0 ||
 		strcmp(argv[0], "local_datas") == 0 ||
+		strcmp(argv[0], "view_local_datas") == 0 ||
 		strcmp(argv[0], "local_datas_remove") == 0)) {
 		send_file(ssl, fd, stdin, buf, sizeof(buf));
 		send_eof(ssl, fd);

testcode/asynclook.c

@@ -182,6 +182,8 @@ struct ext_thr_info {
 	char** argv;
 	/** number of queries to do */
 	int numq;
+	/** list of ids to free once threads are done */
+	struct track_id* id_list;
 };
 
 /** if true, we are testing against 'localhost' and extra checking is done */
@@ -309,6 +311,7 @@ ext_thread(void* arg)
 		for(i=0; i<inf->numq; i++) {
 			lock_basic_init(&async_ids[i].lock);
 		}
+		inf->id_list = async_ids;
 	}
 	for(i=0; i<inf->numq; i++) {
 		if(async_ids) {
@@ -347,14 +350,6 @@ ext_thread(void* arg)
 	/* if these locks are destroyed, or if the async_ids is freed, then
 	   a use-after-free happens in another thread.
 	   The allocation is only part of this test, though. */
-	/*
-	if(async_ids) {
-		for(i=0; i<inf->numq; i++) {
-			lock_basic_destroy(&async_ids[i].lock);
-		}
-	}
-	free(async_ids);
-	*/
 	
 	return NULL;
 }
@@ -375,6 +370,7 @@ ext_test(struct ub_ctx* ctx, int argc, char** argv)
 		inf[i].argc = argc;
 		inf[i].argv = argv;
 		inf[i].numq = 100;
+		inf[i].id_list = NULL;
 		ub_thread_create(&inf[i].tid, ext_thread, &inf[i]);
 	}
 	/* the work happens here */
@@ -382,6 +378,16 @@ ext_test(struct ub_ctx* ctx, int argc, char** argv)
 		ub_thread_join(inf[i].tid);
 	}
 	printf("extended test end\n");
+	/* free the id lists */
+	for(i=0; i<NUMTHR; i++) {
+		if(inf[i].id_list) {
+			int j;
+			for(j=0; j<inf[i].numq; j++) {
+				lock_basic_destroy(&inf[i].id_list[j].lock);
+			}
+			free(inf[i].id_list);
+		}
+	}
 	ub_ctx_delete(ctx);
 	checklock_stop();
 	return 0;

testcode/checklocks.c

@@ -71,6 +71,9 @@ static pid_t check_lock_pid;
 
 /** print all possible debug info on the state of the system */
 static void total_debug_info(void);
+/** print pretty lock error and exit (decl for NORETURN attribute) */
+static void lock_error(struct checked_lock* lock, const char* func,
+	const char* file, int line, const char* err) ATTR_NORETURN;
 
 /** print pretty lock error and exit */
 static void lock_error(struct checked_lock* lock, 

testcode/delayer.c

@@ -788,7 +788,7 @@ service_tcp_relay(struct tcp_proxy** tcp_proxies, struct timeval* now,
 			if(!tcp_relay_write(p->server_s, &p->querylist, 
 				&p->querylast, now))
 				delete_it = 1;
-			if(p->querylist && p->server_s != -1 &&
+			if(p->querylist &&
 				dl_tv_smaller(&p->querylist->wait, now))
 				FD_SET(FD_SET_T p->server_s, worig);
 			else 	FD_CLR(FD_SET_T p->server_s, worig);

testcode/fake_event.c

@@ -385,7 +385,7 @@ answer_callback_from_entry(struct replay_runtime* runtime,
 	repinfo.addrlen = pend->addrlen;
 	memcpy(&repinfo.addr, &pend->addr, pend->addrlen);
 	if(!pend->serviced) {
-		if(entry->reply_list->next &&
+		if(entry && entry->reply_list->next &&
 			pend->tcp_pkt_counter < count_reply_packets(entry)) {
 			/* go to next packet next time */
 			pend->tcp_pkt_counter++;
@@ -509,7 +509,7 @@ fake_pending_callback(struct replay_runtime* runtime,
 	repinfo.addrlen = p->addrlen;
 	memcpy(&repinfo.addr, &p->addr, p->addrlen);
 	if(!p->serviced) {
-		if(todo->match->reply_list->next && !error &&
+		if(todo->match && todo->match->reply_list->next && !error &&
 			p->tcp_pkt_counter < count_reply_packets(todo->match)) {
 			/* go to next packet next time */
 			p->tcp_pkt_counter++;
@@ -1802,4 +1802,24 @@ int outnet_tcp_connect(int ATTR_UNUSED(s), struct sockaddr_storage* ATTR_UNUSED(
 	return 0;
 }
 
+int tcp_req_info_add_meshstate(struct tcp_req_info* ATTR_UNUSED(req),
+        struct mesh_area* ATTR_UNUSED(mesh), struct mesh_state* ATTR_UNUSED(m))
+{
+	log_assert(0);
+	return 0;
+}
+
+void
+tcp_req_info_remove_mesh_state(struct tcp_req_info* ATTR_UNUSED(req),
+	struct mesh_state* ATTR_UNUSED(m))
+{
+	log_assert(0);
+}
+
+size_t
+tcp_req_info_get_stream_buffer_size(void)
+{
+	return 0;
+}
+
 /*********** End of Dummy routines ***********/

testcode/perf.c

@@ -513,10 +513,12 @@ qlist_grow_capacity(struct perfinfo* info)
 	uint8_t** d = (uint8_t**)calloc(sizeof(uint8_t*), newcap);
 	size_t* l = (size_t*)calloc(sizeof(size_t), newcap);
 	if(!d || !l) fatal_exit("out of memory");
-	memcpy(d, info->qlist_data, sizeof(uint8_t*)*
-		info->qlist_capacity);
-	memcpy(l, info->qlist_len, sizeof(size_t)*
-		info->qlist_capacity);
+	if(info->qlist_data && info->qlist_capacity)
+		memcpy(d, info->qlist_data, sizeof(uint8_t*)*
+			info->qlist_capacity);
+	if(info->qlist_len && info->qlist_capacity)
+		memcpy(l, info->qlist_len, sizeof(size_t)*
+			info->qlist_capacity);
 	free(info->qlist_data);
 	free(info->qlist_len);
 	info->qlist_data = d;
@@ -610,7 +612,7 @@ int main(int argc, char* argv[])
 		case 'd':
 			if(atoi(optarg)==0 && strcmp(optarg, "0")!=0) {
 				printf("-d not a number %s", optarg);
-				return 1;
+				exit(1);
 			}
 			info.duration = atoi(optarg);
 			break;
@@ -635,11 +637,11 @@ int main(int argc, char* argv[])
 	}
 	if(!extstrtoaddr(argv[0], &info.dest, &info.destlen)) {
 		printf("Could not parse ip: %s\n", argv[0]);
-		return 1;
+		exit(1);
 	}
 	if(info.qlist_size == 0) {
 		printf("No queries to make, use -f or -a.\n");
-		return 1;
+		exit(1);
 	}
 	
 	/* do the performance test */

testcode/petal.c

@@ -323,9 +323,9 @@ file_name_is_safe(char* s)
 	return 1;
 }
 
-/** adjust host and filename */
+/** adjust host */
 static void
-adjust_host_file(char* host, char* file)
+adjust_host(char* host)
 {
 	size_t i, len;
 	/* remove a port number if present */
@@ -335,6 +335,13 @@ adjust_host_file(char* host, char* file)
 	len = strlen(host);
 	for(i=0; i<len; i++)
 		host[i] = tolower((unsigned char)host[i]);
+}
+
+/** adjust filename */
+static void
+adjust_file(char* file)
+{
+	size_t i, len;
 	len = strlen(file);
 	for(i=0; i<len; i++)
 		file[i] = tolower((unsigned char)file[i]);
@@ -417,7 +424,7 @@ provide_file_10(SSL* ssl, char* fname)
 	}
 	fclose(in);
 	at += len;
-	avail -= len;
+	/* avail -= len; unused */
 	if(SSL_write(ssl, buf, at-buf) <= 0) {
 		/* write failure */
 	}
@@ -506,7 +513,7 @@ provide_file_chunked(SSL* ssl, char* fname)
 			snprintf(at, avail, "\r\n");
 			r = strlen(at);
 			at += r;
-			avail -= r;
+			/* avail -= r; unused */
 		}
 		/* send chunk */
 		if(SSL_write(ssl, buf, at-buf) <= 0) {
@@ -534,7 +541,8 @@ service_ssl(SSL* ssl, struct sockaddr_storage* from, socklen_t falen)
 	if(!read_http_headers(ssl, file, sizeof(file), host, sizeof(host),
 		&vs))
 		return;
-	adjust_host_file(host, file);
+	if(host[0] != 0) adjust_host(host);
+	if(file[0] != 0) adjust_file(file);
 	if(host[0] == 0 || !host_name_is_safe(host))
 		(void)strlcpy(host, "default", sizeof(host));
 	if(!file_name_is_safe(file)) {
@@ -569,7 +577,9 @@ do_service(char* addr, int port, char* key, char* cert)
 	while(go) {
 		struct sockaddr_storage from;
 		socklen_t flen = (socklen_t)sizeof(from);
-		int s = accept(fd, (struct sockaddr*)&from, &flen);
+		int s;
+		memset(&from, 0, sizeof(from));
+		s = accept(fd, (struct sockaddr*)&from, &flen);
 		if(verb) fflush(stdout);
 		if(s != -1) {
 			SSL* ssl = setup_ssl(s, sslctx);
@@ -633,7 +643,7 @@ int main(int argc, char* argv[])
 		}
 	}
 	argc -= optind;
-	argv += optind;
+	/* argv += optind; not using further arguments */
 	if(argc != 0)
 		usage();
 

testcode/replay.c

@@ -458,6 +458,8 @@ replay_scenario_read(FILE* in, const char* name, int* lineno)
 		if(parse_keyword(&parse, ";"))
 			continue; /* comment */
 		if(parse_keyword(&parse, "SCENARIO_BEGIN")) {
+			if(scen)
+				fatal_exit("%d: double SCENARIO_BEGIN", *lineno);
 			scen = make_scenario(parse);
 			if(!scen)
 				fatal_exit("%d: could not make scen", *lineno);
@@ -713,6 +715,7 @@ perform_arith(double x, char op, double y, double* res)
 		*res = x*y;
 		break;
 	default:
+		*res = 0;
 		return 0;
 	}
 
@@ -801,14 +804,19 @@ macro_expand(rbtree_type* store, struct replay_runtime* runtime, char** text)
 
 	/* check for functions */
 	if(strcmp(buf, "time") == 0) {
-		snprintf(buf, sizeof(buf), ARG_LL "d", (long long)runtime->now_secs);
+		if(runtime)
+			snprintf(buf, sizeof(buf), ARG_LL "d", (long long)runtime->now_secs);
+		else
+			snprintf(buf, sizeof(buf), ARG_LL "d", (long long)0);
 		*text += len;
 		return strdup(buf);
 	} else if(strcmp(buf, "timeout") == 0) {
 		time_t res = 0;
-		struct fake_timer* t = first_timer(runtime);
-		if(t && (time_t)t->tv.tv_sec >= runtime->now_secs) 
-			res = (time_t)t->tv.tv_sec - runtime->now_secs;
+		if(runtime) {
+			struct fake_timer* t = first_timer(runtime);
+			if(t && (time_t)t->tv.tv_sec >= runtime->now_secs) 
+				res = (time_t)t->tv.tv_sec - runtime->now_secs;
+		}
 		snprintf(buf, sizeof(buf), ARG_LL "d", (long long)res);
 		*text += len;
 		return strdup(buf);

testcode/streamtcp.1

@@ -44,6 +44,11 @@ Use UDP instead of TCP. No retries are attempted.
 .B \-n
 Do not wait for the answer.
 .TP
+.B \-a
+Print answers on arrival.  This mean queries are sent in sequence without
+waiting for answer but if answers arrive in this time they are printed out. 
+After sending queries the program waits and prints the remainder.
+.TP
 .B \-s
 Use SSL.
 .TP

testcode/streamtcp.c

@@ -73,6 +73,7 @@ static void usage(char* argv[])
 	printf("-f server	what ipaddr@portnr to send the queries to\n");
 	printf("-u 		use UDP. No retries are attempted.\n");
 	printf("-n 		do not wait for an answer.\n");
+	printf("-a 		print answers as they arrive.\n");
 	printf("-d secs		delay after connection before sending query\n");
 	printf("-s		use ssl\n");
 	printf("-h 		this help text\n");
@@ -203,13 +204,22 @@ recv_one(int fd, int udp, SSL* ssl, sldns_buffer* buf)
 	uint16_t len;
 	if(!udp) {
 		if(ssl) {
-			if(SSL_read(ssl, (void*)&len, (int)sizeof(len)) <= 0) {
+			int sr = SSL_read(ssl, (void*)&len, (int)sizeof(len));
+			if(sr == 0) {
+				printf("ssl: stream closed\n");
+				exit(1);
+			}
+			if(sr < 0) {
 				log_crypto_err("could not SSL_read");
 				exit(1);
 			}
 		} else {
-			if(recv(fd, (void*)&len, sizeof(len), 0) <
-				(ssize_t)sizeof(len)) {
+			ssize_t r = recv(fd, (void*)&len, sizeof(len), 0);
+			if(r == 0) {
+				printf("recv: stream closed\n");
+				exit(1);
+			}	
+			if(r < (ssize_t)sizeof(len)) {
 #ifndef USE_WINSOCK
 				perror("read() len failed");
 #else
@@ -267,6 +277,37 @@ recv_one(int fd, int udp, SSL* ssl, sldns_buffer* buf)
 	free(pktstr);
 }
 
+/** see if we can receive any results */
+static void
+print_any_answers(int fd, int udp, SSL* ssl, sldns_buffer* buf,
+	int* num_answers, int wait_all)
+{
+	/* see if the fd can read, if so, print one answer, repeat */
+	int ret;
+	struct timeval tv, *waittv;
+	fd_set rfd;
+	while(*num_answers > 0) {
+		memset(&rfd, 0, sizeof(rfd));
+		memset(&tv, 0, sizeof(tv));
+		FD_ZERO(&rfd);
+		FD_SET(fd, &rfd);
+		if(wait_all) waittv = NULL;
+		else waittv = &tv;
+		ret = select(fd+1, &rfd, NULL, NULL, waittv);
+		if(ret < 0) {
+			if(errno == EINTR || errno == EAGAIN) continue;
+			perror("select() failed");
+			exit(1);
+		}
+		if(ret == 0) {
+			if(wait_all) continue;
+			return;
+		}
+		(*num_answers) -= 1;
+		recv_one(fd, udp, ssl, buf);
+	}
+}
+
 static int get_random(void)
 {
 	int r;
@@ -278,12 +319,12 @@ static int get_random(void)
 
 /** send the TCP queries and print answers */
 static void
-send_em(const char* svr, int udp, int usessl, int noanswer, int delay,
-	int num, char** qs)
+send_em(const char* svr, int udp, int usessl, int noanswer, int onarrival,
+	int delay, int num, char** qs)
 {
 	sldns_buffer* buf = sldns_buffer_new(65553);
 	int fd = open_svr(svr, udp);
-	int i;
+	int i, wait_results = 0;
 	SSL_CTX* ctx = NULL;
 	SSL* ssl = NULL;
 	if(!buf) fatal_exit("out of memory");
@@ -325,9 +366,15 @@ send_em(const char* svr, int udp, int usessl, int noanswer, int delay,
 		write_q(fd, udp, ssl, buf, (uint16_t)get_random(), qs[i],
 			qs[i+1], qs[i+2]);
 		/* print at least one result */
-		if(!noanswer)
+		if(onarrival) {
+			wait_results += 1; /* one more answer to fetch */
+			print_any_answers(fd, udp, ssl, buf, &wait_results, 0);
+		} else if(!noanswer) {
 			recv_one(fd, udp, ssl, buf);
+		}
 	}
+	if(onarrival)
+		print_any_answers(fd, udp, ssl, buf, &wait_results, 1);
 
 	if(usessl) {
 		SSL_shutdown(ssl);
@@ -368,6 +415,7 @@ int main(int argc, char** argv)
 	const char* svr = "127.0.0.1";
 	int udp = 0;
 	int noanswer = 0;
+	int onarrival = 0;
 	int usessl = 0;
 	int delay = 0;
 
@@ -394,11 +442,14 @@ int main(int argc, char** argv)
 	if(argc == 1) {
 		usage(argv);
 	}
-	while( (c=getopt(argc, argv, "f:hnsud:")) != -1) {
+	while( (c=getopt(argc, argv, "af:hnsud:")) != -1) {
 		switch(c) {
 			case 'f':
 				svr = optarg;
 				break;
+			case 'a':
+				onarrival = 1;
+				break;
 			case 'n':
 				noanswer = 1;
 				break;
@@ -446,7 +497,7 @@ int main(int argc, char** argv)
 		(void)OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL);
 #endif
 	}
-	send_em(svr, udp, usessl, noanswer, delay, argc, argv);
+	send_em(svr, udp, usessl, noanswer, onarrival, delay, argc, argv);
 	checklock_stop();
 #ifdef USE_WINSOCK
 	WSACleanup();

testcode/testbound.c

@@ -344,6 +344,7 @@ main(int argc, char* argv[])
 
 	/* we do not want the test to depend on the timezone */
 	(void)putenv("TZ=UTC");
+	memset(pass_argv, 0, sizeof(pass_argv));
 
 	log_init(NULL, 0, NULL);
 	/* determine commandline options for the daemon */
@@ -429,14 +430,14 @@ main(int argc, char* argv[])
 		case 'h':
 		default:
 			testbound_usage();
-			return 1;
+			exit(1);
 		}
 	}
 	argc -= optind;
-	argv += optind;
+	/* argv += optind; not using further arguments */
 	if(argc != 0) {
 		testbound_usage();
-		return 1;
+		exit(1);
 	}
 	log_info("Start of %s testbound program.", PACKAGE_STRING);
 	if(atexit(&remove_configfile) != 0)

testcode/testpkts.c

@@ -46,6 +46,7 @@ enum verbosity_value { NO_VERBOSE=0 };
 #endif
 /** logging routine, provided by caller */
 void verbose(enum verbosity_value lvl, const char* msg, ...) ATTR_FORMAT(printf, 2, 3);
+static void error(const char* msg, ...) ATTR_NORETURN;
 
 /** print error and exit */
 static void error(const char* msg, ...)
@@ -235,6 +236,8 @@ static void adjustline(char* line, struct entry* e,
 			e->copy_query = 1;
 		} else if(str_keyword(&parse, "copy_ednsdata_assume_clientsubnet")) {
 			e->copy_ednsdata_assume_clientsubnet = 1;
+		} else if(str_keyword(&parse, "increment_ecs_scope")) {
+			e->increment_ecs_scope = 1;
 		} else if(str_keyword(&parse, "sleep=")) {
 			e->sleeptime = (unsigned int) strtol(parse, (char**)&parse, 10);
 			while(isspace((unsigned char)*parse)) 
@@ -273,6 +276,7 @@ static struct entry* new_entry(void)
 	e->copy_id = 0;
 	e->copy_query = 0;
 	e->copy_ednsdata_assume_clientsubnet = 0;
+	e->increment_ecs_scope = 0;
 	e->sleeptime = 0;
 	e->next = NULL;
 	return e;
@@ -509,7 +513,8 @@ add_edns(uint8_t* pktbuf, size_t pktsize, int do_flag, uint8_t *ednsdata,
 	if(*pktlen + sizeof(edns) + ednslen > pktsize)
 		error("not enough space for EDNS OPT record");
 	memmove(pktbuf+*pktlen, edns, sizeof(edns));
-	memmove(pktbuf+*pktlen+sizeof(edns), ednsdata, ednslen);
+	if(ednsdata && ednslen)
+		memmove(pktbuf+*pktlen+sizeof(edns), ednsdata, ednslen);
 	sldns_write_uint16(pktbuf+10, LDNS_ARCOUNT(pktbuf)+1);
 	*pktlen += (sizeof(edns) + ednslen);
 }
@@ -1592,6 +1597,9 @@ adjust_packet(struct entry* match, uint8_t** answer_pkt, size_t *answer_len,
 		if(walk_qlen >= 15 && walk_plen >= 15) {
 			walk_p[15] = walk_q[14];
 		}	
+		if(match->increment_ecs_scope) {
+			walk_p[15]++;
+		}
 	}
 
 	if(match->sleeptime > 0) {

testcode/testpkts.h

@@ -208,6 +208,8 @@ struct entry {
 	/** copy ednsdata to reply, assume it is clientsubnet and
 	 * adjust scopemask to match sourcemask */
 	uint8_t copy_ednsdata_assume_clientsubnet;
+	/** increment the ECS scope copied from the sourcemask by one */
+	uint8_t increment_ecs_scope;
 	/** in seconds */
 	unsigned int sleeptime; 
 

testcode/unitauth.c

@@ -42,6 +42,7 @@
 #include "testcode/unitmain.h"
 #include "util/regional.h"
 #include "util/net_help.h"
+#include "util/config_file.h"
 #include "util/data/msgreply.h"
 #include "services/cache/dns.h"
 #include "sldns/str2wire.h"
@@ -522,6 +523,7 @@ addzone(struct auth_zones* az, const char* name, char* fname)
 	struct auth_zone* z;
 	size_t nmlen;
 	uint8_t* nm = sldns_str2wire_dname(name, &nmlen);
+	struct config_file* cfg;
 	if(!nm) fatal_exit("out of memory");
 	lock_rw_wrlock(&az->lock);
 	z = auth_zone_create(az, nm, nmlen, LDNS_RR_CLASS_IN);
@@ -529,12 +531,16 @@ addzone(struct auth_zones* az, const char* name, char* fname)
 	if(!z) fatal_exit("cannot find zone");
 	auth_zone_set_zonefile(z, fname);
 	z->for_upstream = 1;
+	cfg = config_create();
+	free(cfg->chrootdir);
+	cfg->chrootdir = NULL;
 
-	if(!auth_zone_read_zonefile(z)) {
+	if(!auth_zone_read_zonefile(z, cfg)) {
 		fatal_exit("parse failure for auth zone %s", name);
 	}
 	lock_rw_unlock(&z->lock);
 	free(nm);
+	config_delete(cfg);
 	return z;
 }
 

testcode/unitecs.c

@@ -158,7 +158,7 @@ static void consistency_test(void)
 {
 	addrlen_t l;
 	time_t i;
-	unsigned int count;
+	uint32_t count;
 	addrkey_t *k;
 	struct addrtree* t;
 	struct module_env env;

testcode/unitneg.c

@@ -118,6 +118,8 @@ static void get_random_data(char** fromp, char** top, char* zname)
 	int labnum1[10], labnum2[10];
 	int i;
 	char* p;
+	memset(labnum1, 0, sizeof(int)*10);
+	memset(labnum2, 0, sizeof(int)*10);
 
 	*fromp = buf1;
 	*top = buf2;

testdata/07-confroot.tdir/07-confroot.test

@@ -153,7 +153,7 @@ server:
 EOF
 
 $PRE/unbound-checkconf test.conf
-if test $? -ne 1; then
+if test $? -ne 0; then
 	echo "Checkconf of config with chroot inside it failed"
 	exit 1
 fi

testdata/auth_nsec3_wild.rpl

@@ -0,0 +1,203 @@
+; config options
+server:
+	target-fetch-policy: "0 0 0 0 0"
+
+auth-zone:
+	name: "test-ns-signed.dev.internet.nl."
+	## zonefile (or none).
+	## zonefile: "example.com.zone"
+	## master by IP address or hostname
+	## can list multiple masters, each on one line.
+	## master:
+	## url for http fetch
+	## url:
+	## queries from downstream clients get authoritative answers.
+	## for-downstream: yes
+	for-downstream: yes
+	## queries are used to fetch authoritative answers from this zone,
+	## instead of unbound itself sending queries there.
+	## for-upstream: yes
+	for-upstream: yes
+	## on failures with for-upstream, fallback to sending queries to
+	## the authority servers
+	## fallback-enabled: no
+
+	## this line generates zonefile: \n"/tmp/xxx.example.com"\n
+	zonefile:
+TEMPFILE_NAME test-ns-signed.dev.internet.nl
+	## this is the inline file /tmp/xxx.test-ns-signed.dev.internet.nl
+	## the tempfiles are deleted when the testrun is over.
+TEMPFILE_CONTENTS test-ns-signed.dev.internet.nl
+test-ns-signed.dev.internet.nl.	3600	IN	SOA	ns.nlnetlabs.nl. ralph.nlnetlabs.nl. 4 14400 3600 604800 3600
+test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	SOA 8 4 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. ybb0Hc7NC+QOFEEv4cX2+Umlk+miiOAHmeP2Uwvg6lqfxkk+3g7yWBEKMinXjLKz0odWZ6fki6M/3yBPQX8SV0OCRY5gYvAHAjbxAIHozIM+5iwOkRQhNF1DRgQ3BLjL93f6T5e5Z4y1812iOpu4GYswXW/UTOZACXz2UiaCPAg=
+test-ns-signed.dev.internet.nl.	3600	IN	NS	ns.test-ns-signed.dev.internet.nl.
+test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	NS 8 4 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. KqiwTF3hKm1ZHGbgx6MVzZYHlS1p7+Xrikx4izMHFbWiD6ki6lrJBJsnH9j/hH1cwHxjXslOeJh0hdBdbn8la0meZPsebOyUbEjoLPzRLzKNLDBuA4BUJnRGQJy21CX7XooXAMAmR8YFipO8CojI9EogU2m2o9YkfbpacFWQoTk=
+test-ns-signed.dev.internet.nl.	3600	IN	DNSKEY	256 3 8 AwEAAc6c8tpMXBSOFLu/9n4aUUDK43wN4B7A2UDqZi0IOkyptxWCFghleyZeeN5uq6p9MoUt8lS73mFmIYC0ux5zBO3uVaJQ9u+00qRAEVg/RgBwa58y2f/zNtFV/f7mBSPcPTiEjUh0bwHSiTvUn/8JkrvjyAcbQMO0YOsRof5q6tzl ;{id = 32784 (zsk), size = 1024b}
+test-ns-signed.dev.internet.nl.	3600	IN	DNSKEY	257 3 8 AwEAAdC0hBJP1U8lbZ6JFXn0ouK6VipiraN7I8oog62SuEd/fqAupys7A/Ih6WK/UoJorjlnccEL8euNMaS4kNogvoBrFx8ciIWKcbot5mtwc4WDr3cnR+HIZNCUFVkIxsMqE7HCD0yn0zhkB60shED+ZHs8zpyU+cjnsOSizxOnIY+F ;{id = 54502 (ksk), size = 1024b}
+test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	DNSKEY 8 4 3600 20190205132351 20190108132351 54502 test-ns-signed.dev.internet.nl. X3qN+plfjf45FA4pr/tcUqUCR9ajDqwtNe4TS19WOJogVL/Gf/N5/ToOCrs3s+a7VrJl58WvSJquDM8xAS8f4oJggKgHFhopce8tMTGRxkRvJo4y+tt3vCveh/zjHLAnbOaBGA4CJ/IPhRqzHzcX/SjSv0EACWd6XpQIWogRv6c=
+test-ns-signed.dev.internet.nl.	3600	IN	NSEC3PARAM	1 0 1 - 
+test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	NSEC3PARAM 8 4 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. A/1xUGO46uIz+9vjPGfWVD99akwU9bd/UlnVG9LPfoTzG7TMWSoZ4ksg8k8ub8K1TrkDmQokNHSW0Gt6qwoRh17c+p1h/SFlDVL83wgTc4NqG43OQjgGU9RV035XU+VESlO3lavifhlu8rHWBJTlhiXcMGq6H+zvoz4sx9p5GNM=
+93stp7o7i5n9gb83uu7vv6h8qltk14ig.test-ns-signed.dev.internet.nl.	3600	IN	NSEC3	1 0 1 -  fee0c2kfhi6bnljce6vehaenqq3pbupu NS SOA RRSIG DNSKEY NSEC3PARAM 
+93stp7o7i5n9gb83uu7vv6h8qltk14ig.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	NSEC3 8 5 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. YoTRDQ7sSvERcY1WwAH4oRRR7DmaAwA8/H70jdMeSU4wsnM/VM03kDcc2sgq5edmHiZoTWnq7nEb/1Y7Ro0YrqTUQdYFZvXi6UjZQrKI9nqAGnhdXZWlZJHmYpn2+2Emd+bYHkwvKaPnfnnKjUoGVBH8Hly0HBYKPUF1/viquB0=
+kl94uofq16t2vlq0bmampf6e4o9k5hbi.test-ns-signed.dev.internet.nl.	3600	IN	NSEC3	1 0 1 -  7ag3p2pfrvq09dpn63cvga8ub1rnrrg1
+kl94uofq16t2vlq0bmampf6e4o9k5hbi.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	NSEC3 8 5 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. NI5zJ/k1kPVZ1abms5OoME/wazb77Ltduyk6ZevAnt4tKydZYwSsjEd0Ixknw9xnakCABn5rAYEXctARN0KCwCkNHR7TYlTAJT14hlDYjbad2u2HT9L1kzAnfj3BeLZl/LRADeMbTtzrkTSF3Dnezurb94fMnUnKt2hPfQfj560=
+fee0c2kfhi6bnljce6vehaenqq3pbupu.test-ns-signed.dev.internet.nl.	3600	IN	NSEC3	1 0 1 -  i6pi4e3o98e7vtkpjfhqn7g77d3mjcnv
+fee0c2kfhi6bnljce6vehaenqq3pbupu.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	NSEC3 8 5 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. WIb3ISP1nlafbyWoWa4z7sG5IS+V86PyvEMHdD/64hgsFkrCu483XK7VNnBz28SL/631JXA1R19O+UxeWhTUyctp8QSt6cEZcMPY8b7yG97rNFNvhSw75rSXXt+JwgIYHPHQV5oqPtVmEpQM5SfJd+hs+Nn1bJcWB3UaESNNAMQ=
+*.a.b.test-ns-signed.dev.internet.nl.	3600	IN	TXT	"a"
+*.a.b.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	TXT 8 6 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. eNcJkQXdTO1z21od0sXbgqtABhhr/9tNC/Zx8zYbhXkfj7rufN71yk9xqgu6TG0MeJV26ISrqIGRVFJFmTRvO1LLxoKkEPhqe+08nqRztxXZajCV+dDeFoGIDcXJg6tAxB+MJznkKDtZPpIWvyt1WwdYfcMrGtE9AmR3K1/P/xE=
+7ag3p2pfrvq09dpn63cvga8ub1rnrrg1.test-ns-signed.dev.internet.nl.	3600	IN	NSEC3	1 0 1 -  93stp7o7i5n9gb83uu7vv6h8qltk14ig TXT RRSIG 
+7ag3p2pfrvq09dpn63cvga8ub1rnrrg1.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	NSEC3 8 5 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. gtxoiTa3FRUqoRLvkWSxmWQ+DfijVd26gpKH3+GmGIcNB/sr/Cf8kERRwVVHvgzYIcvdJcys5b2LUXnZJwcdAlx7efZPWgNZzWxJrw6ES25LCWJOrp31isWn9FlAZGIbnpyEXxD2apBSmtyPnKbTgU6lHHS9jrsYHu4G8Zouv3k=
+ns.test-ns-signed.dev.internet.nl.	3600	IN	A	185.49.141.11
+ns.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	A 8 5 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. F9sXEVAmlRn+/84WbuvegiCwstNxMDMQLl0Obv2CTPpee4U6psbmXrlzczjjjkE6aLjsIHYdcXCzEWTrmukT+V9jzaGPRJvxNvC0ASWyzggAoh0Z++Hl4cVa9587o6I9ODayehFI9Pgdem+RVdb4zlWuzi9FmKXgeTlgWN54tPg=
+ns.test-ns-signed.dev.internet.nl.	3600	IN	AAAA	2a04:b900:0:100::11
+ns.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	AAAA 8 5 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. F1XRrx/QgfzJ1RS7d0m23QoIPx1G8WL1SrlTOm7pk5vWTL07w7HEw2TETblkjnitJGKfN9ebsIum/cDPUZc3UqLkguP2UCWpePnlllTJuwmG0Z+wyINIR4xF4PQlqttvzThBkD2JKWb/o0W8dQyXTj+jJ1vCZ0NjjA2N4+iJIQE=
+i6pi4e3o98e7vtkpjfhqn7g77d3mjcnv.test-ns-signed.dev.internet.nl.	3600	IN	NSEC3	1 0 1 -  kl94uofq16t2vlq0bmampf6e4o9k5hbi A AAAA RRSIG 
+i6pi4e3o98e7vtkpjfhqn7g77d3mjcnv.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	NSEC3 8 5 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. xLysIqn3r3rdHE3GvwVjZwUyuFClhkhgrQdwyc66RuHKE3MfSuhVr9cHTCJzhipF5TwQTbUpLOr74r99bzdiIY8Xkgjy2M0nc76v1ObSGJdPPjGTevbhDOnavUURwOR/q0NqqO2iPrgFjOVMZ+8uwRJtCty2iAVZfVG+qDzs8hU=
+TEMPFILE_END
+
+stub-zone:
+	name: "."
+	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
+CONFIG_END
+
+SCENARIO_BEGIN Test authority zone with NSEC3 wildcard
+
+; K.ROOT-SERVERS.NET.
+RANGE_BEGIN 0 100
+	ADDRESS 193.0.14.129 
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+. IN NS
+SECTION ANSWER
+. IN NS	K.ROOT-SERVERS.NET.
+SECTION ADDITIONAL
+K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode subdomain
+ADJUST copy_id copy_query
+REPLY QR NOERROR
+SECTION QUESTION
+com. IN NS
+SECTION AUTHORITY
+com.	IN NS	a.gtld-servers.net.
+SECTION ADDITIONAL
+a.gtld-servers.net.	IN 	A	192.5.6.30
+ENTRY_END
+RANGE_END
+
+; a.gtld-servers.net.
+RANGE_BEGIN 0 100
+	ADDRESS 192.5.6.30
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+com. IN NS
+SECTION ANSWER
+com.	IN NS	a.gtld-servers.net.
+SECTION ADDITIONAL
+a.gtld-servers.net.	IN 	A	192.5.6.30
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode subdomain
+ADJUST copy_id copy_query
+REPLY QR NOERROR
+SECTION QUESTION
+example.com. IN NS
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com. IN A 1.2.3.44
+ENTRY_END
+RANGE_END
+
+; ns.example.net.
+RANGE_BEGIN 0 100
+	ADDRESS 1.2.3.44
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+example.net. IN NS
+SECTION ANSWER
+example.net.	IN NS	ns.example.net.
+SECTION ADDITIONAL
+ns.example.net.		IN 	A	1.2.3.44
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.net. IN A
+SECTION ANSWER
+ns.example.net. IN A	1.2.3.44
+SECTION AUTHORITY
+example.net.	IN NS	ns.example.net.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.net. IN AAAA
+SECTION AUTHORITY
+example.net.	IN NS	ns.example.net.
+SECTION ADDITIONAL
+www.example.net. IN A	1.2.3.44
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+example.com. IN NS
+SECTION ANSWER
+example.com.	IN NS	ns.example.net.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+ENTRY_END
+RANGE_END
+
+STEP 1 QUERY
+ENTRY_BEGIN
+REPLY RD DO
+SECTION QUESTION
+something.a.b.test-ns-signed.dev.internet.nl. IN TXT
+ENTRY_END
+
+; recursion happens here.
+STEP 20 CHECK_ANSWER
+ENTRY_BEGIN
+MATCH all
+REPLY QR AA RD RA DO NOERROR
+SECTION QUESTION
+something.a.b.test-ns-signed.dev.internet.nl. IN TXT
+SECTION ANSWER
+something.a.b.test-ns-signed.dev.internet.nl. IN TXT "a"
+something.a.b.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	TXT 8 6 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. eNcJkQXdTO1z21od0sXbgqtABhhr/9tNC/Zx8zYbhXkfj7rufN71yk9xqgu6TG0MeJV26ISrqIGRVFJFmTRvO1LLxoKkEPhqe+08nqRztxXZajCV+dDeFoGIDcXJg6tAxB+MJznkKDtZPpIWvyt1WwdYfcMrGtE9AmR3K1/P/xE=
+SECTION AUTHORITY
+i6pi4e3o98e7vtkpjfhqn7g77d3mjcnv.test-ns-signed.dev.internet.nl. 3600 IN NSEC3 1 0 1 - KL94UOFQ16T2VLQ0BMAMPF6E4O9K5HBI  A AAAA RRSIG
+i6pi4e3o98e7vtkpjfhqn7g77d3mjcnv.test-ns-signed.dev.internet.nl.	3600	IN	RRSIG	NSEC3 8 5 3600 20190205132351 20190108132351 32784 test-ns-signed.dev.internet.nl. xLysIqn3r3rdHE3GvwVjZwUyuFClhkhgrQdwyc66RuHKE3MfSuhVr9cHTCJzhipF5TwQTbUpLOr74r99bzdiIY8Xkgjy2M0nc76v1ObSGJdPPjGTevbhDOnavUURwOR/q0NqqO2iPrgFjOVMZ+8uwRJtCty2iAVZfVG+qDzs8hU=
+ENTRY_END
+
+SCENARIO_END

testdata/black_ds_entry.rpl

@@ -19,7 +19,7 @@ SCENARIO_BEGIN Test validator with blacked key entry for DS and further queries
 ; until the key entry expires.
 
 ; K.ROOT-SERVERS.NET.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 193.0.14.129 
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -47,7 +47,7 @@ ENTRY_END
 RANGE_END
 
 ; a.gtld-servers.net.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 192.5.6.30
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -116,7 +116,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.example.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.3.4
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -228,7 +228,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.blabla.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.3.5
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -336,7 +336,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.sub.example.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.4.6
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -402,7 +402,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.foo.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.4.7
 ENTRY_BEGIN
 MATCH opcode qtype qname

testdata/black_key_entry.rpl

@@ -19,7 +19,7 @@ SCENARIO_BEGIN Test validator with blacked key entry and further queries
 ; until the key entry expires.
 
 ; K.ROOT-SERVERS.NET.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 193.0.14.129 
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -47,7 +47,7 @@ ENTRY_END
 RANGE_END
 
 ; a.gtld-servers.net.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 192.5.6.30
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -116,7 +116,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.example.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.3.4
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -228,7 +228,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.blabla.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.3.5
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -336,7 +336,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.sub.example.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.4.6
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -402,7 +402,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.foo.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.4.7
 ENTRY_BEGIN
 MATCH opcode qtype qname

testdata/black_prime_entry.rpl

@@ -18,7 +18,7 @@ SCENARIO_BEGIN Test validator with blacklist prime gives bad key entry
 ; comes from an 'expired signatures' name server.
 
 ; K.ROOT-SERVERS.NET.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 193.0.14.129 
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -46,7 +46,7 @@ ENTRY_END
 RANGE_END
 
 ; a.gtld-servers.net.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 192.5.6.30
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -96,7 +96,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.example.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.3.4
 ENTRY_BEGIN
 MATCH opcode qtype qname
@@ -177,7 +177,7 @@ ENTRY_END
 RANGE_END
 
 ; ns.blabla.com.
-RANGE_BEGIN 0 100
+RANGE_BEGIN 0 99
 	ADDRESS 1.2.3.5
 ENTRY_BEGIN
 MATCH opcode qtype qname

testdata/clang-analysis.tdir/clang-analysis.dsc

@@ -0,0 +1,15 @@
+BaseName: clang-analysis
+Version: 1.0
+Description: clang analysis
+CreationDate: Wed 12 Sep 16:00:26 CEST 2018
+Maintainer: Wouter Wijngaards
+Category:
+Component:
+Depends:
+Help:
+Pre:
+Post:
+Test: clang-analysis.test
+AuxFiles:
+Passed:
+Failure:

testdata/clang-analysis.tdir/clang-analysis.test

@@ -0,0 +1,82 @@
+# #-- clang-analysis.test --#
+# source the master var file when it's there
+[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master
+# use .tpkg.var.test for in test variable passing
+[ -f .tpkg.var.test ] && source .tpkg.var.test
+# common functions
+. ../common.sh
+
+PRE="../.."
+if test ! -x "`which clang 2>&1`"; then
+	echo "No clang in path"
+	exit 0
+fi
+#echo "have clang"
+# test if assertions are enabled
+if grep "^#define UNBOUND_DEBUG" $PRE/config.h >/dev/null; then
+	:
+else
+	echo "UNBOUND_DEBUG is not enabled, skip test"
+	# no unbound debug means no assertions, and clang analyzer uses
+	# the assertions to make inferences.
+	exit 0
+fi
+
+# read value from Makefile
+# $1: result variable name
+# $2: string on Makefile
+# $3: Makefile location
+read_value () {
+	x=`grep "$2" $3 | sed -e "s/$2//"`
+	eval $1="'""$x""'"
+	# print what we just read
+	#echo $1"="'"'"`eval echo '$'$1`"'"'
+}
+
+# read some values from the Makefile
+read_value srcdir '^srcdir=' $PRE/Makefile
+read_value CPPFLAGS '^CPPFLAGS=' $PRE/Makefile
+read_value LIBOBJS '^LIBOBJS= *' $PRE/Makefile
+read_value DNSCRYPT_SRC '^DNSCRYPT_SRC= *' $PRE/Makefile
+read_value DNSTAP_SRC '^DNSTAP_SRC= *' $PRE/Makefile
+read_value WITH_PYTHONMODULE '^WITH_PYTHONMODULE= *' $PRE/Makefile
+read_value WINAPPS '^WINAPPS= *' $PRE/Makefile
+
+#echo dir is $dir
+# turn libobjs into C files
+compatfiles=`echo "$LIBOBJS" | sed -e 's?..LIBOBJDIR.?compat/?g' -e 's/.U.o/.c/g'`
+#echo compatfiles are $compatfiles
+#echo
+if test "$WITH_PYTHONMODULE" = "yes"; then PYTHONMOD_SRC="pythonmod/*.c"; fi
+if test ! -z "$WINAPPS"; then WIN_SRC="winrc/*.c"; fi
+
+cd $PRE;
+odir=`pwd`
+cd $srcdir
+# check the files in the srcdir
+fail="no"
+for x in cachedb/*.c daemon/*.c dns64/*.c $DNSCRYPT_SRC $DNSTAP_SRC edns-subnet/*.c ipsecmod/*.c iterator/*.c libunbound/*.c $PYTHONMOD_SRC respip/*.c services/*.c services/*/*.c sldns/*.c smallapp/*.c util/*.c util/*/*.c validator/*.c $WIN_SRC $compatfiles testcode/*.c; do
+	if test "$x" = "util/configlexer.c"; then continue; fi
+	if test "$x" = "util/configparser.c"; then continue; fi
+	if test "$x" = "testcode/signit.c"; then continue; fi
+	if test "$x" = "compat/reallocarray.c"; then continue; fi
+	echo clang --analyze $CPPFLAGS $x
+	plist=`basename $x .c`.plist
+	rm -rf $plist
+	#echo "(cd $odir; clang --analyze $CPPFLAGS $srcdir/$x 2>&1 ) | tee tmp.$$"
+	(cd "$odir"; clang --analyze $CPPFLAGS $srcdir/$x 2>&1 ) | tee tmp.$$
+	if grep -e warning -e error tmp.$$ >/dev/null; then
+		fail="yes"
+		fails="$fails $x"
+	fi
+	rm -rf $plist tmp.$$
+done
+
+echo
+if test "$fail" = "yes"; then
+	echo "Failures"
+	echo "create reports in file.plist dir with 	clang --analyze --analyzer-output html $CPPFLAGS""$fails"
+	exit 1
+fi
+echo "OK"
+exit 0

testdata/dlv_ask_higher.rpl

@@ -16,7 +16,7 @@ CONFIG_END
 
 SCENARIO_BEGIN Test validator with DLV where it needs to ask higher up in dlv.
 ; at first negative DLV response, it needs to ask higher.
-; the SOA record in that negative reponse has a big span (if interpreted as NSEC)
+; the SOA record in that negative response has a big span (if interpreted as NSEC)
 ; then a positive response for DLV.
 
 ; K.ROOT-SERVERS.NET.

testdata/edns_cache.tdir/edns_cache.conf

@@ -12,9 +12,6 @@ server:
 stub-zone:
 	name: "example.net"
 	stub-addr: "127.0.0.1@@STUB2_PORT@"
-stub-zone:
-	name: "example.com"
-	stub-addr: "127.0.0.1@@STUB2_PORT@"
 # a k a root hints
 stub-zone:
 	name: "."

testdata/edns_cache.tdir/edns_cache.stub1

@@ -17,17 +17,6 @@ SECTION ADDITIONAL
 root.server.	IN	A	127.0.0.1
 ENTRY_END
 
-; referral to example.com
-ENTRY_BEGIN
-MATCH opcode subdomain
-REPLY QR NOERROR
-ADJUST copy_id copy_query
-SECTION QUESTION
-example.com.	IN	A
-SECTION AUTHORITY
-example.com.	IN 	NS	netdns.example.net.
-ENTRY_END
-
 ; referral to example.net
 ENTRY_BEGIN
 MATCH opcode subdomain

testdata/edns_cache.tdir/edns_cache.stub2

@@ -1,5 +1,5 @@
 ; nameserver test file
-$ORIGIN example.com.
+$ORIGIN example.net.
 $TTL 3600
 
 ENTRY_BEGIN
@@ -7,9 +7,9 @@ MATCH opcode qtype qname noedns
 REPLY QR AA NOERROR
 ADJUST copy_id
 SECTION QUESTION
-www.example.com.	IN	A
+www.example.net.	IN	A
 SECTION ANSWER
-www.example.com.	IN	A	10.20.30.40
+www.example.net.	IN	A	10.20.30.40
 ENTRY_END
 
 ENTRY_BEGIN

testdata/edns_cache.tdir/edns_cache.test

@@ -11,8 +11,8 @@ PRE="../.."
 # do the test
 echo "> dig netdns.example.net."
 dig @::1 -p $UNBOUND_PORT netdns.example.net. | tee outfile
-echo "> dig www.example.com."
-dig @::1 -p $UNBOUND_PORT www.example.com. | tee outfile
+echo "> dig www.example.net."
+dig @::1 -p $UNBOUND_PORT www.example.net. | tee outfile
 echo "> cat stub1.log"
 cat stub1.log 
 echo "> cat stub2.log"

testdata/edns_lame.tdir/edns_lame.conf

@@ -1,15 +0,0 @@
-server:
-	verbosity: 2
-	# num-threads: 1
-	interface: 127.0.0.1
-	port: @PORT@
-	use-syslog: no
-	directory: ""
-	pidfile: "unbound.pid"
-	chroot: ""
-	username: ""
-	do-not-query-localhost: no
-forward-zone:
-	name: "."
-	forward-addr: "127.0.0.1@@TOPORT@"
-

testdata/edns_lame.tdir/edns_lame.dsc

@@ -1,16 +0,0 @@
-BaseName: edns_lame
-Version: 1.0
-Description: Forward UDP but EDNS packets time out
-CreationDate: Mon Sep 29 16:39:15 CEST 2008
-Maintainer: dr. W.C.A. Wijngaards
-Category: 
-Component:
-CmdDepends: 
-Depends: 
-Help:
-Pre: edns_lame.pre
-Post: edns_lame.post
-Test: edns_lame.test
-AuxFiles: 
-Passed:
-Failure:

testdata/edns_lame.tdir/edns_lame.test

@@ -1,24 +0,0 @@
-# #-- edns_lame.test --#
-# source the master var file when it's there
-[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master
-# use .tpkg.var.test for in test variable passing
-[ -f .tpkg.var.test ] && source .tpkg.var.test
-
-PRE="../.."
-# do the test
-echo "> dig www.example.com."
-dig @localhost -p $UNBOUND_PORT www.example.com. | tee outfile
-echo "> dig www.example.com."
-dig @localhost -p $UNBOUND_PORT www.example.com. | tee outfile
-echo "> cat logfiles"
-cat fwd.log 
-cat unbound.log
-echo "> check answer"
-if grep "10.20.30.40" outfile; then
-	echo "OK"
-else
-	echo "Not OK"
-	exit 1
-fi
-
-exit 0

testdata/edns_lame.tdir/edns_lame.testns

@@ -1,14 +0,0 @@
-; nameserver test file
-$ORIGIN example.com.
-$TTL 3600
-
-ENTRY_BEGIN
-MATCH opcode qtype qname noedns
-REPLY QR AA NOERROR
-ADJUST copy_id
-SECTION QUESTION
-www	IN	A
-SECTION ANSWER
-www	IN	A	10.20.30.40
-ENTRY_END
-

testdata/fwd_compress_c00c.tdir/fwd_compress_c00c.post

@@ -8,3 +8,4 @@
 . ../common.sh
 kill_pid $FWD_PID
 kill_pid $UNBOUND_PID
+cat unbound.log

testdata/fwd_compress_c00c.tdir/fwd_compress_c00c.test

@@ -6,9 +6,9 @@
 
 # check what sort of netcat we have
 if nc -h 2>&1 | grep "q secs"; then
-	ncopt="-q 3 -w 2"
+	ncopt="-q 3 -i 2"
 else
-	ncopt="-w 2"
+	ncopt="-i 2"
 fi
 
 PRE="../.."

testdata/fwd_no_cache.rpl

@@ -0,0 +1,78 @@
+; This is a comment.
+; config options go here.
+forward-zone: name: "." forward-addr: 216.0.0.1
+	forward-no-cache: yes
+CONFIG_END
+
+SCENARIO_BEGIN Forward with no_cache set
+RANGE_BEGIN 0 10
+	ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+www.example.com. IN A
+	SECTION ANSWER
+www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+ns.example.com. IN A 10.20.30.50
+	ENTRY_END
+RANGE_END
+RANGE_BEGIN 200 300
+RANGE_END
+
+RANGE_BEGIN 20 100
+	ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+www.example.com. IN A
+	SECTION ANSWER
+www.example.com. IN A 10.20.30.44
+	SECTION AUTHORITY
+www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+ns.example.com. IN A 10.20.30.50
+	ENTRY_END
+RANGE_END
+RANGE_BEGIN 200 300
+RANGE_END
+
+STEP 1 QUERY
+ENTRY_BEGIN
+REPLY RD
+SECTION QUESTION
+www.example.com. IN A
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+REPLY QR RD RA
+MATCH opcode qname qtype all
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A 10.20.30.40
+ENTRY_END
+
+; make some time pass but not enough to timeout a cached record
+STEP 10 TIME_PASSES ELAPSE 10
+
+STEP 20 QUERY
+ENTRY_BEGIN
+REPLY RD
+SECTION QUESTION
+www.example.com. IN A
+ENTRY_END
+STEP 24 CHECK_ANSWER
+ENTRY_BEGIN
+REPLY QR RD RA
+MATCH opcode qname qtype all
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A 10.20.30.44
+ENTRY_END
+SCENARIO_END

testdata/iter_domain_sale.rpl

@@ -241,9 +241,9 @@ SECTION ANSWER
 SECTION AUTHORITY
 ; at TTL 5 because TTL is capped at min-ttl of 5 in rdata of SOA
 example.com.	5 IN SOA	a. b. 1 2 3 4 5
-example.com.	1800 IN NS	ns.example.com.
+;example.com.	1800 IN NS	ns.example.com.
 SECTION ADDITIONAL
-ns.example.com.	1800 	IN 	A	1.2.3.4
+;ns.example.com.	1800 	IN 	A	1.2.3.4
 ENTRY_END
 
 ; after another 1900 seconds the domain must have timed out.