- xfr-soa-tls, fix unit test for detecting changed zonefile and test

for SOA probe sent over TLS.

.github/workflows/analysis_ports.yml

@@ -163,7 +163,7 @@ jobs:
             make: "no"
  
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: false
       - name: test_windows
@@ -175,6 +175,12 @@ jobs:
           cd ..
           export prepath=`pwd`
           echo prepath=${prepath}
+          echo "choco install winflexbison3"
+          choco install winflexbison3
+          echo 'LEX="win_flex"'
+          export LEX="win_flex"
+          echo 'YACC="win_bison -y"'
+          export YACC="win_bison -y"
           #echo "curl cpanm"
           #curl -L -k -s -S -o cpanm https://cpanmin.us/
           #echo "perl cpanm Pod::Usage"

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: configure
       run: ./configure --enable-debug
     - name: make

.gitignore

@@ -10,6 +10,7 @@
 /config.status
 /dnstap/dnstap_config.h
 /dnscrypt/dnscrypt_config.h
+/clubsyms.def
 /doc/example.conf
 /doc/libunbound.3
 /doc/unbound-anchor.8

Makefile.in

@@ -57,7 +57,7 @@ LEX=@LEX@
 STRIP=@STRIP@
 CC=@CC@
 CPPFLAGS=-I. @CPPFLAGS@
-PYTHON_CPPFLAGS=-I. @PYTHON_CPPFLAGS@
+PYTHON_CPPFLAGS=-I. -I$(srcdir) @PYTHON_CPPFLAGS@
 CFLAGS=-DSRCDIR=$(srcdir) @CFLAGS@
 LDFLAGS=@LDFLAGS@
 LIBS=@LIBS@
@@ -122,15 +122,15 @@ iterator/iter_delegpt.c iterator/iter_donotq.c iterator/iter_fwd.c \
 iterator/iter_hints.c iterator/iter_priv.c iterator/iter_resptype.c \
 iterator/iter_scrub.c iterator/iter_utils.c services/listen_dnsport.c \
 services/localzone.c services/mesh.c services/modstack.c services/view.c \
-services/rpz.c \
+services/rpz.c util/rfc_1982.c \
 services/outbound_list.c services/outside_network.c util/alloc.c \
 util/config_file.c util/configlexer.c util/configparser.c \
 util/shm_side/shm_main.c services/authzone.c \
 util/fptr_wlist.c util/locks.c util/log.c util/mini_event.c util/module.c \
 util/netevent.c util/net_help.c util/random.c util/rbtree.c util/regional.c \
-util/rtt.c util/edns.c util/storage/dnstree.c util/storage/lookup3.c \
+util/rtt.c util/siphash.c util/edns.c util/storage/dnstree.c util/storage/lookup3.c \
 util/storage/lruhash.c util/storage/slabhash.c util/tcp_conn_limit.c \
-util/timehist.c util/tube.c \
+util/timehist.c util/tube.c util/proxy_protocol.c util/timeval_func.c \
 util/ub_event.c util/ub_event_pluggable.c util/winsock_event.c \
 validator/autotrust.c validator/val_anchor.c validator/validator.c \
 validator/val_kcache.c validator/val_kentry.c validator/val_neg.c \
@@ -145,14 +145,14 @@ as112.lo msgparse.lo msgreply.lo packed_rrset.lo iterator.lo iter_delegpt.lo \
 iter_donotq.lo iter_fwd.lo iter_hints.lo iter_priv.lo iter_resptype.lo \
 iter_scrub.lo iter_utils.lo localzone.lo mesh.lo modstack.lo view.lo \
 outbound_list.lo alloc.lo config_file.lo configlexer.lo configparser.lo \
-fptr_wlist.lo edns.lo locks.lo log.lo mini_event.lo module.lo net_help.lo \
+fptr_wlist.lo siphash.lo edns.lo locks.lo log.lo mini_event.lo module.lo net_help.lo \
 random.lo rbtree.lo regional.lo rtt.lo dnstree.lo lookup3.lo lruhash.lo \
 slabhash.lo tcp_conn_limit.lo timehist.lo tube.lo winsock_event.lo \
-autotrust.lo val_anchor.lo rpz.lo \
+autotrust.lo val_anchor.lo rpz.lo rfc_1982.lo proxy_protocol.lo \
 validator.lo val_kcache.lo val_kentry.lo val_neg.lo val_nsec3.lo val_nsec.lo \
 val_secalgo.lo val_sigcrypt.lo val_utils.lo dns64.lo $(CACHEDB_OBJ) authzone.lo \
 $(SUBNET_OBJ) $(PYTHONMOD_OBJ) $(CHECKLOCK_OBJ) $(DNSTAP_OBJ) $(DNSCRYPT_OBJ) \
-$(IPSECMOD_OBJ) $(IPSET_OBJ) $(DYNLIBMOD_OBJ) respip.lo
+$(IPSECMOD_OBJ) $(IPSET_OBJ) $(DYNLIBMOD_OBJ) respip.lo timeval_func.lo
 COMMON_OBJ_WITHOUT_UB_EVENT=$(COMMON_OBJ_WITHOUT_NETCALL) netevent.lo listen_dnsport.lo \
 outside_network.lo
 COMMON_OBJ=$(COMMON_OBJ_WITHOUT_UB_EVENT) ub_event.lo
@@ -198,7 +198,7 @@ CHECKCONF_OBJ=unbound-checkconf.lo worker_cb.lo
 CHECKCONF_OBJ_LINK=$(CHECKCONF_OBJ) $(COMMON_OBJ_ALL_SYMBOLS) $(SLDNS_OBJ) \
 $(COMPAT_OBJ) @WIN_CHECKCONF_OBJ_LINK@
 CONTROL_SRC=smallapp/unbound-control.c
-CONTROL_OBJ=unbound-control.lo 
+CONTROL_OBJ=unbound-control.lo
 CONTROL_OBJ_LINK=$(CONTROL_OBJ) worker_cb.lo $(COMMON_OBJ_ALL_SYMBOLS) \
 $(SLDNS_OBJ) $(COMPAT_OBJ) @WIN_CONTROL_OBJ_LINK@
 HOST_SRC=smallapp/unbound-host.c
@@ -344,7 +344,16 @@ longcheck: longtest
 test:	unittest$(EXEEXT) testbound$(EXEEXT)
 	./unittest$(EXEEXT)
 	./testbound$(EXEEXT) -s
-	for x in $(srcdir)/testdata/*.rpl; do printf "%s" "$$x "; if ./testbound$(EXEEXT) -p $$x >/dev/null 2>&1; then echo OK; else echo failed; exit 1; fi done
+	for x in $(srcdir)/testdata/*.rpl; do \
+		output=`./testbound$(EXEEXT) -p $$x -o -vvvvv 2>&1`; \
+		if test $$? -eq 0; then \
+			printf "%s OK\n" "$$x "; \
+		else \
+			printf "%s\n" "$$output "; \
+			printf "%s failed\n" "$$x "; \
+			exit 1; \
+		fi; \
+	done
 	@echo test OK
 
 longtest:	tests
@@ -446,6 +455,7 @@ unbound-dnstap-socket.lo unbound-dnstap-socket.o: $(srcdir)/dnstap/unbound-dnsta
 dynlibmod.lo dynlibdmod.o: $(srcdir)/dynlibmod/dynlibmod.c config.h $(srcdir)/dynlibmod/dynlibmod.h
 cachedb.lo cachedb.o: $(srcdir)/cachedb/cachedb.c config.h $(srcdir)/cachedb/cachedb.h
 redis.lo redis.o: $(srcdir)/cachedb/redis.c config.h $(srcdir)/cachedb/redis.h
+timeval_func.lo timeval_func.o: $(srcdir)/util/timeval_func.c $(srcdir)/util/timeval_func.h
 
 # dnscrypt
 dnscrypt.lo dnscrypt.o: $(srcdir)/dnscrypt/dnscrypt.c config.h \
@@ -489,6 +499,7 @@ util/configlexer.c:  $(srcdir)/util/configlexer.lex util/configparser.h
 		echo "#include \"util/configyyrename.h\"" >> $@ ;\
 		$(LEX) -t $(srcdir)/util/configlexer.lex >> $@ ;\
 	fi
+	@if test ! -f $@; then echo "No $@ : need flex and bison to compile from source repository"; exit 1; fi
 
 util/configparser.c util/configparser.h:  $(srcdir)/util/configparser.y
 	@-if test ! -d util; then $(INSTALL) -d util; fi
@@ -507,7 +518,7 @@ distclean: clean
 	rm -f doc/example.conf doc/libunbound.3 doc/unbound-anchor.8 doc/unbound-checkconf.8 doc/unbound-control.8 doc/unbound.8 doc/unbound.conf.5 doc/unbound-host.1
 	rm -f smallapp/unbound-control-setup.sh dnstap/dnstap_config.h dnscrypt/dnscrypt_config.h contrib/libunbound.pc contrib/unbound.socket contrib/unbound.service
 	rm -f $(TEST_BIN)
-	rm -f Makefile 
+	rm -f Makefile
 
 maintainer-clean: distclean
 	rm -f util/configlexer.c util/configparser.c util/configparser.h
@@ -556,7 +567,7 @@ pythonmod-install:
 
 pyunbound-install:
 	$(INSTALL) -m 755 -d $(DESTDIR)$(PYTHON_SITE_PKG)
-	$(INSTALL) -c -m 644 $(srcdir)/libunbound/python/unbound.py $(DESTDIR)$(PYTHON_SITE_PKG)/unbound.py
+	$(INSTALL) -c -m 644 libunbound/python/unbound.py $(DESTDIR)$(PYTHON_SITE_PKG)/unbound.py
 	$(LIBTOOL) --mode=install cp _unbound.la $(DESTDIR)$(PYTHON_SITE_PKG)
 	$(LIBTOOL) --mode=finish $(DESTDIR)$(PYTHON_SITE_PKG)
 
@@ -583,6 +594,8 @@ install-lib:	lib $(UNBOUND_EVENT_INSTALL)
 		echo ".so man3/libunbound.3" > $(DESTDIR)$(mandir)/man3/$$mpage.3 ; \
 	done
 	$(LIBTOOL) --mode=install cp unbound.h $(DESTDIR)$(includedir)/unbound.h
+	$(INSTALL) -m 755 -d $(DESTDIR)$(libdir)/pkgconfig
+	$(INSTALL) -m 644 contrib/libunbound.pc $(DESTDIR)$(libdir)/pkgconfig
 	$(LIBTOOL) --mode=install cp libunbound.la $(DESTDIR)$(libdir)
 	$(LIBTOOL) --mode=finish $(DESTDIR)$(libdir)
 
@@ -592,8 +605,6 @@ install-all:	all $(PYTHONMOD_INSTALL) $(PYUNBOUND_INSTALL) $(UNBOUND_EVENT_INSTA
 	$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man8
 	$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man5
 	$(INSTALL) -m 755 -d $(DESTDIR)$(mandir)/man1
-	$(INSTALL) -m 755 -d $(DESTDIR)$(libdir)/pkgconfig
-	$(INSTALL) -m 644 contrib/libunbound.pc $(DESTDIR)$(libdir)/pkgconfig
 	$(LIBTOOL) --mode=install cp -f unbound$(EXEEXT) $(DESTDIR)$(sbindir)/unbound$(EXEEXT)
 	$(LIBTOOL) --mode=install cp -f unbound-checkconf$(EXEEXT) $(DESTDIR)$(sbindir)/unbound-checkconf$(EXEEXT)
 	$(LIBTOOL) --mode=install cp -f unbound-control$(EXEEXT) $(DESTDIR)$(sbindir)/unbound-control$(EXEEXT)
@@ -607,7 +618,7 @@ install-all:	all $(PYTHONMOD_INSTALL) $(PYUNBOUND_INSTALL) $(UNBOUND_EVENT_INSTA
 	$(INSTALL) -c -m 644 doc/unbound.conf.5 $(DESTDIR)$(mandir)/man5
 	$(INSTALL) -c -m 644 doc/unbound-host.1 $(DESTDIR)$(mandir)/man1
 	$(INSTALL) -c -m 755 unbound-control-setup $(DESTDIR)$(sbindir)/unbound-control-setup
-	if test ! -e $(DESTDIR)$(configfile); then $(INSTALL) -d `dirname $(DESTDIR)$(configfile)`; $(INSTALL) -c -m 644 doc/example.conf $(DESTDIR)$(configfile); fi
+	if test ! -e "$(DESTDIR)$(configfile)"; then $(INSTALL) -d `dirname "$(DESTDIR)$(configfile)"`; $(INSTALL) -c -m 644 doc/example.conf "$(DESTDIR)$(configfile)"; fi
 
 pythonmod-uninstall:
 	rm -f -- $(DESTDIR)$(PYTHON_SITE_PKG)/unboundmodule.py
@@ -636,11 +647,11 @@ uninstall:	$(PYTHONMOD_UNINSTALL) $(PYUNBOUND_UNINSTALL) $(UNBOUND_EVENT_UNINSTA
 	rm -f -- $(DESTDIR)$(includedir)/unbound.h
 	$(LIBTOOL) --mode=uninstall rm -f $(DESTDIR)$(libdir)/libunbound.la
 	@echo
-	@echo "You still need to remove "`dirname $(DESTDIR)$(configfile)`" , $(DESTDIR)$(configfile) by hand"
+	@echo "You still need to remove "`dirname "$(DESTDIR)$(configfile)"`" , $(DESTDIR)$(configfile) by hand"
 
 iana_update:
 	curl -o port-numbers.tmp https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xml --compressed
-	if file port-numbers.tmp | grep 'gzip' >/dev/null; then zcat port-numbers.tmp; else cat port-numbers.tmp; fi | awk '/<record>/ {p=0;} /<protocol>udp/ {p=1;} /<protocol>[^u]/ {p=0;} /Decomissioned|Decommissioned|Removed|De-registered|unassigned|Unassigned|Reserved/ {u=1;} /<number>/ { if(u==1) {u=0;} else { if(p==1) { match($$0,/[0-9]+/); print substr($$0, RSTART, RLENGTH) ","}}}' | sort -nu > util/iana_ports.inc  
+	if file port-numbers.tmp | grep 'gzip' >/dev/null; then zcat port-numbers.tmp; else cat port-numbers.tmp; fi | awk '/<record>/ {p=0;} /<protocol>udp/ {p=1;} /<protocol>[^u]/ {p=0;} /Decomissioned|Decommissioned|Removed|De-registered|unassigned|Unassigned|Reserved/ {u=1;} /<number>/ { if(u==1) {u=0;} else { if(p==1) { match($$0,/[0-9]+/); print substr($$0, RSTART, RLENGTH) ","}}}' | sort -nu > util/iana_ports.inc
 	rm -f port-numbers.tmp
 
 # dependency generation
@@ -727,7 +738,7 @@ msgencode.lo msgencode.o: $(srcdir)/util/data/msgencode.c config.h $(srcdir)/uti
 msgparse.lo msgparse.o: $(srcdir)/util/data/msgparse.c config.h $(srcdir)/util/data/msgparse.h \
  $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/sldns/pkthdr.h \
  $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \
- $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lookup3.h $(srcdir)/util/regional.h $(srcdir)/sldns/sbuffer.h \
+ $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lookup3.h $(srcdir)/util/regional.h $(srcdir)/util/net_help.h $(srcdir)/sldns/sbuffer.h \
  $(srcdir)/sldns/parseutil.h $(srcdir)/sldns/wire2str.h
 msgreply.lo msgreply.o: $(srcdir)/util/data/msgreply.c config.h $(srcdir)/util/data/msgreply.h \
  $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/packed_rrset.h \
@@ -782,7 +793,7 @@ iter_priv.lo iter_priv.o: $(srcdir)/iterator/iter_priv.c config.h $(srcdir)/iter
  $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/net_help.h \
  $(srcdir)/util/storage/dnstree.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/sbuffer.h
 iter_resptype.lo iter_resptype.o: $(srcdir)/iterator/iter_resptype.c config.h \
- $(srcdir)/iterator/iter_resptype.h $(srcdir)/iterator/iter_delegpt.h $(srcdir)/util/log.h \
+ $(srcdir)/iterator/iter_resptype.h $(srcdir)/iterator/iter_delegpt.h $(srcdir)/iterator/iterator.h $(srcdir)/util/log.h \
  $(srcdir)/services/cache/dns.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \
  $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/net_help.h \
  $(srcdir)/util/data/dname.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/pkthdr.h
@@ -868,7 +879,7 @@ rpz.lo rpz.o: $(srcdir)/services/rpz.c config.h $(srcdir)/services/rpz.h $(srcdi
 outbound_list.lo outbound_list.o: $(srcdir)/services/outbound_list.c config.h \
  $(srcdir)/services/outbound_list.h $(srcdir)/services/outside_network.h $(srcdir)/util/rbtree.h \
  $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h  \
- 
+
 outside_network.lo outside_network.o: $(srcdir)/services/outside_network.c config.h \
  $(srcdir)/services/outside_network.h $(srcdir)/util/rbtree.h $(srcdir)/util/netevent.h \
  $(srcdir)/dnscrypt/dnscrypt.h   \
@@ -906,7 +917,8 @@ config_file.lo config_file.o: $(srcdir)/util/config_file.c config.h $(srcdir)/ut
 configlexer.lo configlexer.o: util/configlexer.c config.h $(srcdir)/util/configyyrename.h \
  $(srcdir)/util/config_file.h util/configparser.h
 configparser.lo configparser.o: util/configparser.c config.h $(srcdir)/util/configyyrename.h \
- $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h $(srcdir)/util/log.h
+ $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h $(srcdir)/util/log.h $(srcdir)/sldns/str2wire.h \
+ $(srcdir)/sldns/rrdef.h
 shm_main.lo shm_main.o: $(srcdir)/util/shm_side/shm_main.c config.h $(srcdir)/util/shm_side/shm_main.h \
  $(srcdir)/libunbound/unbound.h $(srcdir)/daemon/daemon.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \
  $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h  \
@@ -919,7 +931,7 @@ shm_main.lo shm_main.o: $(srcdir)/util/shm_side/shm_main.c config.h $(srcdir)/ut
  $(srcdir)/services/view.h $(srcdir)/util/config_file.h $(srcdir)/services/authzone.h $(srcdir)/respip/respip.h \
  $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/infra.h \
  $(srcdir)/util/rtt.h $(srcdir)/validator/validator.h $(srcdir)/validator/val_utils.h $(srcdir)/util/fptr_wlist.h \
- $(srcdir)/util/tube.h
+ $(srcdir)/util/tube.h $(srcdir)/util/timeval_func.h
 authzone.lo authzone.o: $(srcdir)/services/authzone.c config.h $(srcdir)/services/authzone.h \
  $(srcdir)/util/rbtree.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/services/mesh.h $(srcdir)/util/netevent.h \
  $(srcdir)/dnscrypt/dnscrypt.h  $(srcdir)/util/data/msgparse.h \
@@ -974,7 +986,9 @@ netevent.lo netevent.o: $(srcdir)/util/netevent.c config.h $(srcdir)/util/neteve
  $(srcdir)/services/modstack.h $(srcdir)/services/rpz.h $(srcdir)/services/localzone.h $(srcdir)/services/view.h \
  $(srcdir)/sldns/sbuffer.h $(srcdir)/util/config_file.h $(srcdir)/services/authzone.h $(srcdir)/daemon/stats.h \
  $(srcdir)/util/timehist.h $(srcdir)/libunbound/unbound.h $(srcdir)/respip/respip.h $(srcdir)/sldns/str2wire.h \
- $(srcdir)/dnstap/dnstap.h  $(srcdir)/services/listen_dnsport.h
+ $(srcdir)/dnstap/dnstap.h  $(srcdir)/services/listen_dnsport.h $(srcdir)/util/timeval_func.h
+proxy_protocol.lo proxy_protocol.o: $(srcdir)/util/proxy_protocol.c config.h \
+ $(srcdir)/util/proxy_protocol.h $(srcdir)/sldns/sbuffer.h
 net_help.lo net_help.o: $(srcdir)/util/net_help.c config.h $(srcdir)/util/net_help.h $(srcdir)/util/log.h \
  $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/module.h \
  $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h \
@@ -995,6 +1009,8 @@ rtt.lo rtt.o: $(srcdir)/util/rtt.c config.h $(srcdir)/util/rtt.h $(srcdir)/itera
  $(srcdir)/services/outbound_list.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/storage/lruhash.h \
  $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/module.h \
  $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h
+siphash.lo siphash.o: $(srcdir)/util/siphash.c
+rfc_1982.lo rfc_1982.o: $(srcdir)/util/rfc_1982.c
 edns.lo edns.o: $(srcdir)/util/edns.c config.h $(srcdir)/util/edns.h $(srcdir)/util/storage/dnstree.h \
  $(srcdir)/util/rbtree.h $(srcdir)/util/config_file.h $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h \
   $(srcdir)/util/net_help.h $(srcdir)/util/log.h $(srcdir)/util/regional.h \
@@ -1175,7 +1191,7 @@ unitmain.lo unitmain.o: $(srcdir)/testcode/unitmain.c config.h $(srcdir)/sldns/r
  $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h \
   $(srcdir)/util/random.h $(srcdir)/respip/respip.h \
  $(srcdir)/services/localzone.h $(srcdir)/services/view.h $(srcdir)/sldns/sbuffer.h \
- $(srcdir)/services/outside_network.h 
+ $(srcdir)/services/outside_network.h
 unitmsgparse.lo unitmsgparse.o: $(srcdir)/testcode/unitmsgparse.c config.h $(srcdir)/util/log.h \
  $(srcdir)/testcode/unitmain.h $(srcdir)/util/data/msgparse.h $(srcdir)/util/storage/lruhash.h \
  $(srcdir)/util/locks.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/msgreply.h \
@@ -1248,7 +1264,7 @@ cachedump.lo cachedump.o: $(srcdir)/daemon/cachedump.c config.h $(srcdir)/daemon
  $(srcdir)/util/regional.h $(srcdir)/util/net_help.h $(srcdir)/util/data/dname.h $(srcdir)/iterator/iterator.h \
  $(srcdir)/services/outbound_list.h $(srcdir)/iterator/iter_delegpt.h $(srcdir)/iterator/iter_utils.h \
  $(srcdir)/iterator/iter_resptype.h $(srcdir)/iterator/iter_fwd.h $(srcdir)/iterator/iter_hints.h \
- $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/str2wire.h
+ $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/str2wire.h $(srcdir)/util/config_file.h $(srcdir)/services/outside_network.h
 daemon.lo daemon.o: $(srcdir)/daemon/daemon.c config.h $(srcdir)/daemon/daemon.h $(srcdir)/util/locks.h \
  $(srcdir)/util/log.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h  \
   $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h \
@@ -1310,7 +1326,7 @@ unbound.lo unbound.o: $(srcdir)/daemon/unbound.c config.h $(srcdir)/util/log.h $
 worker.lo worker.o: $(srcdir)/daemon/worker.c config.h $(srcdir)/util/log.h $(srcdir)/util/net_help.h \
  $(srcdir)/util/random.h $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \
  $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \
- $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h  \
+ $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h $(srcdir)/util/timeval_func.h \
  $(srcdir)/util/alloc.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \
  $(srcdir)/sldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/libunbound/unbound.h \
  $(srcdir)/util/module.h $(srcdir)/dnstap/dnstap.h  $(srcdir)/daemon/daemon.h \
@@ -1332,7 +1348,7 @@ testbound.lo testbound.o: $(srcdir)/testcode/testbound.c config.h $(srcdir)/test
  $(srcdir)/daemon/remote.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \
  $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \
  $(srcdir)/util/config_file.h $(srcdir)/sldns/keyraw.h $(srcdir)/daemon/unbound.c $(srcdir)/daemon/daemon.h \
- $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h  \
+ $(srcdir)/util/alloc.h $(srcdir)/util/timeval_func.h $(srcdir)/services/modstack.h  \
  $(srcdir)/util/storage/slabhash.h $(srcdir)/services/listen_dnsport.h $(srcdir)/services/cache/rrset.h \
  $(srcdir)/services/cache/infra.h $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rtt.h \
  $(srcdir)/util/data/msgreply.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/module.h \
@@ -1346,7 +1362,7 @@ testpkts.lo testpkts.o: $(srcdir)/testcode/testpkts.c config.h $(srcdir)/testcod
 worker.lo worker.o: $(srcdir)/daemon/worker.c config.h $(srcdir)/util/log.h $(srcdir)/util/net_help.h \
  $(srcdir)/util/random.h $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \
  $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \
- $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h  \
+ $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h $(srcdir)/util/timeval_func.h \
  $(srcdir)/util/alloc.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \
  $(srcdir)/sldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/libunbound/unbound.h \
  $(srcdir)/util/module.h $(srcdir)/dnstap/dnstap.h  $(srcdir)/daemon/daemon.h \
@@ -1398,7 +1414,7 @@ stats.lo stats.o: $(srcdir)/daemon/stats.c config.h $(srcdir)/daemon/stats.h $(s
  $(srcdir)/validator/val_kcache.h $(srcdir)/validator/val_neg.h
 replay.lo replay.o: $(srcdir)/testcode/replay.c config.h $(srcdir)/util/log.h $(srcdir)/util/net_help.h \
  $(srcdir)/util/config_file.h $(srcdir)/testcode/replay.h $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h \
-  $(srcdir)/testcode/testpkts.h $(srcdir)/util/rbtree.h \
+  $(srcdir)/testcode/testpkts.h $(srcdir)/util/rbtree.h $(srcdir)/util/timeval_func.h \
  $(srcdir)/testcode/fake_event.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/rrdef.h
 fake_event.lo fake_event.o: $(srcdir)/testcode/fake_event.c config.h $(srcdir)/testcode/fake_event.h \
  $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h  \
@@ -1406,7 +1422,7 @@ fake_event.lo fake_event.o: $(srcdir)/testcode/fake_event.c config.h $(srcdir)/t
  $(srcdir)/util/locks.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/msgreply.h \
  $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgencode.h $(srcdir)/util/data/dname.h \
  $(srcdir)/util/edns.h $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h $(srcdir)/util/config_file.h \
- $(srcdir)/services/listen_dnsport.h $(srcdir)/services/outside_network.h \
+ $(srcdir)/services/listen_dnsport.h $(srcdir)/services/outside_network.h $(srcdir)/util/timeval_func.h \
   $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h \
  $(srcdir)/testcode/replay.h $(srcdir)/testcode/testpkts.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/module.h \
  $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h $(srcdir)/services/rpz.h \
@@ -1503,7 +1519,7 @@ asynclook.lo asynclook.o: $(srcdir)/testcode/asynclook.c config.h $(srcdir)/libu
  $(srcdir)/services/modstack.h $(srcdir)/libunbound/unbound-event.h $(srcdir)/util/data/packed_rrset.h \
  $(srcdir)/util/storage/lruhash.h $(srcdir)/sldns/rrdef.h
 streamtcp.lo streamtcp.o: $(srcdir)/testcode/streamtcp.c config.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \
- $(srcdir)/util/net_help.h $(srcdir)/util/data/msgencode.h $(srcdir)/util/data/msgparse.h \
+ $(srcdir)/util/net_help.h $(srcdir)/util/proxy_protocol.h $(srcdir)/util/data/msgencode.h $(srcdir)/util/data/msgparse.h \
  $(srcdir)/util/storage/lruhash.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/msgreply.h \
  $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/dname.h $(srcdir)/sldns/sbuffer.h \
  $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/wire2str.h

README.md

@@ -1,9 +1,10 @@
 # Unbound
 
-[![Travis Build Status](https://travis-ci.org/NLnetLabs/unbound.svg?branch=master)](https://travis-ci.org/NLnetLabs/unbound)
+[![Github Build Status](https://github.com/NLnetLabs/unbound/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/NLnetLabs/unbound/actions)
 [![Packaging status](https://repology.org/badge/tiny-repos/unbound.svg)](https://repology.org/project/unbound/versions)
 [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/unbound.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:unbound)
 [![Documentation Status](https://readthedocs.org/projects/unbound/badge/?version=latest)](https://unbound.readthedocs.io/en/latest/?badge=latest)
+[![Mastodon Follow](https://img.shields.io/mastodon/follow/109262826617293067?domain=https%3A%2F%2Ffosstodon.org&style=social)](https://fosstodon.org/@nlnetlabs)
 
 Unbound is a validating, recursive, caching DNS resolver. It is designed to be
 fast and lean and incorporates modern features based on open standards. If you
@@ -11,7 +12,7 @@ have any feedback, we would love to hear from you. Don’t hesitate to
 [create an issue on Github](https://github.com/NLnetLabs/unbound/issues/new)
 or post a message on the [Unbound mailing list](https://lists.nlnetlabs.nl/mailman/listinfo/unbound-users).
 You can learn more about Unbound by reading our
-[documentation](https://nlnetlabs.nl/documentation/unbound/).
+[documentation](https://unbound.docs.nlnetlabs.nl/).
 
 ## Compiling
 
@@ -33,7 +34,7 @@ support.
 
 All of Unbound's configuration options are described in the man pages, which
 will be installed and are available on the Unbound
-[documentation page](https://nlnetlabs.nl/documentation/unbound/).
+[documentation page](https://unbound.docs.nlnetlabs.nl/).
 
 An example configuration file is located in
 [doc/example.conf](https://github.com/NLnetLabs/unbound/blob/master/doc/example.conf.in).

SECURITY.md

@@ -0,0 +1,31 @@
+# Security Policy
+
+## Supported Versions
+
+NLnet Labs adheres to the straightforward, semantic versioning scheme that is
+commonly used in the software industry.
+
+Support is provided in respect of the latest release, i.e. releases with the
+highest minor and patch version level. We do not backport security fixes to
+older (minor) versions. In the event a new major version is released (e.g.  from
+3.2.18 to 4.0.0), support will also be provided on the latest minor  version of
+the previous major version (3.2.18) for a period of one year from the release of
+the new major version (4.0.0).
+
+In the event that, during this period, a new patch or minor version of the
+previous major version is released, then support on these versions will only be
+provided for the remainder of the one-year-period.
+
+You can find detailed information on our software support policy here:
+
+https://www.nlnetlabs.nl/support/software-support-policy/
+
+## Reporting a Vulnerability
+
+We take security very seriously. If you have discovered a security vulnerability
+in one of our projects and you would like to report it to us, you can send an
+encrypted message to our Security Entry Point.
+
+Details are described here:
+
+https://www.nlnetlabs.nl/security-report/

acx_nlnetlabs.m4

@@ -2,7 +2,10 @@
 # Copyright 2009, Wouter Wijngaards, NLnet Labs.   
 # BSD licensed.
 #
-# Version 43
+# Version 46
+# 2023-05-04 fix to remove unused whitespace.
+# 2023-01-26 fix -Wstrict-prototypes.
+# 2022-09-01 fix checking if nonblocking sockets work on OpenBSD.
 # 2021-08-17 fix sed script in ssldir split handling.
 # 2021-08-17 fix for openssl to detect split version, with ssldir_include
 # 	     and ssldir_lib output directories.
@@ -186,7 +189,7 @@ dnl cache=`echo $1 | sed 'y%.=/+- %___p__%'`
 AC_CACHE_VAL(cv_prog_cc_flag_needed_$cache,
 [
 echo '$2' > conftest.c
-echo 'void f(){}' >>conftest.c
+echo 'void f(void){}' >>conftest.c
 if test -z "`$CC $CPPFLAGS $CFLAGS $ERRFLAG -c conftest.c 2>&1`"; then
 eval "cv_prog_cc_flag_needed_$cache=no"
 else
@@ -232,7 +235,7 @@ dnl DEPFLAG: set to flag that generates dependencies.
 AC_DEFUN([ACX_DEPFLAG],
 [
 AC_MSG_CHECKING([$CC dependency flag])
-echo 'void f(){}' >conftest.c
+echo 'void f(void){}' >conftest.c
 if test "`$CC -MM conftest.c 2>&1`" = "conftest.o: conftest.c"; then
 	DEPFLAG="-MM"
 else 
@@ -271,7 +274,7 @@ ACX_CHECK_COMPILER_FLAG_NEEDED($C99FLAG -D__EXTENSIONS__ -D_BSD_SOURCE -D_DEFAUL
 #include <getopt.h>
 #endif
 
-int test() {
+int test(void) {
 	int a;
 	char **opts = NULL;
 	struct timeval tv;
@@ -308,7 +311,7 @@ ACX_CHECK_COMPILER_FLAG_NEEDED($C99FLAG -D__EXTENSIONS__ -D_BSD_SOURCE -D_DEFAUL
 #include <getopt.h>
 #endif
 
-int test() {
+int test(void) {
 	int a;
 	char **opts = NULL;
 	struct timeval tv;
@@ -334,7 +337,7 @@ ACX_CHECK_COMPILER_FLAG_NEEDED($C99FLAG,
 [
 #include <stdbool.h>
 #include <ctype.h>
-int test() {
+int test(void) {
         int a = 0;
         return a;
 }
@@ -344,7 +347,7 @@ ACX_CHECK_COMPILER_FLAG_NEEDED(-D_BSD_SOURCE -D_DEFAULT_SOURCE,
 [
 #include <ctype.h>
 
-int test() {
+int test(void) {
         int a;
         a = isascii(32);
         return a;
@@ -355,7 +358,7 @@ ACX_CHECK_COMPILER_FLAG_NEEDED(-D_GNU_SOURCE,
 [
 #include <netinet/in.h>
 
-int test() {
+int test(void) {
         struct in6_pktinfo inf;
 	int a = (int)sizeof(inf);
         return a;
@@ -369,7 +372,7 @@ ACX_CHECK_COMPILER_FLAG_NEEDED(-D_GNU_SOURCE -D_FRSRESGID,
 [
 #include <unistd.h>
 
-int test() {
+int test(void) {
 	int a = setresgid(0,0,0);
 	a = setresuid(0,0,0);
         return a;
@@ -384,7 +387,7 @@ ACX_CHECK_COMPILER_FLAG_NEEDED(-D_POSIX_C_SOURCE=200112,
 #endif
 #include <netdb.h>
 
-int test() {
+int test(void) {
         int a = 0;
         char *t;
         time_t time = 0;
@@ -412,7 +415,7 @@ ACX_CHECK_COMPILER_FLAG_NEEDED(-D__EXTENSIONS__,
 #include <getopt.h>
 #endif
 
-int test() {
+int test(void) {
         int a;
         char **opts = NULL;
         struct timeval tv;
@@ -474,7 +477,7 @@ fi
 dnl Setup ATTR_FORMAT config.h parts.
 dnl make sure you call ACX_CHECK_FORMAT_ATTRIBUTE also.
 AC_DEFUN([AHX_CONFIG_FORMAT_ATTRIBUTE],
-[ 
+[
 #ifdef HAVE_ATTR_FORMAT
 #  define ATTR_FORMAT(archetype, string_index, first_to_check) \
     __attribute__ ((format (archetype, string_index, first_to_check)))
@@ -833,7 +836,7 @@ dnl try to see if an additional _LARGEFILE_SOURCE 1 is needed to get fseeko
 ACX_CHECK_COMPILER_FLAG_NEEDED(-D_LARGEFILE_SOURCE=1,
 [
 #include <stdio.h>
-int test() {
+int test(void) {
         int a = fseeko(stdin, 0, 0);
         return a;
 }
@@ -858,7 +861,7 @@ char* (*f) () = getaddrinfo;
 #ifdef __cplusplus
 }
 #endif
-int main() {
+int main(void) {
         ;
         return 0;
 }
@@ -922,7 +925,7 @@ cache=`echo $1 | sed 'y%.=/+-%___p_%'`
 AC_CACHE_VAL(cv_cc_deprecated_$cache,
 [
 echo '$3' >conftest.c
-echo 'void f(){ $2 }' >>conftest.c
+echo 'void f(void){ $2 }' >>conftest.c
 if test -z "`$CC $CPPFLAGS $CFLAGS -c conftest.c 2>&1 | grep -e deprecated -e unavailable`"; then
 eval "cv_cc_deprecated_$cache=no"
 else
@@ -963,6 +966,9 @@ AC_LANG_SOURCE([[
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
 #endif
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
 #ifdef HAVE_SYS_SOCKET_H
 #include <sys/socket.h>
 #endif
@@ -1313,7 +1319,7 @@ AC_DEFUN([AHX_CONFIG_W32_FD_SET_T],
 #ifdef HAVE_WINSOCK2_H
 #define FD_SET_T (u_int)
 #else
-#define FD_SET_T 
+#define FD_SET_T
 #endif
 ])
 
@@ -1351,7 +1357,7 @@ dnl $3: define value, 1
 AC_DEFUN([AHX_CONFIG_FLAG_OMITTED],
 [#if defined($1) && !defined($2)
 #define $2 $3
-[#]endif ])
+[#]endif])
 
 dnl Wrapper for AHX_CONFIG_FLAG_OMITTED for -D style flags
 dnl $1: the -DNAME or -DNAME=value string.

acx_python.m4

@@ -17,28 +17,75 @@ AC_DEFUN([AC_PYTHON_DEVEL],[
 		PYTHON_VERSION=`$PYTHON -c "import sys; \
 			print(sys.version.split()[[0]])"`
 	fi
+	# calculate the version number components.
+	[
+	v="$PYTHON_VERSION"
+	PYTHON_VERSION_MAJOR=`echo $v | sed 's/[^0-9].*//'`
+	if test -z "$PYTHON_VERSION_MAJOR"; then PYTHON_VERSION_MAJOR="0"; fi
+	v=`echo $v | sed -e 's/^[0-9]*$//' -e 's/[0-9]*[^0-9]//'`
+	PYTHON_VERSION_MINOR=`echo $v | sed 's/[^0-9].*//'`
+	if test -z "$PYTHON_VERSION_MINOR"; then PYTHON_VERSION_MINOR="0"; fi
+	v=`echo $v | sed -e 's/^[0-9]*$//' -e 's/[0-9]*[^0-9]//'`
+	PYTHON_VERSION_PATCH=`echo $v | sed 's/[^0-9].*//'`
+	if test -z "$PYTHON_VERSION_PATCH"; then PYTHON_VERSION_PATCH="0"; fi
+	]
 
-        #
-        # Check if you have distutils, else fail
-        #
-        AC_MSG_CHECKING([for the distutils Python package])
-        if ac_distutils_result=`$PYTHON -c "import distutils" 2>&1`; then
+	# For some systems, sysconfig exists, but has the wrong paths,
+	# on Debian 10, for python 2.7 and 3.7. So, we check the version,
+	# and for older versions try distutils.sysconfig first. For newer
+	# versions>=3.10, where distutils.sysconfig is deprecated, use
+	# sysconfig first and then attempt the other one.
+	py_distutils_first="no"
+	if test $PYTHON_VERSION_MAJOR -lt 3; then
+		py_distutils_first="yes"
+	fi
+	if test $PYTHON_VERSION_MAJOR -eq 3 -a $PYTHON_VERSION_MINOR -lt 10; then
+		py_distutils_first="yes"
+	fi
+
+	# Check if you have the first module
+	if test "$py_distutils_first" = "yes"; then m="distutils"; else m="sysconfig"; fi
+	sysconfig_module=""
+	AC_MSG_CHECKING([for the $m Python module])
+        if ac_modulecheck_result1=`$PYTHON -c "import $m" 2>&1`; then
                 AC_MSG_RESULT([yes])
-        else
+		sysconfig_module="$m"
+	else
                 AC_MSG_RESULT([no])
-                AC_MSG_ERROR([cannot import Python module "distutils".
-Please check your Python installation. The error was:
-$ac_distutils_result])
-                PYTHON_VERSION=""
-        fi
+	fi
+
+	# if not found, try the other one.
+	if test -z "$sysconfig_module"; then
+		if test "$py_distutils_first" = "yes"; then m2="sysconfig"; else m2="distutils"; fi
+		AC_MSG_CHECKING([for the $m2 Python module])
+		if ac_modulecheck_result2=`$PYTHON -c "import $m2" 2>&1`; then
+			AC_MSG_RESULT([yes])
+			sysconfig_module="$m2"
+		else
+			AC_MSG_RESULT([no])
+			AC_MSG_ERROR([cannot import Python module "$m", or "$m2".
+	Please check your Python installation. The errors are:
+	$m
+	$ac_modulecheck_result1
+	$m2
+	$ac_modulecheck_result2])
+			PYTHON_VERSION=""
+		fi
+	fi
+	if test "$sysconfig_module" = "distutils"; then sysconfig_module="distutils.sysconfig"; fi
 
         #
         # Check for Python include path
         #
         AC_MSG_CHECKING([for Python include path])
         if test -z "$PYTHON_CPPFLAGS"; then
-                python_path=`$PYTHON -c "import distutils.sysconfig; \
-                        print(distutils.sysconfig.get_python_inc());"`
+		if test "$sysconfig_module" = "sysconfig"; then
+			python_path=`$PYTHON -c 'import sysconfig; \
+				print(sysconfig.get_path("include"));'`
+		else
+			python_path=`$PYTHON -c "import distutils.sysconfig; \
+				print(distutils.sysconfig.get_python_inc());"`
+		fi
                 if test -n "${python_path}"; then
                         python_path="-I$python_path"
                 fi
@@ -52,14 +99,14 @@ $ac_distutils_result])
         #
         AC_MSG_CHECKING([for Python library path])
         if test -z "$PYTHON_LDFLAGS"; then
-                PYTHON_LDFLAGS=`$PYTHON -c "from distutils.sysconfig import *; \
+                PYTHON_LDFLAGS=`$PYTHON -c "from $sysconfig_module import *; \
                         print('-L'+get_config_var('LIBDIR')+' -L'+get_config_var('LIBDEST')+' '+get_config_var('BLDLIBRARY'));"`
         fi
         AC_MSG_RESULT([$PYTHON_LDFLAGS])
         AC_SUBST([PYTHON_LDFLAGS])
 
         if test -z "$PYTHON_LIBDIR"; then
-                PYTHON_LIBDIR=`$PYTHON -c "from distutils.sysconfig import *; \
+                PYTHON_LIBDIR=`$PYTHON -c "from $sysconfig_module import *; \
                         print(get_config_var('LIBDIR'));"`
         fi
 
@@ -68,8 +115,13 @@ $ac_distutils_result])
         #
         AC_MSG_CHECKING([for Python site-packages path])
         if test -z "$PYTHON_SITE_PKG"; then
-                PYTHON_SITE_PKG=`$PYTHON -c "import distutils.sysconfig; \
-                        print(distutils.sysconfig.get_python_lib(1,0));"`
+		if test "$sysconfig_module" = "sysconfig"; then
+			PYTHON_SITE_PKG=`$PYTHON -c 'import sysconfig; \
+				print(sysconfig.get_path("platlib"));'`
+		else
+			PYTHON_SITE_PKG=`$PYTHON -c "import distutils.sysconfig; \
+				print(distutils.sysconfig.get_python_lib(1,0));"`
+		fi
         fi
         AC_MSG_RESULT([$PYTHON_SITE_PKG])
         AC_SUBST([PYTHON_SITE_PKG])

cachedb/cachedb.c

@@ -102,7 +102,6 @@ static int
 testframe_init(struct module_env* env, struct cachedb_env* cachedb_env)
 {
 	struct testframe_moddata* d;
-	(void)env;
 	verbose(VERB_ALGO, "testframe_init");
 	d = (struct testframe_moddata*)calloc(1,
 		sizeof(struct testframe_moddata));
@@ -111,6 +110,15 @@ testframe_init(struct module_env* env, struct cachedb_env* cachedb_env)
 		log_err("out of memory");
 		return 0;
 	}
+	/* Register an EDNS option (65534) to bypass the worker cache lookup
+	 * for testing */
+	if(!edns_register_option(LDNS_EDNS_UNBOUND_CACHEDB_TESTFRAME_TEST,
+		1 /* bypass cache */,
+		0 /* no aggregation */, env)) {
+		log_err("testframe_init, could not register test opcode");
+		free(d);
+		return 0;
+	}
 	lock_basic_init(&d->lock);
 	lock_protect(&d->lock, d, sizeof(*d));
 	return 1;
@@ -218,6 +226,8 @@ static int
 cachedb_apply_cfg(struct cachedb_env* cachedb_env, struct config_file* cfg)
 {
 	const char* backend_str = cfg->cachedb_backend;
+	if(!backend_str || *backend_str==0)
+		return 1;
 	cachedb_env->backend = cachedb_find_backend(backend_str);
 	if(!cachedb_env->backend) {
 		log_err("cachedb: cannot find backend name '%s'", backend_str);
@@ -228,7 +238,7 @@ cachedb_apply_cfg(struct cachedb_env* cachedb_env, struct config_file* cfg)
 	return 1;
 }
 
-int 
+int
 cachedb_init(struct module_env* env, int id)
 {
 	struct cachedb_env* cachedb_env = (struct cachedb_env*)calloc(1,
@@ -255,11 +265,11 @@ cachedb_init(struct module_env* env, int id)
 		return 0;
 	}
 	cachedb_env->enabled = 1;
-	if(env->cfg->serve_expired_reply_ttl)
+	if(env->cfg->serve_expired && env->cfg->serve_expired_reply_ttl)
 		log_warn(
 			"cachedb: serve-expired-reply-ttl is set but not working for data "
-			"originating from the external cache; 0 TLL is used for those.");
-	if(env->cfg->serve_expired_client_timeout)
+			"originating from the external cache; 0 TTL is used for those.");
+	if(env->cfg->serve_expired && env->cfg->serve_expired_client_timeout)
 		log_warn(
 			"cachedb: serve-expired-client-timeout is set but not working for "
 			"data originating from the external cache; expired data are used "
@@ -267,19 +277,16 @@ cachedb_init(struct module_env* env, int id)
 	return 1;
 }
 
-void 
+void
 cachedb_deinit(struct module_env* env, int id)
 {
 	struct cachedb_env* cachedb_env;
 	if(!env || !env->modinfo[id])
 		return;
 	cachedb_env = (struct cachedb_env*)env->modinfo[id];
-	/* free contents */
-	/* TODO */
 	if(cachedb_env->enabled) {
 		(*cachedb_env->backend->deinit)(env, cachedb_env);
 	}
-
 	free(cachedb_env);
 	env->modinfo[id] = NULL;
 }
@@ -390,6 +397,15 @@ prep_data(struct module_qstate* qstate, struct sldns_buffer* buf)
 
 	if(!qstate->return_msg || !qstate->return_msg->rep)
 		return 0;
+	/* do not store failures like SERVFAIL in the cachedb, this avoids
+	 * overwriting expired, valid, content with broken content. */
+	if(FLAGS_GET_RCODE(qstate->return_msg->rep->flags) !=
+		LDNS_RCODE_NOERROR &&
+	   FLAGS_GET_RCODE(qstate->return_msg->rep->flags) !=
+		LDNS_RCODE_NXDOMAIN &&
+	   FLAGS_GET_RCODE(qstate->return_msg->rep->flags) !=
+		LDNS_RCODE_YXDOMAIN)
+		return 0;
 	/* We don't store the reply if its TTL is 0 unless serve-expired is
 	 * enabled.  Such a reply won't be reusable and simply be a waste for
 	 * the backend.  It's also compatible with the default behavior of
@@ -397,6 +413,14 @@ prep_data(struct module_qstate* qstate, struct sldns_buffer* buf)
 	if(qstate->return_msg->rep->ttl == 0 &&
 		!qstate->env->cfg->serve_expired)
 		return 0;
+
+	/* The EDE is added to the out-list so it is encoded in the cached message */
+	if (qstate->env->cfg->ede && qstate->return_msg->rep->reason_bogus != LDNS_EDE_NONE) {
+		edns_opt_list_append_ede(&edns.opt_list_out, qstate->env->scratch,
+					qstate->return_msg->rep->reason_bogus,
+					qstate->return_msg->rep->reason_bogus_str);
+	}
+
 	if(verbosity >= VERB_ALGO)
 		log_dns_msg("cachedb encoding", &qstate->return_msg->qinfo,
 	                qstate->return_msg->rep);
@@ -493,6 +517,7 @@ parse_data(struct module_qstate* qstate, struct sldns_buffer* buf)
 {
 	struct msg_parse* prs;
 	struct edns_data edns;
+	struct edns_option* ede;
 	uint64_t timestamp, expiry;
 	time_t adjust;
 	size_t lim = sldns_buffer_limit(buf);
@@ -530,6 +555,24 @@ parse_data(struct module_qstate* qstate, struct sldns_buffer* buf)
 	if(!qstate->return_msg)
 		return 0;
 	
+	/* We find the EDE in the in-list after parsing */
+	if(qstate->env->cfg->ede &&
+		(ede = edns_opt_list_find(edns.opt_list_in, LDNS_EDNS_EDE))) {
+		if(ede->opt_len >= 2) {
+			qstate->return_msg->rep->reason_bogus =
+				sldns_read_uint16(ede->opt_data);
+		}
+		/* allocate space and store the error string and it's size */
+		if(ede->opt_len > 2) {
+			size_t ede_len = ede->opt_len - 2;
+			qstate->return_msg->rep->reason_bogus_str = regional_alloc(
+				qstate->region, sizeof(char) * (ede_len+1));
+			memcpy(qstate->return_msg->rep->reason_bogus_str,
+				ede->opt_data+2, ede_len);
+			qstate->return_msg->rep->reason_bogus_str[ede_len] = 0;
+		}
+	}
+
 	qstate->return_rcode = LDNS_RCODE_NOERROR;
 
 	/* see how much of the TTL expired, and remove it */
@@ -542,10 +585,16 @@ parse_data(struct module_qstate* qstate, struct sldns_buffer* buf)
 		verbose(VERB_ALGO, "cachedb msg expired");
 		/* If serve-expired is enabled, we still use an expired message
 		 * setting the TTL to 0. */
-		if(qstate->env->cfg->serve_expired)
-			adjust = -1;
-		else
+		if(!qstate->env->cfg->serve_expired ||
+			(FLAGS_GET_RCODE(qstate->return_msg->rep->flags)
+			!= LDNS_RCODE_NOERROR &&
+			FLAGS_GET_RCODE(qstate->return_msg->rep->flags)
+			!= LDNS_RCODE_NXDOMAIN &&
+			FLAGS_GET_RCODE(qstate->return_msg->rep->flags)
+			!= LDNS_RCODE_YXDOMAIN))
 			return 0; /* message expired */
+		else
+			adjust = -1;
 	}
 	verbose(VERB_ALGO, "cachedb msg adjusted down by %d", (int)adjust);
 	adjust_msg_ttl(qstate->return_msg, adjust);
@@ -615,11 +664,15 @@ cachedb_extcache_store(struct module_qstate* qstate, struct cachedb_env* ie)
  * See if unbound's internal cache can answer the query
  */
 static int
-cachedb_intcache_lookup(struct module_qstate* qstate)
+cachedb_intcache_lookup(struct module_qstate* qstate, struct cachedb_env* cde)
 {
 	uint8_t* dpname=NULL;
 	size_t dpnamelen=0;
 	struct dns_msg* msg;
+	/* for testframe bypass this lookup */
+	if(cde->backend == &testframe_backend) {
+		return 0;
+	}
 	if(iter_stub_fwd_no_cache(qstate, &qstate->qinfo,
 		&dpname, &dpnamelen))
 		return 0; /* no cache for these queries */
@@ -662,7 +715,7 @@ cachedb_intcache_store(struct module_qstate* qstate)
 		return;
 	(void)dns_cache_store(qstate->env, &qstate->qinfo,
 		qstate->return_msg->rep, 0, qstate->prefetch_leeway, 0,
-		qstate->region, store_flags);
+		qstate->region, store_flags, qstate->qstarttime);
 }
 
 /**
@@ -678,6 +731,7 @@ cachedb_handle_query(struct module_qstate* qstate,
 	struct cachedb_qstate* ATTR_UNUSED(iq),
 	struct cachedb_env* ie, int id)
 {
+	qstate->is_cachedb_answer = 0;
 	/* check if we are enabled, and skip if so */
 	if(!ie->enabled) {
 		/* pass request to next module */
@@ -694,7 +748,7 @@ cachedb_handle_query(struct module_qstate* qstate,
 
 	/* lookup inside unbound's internal cache.
 	 * This does not look for expired entries. */
-	if(cachedb_intcache_lookup(qstate)) {
+	if(cachedb_intcache_lookup(qstate, ie)) {
 		if(verbosity >= VERB_ALGO) {
 			if(qstate->return_msg->rep)
 				log_dns_msg("cachedb internal cache lookup",
@@ -731,6 +785,7 @@ cachedb_handle_query(struct module_qstate* qstate,
 				qstate->ext_state[id] = module_wait_module;
 				return;
 		}
+		qstate->is_cachedb_answer = 1;
 		/* we are done with the query */
 		qstate->ext_state[id] = module_finished;
 		return;
@@ -753,6 +808,7 @@ static void
 cachedb_handle_response(struct module_qstate* qstate,
 	struct cachedb_qstate* ATTR_UNUSED(iq), struct cachedb_env* ie, int id)
 {
+	qstate->is_cachedb_answer = 0;
 	/* check if we are not enabled or instructed to not cache, and skip */
 	if(!ie->enabled || qstate->no_cache_store) {
 		/* we are done with the query */

cachedb/redis.c

@@ -56,19 +56,43 @@ struct redis_moddata {
 	int numctxs;		/* number of ctx entries */
 	const char* server_host; /* server's IP address or host name */
 	int server_port;	 /* server's TCP port */
+	const char* server_path; /* server's unix path, or "", NULL if unused */
+	const char* server_password; /* server's AUTH password, or "", NULL if unused */
 	struct timeval timeout;	 /* timeout for connection setup and commands */
+	int logical_db;		/* the redis logical database to use */
 };
 
 static redisReply* redis_command(struct module_env*, struct cachedb_env*,
 	const char*, const uint8_t*, size_t);
 
+static void
+moddata_clean(struct redis_moddata** moddata) {
+	if(!moddata || !*moddata)
+		return;
+	if((*moddata)->ctxs) {
+		int i;
+		for(i = 0; i < (*moddata)->numctxs; i++) {
+			if((*moddata)->ctxs[i])
+				redisFree((*moddata)->ctxs[i]);
+		}
+		free((*moddata)->ctxs);
+	}
+	free(*moddata);
+	*moddata = NULL;
+}
+
 static redisContext*
 redis_connect(const struct redis_moddata* moddata)
 {
 	redisContext* ctx;
 
-	ctx = redisConnectWithTimeout(moddata->server_host,
-		moddata->server_port, moddata->timeout);
+	if(moddata->server_path && moddata->server_path[0]!=0) {
+		ctx = redisConnectUnixWithTimeout(moddata->server_path,
+			moddata->timeout);
+	} else {
+		ctx = redisConnectWithTimeout(moddata->server_host,
+			moddata->server_port, moddata->timeout);
+	}
 	if(!ctx || ctx->err) {
 		const char *errstr = "out of memory";
 		if(ctx)
@@ -80,9 +104,31 @@ redis_connect(const struct redis_moddata* moddata)
 		log_err("failed to set redis timeout");
 		goto fail;
 	}
+	if(moddata->server_password && moddata->server_password[0]!=0) {
+		redisReply* rep;
+		rep = redisCommand(ctx, "AUTH %s", moddata->server_password);
+		if(!rep || rep->type == REDIS_REPLY_ERROR) {
+			log_err("failed to authenticate with password");
+			freeReplyObject(rep);
+			goto fail;
+		}
+		freeReplyObject(rep);
+	}
+	if(moddata->logical_db > 0) {
+		redisReply* rep;
+		rep = redisCommand(ctx, "SELECT %d", moddata->logical_db);
+		if(!rep || rep->type == REDIS_REPLY_ERROR) {
+			log_err("failed to set logical database (%d)",
+				moddata->logical_db);
+			freeReplyObject(rep);
+			goto fail;
+		}
+		freeReplyObject(rep);
+	}
+	verbose(VERB_OPS, "Connection to Redis established");
 	return ctx;
 
-  fail:
+fail:
 	if(ctx)
 		redisFree(ctx);
 	return NULL;
@@ -94,28 +140,36 @@ redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
 	int i;
 	struct redis_moddata* moddata = NULL;
 
-	verbose(VERB_ALGO, "redis_init");
+	verbose(VERB_OPS, "Redis initialization");
 
 	moddata = calloc(1, sizeof(struct redis_moddata));
 	if(!moddata) {
 		log_err("out of memory");
-		return 0;
+		goto fail;
 	}
 	moddata->numctxs = env->cfg->num_threads;
 	moddata->ctxs = calloc(env->cfg->num_threads, sizeof(redisContext*));
 	if(!moddata->ctxs) {
 		log_err("out of memory");
-		free(moddata);
-		return 0;
+		goto fail;
 	}
 	/* note: server_host is a shallow reference to configured string.
 	 * we don't have to free it in this module. */
 	moddata->server_host = env->cfg->redis_server_host;
 	moddata->server_port = env->cfg->redis_server_port;
+	moddata->server_path = env->cfg->redis_server_path;
+	moddata->server_password = env->cfg->redis_server_password;
 	moddata->timeout.tv_sec = env->cfg->redis_timeout / 1000;
 	moddata->timeout.tv_usec = (env->cfg->redis_timeout % 1000) * 1000;
-	for(i = 0; i < moddata->numctxs; i++)
-		moddata->ctxs[i] = redis_connect(moddata);
+	moddata->logical_db = env->cfg->redis_logical_db;
+	for(i = 0; i < moddata->numctxs; i++) {
+		redisContext* ctx = redis_connect(moddata);
+		if(!ctx) {
+			log_err("redis_init: failed to init redis");
+			goto fail;
+		}
+		moddata->ctxs[i] = ctx;
+	}
 	cachedb_env->backend_data = moddata;
 	if(env->cfg->redis_expire_records) {
 		redisReply* rep = NULL;
@@ -128,7 +182,7 @@ redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
 			log_err("redis_init: failed to init redis, the "
 				"redis-expire-records option requires the SETEX command "
 				"(redis >= 2.0.0)");
-			return 0;
+			goto fail;
 		}
 		redis_reply_type = rep->type;
 		freeReplyObject(rep);
@@ -140,11 +194,14 @@ redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
 			log_err("redis_init: failed to init redis, the "
 				"redis-expire-records option requires the SETEX command "
 				"(redis >= 2.0.0)");
-			return 0;
+			goto fail;
 		}
 	}
-
 	return 1;
+
+fail:
+	moddata_clean(&moddata);
+	return 0;
 }
 
 static void
@@ -154,19 +211,8 @@ redis_deinit(struct module_env* env, struct cachedb_env* cachedb_env)
 		cachedb_env->backend_data;
 	(void)env;
 
-	verbose(VERB_ALGO, "redis_deinit");
-
-	if(!moddata)
-		return;
-	if(moddata->ctxs) {
-		int i;
-		for(i = 0; i < moddata->numctxs; i++) {
-			if(moddata->ctxs[i])
-				redisFree(moddata->ctxs[i]);
-		}
-		free(moddata->ctxs);
-	}
-	free(moddata);
+	verbose(VERB_OPS, "Redis deinitialization");
+	moddata_clean(&moddata);
 }
 
 /*

compat/arc4random.c

@@ -57,6 +57,8 @@
 #define BLOCKSZ	64
 #define RSBUFSZ	(16*BLOCKSZ)
 
+#define REKEY_BASE	(1024*1024) /* NB. should be a power of 2 */
+
 /* Marked MAP_INHERIT_ZERO, so zero'd out in fork children. */
 static struct {
 	size_t		rs_have;	/* valid bytes at end of rs_buf */
@@ -179,6 +181,7 @@ static void
 _rs_stir(void)
 {
 	u_char rnd[KEYSZ + IVSZ];
+	uint32_t rekey_fuzz = 0;
 
 	if (getentropy(rnd, sizeof rnd) == -1) {
 		if(errno != ENOSYS ||
@@ -201,7 +204,10 @@ _rs_stir(void)
 	rs->rs_have = 0;
 	memset(rsx->rs_buf, 0, sizeof(rsx->rs_buf));
 
-	rs->rs_count = 1600000;
+	/* rekey interval should not be predictable */
+	chacha_encrypt_bytes(&rsx->rs_chacha, (uint8_t *)&rekey_fuzz,
+	    (uint8_t *)&rekey_fuzz, sizeof(rekey_fuzz));
+	rs->rs_count = REKEY_BASE + (rekey_fuzz % REKEY_BASE);
 }
 
 static inline void

compat/getentropy_solaris.c

@@ -47,7 +47,7 @@
 #define SHA512_Update SHA512Update
 #define SHA512_Final SHA512Final
 #else
-#include "openssl/sha.h"
+#include <openssl/sha.h>
 #endif
 
 #include <sys/vfs.h>

config.h.in

@@ -222,6 +222,10 @@
 /* Define to 1 if you have the `EVP_cleanup' function. */
 #undef HAVE_EVP_CLEANUP
 
+/* Define to 1 if you have the `EVP_default_properties_is_fips_enabled'
+   function. */
+#undef HAVE_EVP_DEFAULT_PROPERTIES_IS_FIPS_ENABLED
+
 /* Define to 1 if you have the `EVP_DigestVerify' function. */
 #undef HAVE_EVP_DIGESTVERIFY
 
@@ -294,6 +298,9 @@
 /* Define to 1 if you have the `getrlimit' function. */
 #undef HAVE_GETRLIMIT
 
+/* Define to 1 if you have the `gettid' function. */
+#undef HAVE_GETTID
+
 /* Define to 1 if you have the `glob' function. */
 #undef HAVE_GLOB
 
@@ -357,6 +364,9 @@
 /* Define if we have LibreSSL */
 #undef HAVE_LIBRESSL
 
+/* Define to 1 if you have the <linux/net_tstamp.h> header file. */
+#undef HAVE_LINUX_NET_TSTAMP_H
+
 /* Define to 1 if you have the `localtime_r' function. */
 #undef HAVE_LOCALTIME_R
 
@@ -381,6 +391,9 @@
 /* Define to 1 if you have the <netinet/tcp.h> header file. */
 #undef HAVE_NETINET_TCP_H
 
+/* Define to 1 if you have the <netioapi.h> header file. */
+#undef HAVE_NETIOAPI_H
+
 /* Use libnettle for crypto */
 #undef HAVE_NETTLE
 
@@ -450,6 +463,12 @@
 /* Define to 1 if you have the `OSSL_PARAM_BLD_new' function. */
 #undef HAVE_OSSL_PARAM_BLD_NEW
 
+/* Define to 1 if you have the `poll' function. */
+#undef HAVE_POLL
+
+/* Define to 1 if you have the <poll.h> header file. */
+#undef HAVE_POLL_H
+
 /* Define if you have POSIX threads libraries and header files. */
 #undef HAVE_PTHREAD
 
@@ -793,12 +812,18 @@
 /* Shared data */
 #undef SHARE_DIR
 
+/* The size of `pthread_t', as computed by sizeof. */
+#undef SIZEOF_PTHREAD_T
+
 /* The size of `size_t', as computed by sizeof. */
 #undef SIZEOF_SIZE_T
 
 /* The size of `time_t', as computed by sizeof. */
 #undef SIZEOF_TIME_T
 
+/* The size of `unsigned long', as computed by sizeof. */
+#undef SIZEOF_UNSIGNED_LONG
+
 /* define if (v)snprintf does not return length needed, (but length used) */
 #undef SNPRINTF_RET_BROKEN
 
@@ -968,6 +993,10 @@
 /* Define to 1 if you need to in order for `stat' and other things to work. */
 #undef _POSIX_SOURCE
 
+/* defined to use gcc ansi snprintf and sscanf that understands %lld when
+   compiled for windows. */
+#undef __USE_MINGW_ANSI_STDIO
+
 /* Define to empty if `const' does not conform to ANSI C. */
 #undef const
 
@@ -1042,39 +1071,39 @@
 
 #if defined(OMITTED__D_GNU_SOURCE) && !defined(_GNU_SOURCE)
 #define _GNU_SOURCE 1
-#endif 
+#endif
 
 #if defined(OMITTED__D_BSD_SOURCE) && !defined(_BSD_SOURCE)
 #define _BSD_SOURCE 1
-#endif 
+#endif
 
 #if defined(OMITTED__D_DEFAULT_SOURCE) && !defined(_DEFAULT_SOURCE)
 #define _DEFAULT_SOURCE 1
-#endif 
+#endif
 
 #if defined(OMITTED__D__EXTENSIONS__) && !defined(__EXTENSIONS__)
 #define __EXTENSIONS__ 1
-#endif 
+#endif
 
 #if defined(OMITTED__D_POSIX_C_SOURCE_200112) && !defined(_POSIX_C_SOURCE)
 #define _POSIX_C_SOURCE 200112
-#endif 
+#endif
 
 #if defined(OMITTED__D_XOPEN_SOURCE_600) && !defined(_XOPEN_SOURCE)
 #define _XOPEN_SOURCE 600
-#endif 
+#endif
 
 #if defined(OMITTED__D_XOPEN_SOURCE_EXTENDED_1) && !defined(_XOPEN_SOURCE_EXTENDED)
 #define _XOPEN_SOURCE_EXTENDED 1
-#endif 
+#endif
 
 #if defined(OMITTED__D_ALL_SOURCE) && !defined(_ALL_SOURCE)
 #define _ALL_SOURCE 1
-#endif 
+#endif
 
 #if defined(OMITTED__D_LARGEFILE_SOURCE_1) && !defined(_LARGEFILE_SOURCE)
 #define _LARGEFILE_SOURCE 1
-#endif 
+#endif
 
 
 
@@ -1147,7 +1176,7 @@
 #include <ws2tcpip.h>
 #endif
 
-#ifndef USE_WINSOCK
+#if !defined(USE_WINSOCK) || !defined(HAVE_SNPRINTF) || defined(SNPRINTF_RET_BROKEN) || defined(__USE_MINGW_ANSI_STDIO)
 #define ARG_LL "%ll"
 #else
 #define ARG_LL "%I64"
@@ -1158,7 +1187,7 @@
 #endif
 
 
- 
+
 #ifdef HAVE_ATTR_FORMAT
 #  define ATTR_FORMAT(archetype, string_index, first_to_check) \
     __attribute__ ((format (archetype, string_index, first_to_check)))
@@ -1268,7 +1297,7 @@ void* reallocarray(void *ptr, size_t nmemb, size_t size);
 #ifdef HAVE_WINSOCK2_H
 #define FD_SET_T (u_int)
 #else
-#define FD_SET_T 
+#define FD_SET_T
 #endif
 
 

configure.ac

@@ -10,15 +10,15 @@ sinclude(dnscrypt/dnscrypt.m4)
 
 # must be numbers. ac_defun because of later processing
 m4_define([VERSION_MAJOR],[1])
-m4_define([VERSION_MINOR],[14])
-m4_define([VERSION_MICRO],[0])
+m4_define([VERSION_MINOR],[18])
+m4_define([VERSION_MICRO],[1])
 AC_INIT([unbound],m4_defn([VERSION_MAJOR]).m4_defn([VERSION_MINOR]).m4_defn([VERSION_MICRO]),[unbound-bugs@nlnetlabs.nl or https://github.com/NLnetLabs/unbound/issues],[unbound])
 AC_SUBST(UNBOUND_VERSION_MAJOR, [VERSION_MAJOR])
 AC_SUBST(UNBOUND_VERSION_MINOR, [VERSION_MINOR])
 AC_SUBST(UNBOUND_VERSION_MICRO, [VERSION_MICRO])
 
 LIBUNBOUND_CURRENT=9
-LIBUNBOUND_REVISION=14
+LIBUNBOUND_REVISION=23
 LIBUNBOUND_AGE=1
 # 1.0.0 had 0:12:0
 # 1.0.1 had 0:13:0
@@ -100,6 +100,15 @@ LIBUNBOUND_AGE=1
 # 1.13.1 had 9:12:1
 # 1.13.2 had 9:13:1
 # 1.14.0 had 9:14:1
+# 1.15.0 had 9:15:1
+# 1.16.0 had 9:16:1
+# 1.16.1 had 9:17:1
+# 1.16.2 had 9:18:1
+# 1.16.3 had 9:19:1
+# 1.17.0 had 9:20:1
+# 1.17.1 had 9:21:1
+# 1.18.0 had 9:22:1
+# 1.18.1 had 9:23:1
 
 #   Current  -- the number of the binary API that we're implementing
 #   Revision -- which iteration of the implementation of the binary
@@ -149,7 +158,7 @@ esac
 
 # are we on MinGW?
 if uname -s 2>&1 | grep MINGW >/dev/null; then on_mingw="yes"
-else 
+else
 	if echo $host | grep mingw >/dev/null; then on_mingw="yes"
 	else on_mingw="no"; fi
 fi
@@ -178,9 +187,9 @@ ub_conf_dir=`AS_DIRNAME(["$ub_conf_file"])`
 AC_SUBST(ub_conf_dir)
 
 # Determine run, chroot directory and pidfile locations
-AC_ARG_WITH(run-dir, 
-    AS_HELP_STRING([--with-run-dir=path],[set default directory to chdir to (by default dir part of cfg file)]), 
-    UNBOUND_RUN_DIR="$withval", 
+AC_ARG_WITH(run-dir,
+    AS_HELP_STRING([--with-run-dir=path],[set default directory to chdir to (by default dir part of cfg file)]),
+    UNBOUND_RUN_DIR="$withval",
 if test $on_mingw = no; then
     UNBOUND_RUN_DIR=`dirname "$ub_conf_file"`
 else
@@ -191,9 +200,9 @@ AC_SUBST(UNBOUND_RUN_DIR)
 ACX_ESCAPE_BACKSLASH($UNBOUND_RUN_DIR, hdr_run)
 AC_DEFINE_UNQUOTED(RUN_DIR, ["$hdr_run"], [Directory to chdir to])
 
-AC_ARG_WITH(chroot-dir, 
-    AS_HELP_STRING([--with-chroot-dir=path],[set default directory to chroot to (by default same as run-dir)]), 
-    UNBOUND_CHROOT_DIR="$withval", 
+AC_ARG_WITH(chroot-dir,
+    AS_HELP_STRING([--with-chroot-dir=path],[set default directory to chroot to (by default same as run-dir)]),
+    UNBOUND_CHROOT_DIR="$withval",
 if test $on_mingw = no; then
     UNBOUND_CHROOT_DIR="$UNBOUND_RUN_DIR"
 else
@@ -211,9 +220,9 @@ AC_ARG_WITH(share-dir,
 AC_SUBST(UNBOUND_SHARE_DIR)
 AC_DEFINE_UNQUOTED(SHARE_DIR, ["$UNBOUND_SHARE_DIR"], [Shared data])
 
-AC_ARG_WITH(pidfile, 
-    AS_HELP_STRING([--with-pidfile=filename],[set default pathname to unbound pidfile (default run-dir/unbound.pid)]), 
-    UNBOUND_PIDFILE="$withval", 
+AC_ARG_WITH(pidfile,
+    AS_HELP_STRING([--with-pidfile=filename],[set default pathname to unbound pidfile (default run-dir/unbound.pid)]),
+    UNBOUND_PIDFILE="$withval",
 if test $on_mingw = no; then
     UNBOUND_PIDFILE="$UNBOUND_RUN_DIR/unbound.pid"
 else
@@ -224,9 +233,9 @@ AC_SUBST(UNBOUND_PIDFILE)
 ACX_ESCAPE_BACKSLASH($UNBOUND_PIDFILE, hdr_pid)
 AC_DEFINE_UNQUOTED(PIDFILE, ["$hdr_pid"], [default pidfile location])
 
-AC_ARG_WITH(rootkey-file, 
-    AS_HELP_STRING([--with-rootkey-file=filename],[set default pathname to root key file (default run-dir/root.key). This file is read and written.]), 
-    UNBOUND_ROOTKEY_FILE="$withval", 
+AC_ARG_WITH(rootkey-file,
+    AS_HELP_STRING([--with-rootkey-file=filename],[set default pathname to root key file (default run-dir/root.key). This file is read and written.]),
+    UNBOUND_ROOTKEY_FILE="$withval",
 if test $on_mingw = no; then
     UNBOUND_ROOTKEY_FILE="$UNBOUND_RUN_DIR/root.key"
 else
@@ -237,9 +246,9 @@ AC_SUBST(UNBOUND_ROOTKEY_FILE)
 ACX_ESCAPE_BACKSLASH($UNBOUND_ROOTKEY_FILE, hdr_rkey)
 AC_DEFINE_UNQUOTED(ROOT_ANCHOR_FILE, ["$hdr_rkey"], [default rootkey location])
 
-AC_ARG_WITH(rootcert-file, 
-    AS_HELP_STRING([--with-rootcert-file=filename],[set default pathname to root update certificate file (default run-dir/icannbundle.pem).  This file need not exist if you are content with the builtin.]), 
-    UNBOUND_ROOTCERT_FILE="$withval", 
+AC_ARG_WITH(rootcert-file,
+    AS_HELP_STRING([--with-rootcert-file=filename],[set default pathname to root update certificate file (default run-dir/icannbundle.pem).  This file need not exist if you are content with the builtin.]),
+    UNBOUND_ROOTCERT_FILE="$withval",
 if test $on_mingw = no; then
     UNBOUND_ROOTCERT_FILE="$UNBOUND_RUN_DIR/icannbundle.pem"
 else
@@ -250,9 +259,9 @@ AC_SUBST(UNBOUND_ROOTCERT_FILE)
 ACX_ESCAPE_BACKSLASH($UNBOUND_ROOTCERT_FILE, hdr_rpem)
 AC_DEFINE_UNQUOTED(ROOT_CERT_FILE, ["$hdr_rpem"], [default rootcert location])
 
-AC_ARG_WITH(username, 
-    AS_HELP_STRING([--with-username=user],[set default user that unbound changes to (default user is unbound)]), 
-    UNBOUND_USERNAME="$withval", 
+AC_ARG_WITH(username,
+    AS_HELP_STRING([--with-username=user],[set default user that unbound changes to (default user is unbound)]),
+    UNBOUND_USERNAME="$withval",
     UNBOUND_USERNAME="unbound")
 AC_SUBST(UNBOUND_USERNAME)
 AC_DEFINE_UNQUOTED(UB_USERNAME, ["$UNBOUND_USERNAME"], [default username])
@@ -271,14 +280,14 @@ ACX_CHECK_COMPILER_FLAG(g, [CFLAGS="$CFLAGS -g"])
 ACX_CHECK_COMPILER_FLAG(O2, [CFLAGS="$CFLAGS -O2"])
 default_cflags=yes
 fi
-AC_PROG_CC
+m4_version_prereq([2.70], [AC_PROG_CC], [AC_PROG_CC_STDC])
 ACX_DEPFLAG
 ACX_DETERMINE_EXT_FLAGS_UNBOUND
 
 # debug mode flags warnings
 AC_ARG_ENABLE(checking, AS_HELP_STRING([--enable-checking],[Enable warnings, asserts, makefile-dependencies]))
 AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[same as enable-checking]))
-if test "$enable_debug" = "yes"; then debug_enabled="$enable_debug"; 
+if test "$enable_debug" = "yes"; then debug_enabled="$enable_debug";
 else debug_enabled="$enable_checking"; fi
 AC_SUBST(debug_enabled)
 case "$debug_enabled" in
@@ -381,7 +390,17 @@ fi
 if test "$LEX" != "" -a "$LEX" != ":"; then
 ACX_YYLEX_OPTION
 fi
+if test "$LEX" = "" -o "$LEX" = ":"; then
+	if test ! -f util/configlexer.c; then
+		AC_MSG_ERROR([no lex and no util/configlexer.c: need flex and bison to compile from source repository.])
+	fi
+fi
 AC_PROG_YACC
+if test "$YACC" = "" -o "$YACC" = ":"; then
+	if test ! -f util/configparser.c; then
+		AC_MSG_ERROR([no yacc and no util/configparser.c: need flex and bison to compile from source repository.])
+	fi
+fi
 AC_CHECK_PROG(doxygen, doxygen, doxygen)
 AC_CHECK_TOOL(STRIP, strip)
 ACX_LIBTOOL_C_ONLY
@@ -392,7 +411,7 @@ PKG_PROG_PKG_CONFIG
 fi
 
 # Checks for header files.
-AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h netinet/tcp.h sys/param.h sys/select.h sys/socket.h sys/un.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h sys/endian.h libkern/OSByteOrder.h sys/ipc.h sys/shm.h ifaddrs.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h netinet/tcp.h sys/param.h sys/select.h sys/socket.h sys/un.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h sys/endian.h libkern/OSByteOrder.h sys/ipc.h sys/shm.h ifaddrs.h poll.h],,, [AC_INCLUDES_DEFAULT])
 # net/if.h portability for Darwin see:
 # https://www.gnu.org/software/autoconf/manual/autoconf-2.69/html_node/Header-Portability.html
 AC_CHECK_HEADERS([net/if.h],,, [
@@ -412,6 +431,42 @@ AC_CHECK_HEADERS([net/if.h],,, [
 
 # Check for Apple header. This uncovers TARGET_OS_IPHONE, TARGET_OS_TV or TARGET_OS_WATCH
 AC_CHECK_HEADERS([TargetConditionals.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([netioapi.h],,, [AC_INCLUDES_DEFAULT
+#if HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#ifdef HAVE_NETINET_TCP_H
+#include <netinet/tcp.h>
+#endif
+
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+
+#ifdef HAVE_WS2TCPIP_H
+#include <ws2tcpip.h>
+#endif
+])
+
+# Check for Linux timestamping headers
+AC_CHECK_HEADERS([linux/net_tstamp.h],,, [AC_INCLUDES_DEFAULT])
 
 # check for types.  
 # Using own tests for int64* because autoconf builtin only give 32bit.
@@ -471,7 +526,8 @@ res = strptime("2010-07-15T00:00:00+00:00", "%t%Y%t-%t%m%t-%t%d%tT%t%H%t:%t%M%t:
 if (!res) return 2;
 res = strptime("20070207111842", "%Y%m%d%H%M%S", &tm);
 if (!res) return 1; return 0; }
-]])] , [eval "ac_cv_c_strptime_works=yes"], [eval "ac_cv_c_strptime_works=no"])
+]])] , [eval "ac_cv_c_strptime_works=yes"], [eval "ac_cv_c_strptime_works=no"],
+[eval "ac_cv_c_strptime_works=maybe"])
 else
 eval "ac_cv_c_strptime_works=maybe"
 fi
@@ -508,11 +564,11 @@ sinclude(systemd.m4)
 # Include systemd.m4 - end
 
 # set memory allocation checking if requested
-AC_ARG_ENABLE(alloc-checks, AS_HELP_STRING([--enable-alloc-checks],[ enable to memory allocation statistics, for debug purposes ]), 
+AC_ARG_ENABLE(alloc-checks, AS_HELP_STRING([--enable-alloc-checks],[ enable to memory allocation statistics, for debug purposes ]),
 	, )
-AC_ARG_ENABLE(alloc-lite, AS_HELP_STRING([--enable-alloc-lite],[ enable for lightweight alloc assertions, for debug purposes ]), 
+AC_ARG_ENABLE(alloc-lite, AS_HELP_STRING([--enable-alloc-lite],[ enable for lightweight alloc assertions, for debug purposes ]),
 	, )
-AC_ARG_ENABLE(alloc-nonregional, AS_HELP_STRING([--enable-alloc-nonregional],[ enable nonregional allocs, slow but exposes regional allocations to other memory purifiers, for debug purposes ]), 
+AC_ARG_ENABLE(alloc-nonregional, AS_HELP_STRING([--enable-alloc-nonregional],[ enable nonregional allocs, slow but exposes regional allocations to other memory purifiers, for debug purposes ]),
 	, )
 if test x_$enable_alloc_nonregional = x_yes; then
 	AC_DEFINE(UNBOUND_ALLOC_NONREGIONAL, 1, [use malloc not regions, for debug use])
@@ -545,7 +601,7 @@ if test "$on_mingw" = "yes"; then
 ])],
 	AC_MSG_RESULT(yes)
 	AC_DEFINE(HAVE_WINDOWS_THREADS, 1, [Using Windows threads])
-,	
+,
 	AC_MSG_RESULT(no)
 )
 
@@ -556,7 +612,7 @@ else
 # check this first, so that the pthread lib does not get linked in via
 # libssl or libpython, and thus distorts the tests, and we end up using
 # the non-threadsafe C libraries.
-AC_ARG_WITH(pthreads, AS_HELP_STRING([--with-pthreads],[use pthreads library, or --without-pthreads to disable threading support.]), 
+AC_ARG_WITH(pthreads, AS_HELP_STRING([--with-pthreads],[use pthreads library, or --without-pthreads to disable threading support.]),
  [ ],[ withval="yes" ])
 ub_have_pthreads=no
 if test x_$withval != x_no; then
@@ -569,6 +625,8 @@ if test x_$withval != x_no; then
 		CC="$PTHREAD_CC"
 		ub_have_pthreads=yes
 		AC_CHECK_TYPES([pthread_spinlock_t, pthread_rwlock_t],,,[#include <pthread.h>])
+		AC_CHECK_SIZEOF([unsigned long])
+		AC_CHECK_SIZEOF(pthread_t)
 
 		if echo "$CFLAGS" | $GREP -e "-pthread" >/dev/null; then
 		AC_MSG_CHECKING([if -pthread unused during linking])
@@ -581,7 +639,7 @@ int main(void) {return 0;}
 		# first compile
 		echo "$CC $CFLAGS -c conftest.c -o conftest.o" >&AS_MESSAGE_LOG_FD
 		$CC $CFLAGS -c conftest.c -o conftest.o 2>&AS_MESSAGE_LOG_FD >&AS_MESSAGE_LOG_FD
-		if test $? = 0; then 
+		if test $? = 0; then
 			# then link
 			echo "$CC $CFLAGS -Werror $LDFLAGS $LIBS -o conftest contest.o" >&AS_MESSAGE_LOG_FD
 			$CC $CFLAGS -Werror $LDFLAGS $LIBS -o conftest conftest.o 2>&AS_MESSAGE_LOG_FD >&AS_MESSAGE_LOG_FD
@@ -602,7 +660,7 @@ int main(void) {return 0;}
 		])
 fi
 
-# check solaris thread library 
+# check solaris thread library
 AC_ARG_WITH(solaris-threads, AS_HELP_STRING([--with-solaris-threads],[use solaris native thread library.]), [ ],[ withval="no" ])
 ub_have_sol_threads=no
 if test x_$withval != x_no; then
@@ -616,8 +674,8 @@ if test x_$withval != x_no; then
 		ACX_CHECK_COMPILER_FLAG(mt, [CFLAGS="$CFLAGS -mt"],
 			[CFLAGS="$CFLAGS -D_REENTRANT"])
 		ub_have_sol_threads=yes
-	] , [ 
-		AC_MSG_ERROR([no solaris threads found.]) 
+	] , [
+		AC_MSG_ERROR([no solaris threads found.])
 	])
 	fi
 fi
@@ -692,7 +750,14 @@ if test x_$ub_test_python != x_no; then
    ac_save_LIBS="$LIBS" dnl otherwise AC_PYTHON_DEVEL thrashes $LIBS
    AC_PYTHON_DEVEL
    if test ! -z "$PYTHON_VERSION"; then
-	if test `$PYTHON -c "print('$PYTHON_VERSION' >= '2.4.0')"` = "False"; then
+	badversion="no"
+	if test "$PYTHON_VERSION_MAJOR" -lt 2; then
+		badversion="yes"
+	fi
+	if test "$PYTHON_VERSION_MAJOR" -eq 2 -a "$PYTHON_VERSION_MINOR" -lt 4; then
+		badversion="yes"
+	fi
+	if test "$badversion" = "yes"; then
 		AC_MSG_ERROR([Python version >= 2.4.0 is required])
 	fi
 
@@ -782,7 +847,7 @@ if test "`uname`" = "NetBSD"; then
 	AC_SUBST(NETBSD_LINTFLAGS)
 fi
 
-if test "`uname -o`" = "GNU/Linux"; then
+if test "`uname`" = "Linux"; then
 	# splint cannot parse modern c99 header files
 	GCC_DOCKER_LINTFLAGS='-syntax'
 	AC_SUBST(GCC_DOCKER_LINTFLAGS)
@@ -871,7 +936,7 @@ else
 	AC_MSG_RESULT([no])
 fi
 AC_CHECK_HEADERS([openssl/conf.h openssl/engine.h openssl/bn.h openssl/dh.h openssl/dsa.h openssl/rsa.h openssl/core_names.h openssl/param_build.h],,, [AC_INCLUDES_DEFAULT])
-AC_CHECK_FUNCS([OPENSSL_config EVP_sha1 EVP_sha256 EVP_sha512 FIPS_mode EVP_MD_CTX_new OpenSSL_add_all_digests OPENSSL_init_crypto EVP_cleanup ENGINE_cleanup ERR_load_crypto_strings CRYPTO_cleanup_all_ex_data ERR_free_strings RAND_cleanup DSA_SIG_set0 EVP_dss1 EVP_DigestVerify EVP_aes_256_cbc EVP_EncryptInit_ex HMAC_Init_ex CRYPTO_THREADID_set_callback EVP_MAC_CTX_set_params OSSL_PARAM_BLD_new BIO_set_callback_ex])
+AC_CHECK_FUNCS([OPENSSL_config EVP_sha1 EVP_sha256 EVP_sha512 FIPS_mode EVP_default_properties_is_fips_enabled EVP_MD_CTX_new OpenSSL_add_all_digests OPENSSL_init_crypto EVP_cleanup ENGINE_cleanup ERR_load_crypto_strings CRYPTO_cleanup_all_ex_data ERR_free_strings RAND_cleanup DSA_SIG_set0 EVP_dss1 EVP_DigestVerify EVP_aes_256_cbc EVP_EncryptInit_ex HMAC_Init_ex CRYPTO_THREADID_set_callback EVP_MAC_CTX_set_params OSSL_PARAM_BLD_new BIO_set_callback_ex])
 
 # these check_funcs need -lssl
 BAKLIBS="$LIBS"
@@ -1043,7 +1108,7 @@ int load_gost_id(void)
 	EVP_PKEY_asn1_get0_info(&gost_id, NULL, NULL, NULL, NULL, meth);
 	return gost_id;
 }
-int main(void) { 
+int main(void) {
 	EVP_MD_CTX* ctx;
 	const EVP_MD* md;
 	unsigned char digest[64]; /* its a 256-bit digest, so uses 32 bytes */
@@ -1074,7 +1139,8 @@ int main(void) {
 		return 6;
 	return 0;
 }
-]])] , [eval "ac_cv_c_gost_works=yes"], [eval "ac_cv_c_gost_works=no"])
+]])] , [eval "ac_cv_c_gost_works=yes"], [eval "ac_cv_c_gost_works=no"],
+[eval "ac_cv_c_gost_works=maybe"])
 CFLAGS="$BAKCFLAGS"
 else
 eval "ac_cv_c_gost_works=maybe"
@@ -1464,7 +1530,7 @@ if test x_$enable_static_exe = x_yes; then
 		else
 			LIBS="$LIBS -lgdi32"
 		fi
-		LIBS="$LIBS -lz"
+		AC_CHECK_LIB([z], [compress], [ LIBS="$LIBS -lz" ])
 		LIBS="$LIBS -l:libssp.a"
 	fi
 fi
@@ -1481,13 +1547,13 @@ if test x_$enable_fully_static = x_yes; then
 		else
 			LIBS="$LIBS -lgdi32"
 		fi
-		LIBS="$LIBS -lz"
+		AC_CHECK_LIB([z], [compress], [ LIBS="$LIBS -lz" ])
 		LIBS="$LIBS -l:libssp.a"
 	fi
 fi
 
 # set lock checking if requested
-AC_ARG_ENABLE(lock_checks, AS_HELP_STRING([--enable-lock-checks],[ enable to check lock and unlock calls, for debug purposes ]), 
+AC_ARG_ENABLE(lock_checks, AS_HELP_STRING([--enable-lock-checks],[ enable to check lock and unlock calls, for debug purposes ]),
 	, )
 if test x_$enable_lock_checks = x_yes; then
 	AC_DEFINE(ENABLE_LOCK_CHECKS, 1, [Define if you want to use debug lock checking (slow).])
@@ -1519,6 +1585,7 @@ if test "$USE_WINSOCK" = 1; then
 	AC_SUBST(WIN_CONTROL_OBJ_LINK)
 	WIN_CHECKCONF_OBJ_LINK="rsrc_unbound_checkconf.o"
 	AC_SUBST(WIN_CHECKCONF_OBJ_LINK)
+	AC_DEFINE(__USE_MINGW_ANSI_STDIO, 1, [defined to use gcc ansi snprintf and sscanf that understands %lld when compiled for windows.])
 fi
 if test $ac_cv_func_getaddrinfo = no; then
 	AC_LIBOBJ([fake-rfc2553])
@@ -1605,7 +1672,7 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([
   AC_MSG_RESULT(no))
 
 AC_SEARCH_LIBS([setusercontext], [util])
-AC_CHECK_FUNCS([tzset sigprocmask fcntl getpwnam endpwent getrlimit setrlimit setsid chroot kill chown sleep usleep random srandom recvmsg sendmsg writev socketpair glob initgroups strftime localtime_r setusercontext _beginthreadex endservent endprotoent fsync shmget accept4 getifaddrs if_nametoindex])
+AC_CHECK_FUNCS([tzset sigprocmask fcntl getpwnam endpwent getrlimit setrlimit setsid chroot kill chown sleep usleep random srandom recvmsg sendmsg writev socketpair glob initgroups strftime localtime_r setusercontext _beginthreadex endservent endprotoent fsync shmget accept4 getifaddrs if_nametoindex poll gettid])
 AC_CHECK_FUNCS([setresuid],,[AC_CHECK_FUNCS([setreuid])])
 AC_CHECK_FUNCS([setresgid],,[AC_CHECK_FUNCS([setregid])])
 
@@ -1650,7 +1717,7 @@ int main(void) { return !(snprintf(NULL, 0, "test") == 4); }
 		AC_MSG_RESULT(no)
 		AC_DEFINE([SNPRINTF_RET_BROKEN], [], [define if (v)snprintf does not return length needed, (but length used)])
 		AC_LIBOBJ(snprintf)
-	  ])
+	  ], [AC_MSG_RESULT(maybe)])
     fi
 fi
 AC_REPLACE_FUNCS(strlcat)
@@ -1844,11 +1911,17 @@ case "$enable_ipset" in
 			withval="/usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr"
 		fi
 		for dir in $withval ; do
-			if test -f "$dir/include/libmnl/libmnl.h"; then
+			if test -f "$dir/include/libmnl/libmnl.h" -o -f "$dir/include/libmnl/libmnl/libmnl.h"; then
 				found_libmnl="yes"
 				dnl assume /usr is in default path.
+				extralibmnl=""
+				if test -f "$dir/include/libmnl/libmnl/libmnl.h"; then
+					extralibmnl="/libmnl"
+				fi
+				if test "$dir" != "/usr" -o -n "$extralibmnl"; then
+					CPPFLAGS="$CPPFLAGS -I$dir/include$extralibmnl"
+				fi
 				if test "$dir" != "/usr"; then
-					CPPFLAGS="$CPPFLAGS -I$dir/include"
 					LDFLAGS="$LDFLAGS -L$dir/lib"
 				fi
 				AC_MSG_RESULT(found in $dir)
@@ -1874,7 +1947,7 @@ case "$enable_explicit_port_randomisation" in
 esac
 
 if echo "$host" | $GREP -i -e linux >/dev/null; then
-	AC_ARG_ENABLE(linux-ip-local-port-range, AC_HELP_STRING([--enable-linux-ip-local-port-range], [Define this to enable use of /proc/sys/net/ipv4/ip_local_port_range as a default outgoing port range. This is only for the libunbound on Linux and does not affect unbound resolving daemon itself. This may severely limit the number of available outgoing ports and thus decrease randomness. Define this only when the target system restricts (e.g. some of SELinux enabled distributions) the use of non-ephemeral ports.]))
+	AC_ARG_ENABLE(linux-ip-local-port-range, AS_HELP_STRING([--enable-linux-ip-local-port-range], [Define this to enable use of /proc/sys/net/ipv4/ip_local_port_range as a default outgoing port range. This is only for the libunbound on Linux and does not affect unbound resolving daemon itself. This may severely limit the number of available outgoing ports and thus decrease randomness. Define this only when the target system restricts (e.g. some of SELinux enabled distributions) the use of non-ephemeral ports.]))
 	case "$enable_linux_ip_local_port_range" in
 		yes)
 			AC_DEFINE([USE_LINUX_IP_LOCAL_PORT_RANGE], [1], [Define this to enable use of /proc/sys/net/ipv4/ip_local_port_range as a default outgoing port range. This is only for the libunbound on Linux and does not affect unbound resolving daemon itself. This may severely limit the number of available outgoing ports and thus decrease randomness. Define this only when the target system restricts (e.g. some of SELinux enabled distributions) the use of non-ephemeral ports.])
@@ -1931,11 +2004,11 @@ AC_ARG_WITH(libunbound-only, AS_HELP_STRING([--with-libunbound-only],[do not bui
 	fi
 ])
 if test $ALLTARGET = "alltargets"; then
-	if test $USE_NSS = "yes"; then 
-		AC_MSG_ERROR([--with-nss can only be used in combination with --with-libunbound-only.])	
+	if test $USE_NSS = "yes"; then
+		AC_MSG_ERROR([--with-nss can only be used in combination with --with-libunbound-only.])
 	fi
 	if test $USE_NETTLE = "yes"; then
-		AC_MSG_ERROR([--with-nettle can only be used in combination with --with-libunbound-only.])	
+		AC_MSG_ERROR([--with-nettle can only be used in combination with --with-libunbound-only.])
 	fi
 fi
 
@@ -1946,7 +2019,7 @@ ACX_STRIP_EXT_FLAGS
 if test -n "$LATE_LDFLAGS"; then
   LDFLAGS="$LATE_LDFLAGS $LDFLAGS"
 fi
-# remove start spaces 
+# remove start spaces
 LDFLAGS=`echo "$LDFLAGS"|sed -e 's/^ *//'`
 LIBS=`echo "$LIBS"|sed -e 's/^ *//'`
 
@@ -2026,7 +2099,7 @@ dnl includes
 #include <ws2tcpip.h>
 #endif
 
-#ifndef USE_WINSOCK
+#if !defined(USE_WINSOCK) || !defined(HAVE_SNPRINTF) || defined(SNPRINTF_RET_BROKEN) || defined(__USE_MINGW_ANSI_STDIO)
 #define ARG_LL "%ll"
 #else
 #define ARG_LL "%I64"

contrib/Dockerfile.tests

@@ -1,10 +1,8 @@
 FROM gcc:latest
 WORKDIR /usr/src/unbound
-RUN apt-get update
 # install semantic parser & lexical analyzer
-RUN apt-get install -y bison flex
 # install packages used in tests
-RUN apt-get install -y ldnsutils dnsutils xxd splint doxygen netcat
+RUN apt-get update && apt-get install -y bison flex ldnsutils dnsutils xxd splint doxygen netcat-openbsd
 # accept short rsa keys, which are used in tests
 RUN sed -i 's/SECLEVEL=2/SECLEVEL=1/g' /usr/lib/ssl/openssl.cnf
 

contrib/README

@@ -55,3 +55,6 @@ distribution but may be helpful.
   contributed by Andreas Schulze.
 * metrics.awk: awk script that can convert unbound-control stats to
   Prometheus metrics format output.
+* unbound.init_yocto: An init script to start and stop the server. Put it
+  in /etc/init.d/unbound to use it. It is for the Yocto Project, in
+  embedded systems, contributed by beni-sandu.

contrib/aaaa-filter-iterator.patch

@@ -1,8 +1,8 @@
 diff --git a/doc/unbound.conf.5.in b/doc/unbound.conf.5.in
-index f426ac5f..147fbfa9 100644
+index 5a75e319..c6c6dbe2 100644
 --- a/doc/unbound.conf.5.in
 +++ b/doc/unbound.conf.5.in
-@@ -872,6 +872,13 @@ potentially broken nameservers. A lot of domains will not be resolvable when
+@@ -970,6 +970,13 @@ potentially broken nameservers. A lot of domains will not be resolvable when
  this option in enabled. Only use if you know what you are doing.
  This option only has effect when qname-minimisation is enabled. Default is no.
  .TP
@@ -17,10 +17,10 @@ index f426ac5f..147fbfa9 100644
  Aggressive NSEC uses the DNSSEC NSEC chain to synthesize NXDOMAIN
  and other denials, using information from previous NXDOMAINs answers.
 diff --git a/iterator/iter_scrub.c b/iterator/iter_scrub.c
-index aae934dd..55c55de0 100644
+index f093c1bf..e55a2246 100644
 --- a/iterator/iter_scrub.c
 +++ b/iterator/iter_scrub.c
-@@ -667,6 +667,32 @@ static int sanitize_nsec_is_overreach(struct rrset_parse* rrset,
+@@ -679,6 +679,32 @@ static int sanitize_nsec_is_overreach(sldns_buffer* pkt,
  	return 0;
  }
  
@@ -53,7 +53,7 @@ index aae934dd..55c55de0 100644
  /**
   * Given a response event, remove suspect RRsets from the response.
   * "Suspect" rrsets are potentially poison. Note that this routine expects
-@@ -686,6 +712,7 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
+@@ -698,6 +724,7 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
  	struct query_info* qinfo, uint8_t* zonename, struct module_env* env,
  	struct iter_env* ie)
  {
@@ -61,7 +61,7 @@ index aae934dd..55c55de0 100644
  	int del_addi = 0; /* if additional-holding rrsets are deleted, we
  		do not trust the normalized additional-A-AAAA any more */
  	struct rrset_parse* rrset, *prev;
-@@ -721,6 +748,13 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
+@@ -733,6 +760,13 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
  		rrset = rrset->rrset_all_next;
  	}
  
@@ -75,7 +75,7 @@ index aae934dd..55c55de0 100644
  	/* At this point, we brutally remove ALL rrsets that aren't 
  	 * children of the originating zone. The idea here is that, 
  	 * as far as we know, the server that we contacted is ONLY 
-@@ -732,6 +766,24 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
+@@ -744,6 +778,24 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
  	rrset = msg->rrset_first;
  	while(rrset) {
  
@@ -101,22 +101,22 @@ index aae934dd..55c55de0 100644
  		if( (rrset->type == LDNS_RR_TYPE_A || 
  			rrset->type == LDNS_RR_TYPE_AAAA)) {
 diff --git a/iterator/iter_utils.c b/iterator/iter_utils.c
-index 7bc67da6..e10f547a 100644
+index 2482a1f4..bd5ba243 100644
 --- a/iterator/iter_utils.c
 +++ b/iterator/iter_utils.c
-@@ -175,6 +175,7 @@ iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
- 	}
- 	iter_env->supports_ipv6 = cfg->do_ip6;
- 	iter_env->supports_ipv4 = cfg->do_ip4;
+@@ -177,6 +177,7 @@ iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
+ 	iter_env->outbound_msg_retry = cfg->outbound_msg_retry;
+ 	iter_env->max_sent_count = cfg->max_sent_count;
+ 	iter_env->max_query_restarts = cfg->max_query_restarts;
 +	iter_env->aaaa_filter = cfg->aaaa_filter;
  	return 1;
  }
  
 diff --git a/iterator/iterator.c b/iterator/iterator.c
-index 23b07ea9..ca29b48c 100644
+index 54006940..768fe202 100644
 --- a/iterator/iterator.c
 +++ b/iterator/iterator.c
-@@ -2127,6 +2127,53 @@ processDSNSFind(struct module_qstate* qstate, struct iter_qstate* iq, int id)
+@@ -2155,6 +2155,53 @@ processDSNSFind(struct module_qstate* qstate, struct iter_qstate* iq, int id)
  
  	return 0;
  }
@@ -170,7 +170,7 @@ index 23b07ea9..ca29b48c 100644
  	
  /** 
   * This is the request event state where the request will be sent to one of
-@@ -2186,6 +2233,13 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq,
+@@ -2216,6 +2263,13 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq,
  		return error_response(qstate, id, LDNS_RCODE_SERVFAIL);
  	}
  	
@@ -184,7 +184,7 @@ index 23b07ea9..ca29b48c 100644
  	/* Make sure we have a delegation point, otherwise priming failed
  	 * or another failure occurred */
  	if(!iq->dp) {
-@@ -3574,6 +3628,61 @@ processFinished(struct module_qstate* qstate, struct iter_qstate* iq,
+@@ -3648,6 +3702,61 @@ processFinished(struct module_qstate* qstate, struct iter_qstate* iq,
  	return 0;
  }
  
@@ -246,7 +246,7 @@ index 23b07ea9..ca29b48c 100644
  /*
   * Return priming query results to interested super querystates.
   * 
-@@ -3593,6 +3702,9 @@ iter_inform_super(struct module_qstate* qstate, int id,
+@@ -3667,6 +3776,9 @@ iter_inform_super(struct module_qstate* qstate, int id,
  	else if(super->qinfo.qtype == LDNS_RR_TYPE_DS && ((struct iter_qstate*)
  		super->minfo[id])->state == DSNS_FIND_STATE)
  		processDSNSResponse(qstate, id, super);
@@ -256,7 +256,7 @@ index 23b07ea9..ca29b48c 100644
  	else if(qstate->return_rcode != LDNS_RCODE_NOERROR)
  		error_supers(qstate, id, super);
  	else if(qstate->is_priming)
-@@ -3630,6 +3742,9 @@ iter_handle(struct module_qstate* qstate, struct iter_qstate* iq,
+@@ -3704,6 +3816,9 @@ iter_handle(struct module_qstate* qstate, struct iter_qstate* iq,
  			case INIT_REQUEST_3_STATE:
  				cont = processInitRequest3(qstate, iq, id);
  				break;
@@ -266,7 +266,7 @@ index 23b07ea9..ca29b48c 100644
  			case QUERYTARGETS_STATE:
  				cont = processQueryTargets(qstate, iq, ie, id);
  				break;
-@@ -3961,6 +4076,8 @@ iter_state_to_string(enum iter_state state)
+@@ -4040,6 +4155,8 @@ iter_state_to_string(enum iter_state state)
  		return "INIT REQUEST STATE (stage 2)";
  	case INIT_REQUEST_3_STATE:
  		return "INIT REQUEST STATE (stage 3)";
@@ -275,7 +275,7 @@ index 23b07ea9..ca29b48c 100644
  	case QUERYTARGETS_STATE :
  		return "QUERY TARGETS STATE";
  	case PRIME_RESP_STATE :
-@@ -3985,6 +4102,7 @@ iter_state_is_responsestate(enum iter_state s)
+@@ -4064,6 +4181,7 @@ iter_state_is_responsestate(enum iter_state s)
  		case INIT_REQUEST_STATE :
  		case INIT_REQUEST_2_STATE :
  		case INIT_REQUEST_3_STATE :
@@ -284,10 +284,10 @@ index 23b07ea9..ca29b48c 100644
  		case COLLECT_CLASS_STATE :
  			return 0;
 diff --git a/iterator/iterator.h b/iterator/iterator.h
-index 342ac207..731948d1 100644
+index 8b840528..a61c4195 100644
 --- a/iterator/iterator.h
 +++ b/iterator/iterator.h
-@@ -135,6 +135,9 @@ struct iter_env {
+@@ -133,6 +133,9 @@ struct iter_env {
  	 */
  	int* target_fetch_policy;
  
@@ -297,7 +297,7 @@ index 342ac207..731948d1 100644
  	/** lock on ratelimit counter */
  	lock_basic_type queries_ratelimit_lock;
  	/** number of queries that have been ratelimited */
-@@ -186,6 +189,14 @@ enum iter_state {
+@@ -187,6 +190,14 @@ enum iter_state {
  	 */
  	INIT_REQUEST_3_STATE,
  
@@ -312,7 +312,7 @@ index 342ac207..731948d1 100644
  	/**
  	 * Each time a delegation point changes for a given query or a 
  	 * query times out and/or wakes up, this state is (re)visited. 
-@@ -375,6 +386,13 @@ struct iter_qstate {
+@@ -376,6 +387,13 @@ struct iter_qstate {
  	 */
  	int refetch_glue;
  
@@ -327,10 +327,10 @@ index 342ac207..731948d1 100644
  	struct outbound_list outlist;
  
 diff --git a/pythonmod/interface.i b/pythonmod/interface.i
-index f08b575d..47f1bb2e 100644
+index 1ca8686a..d91b19ec 100644
 --- a/pythonmod/interface.i
 +++ b/pythonmod/interface.i
-@@ -975,6 +975,7 @@ struct config_file {
+@@ -995,6 +995,7 @@ struct config_file {
     int harden_dnssec_stripped;
     int harden_referral_path;
     int use_caps_bits_for_id;
@@ -339,10 +339,10 @@ index f08b575d..47f1bb2e 100644
     struct config_strlist* private_domain;
     size_t unwanted_threshold;
 diff --git a/util/config_file.c b/util/config_file.c
-index 0ab8614a..729fb147 100644
+index 969d664b..8d94b008 100644
 --- a/util/config_file.c
 +++ b/util/config_file.c
-@@ -218,6 +218,7 @@ config_create(void)
+@@ -231,6 +231,7 @@ config_create(void)
  	cfg->harden_referral_path = 0;
  	cfg->harden_algo_downgrade = 0;
  	cfg->use_caps_bits_for_id = 0;
@@ -351,10 +351,10 @@ index 0ab8614a..729fb147 100644
  	cfg->private_address = NULL;
  	cfg->private_domain = NULL;
 diff --git a/util/config_file.h b/util/config_file.h
-index e61257a3..dabaa7bb 100644
+index c7c9a0a4..e3aa15b0 100644
 --- a/util/config_file.h
 +++ b/util/config_file.h
-@@ -260,6 +260,8 @@ struct config_file {
+@@ -285,6 +285,8 @@ struct config_file {
  	int harden_algo_downgrade;
  	/** use 0x20 bits in query as random ID bits */
  	int use_caps_bits_for_id;
@@ -364,38 +364,38 @@ index e61257a3..dabaa7bb 100644
  	struct config_strlist* caps_whitelist;
  	/** strip away these private addrs from answers, no DNS Rebinding */
 diff --git a/util/configlexer.lex b/util/configlexer.lex
-index 79a0edca..4eaec678 100644
+index 34a0e5dd..c890be2a 100644
 --- a/util/configlexer.lex
 +++ b/util/configlexer.lex
-@@ -304,6 +304,7 @@ harden-algo-downgrade{COLON}	{ YDVAR(1, VAR_HARDEN_ALGO_DOWNGRADE) }
- use-caps-for-id{COLON}		{ YDVAR(1, VAR_USE_CAPS_FOR_ID) }
+@@ -317,6 +317,7 @@ use-caps-for-id{COLON}		{ YDVAR(1, VAR_USE_CAPS_FOR_ID) }
  caps-whitelist{COLON}		{ YDVAR(1, VAR_CAPS_WHITELIST) }
+ caps-exempt{COLON}		{ YDVAR(1, VAR_CAPS_WHITELIST) }
  unwanted-reply-threshold{COLON}	{ YDVAR(1, VAR_UNWANTED_REPLY_THRESHOLD) }
 +aaaa-filter{COLON}		{ YDVAR(1, VAR_AAAA_FILTER) }
  private-address{COLON}		{ YDVAR(1, VAR_PRIVATE_ADDRESS) }
  private-domain{COLON}		{ YDVAR(1, VAR_PRIVATE_DOMAIN) }
  prefetch-key{COLON}		{ YDVAR(1, VAR_PREFETCH_KEY) }
 diff --git a/util/configparser.y b/util/configparser.y
-index 1d0e8658..f284dd43 100644
+index d4f965f9..8cc237c6 100644
 --- a/util/configparser.y
 +++ b/util/configparser.y
 @@ -97,6 +97,7 @@ extern struct config_parser_state* cfg_parser;
- %token VAR_STATISTICS_CUMULATIVE VAR_OUTGOING_PORT_PERMIT 
+ %token VAR_STATISTICS_CUMULATIVE VAR_OUTGOING_PORT_PERMIT
  %token VAR_OUTGOING_PORT_AVOID VAR_DLV_ANCHOR_FILE VAR_DLV_ANCHOR
  %token VAR_NEG_CACHE_SIZE VAR_HARDEN_REFERRAL_PATH VAR_PRIVATE_ADDRESS
 +%token VAR_AAAA_FILTER
  %token VAR_PRIVATE_DOMAIN VAR_REMOTE_CONTROL VAR_CONTROL_ENABLE
  %token VAR_CONTROL_INTERFACE VAR_CONTROL_PORT VAR_SERVER_KEY_FILE
  %token VAR_SERVER_CERT_FILE VAR_CONTROL_KEY_FILE VAR_CONTROL_CERT_FILE
-@@ -233,6 +234,7 @@ content_server: server_num_threads | server_verbosity | server_port |
+@@ -247,6 +248,7 @@ content_server: server_num_threads | server_verbosity | server_port |
  	server_dlv_anchor_file | server_dlv_anchor | server_neg_cache_size |
  	server_harden_referral_path | server_private_address |
- 	server_private_domain | server_extended_statistics | 
+ 	server_private_domain | server_extended_statistics |
 +	server_aaaa_filter |
- 	server_local_data_ptr | server_jostle_timeout | 
- 	server_unwanted_reply_threshold | server_log_time_ascii | 
- 	server_domain_insecure | server_val_sig_skew_min | 
-@@ -1563,6 +1565,15 @@ server_caps_whitelist: VAR_CAPS_WHITELIST STRING_ARG
+ 	server_local_data_ptr | server_jostle_timeout |
+ 	server_unwanted_reply_threshold | server_log_time_ascii |
+ 	server_domain_insecure | server_val_sig_skew_min |
+@@ -1754,6 +1756,15 @@ server_caps_whitelist: VAR_CAPS_WHITELIST STRING_ARG
  			yyerror("out of memory");
  	}
  	;

contrib/metrics.awk

@@ -28,6 +28,7 @@ END {
 	print "unbound_hits_queries{type=\"total.num.prefetch\"} " val["total.num.prefetch"];
 	print "unbound_hits_queries{type=\"num.query.tcp\"} " val["num.query.tcp"];
 	print "unbound_hits_queries{type=\"num.query.tcpout\"} " val["num.query.tcpout"];
+	print "unbound_hits_queries{type=\"num.query.udpout\"} " val["num.query.udpout"];
 	print "unbound_hits_queries{type=\"num.query.tls\"} " val["num.query.tls"];
 	print "unbound_hits_queries{type=\"num.query.tls.resume\"} " val["num.query.tls.resume"];
 	print "unbound_hits_queries{type=\"num.query.ipv6\"} " val["num.query.ipv6"];

contrib/unbound.init_yocto

@@ -0,0 +1,139 @@
+#!/bin/sh
+#
+# unbound	This shell script takes care of starting and stopping
+#		unbound (DNS server).
+#
+# chkconfig:   - 14 86
+# description:	unbound is a Domain Name Server (DNS) \
+#		that is used to resolve host names to IP addresses.
+
+### BEGIN INIT INFO
+# Provides: $named unbound
+# Required-Start: $network $local_fs
+# Required-Stop: $network $local_fs
+# Should-Start: $syslog
+# Should-Stop: $syslog
+# Short-Description: unbound recursive Domain Name Server.
+# Description:  unbound is a Domain Name Server (DNS) 
+#		that is used to resolve host names to IP addresses.
+### END INIT INFO
+
+# Source function library.
+. /etc/init.d/functions
+
+exec="/usr/sbin/unbound"
+prog="unbound"
+config="/etc/unbound/unbound.conf"
+pidfile="/var/unbound/unbound.pid"
+rootdir="/var/unbound"
+
+[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog
+
+lockfile=/var/lock/subsys/$prog
+
+start() {
+    [ -x $exec ] || exit 5
+    [ -f $config ] || exit 6
+    echo -n $"Starting $prog: "
+
+    # setup root jail
+    if [ -s /etc/localtime ]; then 
+	[ -d ${rootdir}/etc ] || mkdir -p ${rootdir}/etc ;
+	if [ ! -e ${rootdir}/etc/localtime ] || ! /usr/bin/cmp -s /etc/localtime ${rootdir}/etc/localtime; then
+	    cp -fp /etc/localtime ${rootdir}/etc/localtime
+	fi;
+    fi;
+    if [ -s /etc/resolv.conf ]; then
+	[ -d ${rootdir}/etc ] || mkdir -p ${rootdir}/etc ;
+	if [ ! -e ${rootdir}/etc/resolv.conf ] || ! /usr/bin/cmp -s /etc/resolv.conf ${rootdir}/etc/resolv.conf; then
+	    cp -fp /etc/resolv.conf ${rootdir}/etc/resolv.conf
+	fi;
+    fi;
+    if ! egrep -q '^/[^[:space:]]+[[:space:]]+'${rootdir}'/dev/log' /proc/mounts; then
+	[ -d ${rootdir}/dev ] || mkdir -p ${rootdir}/dev ;
+	[ -e ${rootdir}/dev/log ] || touch ${rootdir}/dev/log
+	mount --bind -n /dev/log ${rootdir}/dev/log >/dev/null 2>&1;
+    fi;
+    if ! egrep -q '^/[^[:space:]]+[[:space:]]+'${rootdir}'/dev/random' /proc/mounts; then
+	[ -d ${rootdir}/dev ] || mkdir -p ${rootdir}/dev ;
+	[ -e ${rootdir}/dev/random ] || touch ${rootdir}/dev/random
+	mount --bind -n /dev/random ${rootdir}/dev/random >/dev/null 2>&1;
+    fi;
+
+    # if not running, start it up here
+    daemonize $exec
+    retval=$?
+    echo
+    [ $retval -eq 0 ] && touch $lockfile
+    return $retval
+}
+
+stop() {
+    echo -n $"Stopping $prog: "
+    # stop it here, often "killproc $prog"
+    killproc $prog
+    retval=$?
+    echo
+    [ $retval -eq 0 ] && rm -f $lockfile
+    if egrep -q '^/[^[:space:]]+[[:space:]]+'${rootdir}'/dev/log' /proc/mounts; then
+	umount ${rootdir}/dev/log >/dev/null 2>&1
+    fi;
+    if egrep -q '^/[^[:space:]]+[[:space:]]+'${rootdir}'/dev/random' /proc/mounts; then
+	umount ${rootdir}/dev/random >/dev/null 2>&1
+    fi;
+    return $retval
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    kill -HUP `cat $pidfile`
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    # run checks to determine if the service is running or use generic status
+    status $prog
+}
+
+rh_status_q() {
+    rh_status -p $pidfile >/dev/null 2>&1
+}
+
+case "$1" in
+    start)
+        rh_status_q && exit 0
+        $1
+        ;;
+    stop)
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        rh_status
+        ;;
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+        exit 2
+esac
+exit $?

contrib/unbound.service.in

@@ -42,9 +42,8 @@
 [Unit]
 Description=Validating, recursive, and caching DNS resolver
 Documentation=man:unbound(8)
-After=network-online.target
-Before=nss-lookup.target
-Wants=network-online.target nss-lookup.target
+After=network.target
+Before=network-online.target nss-lookup.target
 
 [Install]
 WantedBy=multi-user.target

contrib/unbound_munin_

@@ -253,6 +253,7 @@ if test "$1" = "config" ; then
 		p_config "total.num.prefetch" "cache prefetch" "ABSOLUTE"
 		p_config "num.query.tcp" "TCP queries" "ABSOLUTE"
 		p_config "num.query.tcpout" "TCP out queries" "ABSOLUTE"
+		p_config "num.query.udpout" "UDP out queries" "ABSOLUTE"
 		p_config "num.query.tls" "TLS queries" "ABSOLUTE"
 		p_config "num.query.tls.resume" "TLS resumes" "ABSOLUTE"
 		p_config "num.query.ipv6" "IPv6 queries" "ABSOLUTE"
@@ -452,7 +453,7 @@ hits)
 	for x in `grep "^thread[0-9][0-9]*\.num\.queries=" $state |
 		sed -e 's/=.*//'` total.num.queries \
 		total.num.cachehits total.num.prefetch num.query.tcp \
-		num.query.tcpout num.query.tls num.query.tls.resume \
+		num.query.tcpout num.query.udpout num.query.tls num.query.tls.resume \
 		num.query.ipv6 unwanted.queries \
 		unwanted.replies; do
 		if grep "^"$x"=" $state >/dev/null 2>&1; then

daemon/acl_list.c

@@ -46,9 +46,10 @@
 #include "util/config_file.h"
 #include "util/net_help.h"
 #include "services/localzone.h"
+#include "services/listen_dnsport.h"
 #include "sldns/str2wire.h"
 
-struct acl_list* 
+struct acl_list*
 acl_list_create(void)
 {
 	struct acl_list* acl = (struct acl_list*)calloc(1,
@@ -63,10 +64,10 @@ acl_list_create(void)
 	return acl;
 }
 
-void 
+void
 acl_list_delete(struct acl_list* acl)
 {
-	if(!acl) 
+	if(!acl)
 		return;
 	regional_destroy(acl->region);
 	free(acl);
@@ -74,8 +75,8 @@ acl_list_delete(struct acl_list* acl)
 
 /** insert new address into acl_list structure */
 static struct acl_addr*
-acl_list_insert(struct acl_list* acl, struct sockaddr_storage* addr, 
-	socklen_t addrlen, int net, enum acl_access control, 
+acl_list_insert(struct acl_list* acl, struct sockaddr_storage* addr,
+	socklen_t addrlen, int net, enum acl_access control,
 	int complain_duplicates)
 {
 	struct acl_addr* node = regional_alloc_zero(acl->region,
@@ -90,6 +91,33 @@ acl_list_insert(struct acl_list* acl, struct sockaddr_storage* addr,
 	return node;
 }
 
+/** parse str to acl_access enum */
+static int
+parse_acl_access(const char* str, enum acl_access* control)
+{
+	if(strcmp(str, "allow") == 0)
+		*control = acl_allow;
+	else if(strcmp(str, "deny") == 0)
+		*control = acl_deny;
+	else if(strcmp(str, "refuse") == 0)
+		*control = acl_refuse;
+	else if(strcmp(str, "deny_non_local") == 0)
+		*control = acl_deny_non_local;
+	else if(strcmp(str, "refuse_non_local") == 0)
+		*control = acl_refuse_non_local;
+	else if(strcmp(str, "allow_snoop") == 0)
+		*control = acl_allow_snoop;
+	else if(strcmp(str, "allow_setrd") == 0)
+		*control = acl_allow_setrd;
+	else if (strcmp(str, "allow_cookie") == 0)
+		*control = acl_allow_cookie;
+	else {
+		log_err("access control type %s unknown", str);
+		return 0;
+	}
+	return 1;
+}
+
 /** apply acl_list string */
 static int
 acl_list_str_cfg(struct acl_list* acl, const char* str, const char* s2,
@@ -99,29 +127,14 @@ acl_list_str_cfg(struct acl_list* acl, const char* str, const char* s2,
 	int net;
 	socklen_t addrlen;
 	enum acl_access control;
-	if(strcmp(s2, "allow") == 0)
-		control = acl_allow;
-	else if(strcmp(s2, "deny") == 0)
-		control = acl_deny;
-	else if(strcmp(s2, "refuse") == 0)
-		control = acl_refuse;
-	else if(strcmp(s2, "deny_non_local") == 0)
-		control = acl_deny_non_local;
-	else if(strcmp(s2, "refuse_non_local") == 0)
-		control = acl_refuse_non_local;
-	else if(strcmp(s2, "allow_snoop") == 0)
-		control = acl_allow_snoop;
-	else if(strcmp(s2, "allow_setrd") == 0)
-		control = acl_allow_setrd;
-	else {
-		log_err("access control type %s unknown", str);
+	if(!parse_acl_access(s2, &control)) {
 		return 0;
 	}
 	if(!netblockstrtoaddr(str, UNBOUND_DNS_PORT, &addr, &addrlen, &net)) {
 		log_err("cannot parse access control: %s %s", str, s2);
 		return 0;
 	}
-	if(!acl_list_insert(acl, &addr, addrlen, net, control, 
+	if(!acl_list_insert(acl, &addr, addrlen, net, control,
 		complain_duplicates)) {
 		log_err("out of memory");
 		return 0;
@@ -131,19 +144,27 @@ acl_list_str_cfg(struct acl_list* acl, const char* str, const char* s2,
 
 /** find or create node (NULL on parse or error) */
 static struct acl_addr*
-acl_find_or_create(struct acl_list* acl, const char* str)
+acl_find_or_create_str2addr(struct acl_list* acl, const char* str,
+	int is_interface, int port)
 {
 	struct acl_addr* node;
 	struct sockaddr_storage addr;
-	int net;
 	socklen_t addrlen;
-	if(!netblockstrtoaddr(str, UNBOUND_DNS_PORT, &addr, &addrlen, &net)) {
-		log_err("cannot parse netblock: %s", str);
-		return NULL;
+	int net = (str_is_ip6(str)?128:32);
+	if(is_interface) {
+		if(!extstrtoaddr(str, &addr, &addrlen, port)) {
+			log_err("cannot parse interface: %s", str);
+			return NULL;
+		}
+	} else {
+		if(!netblockstrtoaddr(str, UNBOUND_DNS_PORT, &addr, &addrlen, &net)) {
+			log_err("cannot parse netblock: %s", str);
+			return NULL;
+		}
 	}
 	/* find or create node */
 	if(!(node=(struct acl_addr*)addr_tree_find(&acl->tree, &addr,
-		addrlen, net))) {
+		addrlen, net)) && !is_interface) {
 		/* create node, type 'allow' since otherwise tags are
 		 * pointless, can override with specific access-control: cfg */
 		if(!(node=(struct acl_addr*)acl_list_insert(acl, &addr,
@@ -155,14 +176,65 @@ acl_find_or_create(struct acl_list* acl, const char* str)
 	return node;
 }
 
+/** find or create node (NULL on error) */
+static struct acl_addr*
+acl_find_or_create(struct acl_list* acl, struct sockaddr_storage* addr,
+	socklen_t addrlen, enum acl_access control)
+{
+	struct acl_addr* node;
+	int net = (addr_is_ip6(addr, addrlen)?128:32);
+	/* find or create node */
+	if(!(node=(struct acl_addr*)addr_tree_find(&acl->tree, addr,
+		addrlen, net))) {
+		/* create node;
+		 * can override with specific access-control: cfg */
+		if(!(node=(struct acl_addr*)acl_list_insert(acl, addr,
+			addrlen, net, control, 1))) {
+			log_err("out of memory");
+			return NULL;
+		}
+	}
+	return node;
+}
+
+/** apply acl_interface string */
+static int
+acl_interface_str_cfg(struct acl_list* acl_interface, const char* iface,
+	const char* s2, int port)
+{
+	struct acl_addr* node;
+	enum acl_access control;
+	if(!parse_acl_access(s2, &control)) {
+		return 0;
+	}
+	if(!(node=acl_find_or_create_str2addr(acl_interface, iface, 1, port))) {
+		log_err("cannot update ACL on non-configured interface: %s %d",
+			iface, port);
+		return 0;
+	}
+	node->control = control;
+	return 1;
+}
+
+struct acl_addr*
+acl_interface_insert(struct acl_list* acl_interface,
+	struct sockaddr_storage* addr, socklen_t addrlen,
+	enum acl_access control)
+{
+	return acl_find_or_create(acl_interface, addr, addrlen, control);
+}
+
 /** apply acl_tag string */
 static int
 acl_list_tags_cfg(struct acl_list* acl, const char* str, uint8_t* bitmap,
-	size_t bitmaplen)
+	size_t bitmaplen, int is_interface, int port)
 {
 	struct acl_addr* node;
-	if(!(node=acl_find_or_create(acl, str)))
+	if(!(node=acl_find_or_create_str2addr(acl, str, is_interface, port))) {
+		if(is_interface)
+			log_err("non-configured interface: %s", str);
 		return 0;
+	}
 	node->taglen = bitmaplen;
 	node->taglist = regional_alloc_init(acl->region, bitmap, bitmaplen);
 	if(!node->taglist) {
@@ -175,11 +247,14 @@ acl_list_tags_cfg(struct acl_list* acl, const char* str, uint8_t* bitmap,
 /** apply acl_view string */
 static int
 acl_list_view_cfg(struct acl_list* acl, const char* str, const char* str2,
-	struct views* vs)
+	struct views* vs, int is_interface, int port)
 {
 	struct acl_addr* node;
-	if(!(node=acl_find_or_create(acl, str)))
+	if(!(node=acl_find_or_create_str2addr(acl, str, is_interface, port))) {
+		if(is_interface)
+			log_err("non-configured interface: %s", str);
 		return 0;
+	}
 	node->view = views_find_view(vs, str2, 0 /* get read lock*/);
 	if(!node->view) {
 		log_err("no view with name: %s", str2);
@@ -192,13 +267,17 @@ acl_list_view_cfg(struct acl_list* acl, const char* str, const char* str2,
 /** apply acl_tag_action string */
 static int
 acl_list_tag_action_cfg(struct acl_list* acl, struct config_file* cfg,
-	const char* str, const char* tag, const char* action)
+	const char* str, const char* tag, const char* action,
+	int is_interface, int port)
 {
 	struct acl_addr* node;
 	int tagid;
 	enum localzone_type t;
-	if(!(node=acl_find_or_create(acl, str)))
+	if(!(node=acl_find_or_create_str2addr(acl, str, is_interface, port))) {
+		if(is_interface)
+			log_err("non-configured interface: %s", str);
 		return 0;
+	}
 	/* allocate array if not yet */
 	if(!node->tag_actions) {
 		node->tag_actions = (uint8_t*)regional_alloc_zero(acl->region,
@@ -281,13 +360,17 @@ check_data(const char* data, const struct config_strlist* head)
 /** apply acl_tag_data string */
 static int
 acl_list_tag_data_cfg(struct acl_list* acl, struct config_file* cfg,
-	const char* str, const char* tag, const char* data)
+	const char* str, const char* tag, const char* data,
+	int is_interface, int port)
 {
 	struct acl_addr* node;
 	int tagid;
 	char* dupdata;
-	if(!(node=acl_find_or_create(acl, str)))
+	if(!(node=acl_find_or_create_str2addr(acl, str, is_interface, port))) {
+		if(is_interface)
+			log_err("non-configured interface: %s", str);
 		return 0;
+	}
 	/* allocate array if not yet */
 	if(!node->tag_datas) {
 		node->tag_datas = (struct config_strlist**)regional_alloc_zero(
@@ -329,11 +412,11 @@ acl_list_tag_data_cfg(struct acl_list* acl, struct config_file* cfg,
 }
 
 /** read acl_list config */
-static int 
-read_acl_list(struct acl_list* acl, struct config_file* cfg)
+static int
+read_acl_list(struct acl_list* acl, struct config_str2list* acls)
 {
 	struct config_str2list* p;
-	for(p = cfg->acls; p; p = p->next) {
+	for(p = acls; p; p = p->next) {
 		log_assert(p->str && p->str2);
 		if(!acl_list_str_cfg(acl, p->str, p->str2, 1))
 			return 0;
@@ -341,15 +424,38 @@ read_acl_list(struct acl_list* acl, struct config_file* cfg)
 	return 1;
 }
 
-/** read acl tags config */
-static int 
-read_acl_tags(struct acl_list* acl, struct config_file* cfg)
+/** read acl view config */
+static int
+read_acl_view(struct acl_list* acl, struct config_str2list** acl_view,
+	struct views* v)
 {
-	struct config_strbytelist* np, *p = cfg->acl_tags;
-	cfg->acl_tags = NULL;
+	struct config_str2list* np, *p = *acl_view;
+	*acl_view = NULL;
 	while(p) {
 		log_assert(p->str && p->str2);
-		if(!acl_list_tags_cfg(acl, p->str, p->str2, p->str2len)) {
+		if(!acl_list_view_cfg(acl, p->str, p->str2, v, 0, 0)) {
+			config_deldblstrlist(p);
+			return 0;
+		}
+		/* free the items as we go to free up memory */
+		np = p->next;
+		free(p->str);
+		free(p->str2);
+		free(p);
+		p = np;
+	}
+	return 1;
+}
+
+/** read acl tags config */
+static int
+read_acl_tags(struct acl_list* acl, struct config_strbytelist** acl_tags)
+{
+	struct config_strbytelist* np, *p = *acl_tags;
+	*acl_tags = NULL;
+	while(p) {
+		log_assert(p->str && p->str2);
+		if(!acl_list_tags_cfg(acl, p->str, p->str2, p->str2len, 0, 0)) {
 			config_del_strbytelist(p);
 			return 0;
 		}
@@ -363,38 +469,18 @@ read_acl_tags(struct acl_list* acl, struct config_file* cfg)
 	return 1;
 }
 
-/** read acl view config */
-static int 
-read_acl_view(struct acl_list* acl, struct config_file* cfg, struct views* v)
-{
-	struct config_str2list* np, *p = cfg->acl_view;
-	cfg->acl_view = NULL;
-	while(p) {
-		log_assert(p->str && p->str2);
-		if(!acl_list_view_cfg(acl, p->str, p->str2, v)) {
-			return 0;
-		}
-		/* free the items as we go to free up memory */
-		np = p->next;
-		free(p->str);
-		free(p->str2);
-		free(p);
-		p = np;
-	}
-	return 1;
-}
-
 /** read acl tag actions config */
-static int 
-read_acl_tag_actions(struct acl_list* acl, struct config_file* cfg)
+static int
+read_acl_tag_actions(struct acl_list* acl, struct config_file* cfg,
+	struct config_str3list** acl_tag_actions)
 {
 	struct config_str3list* p, *np;
-	p = cfg->acl_tag_actions;
-	cfg->acl_tag_actions = NULL;
+	p = *acl_tag_actions;
+	*acl_tag_actions = NULL;
 	while(p) {
 		log_assert(p->str && p->str2 && p->str3);
 		if(!acl_list_tag_action_cfg(acl, cfg, p->str, p->str2,
-			p->str3)) {
+			p->str3, 0, 0)) {
 			config_deltrplstrlist(p);
 			return 0;
 		}
@@ -410,15 +496,17 @@ read_acl_tag_actions(struct acl_list* acl, struct config_file* cfg)
 }
 
 /** read acl tag datas config */
-static int 
-read_acl_tag_datas(struct acl_list* acl, struct config_file* cfg)
+static int
+read_acl_tag_datas(struct acl_list* acl, struct config_file* cfg,
+	struct config_str3list** acl_tag_datas)
 {
 	struct config_str3list* p, *np;
-	p = cfg->acl_tag_datas;
-	cfg->acl_tag_datas = NULL;
+	p = *acl_tag_datas;
+	*acl_tag_datas = NULL;
 	while(p) {
 		log_assert(p->str && p->str2 && p->str3);
-		if(!acl_list_tag_data_cfg(acl, cfg, p->str, p->str2, p->str3)) {
+		if(!acl_list_tag_data_cfg(acl, cfg, p->str, p->str2, p->str3,
+			0, 0)) {
 			config_deltrplstrlist(p);
 			return 0;
 		}
@@ -433,30 +521,27 @@ read_acl_tag_datas(struct acl_list* acl, struct config_file* cfg)
 	return 1;
 }
 
-int 
+int
 acl_list_apply_cfg(struct acl_list* acl, struct config_file* cfg,
 	struct views* v)
 {
 	regional_free_all(acl->region);
 	addr_tree_init(&acl->tree);
-	if(!read_acl_list(acl, cfg))
+	if(!read_acl_list(acl, cfg->acls))
 		return 0;
-	if(!read_acl_view(acl, cfg, v))
+	if(!read_acl_view(acl, &cfg->acl_view, v))
 		return 0;
-	if(!read_acl_tags(acl, cfg))
+	if(!read_acl_tags(acl, &cfg->acl_tags))
 		return 0;
-	if(!read_acl_tag_actions(acl, cfg))
+	if(!read_acl_tag_actions(acl, cfg, &cfg->acl_tag_actions))
 		return 0;
-	if(!read_acl_tag_datas(acl, cfg))
+	if(!read_acl_tag_datas(acl, cfg, &cfg->acl_tag_datas))
 		return 0;
 	/* insert defaults, with '0' to ignore them if they are duplicates */
-	if(!acl_list_str_cfg(acl, "0.0.0.0/0", "refuse", 0))
-		return 0;
+	/* the 'refuse' defaults for /0 are now done per interface instead */
 	if(!acl_list_str_cfg(acl, "127.0.0.0/8", "allow", 0))
 		return 0;
 	if(cfg->do_ip6) {
-		if(!acl_list_str_cfg(acl, "::0/0", "refuse", 0))
-			return 0;
 		if(!acl_list_str_cfg(acl, "::1", "allow", 0))
 			return 0;
 		if(!acl_list_str_cfg(acl, "::ffff:127.0.0.1", "allow", 0))
@@ -466,7 +551,223 @@ acl_list_apply_cfg(struct acl_list* acl, struct config_file* cfg,
 	return 1;
 }
 
-enum acl_access 
+int
+acl_interface_compare(const void* k1, const void* k2)
+{
+	struct addr_tree_node* n1 = (struct addr_tree_node*)k1;
+	struct addr_tree_node* n2 = (struct addr_tree_node*)k2;
+	return sockaddr_cmp(&n1->addr, n1->addrlen, &n2->addr,
+		n2->addrlen);
+	/* We don't care about comparing node->net. All addresses in the
+	 * acl_interface tree have either 32 (ipv4) or 128 (ipv6). */
+}
+
+void
+acl_interface_init(struct acl_list* acl_interface)
+{
+	regional_free_all(acl_interface->region);
+	/* We want comparison in the tree to include only address and port.
+	 * We don't care about comparing node->net. All addresses in the
+	 * acl_interface->tree should have either 32 (ipv4) or 128 (ipv6).
+	 * Initialise with the appropriate compare function but keep treating
+	 * it as an addr_tree. */
+	addr_tree_addrport_init(&acl_interface->tree);
+}
+
+static int
+read_acl_interface_action(struct acl_list* acl_interface,
+	struct config_str2list* acls, int port)
+{
+	struct config_str2list* p;
+	for(p = acls; p; p = p->next) {
+		char** resif = NULL;
+		int num_resif = 0;
+		int i;
+		log_assert(p->str && p->str2);
+		if(!resolve_interface_names(&p->str, 1, NULL, &resif, &num_resif))
+			return 0;
+		for(i = 0; i<num_resif; i++) {
+			if(!acl_interface_str_cfg(acl_interface, resif[i], p->str2, port)){
+				config_del_strarray(resif, num_resif);
+				return 0;
+			}
+		}
+		config_del_strarray(resif, num_resif);
+	}
+	return 1;
+}
+
+/** read acl view config for interface */
+static int
+read_acl_interface_view(struct acl_list* acl_interface,
+	struct config_str2list** acl_view,
+	struct views* v, int port)
+{
+	struct config_str2list* np, *p = *acl_view;
+	*acl_view = NULL;
+	while(p) {
+		char** resif = NULL;
+		int num_resif = 0;
+		int i;
+		log_assert(p->str && p->str2);
+		if(!resolve_interface_names(&p->str, 1, NULL, &resif, &num_resif)) {
+			config_deldblstrlist(p);
+			return 0;
+		}
+		for(i = 0; i<num_resif; i++) {
+			if(!acl_list_view_cfg(acl_interface, resif[i], p->str2,
+				v, 1, port)) {
+				config_del_strarray(resif, num_resif);
+				config_deldblstrlist(p);
+				return 0;
+			}
+		}
+		config_del_strarray(resif, num_resif);
+		/* free the items as we go to free up memory */
+		np = p->next;
+		free(p->str);
+		free(p->str2);
+		free(p);
+		p = np;
+	}
+	return 1;
+}
+
+/** read acl tags config for interface */
+static int
+read_acl_interface_tags(struct acl_list* acl_interface,
+	struct config_strbytelist** acl_tags, int port)
+{
+	struct config_strbytelist* np, *p = *acl_tags;
+	*acl_tags = NULL;
+	while(p) {
+		char** resif = NULL;
+		int num_resif = 0;
+		int i;
+		log_assert(p->str && p->str2);
+		if(!resolve_interface_names(&p->str, 1, NULL, &resif, &num_resif)) {
+			config_del_strbytelist(p);
+			return 0;
+		}
+		for(i = 0; i<num_resif; i++) {
+			if(!acl_list_tags_cfg(acl_interface, resif[i], p->str2,
+				p->str2len, 1, port)) {
+				config_del_strbytelist(p);
+				config_del_strarray(resif, num_resif);
+				return 0;
+			}
+		}
+		config_del_strarray(resif, num_resif);
+		/* free the items as we go to free up memory */
+		np = p->next;
+		free(p->str);
+		free(p->str2);
+		free(p);
+		p = np;
+	}
+	return 1;
+}
+
+/** read acl tag actions config for interface*/
+static int
+read_acl_interface_tag_actions(struct acl_list* acl_interface,
+	struct config_file* cfg,
+	struct config_str3list** acl_tag_actions, int port)
+{
+	struct config_str3list* p, *np;
+	p = *acl_tag_actions;
+	*acl_tag_actions = NULL;
+	while(p) {
+		char** resif = NULL;
+		int num_resif = 0;
+		int i;
+		log_assert(p->str && p->str2 && p->str3);
+		if(!resolve_interface_names(&p->str, 1, NULL, &resif, &num_resif)) {
+			config_deltrplstrlist(p);
+			return 0;
+		}
+		for(i = 0; i<num_resif; i++) {
+			if(!acl_list_tag_action_cfg(acl_interface, cfg,
+				resif[i], p->str2, p->str3, 1, port)) {
+				config_deltrplstrlist(p);
+				config_del_strarray(resif, num_resif);
+				return 0;
+			}
+		}
+		config_del_strarray(resif, num_resif);
+		/* free the items as we go to free up memory */
+		np = p->next;
+		free(p->str);
+		free(p->str2);
+		free(p->str3);
+		free(p);
+		p = np;
+	}
+	return 1;
+}
+
+/** read acl tag datas config for interface */
+static int
+read_acl_interface_tag_datas(struct acl_list* acl_interface,
+	struct config_file* cfg,
+	struct config_str3list** acl_tag_datas, int port)
+{
+	struct config_str3list* p, *np;
+	p = *acl_tag_datas;
+	*acl_tag_datas = NULL;
+	while(p) {
+		char** resif = NULL;
+		int num_resif = 0;
+		int i;
+		log_assert(p->str && p->str2 && p->str3);
+		if(!resolve_interface_names(&p->str, 1, NULL, &resif, &num_resif)) {
+			config_deltrplstrlist(p);
+			return 0;
+		}
+		for(i = 0; i<num_resif; i++) {
+			if(!acl_list_tag_data_cfg(acl_interface, cfg,
+				resif[i], p->str2, p->str3, 1, port)) {
+				config_deltrplstrlist(p);
+				config_del_strarray(resif, num_resif);
+				return 0;
+			}
+		}
+		config_del_strarray(resif, num_resif);
+		/* free the items as we go to free up memory */
+		np = p->next;
+		free(p->str);
+		free(p->str2);
+		free(p->str3);
+		free(p);
+		p = np;
+	}
+	return 1;
+}
+
+int
+acl_interface_apply_cfg(struct acl_list* acl_interface, struct config_file* cfg,
+	struct views* v)
+{
+	if(!read_acl_interface_action(acl_interface, cfg->interface_actions,
+		cfg->port))
+		return 0;
+	if(!read_acl_interface_view(acl_interface, &cfg->interface_view, v,
+		cfg->port))
+		return 0;
+	if(!read_acl_interface_tags(acl_interface, &cfg->interface_tags,
+		cfg->port))
+		return 0;
+	if(!read_acl_interface_tag_actions(acl_interface, cfg,
+		&cfg->interface_tag_actions, cfg->port))
+		return 0;
+	if(!read_acl_interface_tag_datas(acl_interface, cfg,
+		&cfg->interface_tag_datas, cfg->port))
+		return 0;
+	addr_tree_init_parents(&acl_interface->tree);
+	return 1;
+}
+
+enum acl_access
 acl_get_control(struct acl_addr* acl)
 {
 	if(acl) return acl->control;
@@ -481,9 +782,44 @@ acl_addr_lookup(struct acl_list* acl, struct sockaddr_storage* addr,
 		addr, addrlen);
 }
 
-size_t 
+size_t
 acl_list_get_mem(struct acl_list* acl)
 {
 	if(!acl) return 0;
 	return sizeof(*acl) + regional_get_mem(acl->region);
 }
+
+const char* acl_access_to_str(enum acl_access acl)
+{
+	switch(acl) {
+	case acl_deny: return "deny";
+	case acl_refuse: return "refuse";
+	case acl_deny_non_local: return "deny_non_local";
+	case acl_refuse_non_local: return "refuse_non_local";
+	case acl_allow: return "allow";
+	case acl_allow_snoop: return "allow_snoop";
+	case acl_allow_setrd: return "allow_setrd";
+	default: break;
+	}
+	return "unknown";
+}
+
+void
+log_acl_action(const char* action, struct sockaddr_storage* addr,
+	socklen_t addrlen, enum acl_access acl, struct acl_addr* acladdr)
+{
+	char a[128], n[128];
+	uint16_t port;
+	addr_to_str(addr, addrlen, a, sizeof(a));
+	port = ntohs(((struct sockaddr_in*)addr)->sin_port);
+	if(acladdr) {
+		addr_to_str(&acladdr->node.addr, acladdr->node.addrlen,
+			n, sizeof(n));
+		verbose(VERB_ALGO, "%s query from %s port %d because of "
+			"%s/%d %s", action, a, (int)port, n, acladdr->node.net,
+			acl_access_to_str(acl));
+	} else {
+		verbose(VERB_ALGO, "%s query from %s port %d", action, a,
+			(int)port);
+	}
+}

daemon/acl_list.h

@@ -36,7 +36,7 @@
 /**
  * \file
  *
- * This file keeps track of the list of clients that are allowed to 
+ * This file keeps track of the list of clients that are allowed to
  * access the server.
  */
 
@@ -64,8 +64,12 @@ enum acl_access {
 	acl_allow,
 	/** allow full access for all queries, recursion and cache snooping */
 	acl_allow_snoop,
-	/** allow full access for recursion queries and set RD flag regardless of request */
-	acl_allow_setrd
+	/** allow full access for recursion queries and set RD flag regardless
+	 *  of request */
+	acl_allow_setrd,
+	/** allow full access for recursion (+RD) queries if valid cookie
+	 *  present or stateful transport */
+	acl_allow_cookie
 };
 
 /**
@@ -74,7 +78,7 @@ enum acl_access {
 struct acl_list {
 	/** regional for allocation */
 	struct regional* region;
-	/** 
+	/**
 	 * Tree of the addresses that are allowed/blocked.
 	 * contents of type acl_addr.
 	 */
@@ -108,7 +112,7 @@ struct acl_addr {
 };
 
 /**
- * Create acl structure 
+ * Create acl structure
  * @return new structure or NULL on error.
  */
 struct acl_list* acl_list_create(void);
@@ -119,6 +123,20 @@ struct acl_list* acl_list_create(void);
  */
 void acl_list_delete(struct acl_list* acl);
 
+/**
+ * Insert interface in the acl_list. This should happen when the listening
+ * interface is setup.
+ * @param acl_interface: acl_list to insert to.
+ * @param addr: interface IP.
+ * @param addrlen: length of the interface IP.
+ * @param control: acl_access.
+ * @return new structure or NULL on error.
+ */
+struct acl_addr*
+acl_interface_insert(struct acl_list* acl_interface,
+	struct sockaddr_storage* addr, socklen_t addrlen,
+	enum acl_access control);
+
 /**
  * Process access control config.
  * @param acl: where to store.
@@ -129,6 +147,25 @@ void acl_list_delete(struct acl_list* acl);
 int acl_list_apply_cfg(struct acl_list* acl, struct config_file* cfg,
 	struct views* v);
 
+/** compare ACL interface "addr_tree" nodes (+port) */
+int acl_interface_compare(const void* k1, const void* k2);
+
+/**
+ * Initialise (also clean) the acl_interface struct.
+ * @param acl_interface: where to store.
+ */
+void acl_interface_init(struct acl_list* acl_interface);
+
+/**
+ * Process interface control config.
+ * @param acl_interface: where to store.
+ * @param cfg: config options.
+ * @param v: views structure
+ * @return 0 on error.
+ */
+int acl_interface_apply_cfg(struct acl_list* acl_interface, struct config_file* cfg,
+	struct views* v);
+
 /**
  * Lookup access control status for acl structure.
  * @param acl: structure for acl storage.
@@ -154,4 +191,15 @@ acl_addr_lookup(struct acl_list* acl, struct sockaddr_storage* addr,
  */
 size_t acl_list_get_mem(struct acl_list* acl);
 
+/*
+ * Get string for acl access specification
+ * @param acl: access type value
+ * @return string
+ */
+const char* acl_access_to_str(enum acl_access acl);
+
+/* log acl and addr for action */
+void log_acl_action(const char* action, struct sockaddr_storage* addr,
+	socklen_t addrlen, enum acl_access acl, struct acl_addr* acladdr);
+
 #endif /* DAEMON_ACL_LIST_H */

daemon/cachedump.c

@@ -47,10 +47,12 @@
 #include "services/cache/rrset.h"
 #include "services/cache/dns.h"
 #include "services/cache/infra.h"
+#include "services/outside_network.h"
 #include "util/data/msgreply.h"
 #include "util/regional.h"
 #include "util/net_help.h"
 #include "util/data/dname.h"
+#include "util/config_file.h"
 #include "iterator/iterator.h"
 #include "iterator/iter_delegpt.h"
 #include "iterator/iter_utils.h"
@@ -164,8 +166,7 @@ dump_msg_ref(RES* ssl, struct ub_packed_rrset_key* k)
 
 /** dump message entry */
 static int
-dump_msg(RES* ssl, struct query_info* k, struct reply_info* d, 
-	time_t now)
+dump_msg(RES* ssl, struct query_info* k, struct reply_info* d, time_t now)
 {
 	size_t i;
 	char* nm, *tp, *cl;
@@ -190,13 +191,15 @@ dump_msg(RES* ssl, struct query_info* k, struct reply_info* d,
 	}
 	
 	/* meta line */
-	if(!ssl_printf(ssl, "msg %s %s %s %d %d " ARG_LL "d %d %u %u %u\n",
+	if(!ssl_printf(ssl, "msg %s %s %s %d %d " ARG_LL "d %d %u %u %u %d %s\n",
 			nm, cl, tp,
 			(int)d->flags, (int)d->qdcount, 
 			(long long)(d->ttl-now), (int)d->security,
-			(unsigned)d->an_numrrsets, 
+			(unsigned)d->an_numrrsets,
 			(unsigned)d->ns_numrrsets,
-			(unsigned)d->ar_numrrsets)) {
+			(unsigned)d->ar_numrrsets,
+			(int)d->reason_bogus,
+			d->reason_bogus_str?d->reason_bogus_str:"")) {
 		free(nm);
 		free(tp);
 		free(cl);
@@ -385,7 +388,7 @@ move_into_cache(struct ub_packed_rrset_key* k,
 	struct rrset_ref ref;
 	uint8_t* p;
 
-	ak = alloc_special_obtain(&worker->alloc);
+	ak = alloc_special_obtain(worker->alloc);
 	if(!ak) {
 		log_warn("error out of memory");
 		return 0;
@@ -396,7 +399,7 @@ move_into_cache(struct ub_packed_rrset_key* k,
 	ak->rk.dname = (uint8_t*)memdup(k->rk.dname, k->rk.dname_len);
 	if(!ak->rk.dname) {
 		log_warn("error out of memory");
-		ub_packed_rrset_parsedelete(ak, &worker->alloc);
+		ub_packed_rrset_parsedelete(ak, worker->alloc);
 		return 0;
 	}
 	s = sizeof(*ad) + (sizeof(size_t) + sizeof(uint8_t*) + 
@@ -406,7 +409,7 @@ move_into_cache(struct ub_packed_rrset_key* k,
 	ad = (struct packed_rrset_data*)malloc(s);
 	if(!ad) {
 		log_warn("error out of memory");
-		ub_packed_rrset_parsedelete(ak, &worker->alloc);
+		ub_packed_rrset_parsedelete(ak, worker->alloc);
 		return 0;
 	}
 	p = (uint8_t*)ad;
@@ -429,7 +432,8 @@ move_into_cache(struct ub_packed_rrset_key* k,
 	ref.key = ak;
 	ref.id = ak->id;
 	(void)rrset_cache_update(worker->env.rrset_cache, &ref,
-		&worker->alloc, *worker->env.now);
+		worker->alloc, *worker->env.now);
+
 	return 1;
 }
 
@@ -630,6 +634,9 @@ load_msg(RES* ssl, sldns_buffer* buf, struct worker* worker)
 	long long ttl;
 	size_t i;
 	int go_on = 1;
+	int ede;
+	int consumed = 0;
+	char* ede_str = NULL;
 
 	regional_free_all(region);
 
@@ -644,11 +651,16 @@ load_msg(RES* ssl, sldns_buffer* buf, struct worker* worker)
 	}
 
 	/* read remainder of line */
-	if(sscanf(s, " %u %u " ARG_LL "d %u %u %u %u", &flags, &qdcount, &ttl, 
-		&security, &an, &ns, &ar) != 7) {
+	/* note the last space before any possible EDE text */
+	if(sscanf(s, " %u %u " ARG_LL "d %u %u %u %u %d %n", &flags, &qdcount, &ttl,
+		&security, &an, &ns, &ar, &ede, &consumed) != 8) {
 		log_warn("error cannot parse numbers: %s", s);
 		return 0;
 	}
+	/* there may be EDE text after the numbers */
+	if(consumed > 0 && (size_t)consumed < strlen(s))
+		ede_str = s + consumed;
+	memset(&rep, 0, sizeof(rep));
 	rep.flags = (uint16_t)flags;
 	rep.qdcount = (uint16_t)qdcount;
 	rep.ttl = (time_t)ttl;
@@ -663,6 +675,8 @@ load_msg(RES* ssl, sldns_buffer* buf, struct worker* worker)
 	rep.ns_numrrsets = (size_t)ns;
 	rep.ar_numrrsets = (size_t)ar;
 	rep.rrset_count = (size_t)an+(size_t)ns+(size_t)ar;
+	rep.reason_bogus = (sldns_ede_code)ede;
+	rep.reason_bogus_str = ede_str?(char*)regional_strdup(region, ede_str):NULL;
 	rep.rrsets = (struct ub_packed_rrset_key**)regional_alloc_zero(
 		region, sizeof(struct ub_packed_rrset_key*)*rep.rrset_count);
 
@@ -677,7 +691,8 @@ load_msg(RES* ssl, sldns_buffer* buf, struct worker* worker)
 	if(!go_on) 
 		return 1; /* skip this one, not all references satisfied */
 
-	if(!dns_cache_store(&worker->env, &qinf, &rep, 0, 0, 0, NULL, flags)) {
+	if(!dns_cache_store(&worker->env, &qinf, &rep, 0, 0, 0, NULL, flags,
+		*worker->env.now)) {
 		log_warn("error out of memory");
 		return 0;
 	}
@@ -848,13 +863,16 @@ int print_deleg_lookup(RES* ssl, struct worker* worker, uint8_t* nm,
 	while(1) {
 		dp = dns_cache_find_delegation(&worker->env, nm, nmlen, 
 			qinfo.qtype, qinfo.qclass, region, &msg, 
-			*worker->env.now);
+			*worker->env.now, 0, NULL, 0);
 		if(!dp) {
 			return ssl_printf(ssl, "no delegation from "
 				"cache; goes to configured roots\n");
 		}
 		/* go up? */
-		if(iter_dp_is_useless(&qinfo, BIT_RD, dp)) {
+		if(iter_dp_is_useless(&qinfo, BIT_RD, dp,
+			(worker->env.cfg->do_ip4 && worker->back->num_ip4 != 0),
+			(worker->env.cfg->do_ip6 && worker->back->num_ip6 != 0),
+			worker->env.cfg->do_nat64)) {
 			print_dp_main(ssl, dp, msg);
 			print_dp_details(ssl, worker, dp);
 			if(!ssl_printf(ssl, "cache delegation was "

daemon/daemon.c

@@ -96,6 +96,9 @@
 #ifdef HAVE_SYSTEMD
 #include <systemd/sd-daemon.h>
 #endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
 
 /** How many quit requests happened. */
 static int sig_record_quit = 0;
@@ -271,8 +274,17 @@ daemon_init(void)
 		free(daemon);
 		return NULL;
 	}
+	daemon->acl_interface = acl_list_create();
+	if(!daemon->acl_interface) {
+		acl_list_delete(daemon->acl);
+		edns_known_options_delete(daemon->env);
+		free(daemon->env);
+		free(daemon);
+		return NULL;
+	}
 	daemon->tcl = tcl_list_create();
 	if(!daemon->tcl) {
+		acl_list_delete(daemon->acl_interface);
 		acl_list_delete(daemon->acl);
 		edns_known_options_delete(daemon->env);
 		free(daemon->env);
@@ -284,6 +296,7 @@ daemon_init(void)
 		log_err("gettimeofday: %s", strerror(errno));
 	daemon->time_last_stat = daemon->time_boot;
 	if((daemon->env->auth_zones = auth_zones_create()) == 0) {
+		acl_list_delete(daemon->acl_interface);
 		acl_list_delete(daemon->acl);
 		tcl_list_delete(daemon->tcl);
 		edns_known_options_delete(daemon->env);
@@ -293,6 +306,7 @@ daemon_init(void)
 	}
 	if(!(daemon->env->edns_strings = edns_strings_create())) {
 		auth_zones_delete(daemon->env->auth_zones);
+		acl_list_delete(daemon->acl_interface);
 		acl_list_delete(daemon->acl);
 		tcl_list_delete(daemon->tcl);
 		edns_known_options_delete(daemon->env);
@@ -303,6 +317,29 @@ daemon_init(void)
 	return daemon;	
 }
 
+static int setup_acl_for_ports(struct acl_list* list,
+	struct listen_port* port_list)
+{
+	struct acl_addr* acl_node;
+	struct addrinfo* addr;
+	for(; port_list; port_list=port_list->next) {
+		if(!port_list->socket) {
+			/* This is mainly for testbound where port_list is
+			 * empty. */
+			continue;
+		}
+		addr = port_list->socket->addr;
+		if(!(acl_node = acl_interface_insert(list,
+			(struct sockaddr_storage*)addr->ai_addr,
+			(socklen_t)addr->ai_addrlen,
+			acl_refuse))) {
+			return 0;
+		}
+		port_list->socket->acl = acl_node;
+	}
+	return 1;
+}
+
 int 
 daemon_open_shared_ports(struct daemon* daemon)
 {
@@ -320,6 +357,8 @@ daemon_open_shared_ports(struct daemon* daemon)
 			free(daemon->ports);
 			daemon->ports = NULL;
 		}
+		/* clean acl_interface */
+		acl_interface_init(daemon->acl_interface);
 		if(!resolve_interface_names(daemon->cfg->ifs,
 			daemon->cfg->num_ifs, NULL, &resif, &num_resif))
 			return 0;
@@ -329,7 +368,8 @@ daemon_open_shared_ports(struct daemon* daemon)
 			daemon->reuseport = 1;
 #endif
 		/* try to use reuseport */
-		p0 = listening_ports_open(daemon->cfg, resif, num_resif, &daemon->reuseport);
+		p0 = listening_ports_open(daemon->cfg, resif, num_resif,
+			&daemon->reuseport);
 		if(!p0) {
 			listening_ports_free(p0);
 			config_del_strarray(resif, num_resif);
@@ -350,6 +390,12 @@ daemon_open_shared_ports(struct daemon* daemon)
 			return 0;
 		}
 		daemon->ports[0] = p0;
+		if(!setup_acl_for_ports(daemon->acl_interface,
+		    daemon->ports[0])) {
+			listening_ports_free(p0);
+			config_del_strarray(resif, num_resif);
+			return 0;
+		}
 		if(daemon->reuseport) {
 			/* continue to use reuseport */
 			for(i=1; i<daemon->num_ports; i++) {
@@ -365,6 +411,15 @@ daemon_open_shared_ports(struct daemon* daemon)
 					config_del_strarray(resif, num_resif);
 					return 0;
 				}
+				if(!setup_acl_for_ports(daemon->acl_interface,
+					daemon->ports[i])) {
+					for(i=0; i<daemon->num_ports; i++)
+						listening_ports_free(daemon->ports[i]);
+					free(daemon->ports);
+					daemon->ports = NULL;
+					config_del_strarray(resif, num_resif);
+					return 0;
+				}
 			}
 		}
 		config_del_strarray(resif, num_resif);
@@ -433,6 +488,27 @@ static int daemon_get_shufport(struct daemon* daemon, int* shufport)
 	return avail;
 }
 
+/**
+ * Clear and delete per-worker alloc caches, and free memory maintained in
+ * superalloc.
+ * The rrset and message caches must be empty at the time of call.
+ * @param daemon: the daemon that maintains the alloc caches to be cleared.
+ */
+static void
+daemon_clear_allocs(struct daemon* daemon)
+{
+	int i;
+
+	for(i=0; i<daemon->num; i++) {
+		alloc_clear(daemon->worker_allocs[i]);
+		free(daemon->worker_allocs[i]);
+	}
+	free(daemon->worker_allocs);
+	daemon->worker_allocs = NULL;
+
+	alloc_clear_special(&daemon->superalloc);
+}
+
 /**
  * Allocate empty worker structures. With backptr and thread-number,
  * from 0..numthread initialised. Used as user arguments to new threads.
@@ -485,6 +561,21 @@ daemon_create_workers(struct daemon* daemon)
 			/* the above is not ports/numthr, due to rounding */
 			fatal_exit("could not create worker");
 	}
+	/* create per-worker alloc caches if not reusing existing ones. */
+	if(!daemon->worker_allocs) {
+		daemon->worker_allocs = (struct alloc_cache**)calloc(
+			(size_t)daemon->num, sizeof(struct alloc_cache*));
+		if(!daemon->worker_allocs)
+			fatal_exit("could not allocate worker allocs");
+		for(i=0; i<daemon->num; i++) {
+			struct alloc_cache* alloc = calloc(1,
+				sizeof(struct alloc_cache));
+			if (!alloc)
+				fatal_exit("could not allocate worker alloc");
+			alloc_init(alloc, &daemon->superalloc, i);
+			daemon->worker_allocs[i] = alloc;
+		}
+	}
 	free(shufport);
 }
 
@@ -604,6 +695,9 @@ daemon_fork(struct daemon* daemon)
 
 	if(!acl_list_apply_cfg(daemon->acl, daemon->cfg, daemon->views))
 		fatal_exit("Could not setup access control list");
+	if(!acl_interface_apply_cfg(daemon->acl_interface, daemon->cfg,
+		daemon->views))
+		fatal_exit("Could not setup interface control list");
 	if(!tcl_list_apply_cfg(daemon->tcl, daemon->cfg))
 		fatal_exit("Could not setup TCP connection limits");
 	if(daemon->cfg->dnscrypt) {
@@ -713,6 +807,7 @@ daemon_fork(struct daemon* daemon)
 	/* Shutdown SHM */
 	shm_main_shutdown(daemon);
 
+	daemon->reuse_cache = daemon->workers[0]->reuse_cache;
 	daemon->need_to_exit = daemon->workers[0]->need_to_exit;
 }
 
@@ -727,9 +822,16 @@ daemon_cleanup(struct daemon* daemon)
 	log_thread_set(NULL);
 	/* clean up caches because
 	 * a) RRset IDs will be recycled after a reload, causing collisions
-	 * b) validation config can change, thus rrset, msg, keycache clear */
-	slabhash_clear(&daemon->env->rrset_cache->table);
-	slabhash_clear(daemon->env->msg_cache);
+	 * b) validation config can change, thus rrset, msg, keycache clear
+	 *
+	 * If we are trying to keep the cache as long as possible, we should
+	 * defer the cleanup until we know whether the new configuration allows
+	 * the reuse.  (If we're exiting, cleanup should be done here). */
+	if(!daemon->reuse_cache || daemon->need_to_exit) {
+		slabhash_clear(&daemon->env->rrset_cache->table);
+		slabhash_clear(daemon->env->msg_cache);
+	}
+	daemon->old_num = daemon->num; /* save the current num */
 	local_zones_delete(daemon->local_zones);
 	daemon->local_zones = NULL;
 	respip_set_delete(daemon->respip_set);
@@ -744,8 +846,13 @@ daemon_cleanup(struct daemon* daemon)
 		worker_delete(daemon->workers[i]);
 	free(daemon->workers);
 	daemon->workers = NULL;
+	/* Unless we're trying to keep the cache, worker alloc_caches should be
+	 * cleared and freed here. We do this after deleting workers to
+	 * guarantee that the alloc caches are valid throughout the lifetime
+	 * of workers. */
+	if(!daemon->reuse_cache || daemon->need_to_exit)
+		daemon_clear_allocs(daemon);
 	daemon->num = 0;
-	alloc_clear_special(&daemon->superalloc);
 #ifdef USE_DNSTAP
 	dt_delete(daemon->dtenv);
 	daemon->dtenv = NULL;
@@ -780,6 +887,7 @@ daemon_delete(struct daemon* daemon)
 	ub_randfree(daemon->rand);
 	alloc_clear(&daemon->superalloc);
 	acl_list_delete(daemon->acl);
+	acl_list_delete(daemon->acl_interface);
 	tcl_list_delete(daemon->tcl);
 	listen_desetup_locks();
 	free(daemon->chroot);
@@ -795,7 +903,7 @@ daemon_delete(struct daemon* daemon)
 	ub_c_lex_destroy();
 	/* libcrypto cleanup */
 #ifdef HAVE_SSL
-#  if defined(USE_GOST) && defined(HAVE_LDNS_KEY_EVP_UNLOAD_GOST)
+#  if defined(USE_GOST)
 	sldns_key_EVP_unload_gost();
 #  endif
 #  if HAVE_DECL_SSL_COMP_GET_COMPRESSION_METHODS && HAVE_DECL_SK_SSL_COMP_POP_FREE
@@ -841,8 +949,42 @@ daemon_delete(struct daemon* daemon)
 
 void daemon_apply_cfg(struct daemon* daemon, struct config_file* cfg)
 {
+	int new_num = cfg->num_threads?cfg->num_threads:1;
+
         daemon->cfg = cfg;
 	config_apply(cfg);
+
+	/* If this is a reload and we deferred the decision on whether to
+	 * reuse the alloc, RRset, and message caches, then check to see if
+	 * it's safe to keep the caches:
+	 * - changing the number of threads is obviously incompatible with
+	 *   keeping the per-thread alloc caches. It also means we have to
+	 *   clear RRset and message caches. (note that 'new_num' may be
+	 *   adjusted in daemon_create_workers, but for our purpose we can
+	 *   simply compare it with 'old_num'; if they are equal here,
+	 *   'new_num' won't be adjusted to a different value than 'old_num').
+	 * - changing RRset cache size effectively clears any remaining cache
+	 *   entries. We could keep their keys in alloc caches, but it would
+	 *   be more consistent with the sense of the change to clear allocs
+	 *   and free memory. To do so we also have to clear message cache.
+	 * - only changing message cache size does not necessarily affect
+	 *   RRset or alloc cache. But almost all new subsequent queries will
+	 *   require recursive resolution anyway, so it doesn't help much to
+	 *   just keep RRset and alloc caches. For simplicity we clear/free
+	 *   the other two, too. */
+	if(daemon->worker_allocs &&
+		(new_num != daemon->old_num ||
+		 !slabhash_is_size(daemon->env->msg_cache, cfg->msg_cache_size,
+			cfg->msg_cache_slabs) ||
+		 !slabhash_is_size(&daemon->env->rrset_cache->table,
+			cfg->rrset_cache_size, cfg->rrset_cache_slabs)))
+	{
+		log_warn("cannot reuse caches due to critical config change");
+		slabhash_clear(&daemon->env->rrset_cache->table);
+		slabhash_clear(daemon->env->msg_cache);
+		daemon_clear_allocs(daemon);
+	}
+
 	if(!slabhash_is_size(daemon->env->msg_cache, cfg->msg_cache_size,
 	   	cfg->msg_cache_slabs)) {
 		slabhash_delete(daemon->env->msg_cache);

daemon/daemon.h

@@ -99,8 +99,12 @@ struct daemon {
 	void* listen_sslctx, *connect_sslctx;
 	/** num threads allocated */
 	int num;
+	/** num threads allocated in the previous config or 0 at first */
+	int old_num;
 	/** the worker entries */
 	struct worker** workers;
+	/** per-worker allocation cache */
+	struct alloc_cache **worker_allocs;
 	/** do we need to exit unbound (or is it only a reload?) */
 	int need_to_exit;
 	/** master random table ; used for port div between threads on reload*/
@@ -113,6 +117,8 @@ struct daemon {
 	struct module_stack mods;
 	/** access control, which client IPs are allowed to connect */
 	struct acl_list* acl;
+	/** access control, which interfaces are allowed to connect */
+	struct acl_list* acl_interface;
 	/** TCP connection limit, limit connections from client IPs */
 	struct tcl_list* tcl;
 	/** local authority zones */
@@ -138,6 +144,8 @@ struct daemon {
 	/** the dnscrypt environment */
 	struct dnsc_env* dnscenv;
 #endif
+	/** reuse existing cache on reload if other conditions allow it. */
+	int reuse_cache;
 };
 
 /**

daemon/remote.c

@@ -4,22 +4,22 @@
  * Copyright (c) 2008, NLnet Labs. All rights reserved.
  *
  * This software is open source.
- * 
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
- * 
+ *
  * Redistributions of source code must retain the above copyright notice,
  * this list of conditions and the following disclaimer.
- * 
+ *
  * Redistributions in binary form must reproduce the above copyright notice,
  * this list of conditions and the following disclaimer in the documentation
  * and/or other materials provided with the distribution.
- * 
+ *
  * Neither the name of the NLNET LABS nor the names of its contributors may
  * be used to endorse or promote products derived from this software without
  * specific prior written permission.
- * 
+ *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -38,7 +38,7 @@
  *
  * This file contains the remote control functionality for the daemon.
  * The remote control can be performed using either the commandline
- * unbound-control tool, or a TLS capable web browser. 
+ * unbound-control tool, or a TLS capable web browser.
  * The channel is secured using TLSv1, and certificates.
  * Both the server and the client(control tool) have their own keys.
  */
@@ -87,6 +87,7 @@
 #include "sldns/parseutil.h"
 #include "sldns/wire2str.h"
 #include "sldns/sbuffer.h"
+#include "util/timeval_func.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #  include <sys/types.h>
@@ -105,49 +106,6 @@
 
 /** what to put on statistics lines between var and value, ": " or "=" */
 #define SQ "="
-/** if true, inhibits a lot of =0 lines from the stats output */
-static const int inhibit_zero = 1;
-
-/** subtract timers and the values do not overflow or become negative */
-static void
-timeval_subtract(struct timeval* d, const struct timeval* end, 
-	const struct timeval* start)
-{
-#ifndef S_SPLINT_S
-	time_t end_usec = end->tv_usec;
-	d->tv_sec = end->tv_sec - start->tv_sec;
-	if(end_usec < start->tv_usec) {
-		end_usec += 1000000;
-		d->tv_sec--;
-	}
-	d->tv_usec = end_usec - start->tv_usec;
-#endif
-}
-
-/** divide sum of timers to get average */
-static void
-timeval_divide(struct timeval* avg, const struct timeval* sum, long long d)
-{
-#ifndef S_SPLINT_S
-	size_t leftover;
-	if(d <= 0) {
-		avg->tv_sec = 0;
-		avg->tv_usec = 0;
-		return;
-	}
-	avg->tv_sec = sum->tv_sec / d;
-	avg->tv_usec = sum->tv_usec / d;
-	/* handle fraction from seconds divide */
-	leftover = sum->tv_sec - avg->tv_sec*d;
-	if(leftover <= 0)
-		leftover = 0;
-	avg->tv_usec += (((long long)leftover)*((long long)1000000))/d;
-	if(avg->tv_sec < 0)
-		avg->tv_sec = 0;
-	if(avg->tv_usec < 0)
-		avg->tv_usec = 0;
-#endif
-}
 
 static int
 remote_setup_ctx(struct daemon_remote* rc, struct config_file* cfg)
@@ -203,7 +161,7 @@ remote_setup_ctx(struct daemon_remote* rc, struct config_file* cfg)
 struct daemon_remote*
 daemon_remote_create(struct config_file* cfg)
 {
-	struct daemon_remote* rc = (struct daemon_remote*)calloc(1, 
+	struct daemon_remote* rc = (struct daemon_remote*)calloc(1,
 		sizeof(*rc));
 	if(!rc) {
 		log_err("out of memory in daemon_remote_create");
@@ -300,6 +258,7 @@ add_open(const char* ip, int nr, struct listen_port** list, int noproto_is_err,
 		 */
 		if(fd != -1) {
 #ifdef HAVE_CHOWN
+			chmod(ip, (mode_t)(S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP));
 			if (cfg->username && cfg->username[0] &&
 				cfg_uid != (uid_t)-1) {
 				if(chown(ip, cfg_uid, cfg_gid) == -1)
@@ -307,7 +266,6 @@ add_open(const char* ip, int nr, struct listen_port** list, int noproto_is_err,
 					  (unsigned)cfg_uid, (unsigned)cfg_gid,
 					  ip, strerror(errno));
 			}
-			chmod(ip, (mode_t)(S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP));
 #else
 			(void)cfg;
 #endif
@@ -412,7 +370,7 @@ accept_open(struct daemon_remote* rc, int fd)
 	n->next = rc->accept_list;
 	rc->accept_list = n;
 	/* open commpt */
-	n->com = comm_point_create_raw(rc->worker->base, fd, 0, 
+	n->com = comm_point_create_raw(rc->worker->base, fd, 0,
 		&remote_accept_callback, rc);
 	if(!n->com)
 		return 0;
@@ -421,7 +379,7 @@ accept_open(struct daemon_remote* rc, int fd)
 	return 1;
 }
 
-int daemon_remote_open_accept(struct daemon_remote* rc, 
+int daemon_remote_open_accept(struct daemon_remote* rc,
 	struct listen_port* ports, struct worker* worker)
 {
 	struct listen_port* p;
@@ -439,7 +397,7 @@ void daemon_remote_stop_accept(struct daemon_remote* rc)
 {
 	struct listen_list* p;
 	for(p=rc->accept_list; p; p=p->next) {
-		comm_point_stop_listening(p->com);	
+		comm_point_stop_listening(p->com);
 	}
 }
 
@@ -447,11 +405,11 @@ void daemon_remote_start_accept(struct daemon_remote* rc)
 {
 	struct listen_list* p;
 	for(p=rc->accept_list; p; p=p->next) {
-		comm_point_start_listening(p->com, -1, -1);	
+		comm_point_start_listening(p->com, -1, -1);
 	}
 }
 
-int remote_accept_callback(struct comm_point* c, void* arg, int err, 
+int remote_accept_callback(struct comm_point* c, void* arg, int err,
 	struct comm_reply* ATTR_UNUSED(rep))
 {
 	struct daemon_remote* rc = (struct daemon_remote*)arg;
@@ -483,7 +441,7 @@ int remote_accept_callback(struct comm_point* c, void* arg, int err,
 	}
 	n->fd = newfd;
 	/* start in reading state */
-	n->c = comm_point_create_raw(rc->worker->base, newfd, 0, 
+	n->c = comm_point_create_raw(rc->worker->base, newfd, 0,
 		&remote_control_callback, n);
 	if(!n->c) {
 		log_err("out of memory");
@@ -494,8 +452,8 @@ int remote_accept_callback(struct comm_point* c, void* arg, int err,
 	n->c->do_not_close = 0;
 	comm_point_stop_listening(n->c);
 	comm_point_start_listening(n->c, -1, REMOTE_CONTROL_TCP_TIMEOUT);
-	memcpy(&n->c->repinfo.addr, &addr, addrlen);
-	n->c->repinfo.addrlen = addrlen;
+	memcpy(&n->c->repinfo.remote_addr, &addr, addrlen);
+	n->c->repinfo.remote_addrlen = addrlen;
 	if(rc->use_cert) {
 		n->shake_state = rc_hs_read;
 		n->ssl = SSL_new(rc->ctx);
@@ -523,7 +481,7 @@ int remote_accept_callback(struct comm_point* c, void* arg, int err,
 	rc->busy_list = n;
 	rc->active ++;
 
-	/* perform the first nonblocking read already, for windows, 
+	/* perform the first nonblocking read already, for windows,
 	 * so it can return wouldblock. could be faster too. */
 	(void)remote_control_callback(n->c, n, NETEVENT_NOERROR, NULL);
 	return 0;
@@ -560,17 +518,18 @@ int
 ssl_print_text(RES* res, const char* text)
 {
 	int r;
-	if(!res) 
+	if(!res)
 		return 0;
 	if(res->ssl) {
 		ERR_clear_error();
 		if((r=SSL_write(res->ssl, text, (int)strlen(text))) <= 0) {
-			if(SSL_get_error(res->ssl, r) == SSL_ERROR_ZERO_RETURN) {
+			int r2;
+			if((r2=SSL_get_error(res->ssl, r)) == SSL_ERROR_ZERO_RETURN) {
 				verbose(VERB_QUERY, "warning, in SSL_write, peer "
 					"closed connection");
 				return 0;
 			}
-			log_crypto_err("could not SSL_write");
+			log_crypto_err_io("could not SSL_write", r2);
 			return 0;
 		}
 	} else {
@@ -621,11 +580,12 @@ ssl_read_line(RES* res, char* buf, size_t max)
 		if(res->ssl) {
 			ERR_clear_error();
 			if((r=SSL_read(res->ssl, buf+len, 1)) <= 0) {
-				if(SSL_get_error(res->ssl, r) == SSL_ERROR_ZERO_RETURN) {
+				int r2;
+				if((r2=SSL_get_error(res->ssl, r)) == SSL_ERROR_ZERO_RETURN) {
 					buf[len] = 0;
 					return 1;
 				}
-				log_crypto_err("could not SSL_read");
+				log_crypto_err_io("could not SSL_read", r2);
 				return 0;
 			}
 		} else {
@@ -638,7 +598,7 @@ ssl_read_line(RES* res, char* buf, size_t max)
 					}
 					if(errno == EINTR || errno == EAGAIN)
 						continue;
-					log_err("could not recv: %s",
+					if(rr < 0) log_err("could not recv: %s",
 						sock_strerror(errno));
 					return 0;
 				}
@@ -662,7 +622,7 @@ static char*
 skipwhite(char* str)
 {
 	/* EOS \0 is not a space */
-	while( isspace((unsigned char)*str) ) 
+	while( isspace((unsigned char)*str) )
 		str++;
 	return str;
 }
@@ -684,8 +644,9 @@ do_stop(RES* ssl, struct worker* worker)
 
 /** do the reload command */
 static void
-do_reload(RES* ssl, struct worker* worker)
+do_reload(RES* ssl, struct worker* worker, int reuse_cache)
 {
+	worker->reuse_cache = reuse_cache;
 	worker->need_to_exit = 0;
 	comm_base_exit(worker->base);
 	send_ok(ssl);
@@ -709,20 +670,30 @@ static int
 print_stats(RES* ssl, const char* nm, struct ub_stats_info* s)
 {
 	struct timeval sumwait, avg;
-	if(!ssl_printf(ssl, "%s.num.queries"SQ"%lu\n", nm, 
+	if(!ssl_printf(ssl, "%s.num.queries"SQ"%lu\n", nm,
 		(unsigned long)s->svr.num_queries)) return 0;
 	if(!ssl_printf(ssl, "%s.num.queries_ip_ratelimited"SQ"%lu\n", nm,
 		(unsigned long)s->svr.num_queries_ip_ratelimited)) return 0;
-	if(!ssl_printf(ssl, "%s.num.cachehits"SQ"%lu\n", nm, 
-		(unsigned long)(s->svr.num_queries 
+	if(!ssl_printf(ssl, "%s.num.queries_cookie_valid"SQ"%lu\n", nm,
+		(unsigned long)s->svr.num_queries_cookie_valid)) return 0;
+	if(!ssl_printf(ssl, "%s.num.queries_cookie_client"SQ"%lu\n", nm,
+		(unsigned long)s->svr.num_queries_cookie_client)) return 0;
+	if(!ssl_printf(ssl, "%s.num.queries_cookie_invalid"SQ"%lu\n", nm,
+		(unsigned long)s->svr.num_queries_cookie_invalid)) return 0;
+	if(!ssl_printf(ssl, "%s.num.cachehits"SQ"%lu\n", nm,
+		(unsigned long)(s->svr.num_queries
 			- s->svr.num_queries_missed_cache))) return 0;
-	if(!ssl_printf(ssl, "%s.num.cachemiss"SQ"%lu\n", nm, 
+	if(!ssl_printf(ssl, "%s.num.cachemiss"SQ"%lu\n", nm,
 		(unsigned long)s->svr.num_queries_missed_cache)) return 0;
-	if(!ssl_printf(ssl, "%s.num.prefetch"SQ"%lu\n", nm, 
+	if(!ssl_printf(ssl, "%s.num.prefetch"SQ"%lu\n", nm,
 		(unsigned long)s->svr.num_queries_prefetch)) return 0;
+	if(!ssl_printf(ssl, "%s.num.queries_timed_out"SQ"%lu\n", nm,
+		(unsigned long)s->svr.num_queries_timed_out)) return 0;
+	if(!ssl_printf(ssl, "%s.query.queue_time_us.max"SQ"%lu\n", nm,
+		(unsigned long)s->svr.max_query_time_us)) return 0;
 	if(!ssl_printf(ssl, "%s.num.expired"SQ"%lu\n", nm,
 		(unsigned long)s->svr.ans_expired)) return 0;
-	if(!ssl_printf(ssl, "%s.num.recursivereplies"SQ"%lu\n", nm, 
+	if(!ssl_printf(ssl, "%s.num.recursivereplies"SQ"%lu\n", nm,
 		(unsigned long)s->mesh_replies_sent)) return 0;
 #ifdef USE_DNSCRYPT
 	if(!ssl_printf(ssl, "%s.num.dnscrypt.crypted"SQ"%lu\n", nm,
@@ -756,7 +727,7 @@ print_stats(RES* ssl, const char* nm, struct ub_stats_info* s)
 	timeval_divide(&avg, &sumwait, s->mesh_replies_sent);
 	if(!ssl_printf(ssl, "%s.recursion.time.avg"SQ ARG_LL "d.%6.6d\n", nm,
 		(long long)avg.tv_sec, (int)avg.tv_usec)) return 0;
-	if(!ssl_printf(ssl, "%s.recursion.time.median"SQ"%g\n", nm, 
+	if(!ssl_printf(ssl, "%s.recursion.time.median"SQ"%g\n", nm,
 		s->mesh_time_median)) return 0;
 	if(!ssl_printf(ssl, "%s.tcpusage"SQ"%lu\n", nm,
 		(unsigned long)s->svr.tcp_accept_usage)) return 0;
@@ -781,7 +752,7 @@ print_longnum(RES* ssl, const char* desc, size_t x)
 		/* more than a Gb */
 		size_t front = x / (size_t)1000000;
 		size_t back = x % (size_t)1000000;
-		return ssl_printf(ssl, "%s%u%6.6u\n", desc, 
+		return ssl_printf(ssl, "%s%u%6.6u\n", desc,
 			(unsigned)front, (unsigned)back);
 	} else {
 		return ssl_printf(ssl, "%s%lu\n", desc, (unsigned long)x);
@@ -881,11 +852,11 @@ print_uptime(RES* ssl, struct worker* worker, int reset)
 	timeval_subtract(&dt, &now, &worker->daemon->time_last_stat);
 	if(reset)
 		worker->daemon->time_last_stat = now;
-	if(!ssl_printf(ssl, "time.now"SQ ARG_LL "d.%6.6d\n", 
+	if(!ssl_printf(ssl, "time.now"SQ ARG_LL "d.%6.6d\n",
 		(long long)now.tv_sec, (unsigned)now.tv_usec)) return 0;
-	if(!ssl_printf(ssl, "time.up"SQ ARG_LL "d.%6.6d\n", 
+	if(!ssl_printf(ssl, "time.up"SQ ARG_LL "d.%6.6d\n",
 		(long long)up.tv_sec, (unsigned)up.tv_usec)) return 0;
-	if(!ssl_printf(ssl, "time.elapsed"SQ ARG_LL "d.%6.6d\n", 
+	if(!ssl_printf(ssl, "time.elapsed"SQ ARG_LL "d.%6.6d\n",
 		(long long)dt.tv_sec, (unsigned)dt.tv_usec)) return 0;
 	return 1;
 }
@@ -903,7 +874,7 @@ print_hist(RES* ssl, struct ub_stats_info* s)
 	}
 	timehist_import(hist, s->svr.hist, NUM_BUCKETS_HIST);
 	for(i=0; i<hist->num; i++) {
-		if(!ssl_printf(ssl, 
+		if(!ssl_printf(ssl,
 			"histogram.%6.6d.%6.6d.to.%6.6d.%6.6d=%lu\n",
 			(int)hist->buckets[i].lower.tv_sec,
 			(int)hist->buckets[i].lower.tv_usec,
@@ -920,7 +891,7 @@ print_hist(RES* ssl, struct ub_stats_info* s)
 
 /** print extended stats */
 static int
-print_ext(RES* ssl, struct ub_stats_info* s)
+print_ext(RES* ssl, struct ub_stats_info* s, int inhibit_zero)
 {
 	int i;
 	char nm[32];
@@ -946,11 +917,11 @@ print_ext(RES* ssl, struct ub_stats_info* s)
 		} else {
 			snprintf(nm, sizeof(nm), "TYPE%d", i);
 		}
-		if(!ssl_printf(ssl, "num.query.type.%s"SQ"%lu\n", 
+		if(!ssl_printf(ssl, "num.query.type.%s"SQ"%lu\n",
 			nm, (unsigned long)s->svr.qtype[i])) return 0;
 	}
 	if(!inhibit_zero || s->svr.qtype_big) {
-		if(!ssl_printf(ssl, "num.query.type.other"SQ"%lu\n", 
+		if(!ssl_printf(ssl, "num.query.type.other"SQ"%lu\n",
 			(unsigned long)s->svr.qtype_big)) return 0;
 	}
 	/* CLASS */
@@ -963,11 +934,11 @@ print_ext(RES* ssl, struct ub_stats_info* s)
 		} else {
 			snprintf(nm, sizeof(nm), "CLASS%d", i);
 		}
-		if(!ssl_printf(ssl, "num.query.class.%s"SQ"%lu\n", 
+		if(!ssl_printf(ssl, "num.query.class.%s"SQ"%lu\n",
 			nm, (unsigned long)s->svr.qclass[i])) return 0;
 	}
 	if(!inhibit_zero || s->svr.qclass_big) {
-		if(!ssl_printf(ssl, "num.query.class.other"SQ"%lu\n", 
+		if(!ssl_printf(ssl, "num.query.class.other"SQ"%lu\n",
 			(unsigned long)s->svr.qclass_big)) return 0;
 	}
 	/* OPCODE */
@@ -980,42 +951,44 @@ print_ext(RES* ssl, struct ub_stats_info* s)
 		} else {
 			snprintf(nm, sizeof(nm), "OPCODE%d", i);
 		}
-		if(!ssl_printf(ssl, "num.query.opcode.%s"SQ"%lu\n", 
+		if(!ssl_printf(ssl, "num.query.opcode.%s"SQ"%lu\n",
 			nm, (unsigned long)s->svr.qopcode[i])) return 0;
 	}
 	/* transport */
-	if(!ssl_printf(ssl, "num.query.tcp"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.tcp"SQ"%lu\n",
 		(unsigned long)s->svr.qtcp)) return 0;
-	if(!ssl_printf(ssl, "num.query.tcpout"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.tcpout"SQ"%lu\n",
 		(unsigned long)s->svr.qtcp_outgoing)) return 0;
-	if(!ssl_printf(ssl, "num.query.tls"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.udpout"SQ"%lu\n",
+		(unsigned long)s->svr.qudp_outgoing)) return 0;
+	if(!ssl_printf(ssl, "num.query.tls"SQ"%lu\n",
 		(unsigned long)s->svr.qtls)) return 0;
-	if(!ssl_printf(ssl, "num.query.tls.resume"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.tls.resume"SQ"%lu\n",
 		(unsigned long)s->svr.qtls_resume)) return 0;
-	if(!ssl_printf(ssl, "num.query.ipv6"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.ipv6"SQ"%lu\n",
 		(unsigned long)s->svr.qipv6)) return 0;
 	if(!ssl_printf(ssl, "num.query.https"SQ"%lu\n",
 		(unsigned long)s->svr.qhttps)) return 0;
 	/* flags */
-	if(!ssl_printf(ssl, "num.query.flags.QR"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.flags.QR"SQ"%lu\n",
 		(unsigned long)s->svr.qbit_QR)) return 0;
-	if(!ssl_printf(ssl, "num.query.flags.AA"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.flags.AA"SQ"%lu\n",
 		(unsigned long)s->svr.qbit_AA)) return 0;
-	if(!ssl_printf(ssl, "num.query.flags.TC"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.flags.TC"SQ"%lu\n",
 		(unsigned long)s->svr.qbit_TC)) return 0;
-	if(!ssl_printf(ssl, "num.query.flags.RD"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.flags.RD"SQ"%lu\n",
 		(unsigned long)s->svr.qbit_RD)) return 0;
-	if(!ssl_printf(ssl, "num.query.flags.RA"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.flags.RA"SQ"%lu\n",
 		(unsigned long)s->svr.qbit_RA)) return 0;
-	if(!ssl_printf(ssl, "num.query.flags.Z"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.flags.Z"SQ"%lu\n",
 		(unsigned long)s->svr.qbit_Z)) return 0;
-	if(!ssl_printf(ssl, "num.query.flags.AD"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.flags.AD"SQ"%lu\n",
 		(unsigned long)s->svr.qbit_AD)) return 0;
-	if(!ssl_printf(ssl, "num.query.flags.CD"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.flags.CD"SQ"%lu\n",
 		(unsigned long)s->svr.qbit_CD)) return 0;
-	if(!ssl_printf(ssl, "num.query.edns.present"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.edns.present"SQ"%lu\n",
 		(unsigned long)s->svr.qEDNS)) return 0;
-	if(!ssl_printf(ssl, "num.query.edns.DO"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.edns.DO"SQ"%lu\n",
 		(unsigned long)s->svr.qEDNS_DO)) return 0;
 
 	/* RCODE */
@@ -1029,31 +1002,31 @@ print_ext(RES* ssl, struct ub_stats_info* s)
 		} else {
 			snprintf(nm, sizeof(nm), "RCODE%d", i);
 		}
-		if(!ssl_printf(ssl, "num.answer.rcode.%s"SQ"%lu\n", 
+		if(!ssl_printf(ssl, "num.answer.rcode.%s"SQ"%lu\n",
 			nm, (unsigned long)s->svr.ans_rcode[i])) return 0;
 	}
 	if(!inhibit_zero || s->svr.ans_rcode_nodata) {
-		if(!ssl_printf(ssl, "num.answer.rcode.nodata"SQ"%lu\n", 
+		if(!ssl_printf(ssl, "num.answer.rcode.nodata"SQ"%lu\n",
 			(unsigned long)s->svr.ans_rcode_nodata)) return 0;
 	}
 	/* iteration */
-	if(!ssl_printf(ssl, "num.query.ratelimited"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.ratelimited"SQ"%lu\n",
 		(unsigned long)s->svr.queries_ratelimited)) return 0;
 	/* validation */
-	if(!ssl_printf(ssl, "num.answer.secure"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.answer.secure"SQ"%lu\n",
 		(unsigned long)s->svr.ans_secure)) return 0;
-	if(!ssl_printf(ssl, "num.answer.bogus"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.answer.bogus"SQ"%lu\n",
 		(unsigned long)s->svr.ans_bogus)) return 0;
-	if(!ssl_printf(ssl, "num.rrset.bogus"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.rrset.bogus"SQ"%lu\n",
 		(unsigned long)s->svr.rrset_bogus)) return 0;
-	if(!ssl_printf(ssl, "num.query.aggressive.NOERROR"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.aggressive.NOERROR"SQ"%lu\n",
 		(unsigned long)s->svr.num_neg_cache_noerror)) return 0;
-	if(!ssl_printf(ssl, "num.query.aggressive.NXDOMAIN"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "num.query.aggressive.NXDOMAIN"SQ"%lu\n",
 		(unsigned long)s->svr.num_neg_cache_nxdomain)) return 0;
 	/* threat detection */
-	if(!ssl_printf(ssl, "unwanted.queries"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "unwanted.queries"SQ"%lu\n",
 		(unsigned long)s->svr.unwanted_queries)) return 0;
-	if(!ssl_printf(ssl, "unwanted.replies"SQ"%lu\n", 
+	if(!ssl_printf(ssl, "unwanted.replies"SQ"%lu\n",
 		(unsigned long)s->svr.unwanted_replies)) return 0;
 	/* cache counts */
 	if(!ssl_printf(ssl, "msg.cache.count"SQ"%u\n",
@@ -1064,6 +1037,11 @@ print_ext(RES* ssl, struct ub_stats_info* s)
 		(unsigned)s->svr.infra_cache_count)) return 0;
 	if(!ssl_printf(ssl, "key.cache.count"SQ"%u\n",
 		(unsigned)s->svr.key_cache_count)) return 0;
+	/* max collisions */
+	if(!ssl_printf(ssl, "msg.cache.max_collisions"SQ"%u\n",
+		(unsigned)s->svr.msg_cache_max_collisions)) return 0;
+	if(!ssl_printf(ssl, "rrset.cache.max_collisions"SQ"%u\n",
+		(unsigned)s->svr.rrset_cache_max_collisions)) return 0;
 	/* applied RPZ actions */
 	for(i=0; i<UB_STATS_RPZ_ACTION_NUM; i++) {
 		if(i == RPZ_NO_OVERRIDE_ACTION)
@@ -1094,6 +1072,10 @@ print_ext(RES* ssl, struct ub_stats_info* s)
 	if(!ssl_printf(ssl, "num.query.subnet_cache"SQ"%lu\n",
 		(unsigned long)s->svr.num_query_subnet_cache)) return 0;
 #endif /* CLIENT_SUBNET */
+#ifdef USE_CACHEDB
+	if(!ssl_printf(ssl, "num.query.cachedb"SQ"%lu\n",
+		(unsigned long)s->svr.num_query_cachedb)) return 0;
+#endif /* USE_CACHEDB */
 	return 1;
 }
 
@@ -1118,7 +1100,7 @@ do_stats(RES* ssl, struct worker* worker, int reset)
 	}
 	/* print the thread statistics */
 	total.mesh_time_median /= (double)daemon->num;
-	if(!print_stats(ssl, "total", &total)) 
+	if(!print_stats(ssl, "total", &total))
 		return;
 	if(!print_uptime(ssl, worker, reset))
 		return;
@@ -1127,7 +1109,7 @@ do_stats(RES* ssl, struct worker* worker, int reset)
 			return;
 		if(!print_hist(ssl, &total))
 			return;
-		if(!print_ext(ssl, &total))
+		if(!print_ext(ssl, &total, daemon->cfg->stat_inhibit_zero))
 			return;
 	}
 }
@@ -1207,7 +1189,7 @@ perform_zone_add(RES* ssl, struct local_zones* zones, char* arg)
 		return 0;
 	}
 	lock_rw_wrlock(&zones->lock);
-	if((z=local_zones_find(zones, nm, nmlen, 
+	if((z=local_zones_find(zones, nm, nmlen,
 		nmlabs, LDNS_RR_CLASS_IN))) {
 		/* already present in tree */
 		lock_rw_wrlock(&z->lock);
@@ -1217,7 +1199,7 @@ perform_zone_add(RES* ssl, struct local_zones* zones, char* arg)
 		lock_rw_unlock(&zones->lock);
 		return 1;
 	}
-	if(!local_zones_add_zone(zones, nm, nmlen, 
+	if(!local_zones_add_zone(zones, nm, nmlen,
 		nmlabs, LDNS_RR_CLASS_IN, t)) {
 		lock_rw_unlock(&zones->lock);
 		ssl_printf(ssl, "error out of memory\n");
@@ -1243,8 +1225,8 @@ do_zones_add(RES* ssl, struct local_zones* zones)
 	char buf[2048];
 	int num = 0;
 	while(ssl_read_line(ssl, buf, sizeof(buf))) {
-		if(buf[0] == 0x04 && buf[1] == 0)
-			break; /* end of transmission */
+		if(buf[0] == 0 || (buf[0] == 0x04 && buf[1] == 0))
+			break; /* zero byte line or end of transmission */
 		if(!perform_zone_add(ssl, zones, buf)) {
 			if(!ssl_printf(ssl, "error for input line: %s\n", buf))
 				return;
@@ -1266,7 +1248,7 @@ perform_zone_remove(RES* ssl, struct local_zones* zones, char* arg)
 	if(!parse_arg_name(ssl, arg, &nm, &nmlen, &nmlabs))
 		return 0;
 	lock_rw_wrlock(&zones->lock);
-	if((z=local_zones_find(zones, nm, nmlen, 
+	if((z=local_zones_find(zones, nm, nmlen,
 		nmlabs, LDNS_RR_CLASS_IN))) {
 		/* present in tree */
 		local_zones_del_zone(zones, z);
@@ -1292,8 +1274,8 @@ do_zones_remove(RES* ssl, struct local_zones* zones)
 	char buf[2048];
 	int num = 0;
 	while(ssl_read_line(ssl, buf, sizeof(buf))) {
-		if(buf[0] == 0x04 && buf[1] == 0)
-			break; /* end of transmission */
+		if(buf[0] == 0 || (buf[0] == 0x04 && buf[1] == 0))
+			break; /* zero byte line or end of transmission */
 		if(!perform_zone_remove(ssl, zones, buf)) {
 			if(!ssl_printf(ssl, "error for input line: %s\n", buf))
 				return;
@@ -1356,8 +1338,8 @@ do_datas_add(RES* ssl, struct local_zones* zones)
 	char buf[2048];
 	int num = 0, line = 0;
 	while(ssl_read_line(ssl, buf, sizeof(buf))) {
-		if(buf[0] == 0x04 && buf[1] == 0)
-			break; /* end of transmission */
+		if(buf[0] == 0 || (buf[0] == 0x04 && buf[1] == 0))
+			break; /* zero byte line or end of transmission */
 		line++;
 		if(perform_data_add(ssl, zones, buf, line))
 			num++;
@@ -1396,8 +1378,8 @@ do_datas_remove(RES* ssl, struct local_zones* zones)
 	char buf[2048];
 	int num = 0;
 	while(ssl_read_line(ssl, buf, sizeof(buf))) {
-		if(buf[0] == 0x04 && buf[1] == 0)
-			break; /* end of transmission */
+		if(buf[0] == 0 || (buf[0] == 0x04 && buf[1] == 0))
+			break; /* zero byte line or end of transmission */
 		if(!perform_data_remove(ssl, zones, buf)) {
 			if(!ssl_printf(ssl, "error for input line: %s\n", buf))
 				return;
@@ -1607,8 +1589,11 @@ do_flush_type(RES* ssl, struct worker* worker, char* arg)
 	if(!parse_arg_name(ssl, arg, &nm, &nmlen, &nmlabs))
 		return;
 	t = sldns_get_rr_type_by_name(arg2);
+	if(t == 0 && strcmp(arg2, "TYPE0") != 0) {
+		return;
+	}
 	do_cache_remove(worker, nm, nmlen, t, LDNS_RR_CLASS_IN);
-	
+
 	free(nm);
 	send_ok(ssl);
 }
@@ -1718,7 +1703,7 @@ zone_del_rrset(struct lruhash_entry* e, void* arg)
 	struct del_info* inf = (struct del_info*)arg;
 	struct ub_packed_rrset_key* k = (struct ub_packed_rrset_key*)e->key;
 	if(dname_subdomain_c(k->rk.dname, inf->name)) {
-		struct packed_rrset_data* d = 
+		struct packed_rrset_data* d =
 			(struct packed_rrset_data*)e->data;
 		if(d->ttl > inf->expired) {
 			d->ttl = inf->expired;
@@ -1782,21 +1767,21 @@ do_flush_zone(RES* ssl, struct worker* worker, char* arg)
 	inf.num_rrsets = 0;
 	inf.num_msgs = 0;
 	inf.num_keys = 0;
-	slabhash_traverse(&worker->env.rrset_cache->table, 1, 
+	slabhash_traverse(&worker->env.rrset_cache->table, 1,
 		&zone_del_rrset, &inf);
 
 	slabhash_traverse(worker->env.msg_cache, 1, &zone_del_msg, &inf);
 
 	/* and validator cache */
 	if(worker->env.key_cache) {
-		slabhash_traverse(worker->env.key_cache->slab, 1, 
+		slabhash_traverse(worker->env.key_cache->slab, 1,
 			&zone_del_kcache, &inf);
 	}
 
 	free(nm);
 
 	(void)ssl_printf(ssl, "ok removed %lu rrsets, %lu messages "
-		"and %lu key entries\n", (unsigned long)inf.num_rrsets, 
+		"and %lu key entries\n", (unsigned long)inf.num_rrsets,
 		(unsigned long)inf.num_msgs, (unsigned long)inf.num_keys);
 }
 
@@ -1851,19 +1836,19 @@ do_flush_bogus(RES* ssl, struct worker* worker)
 	inf.num_rrsets = 0;
 	inf.num_msgs = 0;
 	inf.num_keys = 0;
-	slabhash_traverse(&worker->env.rrset_cache->table, 1, 
+	slabhash_traverse(&worker->env.rrset_cache->table, 1,
 		&bogus_del_rrset, &inf);
 
 	slabhash_traverse(worker->env.msg_cache, 1, &bogus_del_msg, &inf);
 
 	/* and validator cache */
 	if(worker->env.key_cache) {
-		slabhash_traverse(worker->env.key_cache->slab, 1, 
+		slabhash_traverse(worker->env.key_cache->slab, 1,
 			&bogus_del_kcache, &inf);
 	}
 
 	(void)ssl_printf(ssl, "ok removed %lu rrsets, %lu messages "
-		"and %lu key entries\n", (unsigned long)inf.num_rrsets, 
+		"and %lu key entries\n", (unsigned long)inf.num_rrsets,
 		(unsigned long)inf.num_msgs, (unsigned long)inf.num_keys);
 }
 
@@ -1926,19 +1911,19 @@ do_flush_negative(RES* ssl, struct worker* worker)
 	inf.num_rrsets = 0;
 	inf.num_msgs = 0;
 	inf.num_keys = 0;
-	slabhash_traverse(&worker->env.rrset_cache->table, 1, 
+	slabhash_traverse(&worker->env.rrset_cache->table, 1,
 		&negative_del_rrset, &inf);
 
 	slabhash_traverse(worker->env.msg_cache, 1, &negative_del_msg, &inf);
 
 	/* and validator cache */
 	if(worker->env.key_cache) {
-		slabhash_traverse(worker->env.key_cache->slab, 1, 
+		slabhash_traverse(worker->env.key_cache->slab, 1,
 			&negative_del_kcache, &inf);
 	}
 
 	(void)ssl_printf(ssl, "ok removed %lu rrsets, %lu messages "
-		"and %lu key entries\n", (unsigned long)inf.num_rrsets, 
+		"and %lu key entries\n", (unsigned long)inf.num_rrsets,
 		(unsigned long)inf.num_msgs, (unsigned long)inf.num_keys);
 }
 
@@ -1961,7 +1946,9 @@ do_flush_name(RES* ssl, struct worker* w, char* arg)
 	do_cache_remove(w, nm, nmlen, LDNS_RR_TYPE_PTR, LDNS_RR_CLASS_IN);
 	do_cache_remove(w, nm, nmlen, LDNS_RR_TYPE_SRV, LDNS_RR_CLASS_IN);
 	do_cache_remove(w, nm, nmlen, LDNS_RR_TYPE_NAPTR, LDNS_RR_CLASS_IN);
-	
+	do_cache_remove(w, nm, nmlen, LDNS_RR_TYPE_SVCB, LDNS_RR_CLASS_IN);
+	do_cache_remove(w, nm, nmlen, LDNS_RR_TYPE_HTTPS, LDNS_RR_CLASS_IN);
+
 	free(nm);
 	send_ok(ssl);
 }
@@ -2015,7 +2002,7 @@ print_root_fwds(RES* ssl, struct iter_forwards* fwds, uint8_t* root)
 
 /** parse args into delegpt */
 static struct delegpt*
-parse_delegpt(RES* ssl, char* args, uint8_t* nm, int allow_names)
+parse_delegpt(RES* ssl, char* args, uint8_t* nm)
 {
 	/* parse args and add in */
 	char* p = args;
@@ -2037,40 +2024,35 @@ parse_delegpt(RES* ssl, char* args, uint8_t* nm, int allow_names)
 		}
 		/* parse address */
 		if(!authextstrtoaddr(todo, &addr, &addrlen, &auth_name)) {
-			if(allow_names) {
-				uint8_t* n = NULL;
-				size_t ln;
-				int lb;
-				if(!parse_arg_name(ssl, todo, &n, &ln, &lb)) {
-					(void)ssl_printf(ssl, "error cannot "
-						"parse IP address or name "
-						"'%s'\n", todo);
-					delegpt_free_mlc(dp);
-					return NULL;
-				}
-				if(!delegpt_add_ns_mlc(dp, n, 0)) {
-					(void)ssl_printf(ssl, "error out of memory\n");
-					free(n);
-					delegpt_free_mlc(dp);
-					return NULL;
-				}
-				free(n);
-
-			} else {
+			uint8_t* dname= NULL;
+			int port;
+			dname = authextstrtodname(todo, &port, &auth_name);
+			if(!dname) {
 				(void)ssl_printf(ssl, "error cannot parse"
-					" IP address '%s'\n", todo);
+					" '%s'\n", todo);
+				delegpt_free_mlc(dp);
+				return NULL;
+			}
+#if ! defined(HAVE_SSL_SET1_HOST) && ! defined(HAVE_X509_VERIFY_PARAM_SET1_HOST)
+			if(auth_name)
+				log_err("no name verification functionality in "
+				"ssl library, ignored name for %s", todo);
+#endif
+			if(!delegpt_add_ns_mlc(dp, dname, 0, auth_name, port)) {
+				(void)ssl_printf(ssl, "error out of memory\n");
+				free(dname);
 				delegpt_free_mlc(dp);
 				return NULL;
 			}
 		} else {
 #if ! defined(HAVE_SSL_SET1_HOST) && ! defined(HAVE_X509_VERIFY_PARAM_SET1_HOST)
 			if(auth_name)
-			  log_err("no name verification functionality in "
+				log_err("no name verification functionality in "
 				"ssl library, ignored name for %s", todo);
 #endif
 			/* add address */
 			if(!delegpt_add_addr_mlc(dp, &addr, addrlen, 0, 0,
-				auth_name)) {
+				auth_name, -1)) {
 				(void)ssl_printf(ssl, "error out of memory\n");
 				delegpt_free_mlc(dp);
 				return NULL;
@@ -2103,7 +2085,7 @@ do_forward(RES* ssl, struct worker* worker, char* args)
 		forwards_delete_zone(fwd, LDNS_RR_CLASS_IN, root);
 	} else {
 		struct delegpt* dp;
-		if(!(dp = parse_delegpt(ssl, args, root, 0)))
+		if(!(dp = parse_delegpt(ssl, args, root)))
 			return;
 		if(!forwards_add_zone(fwd, LDNS_RR_CLASS_IN, dp)) {
 			(void)ssl_printf(ssl, "error out of memory\n");
@@ -2149,7 +2131,7 @@ parse_fs_args(RES* ssl, char* args, uint8_t** nm, struct delegpt** dp,
 
 	/* parse dp */
 	if(dp) {
-		if(!(*dp = parse_delegpt(ssl, args, *nm, 1))) {
+		if(!(*dp = parse_delegpt(ssl, args, *nm))) {
 			free(*nm);
 			return 0;
 		}
@@ -2336,7 +2318,7 @@ do_status(RES* ssl, struct worker* worker)
 	uptime = (time_t)time(NULL) - (time_t)worker->daemon->time_boot.tv_sec;
 	if(!ssl_printf(ssl, "uptime: " ARG_LL "d seconds\n", (long long)uptime))
 		return;
-	if(!ssl_printf(ssl, "options:%s%s%s%s\n" , 
+	if(!ssl_printf(ssl, "options:%s%s%s%s\n" ,
 		(worker->daemon->reuseport?" reuseport":""),
 		(worker->daemon->rc->accept_list?" control":""),
 		(worker->daemon->rc->accept_list && worker->daemon->rc->use_cert?"(ssl)":""),
@@ -2350,7 +2332,7 @@ do_status(RES* ssl, struct worker* worker)
 
 /** get age for the mesh state */
 static void
-get_mesh_age(struct mesh_state* m, char* buf, size_t len, 
+get_mesh_age(struct mesh_state* m, char* buf, size_t len,
 	struct module_env* env)
 {
 	if(m->reply_list) {
@@ -2369,7 +2351,7 @@ get_mesh_age(struct mesh_state* m, char* buf, size_t len,
 
 /** get status of a mesh state */
 static void
-get_mesh_status(struct mesh_area* mesh, struct mesh_state* m, 
+get_mesh_status(struct mesh_area* mesh, struct mesh_state* m,
 	char* buf, size_t len)
 {
 	enum module_ext_state s = m->s.ext_state[m->s.curmod];
@@ -2391,7 +2373,7 @@ get_mesh_status(struct mesh_area* mesh, struct mesh_state* m,
 			snprintf(buf, len, " ");
 			l = strlen(buf);
 			buf += l; len -= l;
-			addr_to_str(&e->qsent->addr, e->qsent->addrlen, 
+			addr_to_str(&e->qsent->addr, e->qsent->addrlen,
 				buf, len);
 			l = strlen(buf);
 			buf += l; len -= l;
@@ -2444,7 +2426,7 @@ do_dump_requestlist(RES* ssl, struct worker* worker)
 		dname_str(m->s.qinfo.qname, buf);
 		get_mesh_age(m, timebuf, sizeof(timebuf), &worker->env);
 		get_mesh_status(mesh, m, statbuf, sizeof(statbuf));
-		if(!ssl_printf(ssl, "%3d %4s %2s %s %s %s\n", 
+		if(!ssl_printf(ssl, "%3d %4s %2s %s %s %s\n",
 			num, (t?t:"TYPE??"), (c?c:"CLASS??"), buf, timebuf,
 			statbuf)) {
 			free(t);
@@ -2634,7 +2616,7 @@ do_auth_zone_transfer(RES* ssl, struct worker* worker, char* arg)
 	free(nm);
 	send_ok(ssl);
 }
-	
+
 /** do the set_option command */
 static void
 do_set_option(RES* ssl, struct worker* worker, char* arg)
@@ -2772,7 +2754,7 @@ do_list_local_zones(RES* ssl, struct local_zones* zones)
 	RBTREE_FOR(z, struct local_zone*, &zones->ztree) {
 		lock_rw_rdlock(&z->lock);
 		dname_str(z->name, buf);
-		if(!ssl_printf(ssl, "%s %s\n", buf, 
+		if(!ssl_printf(ssl, "%s %s\n", buf,
 			local_zone_type2str(z->type))) {
 			/* failure to print */
 			lock_rw_unlock(&z->lock);
@@ -2865,6 +2847,8 @@ struct ratelimit_list_arg {
 	int all;
 	/** current time */
 	time_t now;
+	/** if backoff is enabled */
+	int backoff;
 };
 
 #define ip_ratelimit_list_arg ratelimit_list_arg
@@ -2878,7 +2862,7 @@ rate_list(struct lruhash_entry* e, void* arg)
 	struct rate_data* d = (struct rate_data*)e->data;
 	char buf[257];
 	int lim = infra_find_ratelimit(a->infra, k->name, k->namelen);
-	int max = infra_rate_max(d, a->now);
+	int max = infra_rate_max(d, a->now, a->backoff);
 	if(a->all == 0) {
 		if(max < lim)
 			return;
@@ -2896,7 +2880,7 @@ ip_rate_list(struct lruhash_entry* e, void* arg)
 	struct ip_rate_key* k = (struct ip_rate_key*)e->key;
 	struct ip_rate_data* d = (struct ip_rate_data*)e->data;
 	int lim = infra_ip_ratelimit;
-	int max = infra_rate_max(d, a->now);
+	int max = infra_rate_max(d, a->now, a->backoff);
 	if(a->all == 0) {
 		if(max < lim)
 			return;
@@ -2914,6 +2898,7 @@ do_ratelimit_list(RES* ssl, struct worker* worker, char* arg)
 	a.infra = worker->env.infra_cache;
 	a.now = *worker->env.now;
 	a.ssl = ssl;
+	a.backoff = worker->env.cfg->ratelimit_backoff;
 	arg = skipwhite(arg);
 	if(strcmp(arg, "+a") == 0)
 		a.all = 1;
@@ -2932,6 +2917,7 @@ do_ip_ratelimit_list(RES* ssl, struct worker* worker, char* arg)
 	a.infra = worker->env.infra_cache;
 	a.now = *worker->env.now;
 	a.ssl = ssl;
+	a.backoff = worker->env.cfg->ip_ratelimit_backoff;
 	arg = skipwhite(arg);
 	if(strcmp(arg, "+a") == 0)
 		a.all = 1;
@@ -2997,7 +2983,7 @@ static void
 distribute_cmd(struct daemon_remote* rc, RES* ssl, char* cmd)
 {
 	int i;
-	if(!cmd || !ssl) 
+	if(!cmd || !ssl)
 		return;
 	/* skip i=0 which is me */
 	for(i=1; i<rc->worker->daemon->num; i++) {
@@ -3020,7 +3006,7 @@ cmdcmp(char* p, const char* cmd, size_t len)
 
 /** execute a remote control command */
 static void
-execute_cmd(struct daemon_remote* rc, RES* ssl, char* cmd, 
+execute_cmd(struct daemon_remote* rc, RES* ssl, char* cmd,
 	struct worker* worker)
 {
 	char* p = skipwhite(cmd);
@@ -3028,8 +3014,11 @@ execute_cmd(struct daemon_remote* rc, RES* ssl, char* cmd,
 	if(cmdcmp(p, "stop", 4)) {
 		do_stop(ssl, worker);
 		return;
+	} else if(cmdcmp(p, "reload_keep_cache", 17)) {
+		do_reload(ssl, worker, 1);
+		return;
 	} else if(cmdcmp(p, "reload", 6)) {
-		do_reload(ssl, worker);
+		do_reload(ssl, worker, 0);
 		return;
 	} else if(cmdcmp(p, "stats_noreset", 13)) {
 		do_stats(ssl, worker, 0);
@@ -3201,7 +3190,7 @@ execute_cmd(struct daemon_remote* rc, RES* ssl, char* cmd,
 	}
 }
 
-void 
+void
 daemon_remote_exec(struct worker* worker)
 {
 	/* read the cmd string */
@@ -3235,9 +3224,10 @@ handle_req(struct daemon_remote* rc, struct rc_state* s, RES* res)
 	if(res->ssl) {
 		ERR_clear_error();
 		if((r=SSL_read(res->ssl, magic, (int)sizeof(magic)-1)) <= 0) {
-			if(SSL_get_error(res->ssl, r) == SSL_ERROR_ZERO_RETURN)
+			int r2;
+			if((r2=SSL_get_error(res->ssl, r)) == SSL_ERROR_ZERO_RETURN)
 				return;
-			log_crypto_err("could not SSL_read");
+			log_crypto_err_io("could not SSL_read", r2);
 			return;
 		}
 	} else {
@@ -3303,14 +3293,14 @@ remote_handshake_later(struct daemon_remote* rc, struct rc_state* s,
 		if(r == 0)
 			log_err("remote control connection closed prematurely");
 		log_addr(VERB_OPS, "failed connection from",
-			&s->c->repinfo.addr, s->c->repinfo.addrlen);
-		log_crypto_err("remote control failed ssl");
+			&s->c->repinfo.remote_addr, s->c->repinfo.remote_addrlen);
+		log_crypto_err_io("remote control failed ssl", r2);
 		clean_point(rc, s);
 	}
 	return 0;
 }
 
-int remote_control_callback(struct comm_point* c, void* arg, int err, 
+int remote_control_callback(struct comm_point* c, void* arg, int err,
 	struct comm_reply* ATTR_UNUSED(rep))
 {
 	RES res;
@@ -3318,7 +3308,7 @@ int remote_control_callback(struct comm_point* c, void* arg, int err,
 	struct daemon_remote* rc = s->rc;
 	int r;
 	if(err != NETEVENT_NOERROR) {
-		if(err==NETEVENT_TIMEOUT) 
+		if(err==NETEVENT_TIMEOUT)
 			log_err("remote control timed out");
 		clean_point(rc, s);
 		return 0;

daemon/remote.h

@@ -46,7 +46,7 @@
 #ifndef DAEMON_REMOTE_H
 #define DAEMON_REMOTE_H
 #ifdef HAVE_OPENSSL_SSL_H
-#include "openssl/ssl.h"
+#include <openssl/ssl.h>
 #endif
 struct config_file;
 struct listen_list;

daemon/stats.c

@@ -4,22 +4,22 @@
  * Copyright (c) 2007, NLnet Labs. All rights reserved.
  *
  * This software is open source.
- * 
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
- * 
+ *
  * Redistributions of source code must retain the above copyright notice,
  * this list of conditions and the following disclaimer.
- * 
+ *
  * Redistributions in binary form must reproduce the above copyright notice,
  * this list of conditions and the following disclaimer in the documentation
  * and/or other materials provided with the distribution.
- * 
+ *
  * Neither the name of the NLNET LABS nor the names of its contributors may
  * be used to endorse or promote products derived from this software without
  * specific prior written permission.
- * 
+ *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -70,6 +70,9 @@
 #include <openssl/ssl.h>
 #endif
 
+/** How long to wait for threads to transmit statistics, in msec. */
+#define STATS_THREAD_WAIT 60000
+
 /** add timers and the values do not overflow or become negative */
 static void
 stats_timeval_add(long long* d_sec, long long* d_usec, long long add_sec, long long add_usec)
@@ -113,8 +116,8 @@ void server_stats_log(struct ub_server_stats* stats, struct worker* worker,
 	log_info("server stats for thread %d: %u queries, "
 		"%u answers from cache, %u recursions, %u prefetch, %u rejected by "
 		"ip ratelimiting",
-		threadnum, (unsigned)stats->num_queries, 
-		(unsigned)(stats->num_queries - 
+		threadnum, (unsigned)stats->num_queries,
+		(unsigned)(stats->num_queries -
 			stats->num_queries_missed_cache),
 		(unsigned)stats->num_queries_missed_cache,
 		(unsigned)stats->num_queries_prefetch,
@@ -276,11 +279,12 @@ server_stats_compile(struct worker* worker, struct ub_stats_info* s, int reset)
 		s->svr.ans_rcode[i] += (long long)worker->env.mesh->ans_rcode[i];
 	for(i=0; i<UB_STATS_RPZ_ACTION_NUM; i++)
 		s->svr.rpz_action[i] += (long long)worker->env.mesh->rpz_action[i];
-	timehist_export(worker->env.mesh->histogram, s->svr.hist, 
+	timehist_export(worker->env.mesh->histogram, s->svr.hist,
 		NUM_BUCKETS_HIST);
 	/* values from outside network */
 	s->svr.unwanted_replies = (long long)worker->back->unwanted_replies;
 	s->svr.qtcp_outgoing = (long long)worker->back->num_tcp_outgoing;
+	s->svr.qudp_outgoing = (long long)worker->back->num_udp_outgoing;
 
 	/* get and reset validator rrset bogus number */
 	s->svr.rrset_bogus = (long long)get_rrset_bogus(worker, reset);
@@ -289,8 +293,10 @@ server_stats_compile(struct worker* worker, struct ub_stats_info* s, int reset)
 	s->svr.queries_ratelimited = (long long)get_queries_ratelimit(worker, reset);
 
 	/* get cache sizes */
-	s->svr.msg_cache_count = (long long)count_slabhash_entries(worker->env.msg_cache);
-	s->svr.rrset_cache_count = (long long)count_slabhash_entries(&worker->env.rrset_cache->table);
+	get_slabhash_stats(worker->env.msg_cache,
+		&s->svr.msg_cache_count, &s->svr.msg_cache_max_collisions);
+	get_slabhash_stats(&worker->env.rrset_cache->table,
+		&s->svr.rrset_cache_count, &s->svr.rrset_cache_max_collisions);
 	s->svr.infra_cache_count = (long long)count_slabhash_entries(worker->env.infra_cache->hosts);
 	if(worker->env.key_cache)
 		s->svr.key_cache_count = (long long)count_slabhash_entries(worker->env.key_cache->slab);
@@ -350,6 +356,11 @@ server_stats_compile(struct worker* worker, struct ub_stats_info* s, int reset)
 	s->svr.num_query_subnet = 0;
 	s->svr.num_query_subnet_cache = 0;
 #endif
+#ifdef USE_CACHEDB
+	s->svr.num_query_cachedb = (long long)worker->env.mesh->ans_cachedb;
+#else
+	s->svr.num_query_cachedb = 0;
+#endif
 
 	/* get tcp accept usage */
 	s->svr.tcp_accept_usage = 0;
@@ -379,6 +390,28 @@ void server_stats_obtain(struct worker* worker, struct worker* who,
 		worker_send_cmd(who, worker_cmd_stats);
 	else 	worker_send_cmd(who, worker_cmd_stats_noreset);
 	verbose(VERB_ALGO, "wait for stats reply");
+	if(tube_wait_timeout(worker->cmd, STATS_THREAD_WAIT) == 0) {
+		verbose(VERB_OPS, "no response from thread %d"
+#ifdef HAVE_GETTID
+			" LWP %u"
+#endif
+#if defined(HAVE_PTHREAD) && defined(SIZEOF_PTHREAD_T) && defined(SIZEOF_UNSIGNED_LONG)
+#  if SIZEOF_PTHREAD_T == SIZEOF_UNSIGNED_LONG
+			" pthread 0x%lx"
+#  endif
+#endif
+			,
+			who->thread_num
+#ifdef HAVE_GETTID
+			, (unsigned)who->thread_tid
+#endif
+#if defined(HAVE_PTHREAD) && defined(SIZEOF_PTHREAD_T) && defined(SIZEOF_UNSIGNED_LONG)
+#  if SIZEOF_PTHREAD_T == SIZEOF_UNSIGNED_LONG
+			, (unsigned long)*((unsigned long*)&who->thr_id)
+#  endif
+#endif
+			);
+	}
 	if(!tube_read_msg(worker->cmd, &reply, &len, 0))
 		fatal_exit("failed to read stats over cmd channel");
 	if(len != (uint32_t)sizeof(*s))
@@ -393,7 +426,7 @@ void server_stats_reply(struct worker* worker, int reset)
 	struct ub_stats_info s;
 	server_stats_compile(worker, &s, reset);
 	verbose(VERB_ALGO, "write stats replymsg");
-	if(!tube_write_msg(worker->daemon->workers[0]->cmd, 
+	if(!tube_write_msg(worker->daemon->workers[0]->cmd,
 		(uint8_t*)&s, sizeof(s), 0))
 		fatal_exit("could not write stat values over cmd channel");
 }
@@ -402,8 +435,14 @@ void server_stats_add(struct ub_stats_info* total, struct ub_stats_info* a)
 {
 	total->svr.num_queries += a->svr.num_queries;
 	total->svr.num_queries_ip_ratelimited += a->svr.num_queries_ip_ratelimited;
+	total->svr.num_queries_cookie_valid += a->svr.num_queries_cookie_valid;
+	total->svr.num_queries_cookie_client += a->svr.num_queries_cookie_client;
+	total->svr.num_queries_cookie_invalid += a->svr.num_queries_cookie_invalid;
 	total->svr.num_queries_missed_cache += a->svr.num_queries_missed_cache;
 	total->svr.num_queries_prefetch += a->svr.num_queries_prefetch;
+	total->svr.num_queries_timed_out += a->svr.num_queries_timed_out;
+	if (total->svr.max_query_time_us < a->svr.max_query_time_us)
+		total->svr.max_query_time_us = a->svr.max_query_time_us;
 	total->svr.sum_query_list_size += a->svr.sum_query_list_size;
 	total->svr.ans_expired += a->svr.ans_expired;
 #ifdef USE_DNSCRYPT
@@ -424,6 +463,7 @@ void server_stats_add(struct ub_stats_info* total, struct ub_stats_info* a)
 		total->svr.qclass_big += a->svr.qclass_big;
 		total->svr.qtcp += a->svr.qtcp;
 		total->svr.qtcp_outgoing += a->svr.qtcp_outgoing;
+		total->svr.qudp_outgoing += a->svr.qudp_outgoing;
 		total->svr.qtls += a->svr.qtls;
 		total->svr.qtls_resume += a->svr.qtls_resume;
 		total->svr.qhttps += a->svr.qhttps;
@@ -444,6 +484,9 @@ void server_stats_add(struct ub_stats_info* total, struct ub_stats_info* a)
 		total->svr.unwanted_replies += a->svr.unwanted_replies;
 		total->svr.unwanted_queries += a->svr.unwanted_queries;
 		total->svr.tcp_accept_usage += a->svr.tcp_accept_usage;
+#ifdef USE_CACHEDB
+		total->svr.num_query_cachedb += a->svr.num_query_cachedb;
+#endif
 		for(i=0; i<UB_STATS_QTYPE_NUM; i++)
 			total->svr.qtype[i] += a->svr.qtype[i];
 		for(i=0; i<UB_STATS_QCLASS_NUM; i++)
@@ -487,14 +530,14 @@ void server_stats_insquery(struct ub_server_stats* stats, struct comm_point* c,
 		if(c->ssl != NULL) {
 			stats->qtls++;
 #ifdef HAVE_SSL
-			if(SSL_session_reused(c->ssl)) 
+			if(SSL_session_reused(c->ssl))
 				stats->qtls_resume++;
 #endif
 			if(c->type == comm_http)
 				stats->qhttps++;
 		}
 	}
-	if(repinfo && addr_is_ip6(&repinfo->addr, repinfo->addrlen))
+	if(repinfo && addr_is_ip6(&repinfo->remote_addr, repinfo->remote_addrlen))
 		stats->qipv6++;
 	if( (flags&BIT_QR) )
 		stats->qbit_QR++;
@@ -528,3 +571,16 @@ void server_stats_insrcode(struct ub_server_stats* stats, sldns_buffer* buf)
 			stats->ans_rcode_nodata ++;
 	}
 }
+
+void server_stats_downstream_cookie(struct ub_server_stats* stats,
+	struct edns_data* edns)
+{
+	if(!(edns->edns_present && edns->cookie_present)) return;
+	if(edns->cookie_valid) {
+		stats->num_queries_cookie_valid++;
+	} else if(edns->cookie_client) {
+		stats->num_queries_cookie_client++;
+	} else {
+		stats->num_queries_cookie_invalid++;
+	}
+}

daemon/stats.h

@@ -126,4 +126,11 @@ void server_stats_insquery(struct ub_server_stats* stats, struct comm_point* c,
  */
 void server_stats_insrcode(struct ub_server_stats* stats, struct sldns_buffer* buf);
 
+/**
+ * Add DNS Cookie stats for this query
+ * @param stats: the stats
+ * @param edns: edns record
+ */
+void server_stats_downstream_cookie(struct ub_server_stats* stats,
+	struct edns_data* edns);
 #endif /* DAEMON_STATS_H */

daemon/worker.h

@@ -86,6 +86,10 @@ struct worker {
 	struct daemon* daemon;
 	/** thread id */
 	ub_thread_type thr_id;
+#ifdef HAVE_GETTID
+	/** thread tid, the LWP id. */
+	pid_t thread_tid;
+#endif
 	/** pipe, for commands for this worker */
 	struct tube* cmd;
 	/** the event base this worker works with */
@@ -114,7 +118,7 @@ struct worker {
 	/** do we need to restart or quit (on signal) */
 	int need_to_exit;
 	/** allocation cache for this thread */
-	struct alloc_cache alloc;
+	struct alloc_cache *alloc;
 	/** per thread statistics */
 	struct ub_server_stats stats;
 	/** thread scratch regional */
@@ -127,6 +131,8 @@ struct worker {
 	/** dnstap environment, changed for this thread */
 	struct dt_env dtenv;
 #endif
+	/** reuse existing cache on reload if other conditions allow it. */
+	int reuse_cache;
 };
 
 /**

dns64/dns64.c

@@ -59,7 +59,7 @@
  ******************************************************************************/
 
 /**
- * This is the default DNS64 prefix that is used whent he dns64 module is listed
+ * This is the default DNS64 prefix that is used when the dns64 module is listed
  * in module-config but when the dns64-prefix variable is not present.
  */
 static const char DEFAULT_DNS64_PREFIX[] = "64:ff9b::/96";
@@ -573,28 +573,29 @@ static enum module_ext_state
 handle_event_pass(struct module_qstate* qstate, int id)
 {
 	struct dns64_qstate* iq = (struct dns64_qstate*)qstate->minfo[id];
-	if (iq && iq->state == DNS64_NEW_QUERY
-            && qstate->qinfo.qtype == LDNS_RR_TYPE_PTR
-            && qstate->qinfo.qname_len == 74
-            && !strcmp((char*)&qstate->qinfo.qname[64], "\03ip6\04arpa"))
-        /* Handle PTR queries for IPv6 addresses. */
-        return handle_ipv6_ptr(qstate, id);
+	int synth_all_cfg = qstate->env->cfg->dns64_synthall;
+	int synth_qname = 0;
 
-	if (qstate->env->cfg->dns64_synthall &&
-	    iq && iq->state == DNS64_NEW_QUERY
-	    && qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA)
-		return generate_type_A_query(qstate, id);
+	if(iq && iq->state == DNS64_NEW_QUERY
+		&& qstate->qinfo.qtype == LDNS_RR_TYPE_PTR
+		&& qstate->qinfo.qname_len == 74
+		&& !strcmp((char*)&qstate->qinfo.qname[64], "\03ip6\04arpa")) {
+		/* Handle PTR queries for IPv6 addresses. */
+		return handle_ipv6_ptr(qstate, id);
+	}
 
-	if(dns64_always_synth_for_qname(qstate, id) &&
-	    iq && iq->state == DNS64_NEW_QUERY
-	    && !(qstate->query_flags & BIT_CD)
-	    && qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA) {
-		verbose(VERB_ALGO, "dns64: ignore-aaaa and synthesize anyway");
+	if(iq && iq->state == DNS64_NEW_QUERY &&
+		qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA &&
+		(synth_all_cfg ||
+		(synth_qname=(dns64_always_synth_for_qname(qstate, id)
+			&& !(qstate->query_flags & BIT_CD))))) {
+		if(synth_qname)
+			verbose(VERB_ALGO, "dns64: ignore-aaaa and synthesize anyway");
 		return generate_type_A_query(qstate, id);
 	}
 
 	/* We are finished when our sub-query is finished. */
-	if (iq && iq->state == DNS64_SUBQUERY_FINISHED)
+	if(iq && iq->state == DNS64_SUBQUERY_FINISHED)
 		return module_finished;
 
 	/* Otherwise, pass request to next module. */
@@ -627,32 +628,37 @@ handle_event_moddone(struct module_qstate* qstate, int id)
      *        synthesize in (sec 5.1.2 of RFC6147).
      *   - A successful AAAA query with an answer.
      */
-	if((!iq || iq->state != DNS64_INTERNAL_QUERY)
-            && qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA
-	    && !(qstate->query_flags & BIT_CD)
-	    && !(qstate->return_msg &&
-		    qstate->return_msg->rep &&
-		    reply_find_answer_rrset(&qstate->qinfo,
-			    qstate->return_msg->rep)))
-		/* not internal, type AAAA, not CD, and no answer RRset,
-		 * So, this is a AAAA noerror/nodata answer */
-		return generate_type_A_query(qstate, id);
 
-	if((!iq || iq->state != DNS64_INTERNAL_QUERY)
-	    && qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA
-	    && !(qstate->query_flags & BIT_CD)
-	    && dns64_always_synth_for_qname(qstate, id)) {
-		/* if it is not internal, AAAA, not CD and listed domain,
-		 * generate from A record and ignore AAAA */
-		verbose(VERB_ALGO, "dns64: ignore-aaaa and synthesize anyway");
+	/* When an AAAA query completes check if we want to perform DNS64
+	 * synthesis. We skip queries with DNSSEC enabled (!CD) and
+	 * ones generated by us to retrive the A/PTR record to use for
+	 * synth. */
+	int could_synth =
+		qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA &&
+		(!iq || iq->state != DNS64_INTERNAL_QUERY) &&
+		!(qstate->query_flags & BIT_CD);
+	int has_data = /* whether query returned non-empty rrset */
+		qstate->return_msg &&
+		qstate->return_msg->rep &&
+		reply_find_answer_rrset(&qstate->qinfo, qstate->return_msg->rep);
+	int synth_qname = 0;
+
+	if(could_synth &&
+		(!has_data ||
+		(synth_qname=dns64_always_synth_for_qname(qstate, id)))) {
+		if(synth_qname)
+			verbose(VERB_ALGO, "dns64: ignore-aaaa and synthesize anyway");
 		return generate_type_A_query(qstate, id);
 	}
 
 	/* Store the response in cache. */
-	if ( (!iq || !iq->started_no_cache_store) &&
-		qstate->return_msg && qstate->return_msg->rep &&
-		!dns_cache_store(qstate->env, &qstate->qinfo, qstate->return_msg->rep,
-		0, 0, 0, NULL, qstate->query_flags))
+	if( (!iq || !iq->started_no_cache_store) &&
+		qstate->return_msg &&
+		qstate->return_msg->rep &&
+		!dns_cache_store(
+			qstate->env, &qstate->qinfo, qstate->return_msg->rep,
+			0, 0, 0, NULL,
+			qstate->query_flags, qstate->qstarttime))
 		log_err("out of memory");
 
 	/* do nothing */
@@ -841,7 +847,7 @@ dns64_adjust_a(int id, struct module_qstate* super, struct module_qstate* qstate
 	cp = construct_reply_info_base(super->region, rep->flags, rep->qdcount,
 		rep->ttl, rep->prefetch_ttl, rep->serve_expired_ttl,
 		rep->an_numrrsets, rep->ns_numrrsets, rep->ar_numrrsets,
-		rep->rrset_count, rep->security);
+		rep->rrset_count, rep->security, LDNS_EDE_NONE);
 	if(!cp)
 		return;
 
@@ -969,10 +975,19 @@ dns64_inform_super(struct module_qstate* qstate, int id,
 	}
 	super_dq->state = DNS64_SUBQUERY_FINISHED;
 
-	/* If there is no successful answer, we're done. */
-	if (qstate->return_rcode != LDNS_RCODE_NOERROR
-	    || !qstate->return_msg
-	    || !qstate->return_msg->rep) {
+	/* If there is no successful answer, we're done.
+	 * Guarantee that we have at least a NOERROR reply further on. */
+	if(qstate->return_rcode != LDNS_RCODE_NOERROR
+		|| !qstate->return_msg
+		|| !qstate->return_msg->rep) {
+		return;
+	}
+
+	/* When no A record is found for synthesis fall back to AAAA again. */
+	if(qstate->qinfo.qtype == LDNS_RR_TYPE_A &&
+		!reply_find_answer_rrset(&qstate->qinfo,
+			qstate->return_msg->rep)) {
+		super_dq->state = DNS64_INTERNAL_QUERY;
 		return;
 	}
 
@@ -991,7 +1006,7 @@ dns64_inform_super(struct module_qstate* qstate, int id,
 	/* Store the generated response in cache. */
 	if ( (!super_dq || !super_dq->started_no_cache_store) &&
 		!dns_cache_store(super->env, &super->qinfo, super->return_msg->rep,
-		0, 0, 0, NULL, super->query_flags))
+		0, 0, 0, NULL, super->query_flags, qstate->qstarttime))
 		log_err("out of memory");
 }
 

dnstap/dnstap.c

@@ -388,12 +388,15 @@ dt_msg_send_client_query(struct dt_env *env,
 			 struct sockaddr_storage *qsock,
 			 struct sockaddr_storage *rsock,
 			 enum comm_point_type cptype,
-			 sldns_buffer *qmsg)
+			 sldns_buffer *qmsg,
+			 struct timeval* tstamp)
 {
 	struct dt_msg dm;
 	struct timeval qtime;
 
-	gettimeofday(&qtime, NULL);
+	if(tstamp)
+		memcpy(&qtime, tstamp, sizeof(qtime));
+	else 	gettimeofday(&qtime, NULL);
 
 	/* type */
 	dt_msg_init(env, &dm, DNSTAP__MESSAGE__TYPE__CLIENT_QUERY);

dnstap/dnstap.h

@@ -126,13 +126,15 @@ dt_delete(struct dt_env *env);
  * @param rsock: local (service) address/port.
  * @param cptype: comm_udp or comm_tcp.
  * @param qmsg: query message.
+ * @param tstamp: timestamp or NULL if none provided.
  */
 void
 dt_msg_send_client_query(struct dt_env *env,
 			 struct sockaddr_storage *qsock,
 			 struct sockaddr_storage *rsock,
 			 enum comm_point_type cptype,
-			 struct sldns_buffer *qmsg);
+			 struct sldns_buffer *qmsg,
+			 struct timeval* tstamp);
 
 /**
  * Create and send a new dnstap "Message" event of type CLIENT_RESPONSE.

dnstap/dtstream.c

@@ -188,9 +188,9 @@ mq_wakeup_cb(void* arg)
 
 /** start timer to wakeup dtio because there is content in the queue */
 static void
-dt_msg_queue_start_timer(struct dt_msg_queue* mq)
+dt_msg_queue_start_timer(struct dt_msg_queue* mq, int wakeupnow)
 {
-	struct timeval tv;
+	struct timeval tv = {0};
 	/* Start a timer to process messages to be logged.
 	 * If we woke up the dtio thread for every message, the wakeup
 	 * messages take up too much processing power.  If the queue
@@ -204,19 +204,26 @@ dt_msg_queue_start_timer(struct dt_msg_queue* mq)
 
 	/* do not start the timer if a timer already exists, perhaps
 	 * in another worker.  So this variable is protected by a lock in
-	 * dtio */
+	 * dtio. */
+
+	/* If we need to wakeupnow, 0 the timer to force the callback. */
 	lock_basic_lock(&mq->dtio->wakeup_timer_lock);
 	if(mq->dtio->wakeup_timer_enabled) {
+		if(wakeupnow) {
+			comm_timer_set(mq->wakeup_timer, &tv);
+		}
 		lock_basic_unlock(&mq->dtio->wakeup_timer_lock);
 		return;
 	}
 	mq->dtio->wakeup_timer_enabled = 1; /* we are going to start one */
-	lock_basic_unlock(&mq->dtio->wakeup_timer_lock);
 
 	/* start the timer, in mq, in the event base of our worker */
-	tv.tv_sec = 1;
-	tv.tv_usec = 0;
+	if(!wakeupnow) {
+		tv.tv_sec = 1;
+		tv.tv_usec = 0;
+	}
 	comm_timer_set(mq->wakeup_timer, &tv);
+	lock_basic_unlock(&mq->dtio->wakeup_timer_lock);
 }
 
 void
@@ -283,10 +290,8 @@ dt_msg_queue_submit(struct dt_msg_queue* mq, void* buf, size_t len)
 	/* release lock */
 	lock_basic_unlock(&mq->lock);
 
-	if(wakeupnow) {
-		dtio_wakeup(mq->dtio);
-	} else if(wakeupstarttimer) {
-		dt_msg_queue_start_timer(mq);
+	if(wakeupnow || wakeupstarttimer) {
+		dt_msg_queue_start_timer(mq, wakeupnow);
 	}
 }
 
@@ -783,7 +788,7 @@ static int dtio_write_ssl(struct dt_io_thread* dtio, uint8_t* buf,
 			}
 			return -1;
 		}
-		log_crypto_err("dnstap io, could not SSL_write");
+		log_crypto_err_io("dnstap io, could not SSL_write", want);
 		return -1;
 	}
 	return r;
@@ -949,7 +954,7 @@ static int dtio_write_more(struct dt_io_thread* dtio)
  * -1: continue, >0: number of bytes read into buffer */
 static ssize_t receive_bytes(struct dt_io_thread* dtio, void* buf, size_t len) {
 	ssize_t r;
-	r = recv(dtio->fd, (void*)buf, len, 0);
+	r = recv(dtio->fd, (void*)buf, len, MSG_DONTWAIT);
 	if(r == -1) {
 		char* to = dtio->socket_path;
 		if(!to) to = dtio->ip_str;
@@ -1024,7 +1029,7 @@ static int ssl_read_bytes(struct dt_io_thread* dtio, void* buf, size_t len)
 				"other side");
 			return 0;
 		}
-		log_crypto_err("could not SSL_read");
+		log_crypto_err_io("could not SSL_read", want);
 		verbose(VERB_DETAIL, "dnstap io: output closed by the "
 				"other side");
 		return 0;
@@ -1426,8 +1431,8 @@ static int dtio_ssl_handshake(struct dt_io_thread* dtio,
 		} else {
 			unsigned long err = ERR_get_error();
 			if(!squelch_err_ssl_handshake(err)) {
-				log_crypto_err_code("dnstap io, ssl handshake failed",
-					err);
+				log_crypto_err_io_code("dnstap io, ssl handshake failed",
+					want, err);
 				verbose(VERB_OPS, "dnstap io, ssl handshake failed "
 					"from %s", dtio->ip_str);
 			}
@@ -1955,7 +1960,7 @@ static int dtio_open_output_tcp(struct dt_io_thread* dtio)
 	memset(&addr, 0, sizeof(addr));
 	addrlen = (socklen_t)sizeof(addr);
 
-	if(!extstrtoaddr(dtio->ip_str, &addr, &addrlen)) {
+	if(!extstrtoaddr(dtio->ip_str, &addr, &addrlen, UNBOUND_DNS_PORT)) {
 		log_err("could not parse IP '%s'", dtio->ip_str);
 		return 0;
 	}

dnstap/unbound-dnstap-socket.c

@@ -61,6 +61,7 @@
 #include "services/listen_dnsport.h"
 #include "sldns/sbuffer.h"
 #include "sldns/wire2str.h"
+#include "sldns/pkthdr.h"
 #ifdef USE_DNSTAP
 #include <protobuf-c/protobuf-c.h>
 #include "dnstap/dnstap.pb-c.h"
@@ -272,7 +273,7 @@ static int make_tcp_accept(char* ip)
 
 	memset(&addr, 0, sizeof(addr));
 	len = (socklen_t)sizeof(addr);
-	if(!extstrtoaddr(ip, &addr, &len)) {
+	if(!extstrtoaddr(ip, &addr, &len, UNBOUND_DNS_PORT)) {
 		log_err("could not parse IP '%s'", ip);
 		return -1;
 	}
@@ -448,6 +449,7 @@ static char* q_of_msg(ProtobufCBinaryData message)
 	char buf[300];
 	/* header, name, type, class minimum to get the query tuple */
 	if(message.len < 12 + 1 + 4 + 4) return NULL;
+	if(LDNS_QDCOUNT(message.data) < 1) return NULL;
 	if(sldns_wire2str_rrquestion_buf(message.data+12, message.len-12,
 		buf, sizeof(buf)) != 0) {
 		/* remove trailing newline, tabs to spaces */
@@ -502,7 +504,7 @@ static char* tv_to_str(protobuf_c_boolean has_time_sec, uint64_t time_sec,
 	time_t time_t_sec;
 	memset(&tv, 0, sizeof(tv));
 	if(has_time_sec) tv.tv_sec = time_sec;
-	if(has_time_nsec) tv.tv_usec = time_nsec;
+	if(has_time_nsec) tv.tv_usec = time_nsec/1000;
 
 	buf[0]=0;
 	time_t_sec = tv.tv_sec;
@@ -617,7 +619,7 @@ static void log_data_frame(uint8_t* pkt, size_t len)
 static ssize_t receive_bytes(struct tap_data* data, int fd, void* buf,
 	size_t len)
 {
-	ssize_t ret = recv(fd, buf, len, 0);
+	ssize_t ret = recv(fd, buf, len, MSG_DONTWAIT);
 	if(ret == 0) {
 		/* closed */
 		if(verbosity) log_info("dnstap client stream closed from %s",
@@ -706,7 +708,7 @@ static ssize_t ssl_read_bytes(struct tap_data* data, void* buf, size_t len)
 				(data->id?data->id:""));
 			return 0;
 		}
-		log_crypto_err("could not SSL_read");
+		log_crypto_err_io("could not SSL_read", want);
 		if(verbosity) log_info("dnstap client stream closed from %s",
 			(data->id?data->id:""));
 		return 0;
@@ -758,10 +760,11 @@ static int reply_with_accept(struct tap_data* data)
 	fd_set_block(data->fd);
 	if(data->ssl) {
 		if((r=SSL_write(data->ssl, acceptframe, len)) <= 0) {
-			if(SSL_get_error(data->ssl, r) == SSL_ERROR_ZERO_RETURN)
+			int r2;
+			if((r2=SSL_get_error(data->ssl, r)) == SSL_ERROR_ZERO_RETURN)
 				log_err("SSL_write, peer closed connection");
 			else
-				log_err("could not SSL_write");
+				log_crypto_err_io("could not SSL_write", r2);
 			fd_set_nonblock(data->fd);
 			free(acceptframe);
 			return 0;
@@ -789,7 +792,7 @@ static int reply_with_accept(struct tap_data* data)
 
 /** reply with FINISH control frame to bidirectional client,
  * returns 0 on error */
-static int reply_with_finish(int fd)
+static int reply_with_finish(struct tap_data* data)
 {
 #ifdef USE_DNSTAP
 	size_t len = 0;
@@ -799,21 +802,35 @@ static int reply_with_finish(int fd)
 		return 0;
 	}
 
-	fd_set_block(fd);
-	if(send(fd, finishframe, len, 0) == -1) {
-		log_err("send failed: %s", sock_strerror(errno));
-		fd_set_nonblock(fd);
-		free(finishframe);
-		return 0;
+	fd_set_block(data->fd);
+	if(data->ssl) {
+		int r;
+		if((r=SSL_write(data->ssl, finishframe, len)) <= 0) {
+			int r2;
+			if((r2=SSL_get_error(data->ssl, r)) == SSL_ERROR_ZERO_RETURN)
+				log_err("SSL_write, peer closed connection");
+			else
+				log_crypto_err_io("could not SSL_write", r2);
+			fd_set_nonblock(data->fd);
+			free(finishframe);
+			return 0;
+		}
+	} else {
+		if(send(data->fd, finishframe, len, 0) == -1) {
+			log_err("send failed: %s", sock_strerror(errno));
+			fd_set_nonblock(data->fd);
+			free(finishframe);
+			return 0;
+		}
 	}
 	if(verbosity) log_info("sent control frame(finish)");
 
-	fd_set_nonblock(fd);
+	fd_set_nonblock(data->fd);
 	free(finishframe);
 	return 1;
 #else
 	log_err("no dnstap compiled, no reply");
-	(void)fd;
+	(void)data;
 	return 0;
 #endif
 }
@@ -933,7 +950,7 @@ static int tap_handshake(struct tap_data* data)
 #endif /* HAVE_SSL */
 
 /** callback for dnstap listener */
-void dtio_tap_callback(int fd, short ATTR_UNUSED(bits), void* arg)
+void dtio_tap_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(bits), void* arg)
 {
 	struct tap_data* data = (struct tap_data*)arg;
 	if(verbosity>=3) log_info("tap callback");
@@ -1016,7 +1033,7 @@ void dtio_tap_callback(int fd, short ATTR_UNUSED(bits), void* arg)
 		}
 	} else if(data->len >= 4 && sldns_read_uint32(data->frame) ==
 		FSTRM_CONTROL_FRAME_STOP && data->is_bidirectional) {
-		if(!reply_with_finish(fd)) {
+		if(!reply_with_finish(data)) {
 			tap_data_free(data);
 			return;
 		}
@@ -1413,11 +1430,12 @@ void worker_sighandler(int ATTR_UNUSED(sig), void* ATTR_UNUSED(arg))
 struct outbound_entry* worker_send_query(
 	struct query_info* ATTR_UNUSED(qinfo), uint16_t ATTR_UNUSED(flags),
 	int ATTR_UNUSED(dnssec), int ATTR_UNUSED(want_dnssec),
-	int ATTR_UNUSED(nocaps), struct sockaddr_storage* ATTR_UNUSED(addr),
+	int ATTR_UNUSED(nocaps), int ATTR_UNUSED(check_ratelimit),
+	struct sockaddr_storage* ATTR_UNUSED(addr),
 	socklen_t ATTR_UNUSED(addrlen), uint8_t* ATTR_UNUSED(zone),
 	size_t ATTR_UNUSED(zonelen), int ATTR_UNUSED(tcp_upstream),
 	int ATTR_UNUSED(ssl_upstream), char* ATTR_UNUSED(tls_auth_name),
-	struct module_qstate* ATTR_UNUSED(q))
+	struct module_qstate* ATTR_UNUSED(q), int* ATTR_UNUSED(was_ratelimited))
 {
 	log_assert(0);
 	return 0;
@@ -1446,11 +1464,12 @@ worker_alloc_cleanup(void* ATTR_UNUSED(arg))
 struct outbound_entry* libworker_send_query(
 	struct query_info* ATTR_UNUSED(qinfo), uint16_t ATTR_UNUSED(flags),
 	int ATTR_UNUSED(dnssec), int ATTR_UNUSED(want_dnssec),
-	int ATTR_UNUSED(nocaps), struct sockaddr_storage* ATTR_UNUSED(addr),
+	int ATTR_UNUSED(nocaps), int ATTR_UNUSED(check_ratelimit),
+	struct sockaddr_storage* ATTR_UNUSED(addr),
 	socklen_t ATTR_UNUSED(addrlen), uint8_t* ATTR_UNUSED(zone),
 	size_t ATTR_UNUSED(zonelen), int ATTR_UNUSED(tcp_upstream),
 	int ATTR_UNUSED(ssl_upstream), char* ATTR_UNUSED(tls_auth_name),
-	struct module_qstate* ATTR_UNUSED(q))
+	struct module_qstate* ATTR_UNUSED(q), int* ATTR_UNUSED(was_ratelimited))
 {
 	log_assert(0);
 	return 0;

doc/README.DNS64

@@ -28,3 +28,23 @@ prefix. For example:
     ;; ANSWER SECTION:
     jazz-v4.viagenie.ca.        86400   IN      AAAA    64:ff9b::ce7b:1f02
 
+
+NAT64 support was added by David Lamparter in 2022; license(s) of the
+surrounding code apply.  Note that NAT64 is closely related but functionally
+orthogonal to DNS64;  it allows Unbound to send outgoing queries to IPv4-only
+servers over IPv6 through the configured NAT64 prefix.  This allows running
+an Unbound instance on an IPv6-only host without breaking every single domain
+that only has IPv4 servers.  Whether that Unbound instance also does DNS64 is
+an independent choice.
+
+To enable NAT64 in Unbound, add to unbound.conf's "server" section:
+
+    do-nat64: yes
+
+The NAT64 prefix defaults to the DNS64 prefix, which in turn defaults to the
+standard 64:FF9B::/96 prefix.  You can reconfigure it with:
+
+    nat64-prefix: 64:FF9B::/96
+
+To test NAT64 operation, pick a domain that only has IPv4 reachability for its
+nameservers and try resolving any names in that domain.

doc/example.conf.in

@@ -17,7 +17,7 @@ server:
 	# whitespace is not necessary, but looks cleaner.
 
 	# verbosity number, 0 is least verbose. 1 is default.
-	verbosity: 1
+	# verbosity: 1
 
 	# print statistics to the log (for every thread) every N seconds.
 	# Set to "" or 0 to disable. Default is disabled.
@@ -35,9 +35,14 @@ server:
 	# statistics-cumulative: no
 
 	# enable extended statistics (query types, answer codes, status)
-	# printed from unbound-control. default off, because of speed.
+	# printed from unbound-control. Default off, because of speed.
 	# extended-statistics: no
 
+	# Inhibits selected extended statistics (qtype, qclass, qopcode, rcode,
+	# rpz-actions) from printing if their value is 0.
+	# Default on.
+	# statistics-inhibit-zero: yes
+
 	# number of threads to create. 1 disables threading.
 	# num-threads: 1
 
@@ -50,11 +55,16 @@ server:
 	# interface: 192.0.2.154
 	# interface: 192.0.2.154@5003
 	# interface: 2001:DB8::5
+	# interface: eth0@5003
 
 	# enable this feature to copy the source address of queries to reply.
 	# Socket options are not supported on all platforms. experimental.
 	# interface-automatic: no
 
+	# instead of the default port, open additional ports separated by
+	# spaces when interface-automatic is enabled, by listing them here.
+	# interface-automatic-ports: ""
+
 	# port to answer queries from
 	# port: 53
 
@@ -82,13 +92,13 @@ server:
 	# num-queries-per-thread, or, use as many as the OS will allow you.
 	# outgoing-range: 4096
 
-	# permit unbound to use this port number or port range for
+	# permit Unbound to use this port number or port range for
 	# making outgoing queries, using an outgoing interface.
 	# outgoing-port-permit: 32768
 
-	# deny unbound the use this of port number or port range for
+	# deny Unbound the use this of port number or port range for
 	# making outgoing queries, using an outgoing interface.
-	# Use this to make sure unbound does not grab a UDP port that some
+	# Use this to make sure Unbound does not grab a UDP port that some
 	# other server on this computer needs. The default is to avoid
 	# IANA-assigned port numbers.
 	# If multiple outgoing-port-permit and outgoing-port-avoid options
@@ -133,8 +143,8 @@ server:
 	# edns-buffer-size: 1232
 
 	# Maximum UDP response size (not applied to TCP response).
-	# Suggested values are 512 to 4096. Default is 4096. 65536 disables it.
-	# max-udp-size: 4096
+	# Suggested values are 512 to 4096. Default is 1232. 65536 disables it.
+	# max-udp-size: 1232
 
 	# max memory to use for stream(tcp and tls) waiting result buffers.
 	# stream-wait-size: 4m
@@ -164,9 +174,19 @@ server:
 	# perform connect for UDP sockets to mitigate ICMP side channel.
 	# udp-connect: yes
 
-	# The number of retries when a non-positive response is received.
+	# The number of retries, per upstream nameserver in a delegation, when
+	# a throwaway response (also timeouts) is received.
 	# outbound-msg-retry: 5
 
+	# Hard limit on the number of outgoing queries Unbound will make while
+	# resolving a name, making sure large NS sets do not loop.
+	# It resets on query restarts (e.g., CNAME) and referrals.
+	# max-sent-count: 32
+
+	# Hard limit on the number of times Unbound is allowed to restart a
+	# query upon encountering a CNAME record.
+	# max-query-restarts: 11
+
 	# msec for waiting for an unknown server to reply.  Increase if you
 	# are behind a slow satellite link, to eg. 1128.
 	# unknown-server-time-limit: 376
@@ -198,6 +218,9 @@ server:
 	# minimum wait time for responses, increase if uplink is long. In msec.
 	# infra-cache-min-rtt: 50
 
+	# maximum wait time for responses. In msec.
+	# infra-cache-max-rtt: 120000
+
 	# enable to make server probe down hosts more frequently.
 	# infra-keep-probing: no
 
@@ -209,7 +232,8 @@ server:
 	# the maximum number of hosts that are cached (roundtrip, EDNS, lame).
 	# infra-cache-numhosts: 10000
 
-	# define a number of tags here, use with local-zone, access-control.
+	# define a number of tags here, use with local-zone, access-control,
+	# interface-*.
 	# repeat the define-tag statement to add additional tags.
 	# define-tag: "tag1 tag2 tag3"
 
@@ -219,6 +243,18 @@ server:
 	# Enable IPv6, "yes" or "no".
 	# do-ip6: yes
 
+	# If running unbound on an IPv6-only host, domains that only have
+	# IPv4 servers would become unresolveable.  If NAT64 is available in
+	# the network, unbound can use NAT64 to reach these servers with
+	# the following option.  This is NOT needed for enabling DNS64 on a
+	# system that has IPv4 connectivity.
+	# Consider also enabling prefer-ip6 to prefer native IPv6 connections
+	# to nameservers.
+	# do-nat64: no
+
+	# NAT64 prefix.  Defaults to using dns64-prefix value.
+	# nat64-prefix: 64:ff9b::0/96
+
 	# Enable UDP, "yes" or "no".
 	# do-udp: yes
 
@@ -250,11 +286,15 @@ server:
 	# Timeout for EDNS TCP keepalive, in msec.
 	# edns-tcp-keepalive-timeout: 120000
 
+	# UDP queries that have waited in the socket buffer for a long time
+	# can be dropped. Default is 0, disabled. In seconds, such as 3.
+	# sock-queue-timeout: 0
+
 	# Use systemd socket activation for UDP, TCP, and control sockets.
 	# use-systemd: no
 
 	# Detach from the terminal, run in background, "yes" or "no".
-	# Set the value to "no" when unbound runs as systemd service.
+	# Set the value to "no" when Unbound runs as systemd service.
 	# do-daemonize: yes
 
 	# control which clients are allowed to make (recursive) queries
@@ -265,9 +305,7 @@ server:
 	# allow_snoop (recursive and nonrecursive ok)
 	# deny_non_local (drop queries unless can be answered from local-data)
 	# refuse_non_local (like deny_non_local but polite error reply).
-	# access-control: 0.0.0.0/0 refuse
 	# access-control: 127.0.0.0/8 allow
-	# access-control: ::0/0 refuse
 	# access-control: ::1 allow
 	# access-control: ::ffff:127.0.0.1 allow
 
@@ -276,7 +314,7 @@ server:
 	# are tagged with one of these tags.
 	# access-control-tag: 192.0.2.0/24 "tag2 tag3"
 
-	# set action for particular tag for given access control element
+	# set action for particular tag for given access control element.
 	# if you have multiple tag values, the tag used to lookup the action
 	# is the first tag match between access-control-tag and local-zone-tag
 	# where "first" comes from the order of the define-tag values.
@@ -288,6 +326,58 @@ server:
 	# Set view for access control element
 	# access-control-view: 192.0.2.0/24 viewname
 
+	# Similar to 'access-control:' but for interfaces.
+	# Control which listening interfaces are allowed to accept (recursive)
+	# queries for this server.
+	# The specified interfaces should be the same as the ones specified in
+	# 'interface:' followed by the action.
+	# The actions are the same as 'access-control:' above.
+	# By default all the interfaces configured are refused.
+	# Note: any 'access-control*:' setting overrides all 'interface-*:'
+	# settings for targeted clients.
+	# interface-action: 192.0.2.153 allow
+	# interface-action: 192.0.2.154 allow
+	# interface-action: 192.0.2.154@5003 allow
+	# interface-action: 2001:DB8::5 allow
+	# interface-action: eth0@5003 allow
+
+	# Similar to 'access-control-tag:' but for interfaces.
+	# Tag interfaces with a list of tags (in "" with spaces between).
+	# Interfaces using these tags use localzones that are tagged with one
+	# of these tags.
+	# The specified interfaces should be the same as the ones specified in
+	# 'interface:' followed by the list of tags.
+	# Note: any 'access-control*:' setting overrides all 'interface-*:'
+	# settings for targeted clients.
+	# interface-tag: eth0@5003 "tag2 tag3"
+
+	# Similar to 'access-control-tag-action:' but for interfaces.
+	# Set action for particular tag for a given interface element.
+	# If you have multiple tag values, the tag used to lookup the action
+	# is the first tag match between interface-tag and local-zone-tag
+	# where "first" comes from the order of the define-tag values.
+	# The specified interfaces should be the same as the ones specified in
+	# 'interface:' followed by the tag and action.
+	# Note: any 'access-control*:' setting overrides all 'interface-*:'
+	# settings for targeted clients.
+	# interface-tag-action: eth0@5003 tag3 refuse
+
+	# Similar to 'access-control-tag-data:' but for interfaces.
+	# Set redirect data for a particular tag for an interface element.
+	# The specified interfaces should be the same as the ones specified in
+	# 'interface:' followed by the tag and the redirect data.
+	# Note: any 'access-control*:' setting overrides all 'interface-*:'
+	# settings for targeted clients.
+	# interface-tag-data: eth0@5003 tag2 "A 127.0.0.1"
+
+	# Similar to 'access-control-view:' but for interfaces.
+	# Set view for an interface element.
+	# The specified interfaces should be the same as the ones specified in
+	# 'interface:' followed by the view name.
+	# Note: any 'access-control*:' setting overrides all 'interface-*:'
+	# settings for targeted clients.
+	# interface-view: eth0@5003 viewname
+
 	# if given, a chroot(2) is done to the given directory.
 	# i.e. you can chroot to the working directory, for example,
 	# for extra security, but make sure all files are in that directory.
@@ -307,7 +397,7 @@ server:
 	# The pid file can be absolute and outside of the chroot, it is
 	# written just prior to performing the chroot and dropping permissions.
 	#
-	# Additionally, unbound may need to access /dev/urandom (for entropy).
+	# Additionally, Unbound may need to access /dev/urandom (for entropy).
 	# How to do this is specific to your OS.
 	#
 	# If you give "" no chroot is performed. The path must not end in a /.
@@ -429,6 +519,10 @@ server:
 	# to validate the zone.
 	# harden-algo-downgrade: no
 
+	# Harden against unknown records in the authority section and the
+	# additional section.
+	# harden-unknown-additional: no
+
 	# Sent minimum amount of information to upstream servers to enhance
 	# privacy. Only sent minimum required labels of the QNAME and set QTYPE
 	# to A when possible.
@@ -442,7 +536,7 @@ server:
 
 	# Aggressive NSEC uses the DNSSEC NSEC chain to synthesize NXDOMAIN
 	# and other denials, using information from previous NXDOMAINs answers.
-	# aggressive-nsec: no
+	# aggressive-nsec: yes
 
 	# Use 0x20-encoded random bits in the query to foil spoof attempts.
 	# This feature is an experimental implementation of draft dns-0x20.
@@ -517,7 +611,7 @@ server:
 	# Use several entries, one per domain name, to track multiple zones.
 	#
 	# If you want to perform DNSSEC validation, run unbound-anchor before
-	# you start unbound (i.e. in the system boot scripts).
+	# you start Unbound (i.e. in the system boot scripts).
 	# And then enable the auto-trust-anchor-file config item.
 	# Please note usage of unbound-anchor root anchor is at your own risk
 	# and under the terms of our LICENSE (see that file in the source).
@@ -585,10 +679,15 @@ server:
 	# val-permissive-mode: no
 
 	# Ignore the CD flag in incoming queries and refuse them bogus data.
-	# Enable it if the only clients of unbound are legacy servers (w2008)
+	# Enable it if the only clients of Unbound are legacy servers (w2008)
 	# that set CD but cannot validate themselves.
 	# ignore-cd-flag: no
 
+	# Disable the DO flag in outgoing requests. It is helpful for upstream
+	# devices that cannot handle DNSSEC information. But do not enable it
+	# otherwise, because it would stop DNSSEC validation.
+	# disable-edns-do: no
+
 	# Serve expired responses from cache, with serve-expired-reply-ttl in
 	# the response, and then attempt to fetch the data afresh.
 	# serve-expired: no
@@ -615,7 +714,7 @@ server:
 
 	# Return the original TTL as received from the upstream name server rather
 	# than the decrementing TTL as stored in the cache.  Enabling this feature
-	# does not impact cache expiry, it only changes the TTL unbound embeds in
+	# does not impact cache expiry, it only changes the TTL Unbound embeds in
 	# responses to queries. Note that enabling this feature implicitly disables
 	# enforcement of the configured minimum and maximum TTL.
 	# serve-original-ttl: no
@@ -709,9 +808,9 @@ server:
 	# Add example.com into ipset
 	# local-zone: "example.com" ipset
 
-	# If unbound is running service for the local host then it is useful
+	# If Unbound is running service for the local host then it is useful
 	# to perform lan-wide lookups to the upstream, and unblock the
-	# long list of local-zones above.  If this unbound is a dns server
+	# long list of local-zones above.  If this Unbound is a dns server
 	# for a network of computers, disabled is better and stops information
 	# leakage of local lan information.
 	# unblock-lan-zones: no
@@ -736,6 +835,8 @@ server:
 	# o always_transparent, always_refuse, always_nxdomain, always_nodata,
 	#   always_deny resolve in that way but ignore local data for
 	#   that name
+	# o block_a resolves all records normally but returns
+	#   NODATA for A queries and ignores local data for that name
 	# o always_null returns 0.0.0.0 or ::0 for any name in the zone.
 	# o noview breaks out of that view towards global local-zones.
 	#
@@ -811,6 +912,8 @@ server:
 
 	# Add system certs to the cert bundle, from the Windows Cert Store
 	# tls-win-cert: no
+	# and on other systems, the default openssl certificates
+	# tls-system-cert: no
 
 	# Pad queries over TLS upstreams
 	# pad-queries: yes
@@ -840,6 +943,10 @@ server:
 	# Disable TLS for DNS-over-HTTP downstream service.
 	# http-notls-downstream: no
 
+	# The interfaces that use these listed port numbers will support and
+	# expect PROXYv2. For UDP and TCP/TLS interfaces.
+	# proxy-protocol-port: portno for each of the port numbers.
+
 	# DNS64 prefix. Must be specified when DNS64 is use.
 	# Enable dns64 in module-config.  Used to synthesize IPv6 from IPv4.
 	# dns64-prefix: 64:ff9b::0/96
@@ -860,6 +967,10 @@ server:
 	# 0 blocks when ratelimited, otherwise let 1/xth traffic through
 	# ratelimit-factor: 10
 
+	# Aggressive rate limit when the limit is reached and until demand has
+	# decreased in a 2 second rate window.
+	# ratelimit-backoff: no
+
 	# override the ratelimit for a specific domain name.
 	# give this setting multiple times to have multiple overrides.
 	# ratelimit-for-domain: example.com 1000
@@ -880,6 +991,10 @@ server:
 	# 0 blocks when ip is ratelimited, otherwise let 1/xth traffic through
 	# ip-ratelimit-factor: 10
 
+	# Aggressive rate limit when the limit is reached and until demand has
+	# decreased in a 2 second rate window.
+	# ip-ratelimit-backoff: no
+
 	# Limit the number of connections simultaneous from a netblock
 	# tcp-connection-limit: 192.0.2.0/24 12
 
@@ -889,7 +1004,15 @@ server:
 	# the number of servers that will be used in the fast server selection.
 	# fast-server-num: 3
 
-	# Specific options for ipsecmod. unbound needs to be configured with
+	# Enable to attach Extended DNS Error codes (RFC8914) to responses.
+	# ede: no
+
+	# Enable to attach an Extended DNS Error (RFC8914) Code 3 - Stale
+	# Answer as EDNS0 option to expired responses.
+	# Note that the ede option above needs to be enabled for this to work.
+	# ede-serve-expired: no
+
+	# Specific options for ipsecmod. Unbound needs to be configured with
 	# --enable-ipsecmod for these to take effect.
 	#
 	# Enable or disable ipsecmod (it still needs to be defined in
@@ -901,7 +1024,7 @@ server:
 	# listed in module-config (above).
 	# ipsecmod-hook: "./my_executable"
 	#
-	# When enabled unbound will reply with SERVFAIL if the return value of
+	# When enabled Unbound will reply with SERVFAIL if the return value of
 	# the ipsecmod-hook is not 0.
 	# ipsecmod-strict: no
 	#
@@ -966,10 +1089,10 @@ remote-control:
 	# For local sockets this option is ignored, and TLS is not used.
 	# control-use-cert: "yes"
 
-	# unbound server key file.
+	# Unbound server key file.
 	# server-key-file: "@UNBOUND_RUN_DIR@/unbound_server.key"
 
-	# unbound server certificate file.
+	# Unbound server certificate file.
 	# server-cert-file: "@UNBOUND_RUN_DIR@/unbound_server.pem"
 
 	# unbound-control key file.
@@ -1023,8 +1146,8 @@ remote-control:
 # has a copy of the root for local usage.  The second serves example.org
 # authoritatively.  zonefile: reads from file (and writes to it if you also
 # download it), primary: fetches with AXFR and IXFR, or url to zonefile.
-# With allow-notify: you can give additional (apart from primaries) sources of
-# notifies.
+# With allow-notify: you can give additional (apart from primaries and urls)
+# sources of notifies.
 # auth-zone:
 #	name: "."
 #	primary: 199.9.14.201         # b.root-servers.net
@@ -1072,8 +1195,9 @@ remote-control:
 #	local-zone: "example.com" refuse
 
 # DNSCrypt
+# To enable, use --enable-dnscrypt to configure before compiling.
 # Caveats:
-# 1. the keys/certs cannot be produced by unbound. You can use dnscrypt-wrapper
+# 1. the keys/certs cannot be produced by Unbound. You can use dnscrypt-wrapper
 #   for this: https://github.com/cofyc/dnscrypt-wrapper/blob/master/README.md#usage
 # 2. dnscrypt channel attaches to an interface. you MUST set interfaces to
 #   listen on `dnscrypt-port` with the follo0wing snippet:
@@ -1092,7 +1216,9 @@ remote-control:
 #     dnscrypt-provider-cert: /path/unbound-conf/keys2/1.cert
 
 # CacheDB
-# Enable external backend DB as auxiliary cache.  Specify the backend name
+# External backend DB as auxiliary cache.
+# To enable, use --enable-cachedb to configure before compiling.
+# Specify the backend name
 # (default is "testframe", which has no use other than for debugging and
 # testing) and backend-specific options.  The 'cachedb' module must be
 # included in module-config, just before the iterator module.
@@ -1102,18 +1228,27 @@ remote-control:
 #     secret-seed: "default"
 #
 #     # For "redis" backend:
+#     # (to enable, use --with-libhiredis to configure before compiling)
 #     # redis server's IP address or host name
 #     redis-server-host: 127.0.0.1
 #     # redis server's TCP port
 #     redis-server-port: 6379
+#     # if the server uses a unix socket, set its path, or "" when not used.
+#     # redis-server-path: "/var/lib/redis/redis-server.sock"
+#     # if the server uses an AUTH password, specify here, or "" when not used.
+#     # redis-server-password: ""
 #     # timeout (in ms) for communication with the redis server
 #     redis-timeout: 100
 #     # set timeout on redis records based on DNS response TTL
 #     redis-expire-records: no
+#     # redis logical database to use, 0 is the default database.
+#     redis-logical-db: 0
 
 # IPSet
 # Add specify domain into set via ipset.
-# Note: To enable ipset unbound needs to run as root user.
+# To enable:
+# o use --enable-ipset to configure before compiling;
+# o Unbound then needs to run as root user.
 # ipset:
 #     # set name for ip v4 addresses
 #     name-v4: "list-v4"
@@ -1121,9 +1256,10 @@ remote-control:
 #     name-v6: "list-v6"
 #
 
-# Dnstap logging support, if compiled in.  To enable, set the dnstap-enable
-# to yes and also some of dnstap-log-..-messages to yes.  And select an
-# upstream log destination, by socket path, TCP or TLS destination.
+# Dnstap logging support, if compiled in by using --enable-dnstap to configure.
+# To enable, set the dnstap-enable to yes and also some of
+# dnstap-log-..-messages to yes.  And select an upstream log destination, by
+# socket path, TCP or TLS destination.
 # dnstap:
 # 	dnstap-enable: no
 # 	# if set to yes frame streams will be used in bidirectional mode
@@ -1136,7 +1272,7 @@ remote-control:
 # 	dnstap-tls: yes
 # 	# name for authenticating the upstream server. or "" disabled.
 # 	dnstap-tls-server-name: ""
-# 	# if "", it uses the cert bundle from the main unbound config.
+# 	# if "", it uses the cert bundle from the main Unbound config.
 # 	dnstap-tls-cert-bundle: ""
 # 	# key file for client authentication, or "" disabled.
 # 	dnstap-tls-client-key-file: ""
@@ -1172,4 +1308,6 @@ remote-control:
 #     rpz-cname-override: www.example.org
 #     rpz-log: yes
 #     rpz-log-name: "example policy"
+#     rpz-signal-nxdomain-ra: no
+#     for-downstream: no
 #     tags: "example"

doc/unbound-anchor.8.in

@@ -41,7 +41,7 @@ update certificate files.
 .P
 It tests if the root anchor file works, and if not, and an update is possible,
 attempts to update the root anchor using the root update certificate.
-It performs a https fetch of root-anchors.xml and checks the results (RFC7958), 
+It performs a https fetch of root-anchors.xml and checks the results (RFC7958),
 if all checks are successful, it updates the root anchor file.  Otherwise
 the root anchor file is unchanged.  It performs RFC5011 tracking if the
 DNSSEC information available via the DNS makes that possible.
@@ -159,7 +159,7 @@ Or something more suitable for your operational environment.
 The root keys and update certificate included in this tool
 are provided for convenience and under the terms of our
 license (see the LICENSE file in the source distribution or
-http://unbound.nlnetlabs.nl/svn/trunk/LICENSE) and might be stale or
+https://github.com/NLnetLabs/unbound/blob/master/LICENSE) and might be stale or
 not suitable to your purpose.
 .P
 By running "unbound\-anchor \-l" the  keys and certificate that are
@@ -185,5 +185,5 @@ Source for the root key information.
 .I https://data.iana.org/root\-anchors/root\-anchors.p7s
 Signature on the root key information.
 .SH "SEE ALSO"
-\fIunbound.conf\fR(5), 
+\fIunbound.conf\fR(5),
 \fIunbound\fR(8).

doc/unbound-checkconf.8.in

@@ -9,7 +9,7 @@
 .\"
 .SH "NAME"
 unbound\-checkconf
-\- Check unbound configuration file for errors.
+\- Check Unbound configuration file for errors.
 .SH "SYNOPSIS"
 .B unbound\-checkconf
 .RB [ \-h ]
@@ -21,8 +21,8 @@ unbound\-checkconf
 .B Unbound\-checkconf
 checks the configuration file for the
 \fIunbound\fR(8)
-DNS resolver for syntax and other errors. 
-The config file syntax is described in 
+DNS resolver for syntax and other errors.
+The config file syntax is described in
 \fIunbound.conf\fR(5).
 .P
 The available options are:
@@ -34,19 +34,19 @@ Show the version and commandline option help.
 Print full pathname, with chroot applied to it.  Use with the \-o option.
 .TP
 .B \-o\fI option
-If given, after checking the config file the value of this option is 
+If given, after checking the config file the value of this option is
 printed to stdout.  For "" (disabled) options an empty line is printed.
 .TP
 .I cfgfile
-The config file to read with settings for unbound. It is checked.
+The config file to read with settings for Unbound. It is checked.
 If omitted, the config file at the default location is checked.
 .SH "EXIT CODE"
-The unbound\-checkconf program exits with status code 1 on error, 
+The unbound\-checkconf program exits with status code 1 on error,
 0 for a correct config file.
 .SH "FILES"
 .TP
 .I @ub_conf_file@
-unbound configuration file.
+Unbound configuration file.
 .SH "SEE ALSO"
-\fIunbound.conf\fR(5), 
+\fIunbound.conf\fR(5),
 \fIunbound\fR(8).

doc/unbound-control.8.in

@@ -14,15 +14,15 @@
 .SH "SYNOPSIS"
 .B unbound\-control
 .RB [ \-hq ]
-.RB [ \-c 
+.RB [ \-c
 .IR cfgfile ]
-.RB [ \-s 
+.RB [ \-s
 .IR server ]
 .IR command
 .SH "DESCRIPTION"
 .B Unbound\-control
 performs remote administration on the \fIunbound\fR(8) DNS server.
-It reads the configuration file, contacts the unbound server over SSL
+It reads the configuration file, contacts the Unbound server over SSL
 sends the command and displays the result.
 .P
 The available options are:
@@ -44,8 +44,8 @@ quiet, if the option is given it does not print anything if it works ok.
 There are several commands that the server understands.
 .TP
 .B start
-Start the server. Simply execs \fIunbound\fR(8).  The unbound executable 
-is searched for in the \fBPATH\fR set in the environment.  It is started 
+Start the server. Simply execs \fIunbound\fR(8).  The Unbound executable
+is searched for in the \fBPATH\fR set in the environment.  It is started
 with the config file specified using \fI\-c\fR or the default config file.
 .TP
 .B stop
@@ -54,6 +54,12 @@ Stop the server. The server daemon exits.
 .B reload
 Reload the server. This flushes the cache and reads the config file fresh.
 .TP
+.B reload_keep_cache
+Reload the server but try to keep the RRset and message cache if
+(re)configuration allows for it.
+That means the caches sizes and the number of threads must not change between
+reloads.
+.TP
 .B verbosity \fInumber
 Change verbosity value for logging. Same values as \fBverbosity\fR keyword in
 \fIunbound.conf\fR(5).  This new setting lasts until the server is issued
@@ -65,8 +71,8 @@ daemon release the file it is logging to.  If you are using syslog it will
 attempt to close and open the syslog (which may not work if chrooted).
 .TP
 .B stats
-Print statistics. Resets the internal counters to zero, this can be 
-controlled using the \fBstatistics\-cumulative\fR config statement. 
+Print statistics. Resets the internal counters to zero, this can be
+controlled using the \fBstatistics\-cumulative\fR config statement.
 Statistics are printed with one [name]: [value] per line.
 .TP
 .B stats_noreset
@@ -74,7 +80,7 @@ Peek at statistics. Prints them like the \fBstats\fR command does, but does not
 reset the internal counters to zero.
 .TP
 .B status
-Display server status. Exit code 3 if not running (the connection to the 
+Display server status. Exit code 3 if not running (the connection to the
 port is refused), 1 on error, 0 if running.
 .TP
 .B local_zone \fIname\fR \fItype
@@ -88,14 +94,14 @@ it.  If the zone does not exist, the command succeeds.
 .B local_data \fIRR data...
 Add new local data, the given resource record. Like \fBlocal\-data\fR
 config statement, except for when no covering zone exists.  In that case
-this remote control command creates a transparent zone with the same 
+this remote control command creates a transparent zone with the same
 name as this record.
 .TP
 .B local_data_remove \fIname
 Remove all RR data from local name.  If the name already has no items,
 nothing happens.  Often results in NXDOMAIN for the name (in a static zone),
-but if the name has become an empty nonterminal (there is still data in 
-domain names below the removed name), NOERROR nodata answers are the 
+but if the name has become an empty nonterminal (there is still data in
+domain names below the removed name), NOERROR nodata answers are the
 result for that name.
 .TP
 .B local_zones
@@ -125,22 +131,22 @@ in old or wrong data returned to clients.  Loading data into the cache
 in this way is supported in order to aid with debugging.
 .TP
 .B lookup \fIname
-Print to stdout the name servers that would be used to look up the 
+Print to stdout the name servers that would be used to look up the
 name specified.
 .TP
 .B flush \fIname
 Remove the name from the cache. Removes the types
-A, AAAA, NS, SOA, CNAME, DNAME, MX, PTR, SRV and NAPTR.
-Because that is fast to do. Other record types can be removed using 
-.B flush_type 
-or 
+A, AAAA, NS, SOA, CNAME, DNAME, MX, PTR, SRV, NAPTR, SVCB and HTTPS.
+Because that is fast to do. Other record types can be removed using
+.B flush_type
+or
 .B flush_zone\fR.
 .TP
 .B flush_type \fIname\fR \fItype
 Remove the name, type information from the cache.
 .TP
 .B flush_zone \fIname
-Remove all information at or below the name from the cache. 
+Remove all information at or below the name from the cache.
 The rrsets and key entries are removed so that new lookups will be performed.
 This needs to walk and inspect the entire cache, and is a slow operation.
 The entries are set to expired in the implementation of this command (so,
@@ -187,7 +193,7 @@ therefore not flushed.  The option must end with a ':' and whitespace
 must be between the option and the value.  Some values may not have an
 effect if set this way, the new values are not written to the config file,
 not all options are supported.  This is different from the set_option call
-in libunbound, where all values work because unbound has not been initialized.
+in libunbound, where all values work because Unbound has not been initialized.
 .IP
 The values that work are: statistics\-interval, statistics\-cumulative,
 do\-not\-query\-localhost, harden\-short\-bufsize, harden\-large\-queries,
@@ -227,36 +233,36 @@ List the local data RRs in use.  The resource records are printed.
 .TP
 .B insecure_add \fIzone
 Add a \fBdomain\-insecure\fR for the given zone, like the statement in unbound.conf.
-Adds to the running unbound without affecting the cache contents (which may
+Adds to the running Unbound without affecting the cache contents (which may
 still be bogus, use \fBflush_zone\fR to remove it), does not affect the config file.
 .TP
 .B insecure_remove \fIzone
 Removes domain\-insecure for the given zone.
 .TP
 .B forward_add \fR[\fI+i\fR] \fIzone addr ...
-Add a new forward zone to running unbound.  With +i option also adds a
+Add a new forward zone to running Unbound.  With +i option also adds a
 \fIdomain\-insecure\fR for the zone (so it can resolve insecurely if you have
 a DNSSEC root trust anchor configured for other names).
 The addr can be IP4, IP6 or nameserver names, like \fIforward-zone\fR config
 in unbound.conf.
 .TP
 .B forward_remove \fR[\fI+i\fR] \fIzone
-Remove a forward zone from running unbound.  The +i also removes a
+Remove a forward zone from running Unbound.  The +i also removes a
 \fIdomain\-insecure\fR for the zone.
 .TP
 .B stub_add \fR[\fI+ip\fR] \fIzone addr ...
-Add a new stub zone to running unbound.  With +i option also adds a
+Add a new stub zone to running Unbound.  With +i option also adds a
 \fIdomain\-insecure\fR for the zone.  With +p the stub zone is set to prime,
 without it it is set to notprime.  The addr can be IP4, IP6 or nameserver
 names, like the \fIstub-zone\fR config in unbound.conf.
 .TP
 .B stub_remove \fR[\fI+i\fR] \fIzone
-Remove a stub zone from running unbound.  The +i also removes a
+Remove a stub zone from running Unbound.  The +i also removes a
 \fIdomain\-insecure\fR for the zone.
 .TP
 .B forward \fR[\fIoff\fR | \fIaddr ...\fR ]
 Setup forwarding mode.  Configures if the server should ask other upstream
-nameservers, should go to the internet root nameservers itself, or show 
+nameservers, should go to the internet root nameservers itself, or show
 the current config.  You could pass the nameservers after a DHCP update.
 .IP
 Without arguments the current list of addresses used to forward all queries
@@ -289,20 +295,22 @@ just the ratelimited ips, with their estimated qps.  The ratelimited
 ips are dropped before checking the cache.
 .TP
 .B list_auth_zones
-List the auth zones that are configured.  Printed one per line with a
-status, indicating if the zone is expired and current serial number.
+List the auth zones that are configured.  Printed one per line with a status,
+indicating if the zone is expired and current serial number.  Configured RPZ
+zones are included.
 .TP
 .B auth_zone_reload \fIzone\fR
-Reload the auth zone from zonefile.  The zonefile is read in overwriting
-the current contents of the zone in memory.  This changes the auth zone
-contents itself, not the cache contents.  Such cache contents exists if
-you set unbound to validate with for-upstream yes and that can be cleared
-with \fBflush_zone\fR \fIzone\fR.
+Reload the auth zone (or RPZ zone) from zonefile.  The zonefile is read in
+overwriting the current contents of the zone in memory.  This changes the auth
+zone contents itself, not the cache contents.  Such cache contents exists if
+you set Unbound to validate with for-upstream yes and that can be cleared with
+\fBflush_zone\fR \fIzone\fR.
 .TP
 .B auth_zone_transfer \fIzone\fR
-Transfer the auth zone from master.  The auth zone probe sequence is started,
-where the masters are probed to see if they have an updated zone (with the SOA
-serial check).  And then the zone is transferred for a newer zone version.
+Transfer the auth zone (or RPZ zone) from master.  The auth zone probe sequence
+is started, where the masters are probed to see if they have an updated zone
+(with the SOA serial check).  And then the zone is transferred for a newer zone
+version.
 .TP
 .B rpz_enable \fIzone\fR
 Enable the RPZ zone if it had previously been disabled.
@@ -336,7 +344,7 @@ Add a list of \fIlocal_data\fR for given view from stdin.  Like local_datas.
 .SH "EXIT CODE"
 The unbound\-control program exits with status code 1 on error, 0 on success.
 .SH "SET UP"
-The setup requires a self\-signed certificate and private keys for both 
+The setup requires a self\-signed certificate and private keys for both
 the server and client.  The script \fIunbound\-control\-setup\fR generates
 these in the default run directory, or with \-d in another directory.
 If you change the access control permissions on the key files you can decide
@@ -350,7 +358,7 @@ If you have not configured
 a username in unbound.conf, the keys need read permission for the user
 credentials under which the daemon is started.
 The script preserves private keys present in the directory.
-After running the script as root, turn on \fBcontrol\-enable\fR in 
+After running the script as root, turn on \fBcontrol\-enable\fR in
 \fIunbound.conf\fR.
 .SH "STATISTIC COUNTERS"
 The \fIstats\fR command shows a number of statistic counters.
@@ -361,6 +369,15 @@ number of queries received by thread
 .I threadX.num.queries_ip_ratelimited
 number of queries rate limited by thread
 .TP
+.I threadX.num.queries_cookie_valid
+number of queries with a valid DNS Cookie by thread
+.TP
+.I threadX.num.queries_cookie_client
+number of queries with a client part only DNS Cookie by thread
+.TP
+.I threadX.num.queries_cookie_invalid
+number of queries with an invalid DNS Cookie by thread
+.TP
 .I threadX.num.cachehits
 number of queries that were successfully answered using a cache lookup
 .TP
@@ -390,6 +407,14 @@ as a cache response was sent.
 .I threadX.num.expired
 number of replies that served an expired cache entry.
 .TP
+.I threadX.num.queries_timed_out
+number of queries that are dropped because they waited in the UDP socket buffer
+for too long.
+.TP
+.I threadX.query.queue_time_us.max
+The maximum wait time for packets in the socket buffer, in microseconds. This
+is only reported when sock-queue-timeout is enabled.
+.TP
 .I threadX.num.recursivereplies
 The number of replies sent to queries that needed recursive processing. Could be smaller than threadX.num.cachemiss if due to timeouts no replies were sent for some queries.
 .TP
@@ -417,8 +442,8 @@ Average time it took to answer queries that needed recursive processing. Note th
 .TP
 .I threadX.recursion.time.median
 The median of the time it took to answer queries that needed recursive
-processing.  The median means that 50% of the user queries were answered in 
-less than this time.  Because of big outliers (usually queries to non 
+processing.  The median means that 50% of the user queries were answered in
+less than this time.  Because of big outliers (usually queries to non
 responsive servers), the average can be bigger than the median.  This median
 has been calculated by interpolation from a histogram.
 .TP
@@ -430,6 +455,18 @@ buffers are full.
 .I total.num.queries
 summed over threads.
 .TP
+.I total.num.queries_ip_ratelimited
+summed over threads.
+.TP
+.I total.num.queries_cookie_valid
+summed over threads.
+.TP
+.I total.num.queries_cookie_client
+summed over threads.
+.TP
+.I total.num.queries_cookie_invalid
+summed over threads.
+.TP
 .I total.num.cachehits
 summed over threads.
 .TP
@@ -454,6 +491,12 @@ summed over threads.
 .I total.num.expired
 summed over threads.
 .TP
+.I total.num.queries_timed_out
+summed over threads.
+.TP
+.I total.query.queue_time_us.max
+the maximum of the thread values.
+.TP
 .I total.num.recursivereplies
 summed over threads.
 .TP
@@ -544,32 +587,36 @@ The total number of queries over all threads with query opcode QUERY.
 Also printed for other opcodes, UPDATE, ...
 .TP
 .I num.query.tcp
-Number of queries that were made using TCP towards the unbound server.
+Number of queries that were made using TCP towards the Unbound server.
 .TP
 .I num.query.tcpout
-Number of queries that the unbound server made using TCP outgoing towards
+Number of queries that the Unbound server made using TCP outgoing towards
+other servers.
+.TP
+.I num.query.udpout
+Number of queries that the Unbound server made using UDP outgoing towards
 other servers.
 .TP
 .I num.query.tls
-Number of queries that were made using TLS towards the unbound server.
+Number of queries that were made using TLS towards the Unbound server.
 These are also counted in num.query.tcp, because TLS uses TCP.
 .TP
 .I num.query.tls.resume
 Number of TLS session resumptions, these are queries over TLS towards
-the unbound server where the client negotiated a TLS session resumption key.
+the Unbound server where the client negotiated a TLS session resumption key.
 .TP
 .I num.query.https
-Number of queries that were made using HTTPS towards the unbound server.
+Number of queries that were made using HTTPS towards the Unbound server.
 These are also counted in num.query.tcp and num.query.tls, because HTTPS
 uses TLS and TCP.
 .TP
 .I num.query.ipv6
-Number of queries that were made using IPv6 towards the unbound server.
+Number of queries that were made using IPv6 towards the Unbound server.
 .TP
 .I num.query.flags.RD
 The number of queries that had the RD flag set in the header.
 Also printed for flags QR, AA, TC, RA, Z, AD, CD.
-Note that queries with flags QR, AA or TC may have been rejected 
+Note that queries with flags QR, AA or TC may have been rejected
 because of that.
 .TP
 .I num.query.edns.present
@@ -585,7 +632,7 @@ ratelimiting.
 .TP
 .I num.query.dnscrypt.shared_secret.cachemiss
 The number of dnscrypt queries that did not find a shared secret in the cache.
-The can be use to compute the shared secret hitrate.
+This can be used to compute the shared secret hitrate.
 .TP
 .I num.query.dnscrypt.replay
 The number of dnscrypt queries that found a nonce hit in the nonce cache and
@@ -603,7 +650,7 @@ These queries are also included in the num.answer.rcode.NOERROR number.
 Common for AAAA lookups when an A record exists, and no AAAA.
 .TP
 .I num.answer.secure
-Number of answers that were secure.  The answer validated correctly. 
+Number of answers that were secure.  The answer validated correctly.
 The AD bit might have been set in some of these answers, where the client
 signalled (with DO or AD bit in the query) that they were ready to accept
 the AD bit in the answer.
@@ -641,10 +688,22 @@ timing and protocol support information.
 The number of items in the key cache.  These are DNSSEC keys, one item
 per delegation point, and their validation status.
 .TP
+.I msg.cache.max_collisions
+The maximum number of hash table collisions in the msg cache. This is the
+number of hashes that are identical when a new element is inserted in the
+hash table. If the value is very large, like hundreds, something is wrong
+with the performance of the hash table, hash values are incorrect or malicious.
+.TP
+.I rrset.cache.max_collisions
+The maximum number of hash table collisions in the rrset cache. This is the
+number of hashes that are identical when a new element is inserted in the
+hash table. If the value is very large, like hundreds, something is wrong
+with the performance of the hash table, hash values are incorrect or malicious.
+.TP
 .I dnscrypt_shared_secret.cache.count
 The number of items in the shared secret cache. These are precomputed shared
 secrets for a given client public key/server secret key pair. Shared secrets
-are CPU intensive and this cache allows unbound to avoid recomputing the
+are CPU intensive and this cache allows Unbound to avoid recomputing the
 shared secret when multiple dnscrypt queries are sent from the same client.
 .TP
 .I dnscrypt_nonce.cache.count
@@ -680,7 +739,12 @@ Number of queries that got an answer that contained EDNS client subnet data.
 .I num.query.subnet_cache
 Number of queries answered from the edns client subnet cache.  These are
 counted as cachemiss by the main counters, but hit the client subnet
-specific cache, after getting processed by the edns client subnet module.
+specific cache after getting processed by the edns client subnet module.
+.TP
+.I num.query.cachedb
+Number of queries answered from the external cache of cachedb.
+These are counted as cachemiss by the main counters, but hit the cachedb
+external cache after getting processed by the cachedb module.
 .TP
 .I num.rpz.action.<rpz_action>
 Number of queries answered using configured RPZ policy, per RPZ action type.
@@ -689,11 +753,11 @@ disabled, and cname\-override.
 .SH "FILES"
 .TP
 .I @ub_conf_file@
-unbound configuration file.
+Unbound configuration file.
 .TP
 .I @UNBOUND_RUN_DIR@
 directory with private keys (unbound_server.key and unbound_control.key) and
 self\-signed certificates (unbound_server.pem and unbound_control.pem).
 .SH "SEE ALSO"
-\fIunbound.conf\fR(5), 
+\fIunbound.conf\fR(5),
 \fIunbound\fR(8).

doc/unbound-host.1.in

@@ -15,7 +15,7 @@
 .RB [ \-C
 .IR configfile ]
 .RB [ \-vdhr46D ]
-.RB [ \-c 
+.RB [ \-c
 .IR class ]
 .RB [ \-t
 .IR type ]
@@ -28,12 +28,12 @@
 .I hostname
 .SH "DESCRIPTION"
 .B Unbound\-host
-uses the unbound validating resolver to query for the hostname and display
-results. With the \fB\-v\fR option it displays validation 
+uses the Unbound validating resolver to query for the hostname and display
+results. With the \fB\-v\fR option it displays validation
 status: secure, insecure, bogus (security failure).
 .P
 By default it reads no configuration file whatsoever.  It attempts to reach
-the internet root servers.  With \fB\-C\fR an unbound config file and with
+the internet root servers.  With \fB\-C\fR an Unbound config file and with
 \fB\-r\fR resolv.conf can be read.
 .P
 The available options are:
@@ -73,7 +73,7 @@ For example \-y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546
 .TP
 .B \-D
 Enables DNSSEC validation.  Reads the root anchor from the default configured
-root anchor at the default location, \fI@UNBOUND_ROOTKEY_FILE@\fR. 
+root anchor at the default location, \fI@UNBOUND_ROOTKEY_FILE@\fR.
 .TP
 .B \-f \fIkeyfile
 Reads keys from a file. Every line has a DS or DNSKEY record, in the format
@@ -110,9 +110,9 @@ $ unbound\-host \-v \-y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325
 .P
 $ unbound\-host \-v \-y "example.com DS 31560 5 1 1CFED84787E6E19CCF9372C1187325972FE546CD" 192.0.2.153
 .SH "EXIT CODE"
-The unbound\-host program exits with status code 1 on error, 
+The unbound\-host program exits with status code 1 on error,
 0 on no error. The data may not be available on exit code 0, exit code 1
 means the lookup encountered a fatal error.
 .SH "SEE ALSO"
-\fIunbound.conf\fR(5), 
+\fIunbound.conf\fR(5),
 \fIunbound\fR(8).

doc/unbound.8.in

@@ -57,7 +57,7 @@ The available options are:
 Show the version number and commandline option help, and exit.
 .TP
 .B \-c\fI cfgfile
-Set the config file with settings for unbound to read instead of reading the
+Set the config file with settings for Unbound to read instead of reading the
 file at the default location, @ub_conf_file@. The syntax is
 described in \fIunbound.conf\fR(5).
 .TP
@@ -70,12 +70,12 @@ or to syslog, but the log messages are printed to stderr all the time.
 .TP
 .B \-p
 Don't use a pidfile.  This argument should only be used by supervision
-systems which can ensure that only one instance of unbound will run
+systems which can ensure that only one instance of Unbound will run
 concurrently.
 .TP
 .B \-v
 Increase verbosity. If given multiple times, more information is logged.
-This is in addition to the verbosity (if any) from the config file.
+This is added to the verbosity (if any) from the config file.
 .TP
 .B \-V
 Show the version number and build options, and exit.

dynlibmod/dynlibmod.c

@@ -75,6 +75,7 @@ int dynlibmod_init(struct module_env* env, int id) {
     struct config_strlist* cfg_item = env->cfg->dynlib_file;
     struct dynlibmod_env* de = (struct dynlibmod_env*)calloc(1, sizeof(struct dynlibmod_env));
     __DYNMOD dynamic_library;
+    int i;
     if (!de)
     {
         log_err("dynlibmod[%d]: malloc failure", dynlib_mod_idx);
@@ -84,7 +85,7 @@ int dynlibmod_init(struct module_env* env, int id) {
     env->modinfo[id] = (void*) de;
 
     de->fname = NULL;
-    for(int i = dynlib_mod_idx;
+    for(i = dynlib_mod_idx;
         i != 0 && cfg_item != NULL;
         i--, cfg_item = cfg_item->next) {}
 

edns-subnet/addrtree.c

@@ -97,6 +97,7 @@ node_create(struct addrtree *tree, void *elem, addrlen_t scope,
 	tree->node_count++;
 	node->scope = scope;
 	node->ttl = ttl;
+	node->only_match_scope_zero = 0;
 	node->edge[0] = NULL;
 	node->edge[1] = NULL;
 	node->parent_edge = NULL;
@@ -155,6 +156,7 @@ clean_node(struct addrtree *tree, struct addrnode *node)
 	if (!node->elem) return;
 	tree->size_bytes -= tree->sizefunc(node->elem);
 	tree->delfunc(tree->env, node->elem);
+	node->only_match_scope_zero = 0;
 	node->elem = NULL;
 }
 
@@ -358,7 +360,7 @@ issub(const addrkey_t *s1, addrlen_t l1,
 void
 addrtree_insert(struct addrtree *tree, const addrkey_t *addr, 
 	addrlen_t sourcemask, addrlen_t scope, void *elem, time_t ttl, 
-	time_t now)
+	time_t now, int only_match_scope_zero)
 {
 	struct addrnode *newnode, *node;
 	struct addredge *edge;
@@ -381,6 +383,7 @@ addrtree_insert(struct addrtree *tree, const addrkey_t *addr,
 			/* update this node's scope and data */
 			clean_node(tree, node);
 			node->ttl = ttl;
+			node->only_match_scope_zero = only_match_scope_zero;
 			node->elem = elem;
 			node->scope = scope;
 			tree->size_bytes += tree->sizefunc(elem);
@@ -447,6 +450,7 @@ addrtree_insert(struct addrtree *tree, const addrkey_t *addr,
 			newnode->elem = elem;
 			newnode->scope = scope;
 			newnode->ttl = ttl;
+			newnode->only_match_scope_zero = only_match_scope_zero;
 		} 
 		
 		tree->size_bytes += node_size(tree, newnode);
@@ -483,7 +487,8 @@ addrtree_find(struct addrtree *tree, const addrkey_t *addr,
 		/* Current node more specific then question. */
 		log_assert(depth <= sourcemask);
 		/* does this node have data? if yes, see if we have a match */
-		if (node->elem && node->ttl >= now) {
+		if (node->elem && node->ttl >= now &&
+			!(sourcemask != 0 && node->only_match_scope_zero)) {
 			/* saved at wrong depth */;
 			log_assert(node->scope >= depth);
 			if (depth == node->scope ||

edns-subnet/addrtree.h

@@ -95,6 +95,10 @@ struct addrnode {
 	time_t ttl;
 	/** Number of significant bits in address. */
 	addrlen_t scope;
+	/** Only use the element for queries for subnet/0. Set if the query
+	 * for /0 was answered with scope 0. For query /x answer scope 0,
+	 * they can match anything and this is false. */
+	int only_match_scope_zero;
 	/** A node can have 0-2 edges, set to NULL for unused */
 	struct addredge *edge[2];
 	/** edge between this node and parent */
@@ -157,11 +161,12 @@ void addrtree_delete(struct addrtree *tree);
  * @param scope: Number of significant bits in addr.
  * @param elem: data to store in the tree.
  * @param ttl: elem is valid up to this time, seconds.
+ * @param only_match_scope_zero: set for when query /0 has scope /0 answer.
  * @param now: Current time in seconds.
  */
 void addrtree_insert(struct addrtree *tree, const addrkey_t *addr, 
 	addrlen_t sourcemask, addrlen_t scope, void *elem, time_t ttl, 
-	time_t now);
+	time_t now, int only_match_scope_zero);
 
 /**
  * Find a node containing an element in the tree.

edns-subnet/subnetmod.c

@@ -55,6 +55,7 @@
 #include "util/config_file.h"
 #include "util/data/msgreply.h"
 #include "sldns/sbuffer.h"
+#include "sldns/wire2str.h"
 #include "iterator/iter_utils.h"
 
 /** externally called */
@@ -93,13 +94,14 @@ subnet_new_qstate(struct module_qstate *qstate, int id)
 	qstate->minfo[id] = sq;
 	memset(sq, 0, sizeof(*sq));
 	sq->started_no_cache_store = qstate->no_cache_store;
+	sq->started_no_cache_lookup = qstate->no_cache_lookup;
 	return 1;
 }
 
 /** Add ecs struct to edns list, after parsing it to wire format. */
-static void
-ecs_opt_list_append(struct ecs_data* ecs, struct edns_option** list,
-	struct module_qstate *qstate)
+void
+subnet_ecs_opt_list_append(struct ecs_data* ecs, struct edns_option** list,
+	struct module_qstate *qstate, struct regional *region)
 {
 	size_t sn_octs, sn_octs_remainder;
 	sldns_buffer* buf = qstate->env->scratch_buffer;
@@ -131,7 +133,7 @@ ecs_opt_list_append(struct ecs_data* ecs, struct edns_option** list,
 		edns_opt_list_append(list,
 				qstate->env->cfg->client_subnet_opcode,
 				sn_octs + sn_octs_remainder + 4,
-				sldns_buffer_begin(buf), qstate->region);
+				sldns_buffer_begin(buf), region);
 	}
 }
 
@@ -139,7 +141,7 @@ int ecs_whitelist_check(struct query_info* qinfo,
 	uint16_t ATTR_UNUSED(flags), struct module_qstate* qstate,
 	struct sockaddr_storage* addr, socklen_t addrlen,
 	uint8_t* ATTR_UNUSED(zone), size_t ATTR_UNUSED(zonelen),
-	struct regional* ATTR_UNUSED(region), int id, void* ATTR_UNUSED(cbargs))
+	struct regional *region, int id, void* ATTR_UNUSED(cbargs))
 {
 	struct subnet_qstate *sq;
 	struct subnet_env *sn_env;
@@ -154,6 +156,7 @@ int ecs_whitelist_check(struct query_info* qinfo,
 		qstate->no_cache_store = 0;
 	}
 
+	sq->subnet_sent_no_subnet = 0;
 	if(sq->ecs_server_out.subnet_validdata && ((sq->subnet_downstream &&
 		qstate->env->cfg->client_subnet_always_forward) ||
 		ecs_is_whitelisted(sn_env->whitelist, 
@@ -162,17 +165,29 @@ int ecs_whitelist_check(struct query_info* qinfo,
 		/* Address on whitelist or client query contains ECS option, we
 		 * want to sent out ECS. Only add option if it is not already
 		 * set. */
-		if(!(sq->subnet_sent)) {
-			ecs_opt_list_append(&sq->ecs_server_out,
-				&qstate->edns_opts_back_out, qstate);
-			sq->subnet_sent = 1;
+		if(!edns_opt_list_find(qstate->edns_opts_back_out,
+			qstate->env->cfg->client_subnet_opcode)) {
+			/* if the client is not wanting an EDNS subnet option,
+			 * omit it and store that we omitted it but actually
+			 * are doing EDNS subnet to the server. */
+			if(sq->ecs_server_out.subnet_source_mask == 0) {
+				sq->subnet_sent_no_subnet = 1;
+				sq->subnet_sent = 0;
+				return 1;
+			}
+			subnet_ecs_opt_list_append(&sq->ecs_server_out,
+				&qstate->edns_opts_back_out, qstate, region);
 		}
+		sq->subnet_sent = 1;
 	}
-	else if(sq->subnet_sent) {
+	else {
 		/* Outgoing ECS option is set, but we don't want to sent it to
 		 * this address, remove option. */
-		edns_opt_list_remove(&qstate->edns_opts_back_out,
-			qstate->env->cfg->client_subnet_opcode);
+		if(edns_opt_list_find(qstate->edns_opts_back_out,
+			qstate->env->cfg->client_subnet_opcode)) {
+			edns_opt_list_remove(&qstate->edns_opts_back_out,
+				qstate->env->cfg->client_subnet_opcode);
+		}
 		sq->subnet_sent = 0;
 	}
 	return 1;
@@ -198,6 +213,17 @@ subnetmod_init(struct module_env *env, int id)
 	}
 	alloc_init(&sn_env->alloc, NULL, 0);
 	env->modinfo[id] = (void*)sn_env;
+
+	/* Warn that serve-expired and prefetch do not work with the subnet
+	 * module cache. */
+	if(env->cfg->serve_expired)
+		log_warn(
+			"subnetcache: serve-expired is set but not working "
+			"for data originating from the subnet module cache.");
+	if(env->cfg->prefetch)
+		log_warn(
+			"subnetcache: prefetch is set but not working "
+			"for data originating from the subnet module cache.");
 	/* Copy msg_cache settings */
 	sn_env->subnet_msg_cache = slabhash_create(env->cfg->msg_cache_slabs,
 		HASH_DEFAULT_STARTARRAY, env->cfg->msg_cache_size,
@@ -227,7 +253,7 @@ subnetmod_init(struct module_env *env, int id)
 	env->unique_mesh = 1;
 	if(!edns_register_option(env->cfg->client_subnet_opcode,
 		env->cfg->client_subnet_always_forward /* bypass cache */,
-		0 /* no aggregation */, env)) {
+		1 /* no aggregation */, env)) {
 		log_err("subnetcache: could not register opcode");
 		ecs_whitelist_delete(sn_env->whitelist);
 		slabhash_delete(sn_env->subnet_msg_cache);
@@ -326,13 +352,16 @@ update_cache(struct module_qstate *qstate, int id)
 	struct slabhash *subnet_msg_cache = sne->subnet_msg_cache;
 	struct ecs_data *edns = &sq->ecs_client_in;
 	size_t i;
+	int only_match_scope_zero;
 
-	/* We already calculated hash upon lookup */
-	hashvalue_type h = qstate->minfo[id] ? 
-		((struct subnet_qstate*)qstate->minfo[id])->qinfo_hash : 
+	/* We already calculated hash upon lookup (lookup_and_reply) if we were
+	 * allowed to look in the ECS cache */
+	hashvalue_type h = qstate->minfo[id] &&
+		((struct subnet_qstate*)qstate->minfo[id])->qinfo_hash_calculated?
+		((struct subnet_qstate*)qstate->minfo[id])->qinfo_hash :
 		query_info_hash(&qstate->qinfo, qstate->query_flags);
 	/* Step 1, general qinfo lookup */
-	struct lruhash_entry *lru_entry = slabhash_lookup(subnet_msg_cache, h,
+	struct lruhash_entry* lru_entry = slabhash_lookup(subnet_msg_cache, h,
 		&qstate->qinfo, 1);
 	int need_to_insert = (lru_entry == NULL);
 	if (!lru_entry) {
@@ -376,7 +405,7 @@ update_cache(struct module_qstate *qstate, int id)
 		log_err("subnetcache: cache insertion failed");
 		return;
 	}
-	
+
 	/* store RRsets */
 	for(i=0; i<rep->rrset_count; i++) {
 		rep->ref[i].key = rep->rrsets[i];
@@ -385,9 +414,12 @@ update_cache(struct module_qstate *qstate, int id)
 	reply_info_set_ttls(rep, *qstate->env->now);
 	rep->flags |= (BIT_RA | BIT_QR); /* fix flags to be sensible for */
 	rep->flags &= ~(BIT_AA | BIT_CD);/* a reply based on the cache   */
+	if(edns->subnet_source_mask == 0 && edns->subnet_scope_mask == 0)
+		only_match_scope_zero = 1;
+	else only_match_scope_zero = 0;
 	addrtree_insert(tree, (addrkey_t*)edns->subnet_addr, 
 		edns->subnet_source_mask, sq->max_scope, rep,
-		rep->ttl, *qstate->env->now);
+		rep->ttl, *qstate->env->now, only_match_scope_zero);
 
 	lock_rw_unlock(&lru_entry->lock);
 	if (need_to_insert) {
@@ -398,7 +430,7 @@ update_cache(struct module_qstate *qstate, int id)
 
 /** Lookup in cache and reply true iff reply is sent. */
 static int
-lookup_and_reply(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
+lookup_and_reply(struct module_qstate *qstate, int id, struct subnet_qstate *sq, int prefetch)
 {
 	struct lruhash_entry *e;
 	struct module_env *env = qstate->env;
@@ -412,7 +444,10 @@ lookup_and_reply(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 
 	memset(&sq->ecs_client_out, 0, sizeof(sq->ecs_client_out));
 
-	if (sq) sq->qinfo_hash = h; /* Might be useful on cache miss */
+	if (sq) {
+		sq->qinfo_hash = h; /* Might be useful on cache miss */
+		sq->qinfo_hash_calculated = 1;
+	}
 	e = slabhash_lookup(sne->subnet_msg_cache, h, &qstate->qinfo, 1);
 	if (!e) return 0; /* qinfo not in cache */
 	data = e->data;
@@ -447,6 +482,10 @@ lookup_and_reply(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 			INET6_SIZE);
 		sq->ecs_client_out.subnet_validdata = 1;
 	}
+
+	if (prefetch && *qstate->env->now >= ((struct reply_info *)node->elem)->prefetch_ttl) {
+		qstate->need_refetch = 1;
+	}
 	return 1;
 }
 
@@ -483,18 +522,18 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 		 * module_finished */
 		return module_finished;
 	}
-	
+
 	/* We have not asked for subnet data */
-	if (!sq->subnet_sent) {
+	if (!sq->subnet_sent && !sq->subnet_sent_no_subnet) {
 		if (s_in->subnet_validdata)
 			verbose(VERB_QUERY, "subnetcache: received spurious data");
 		if (sq->subnet_downstream) /* Copy back to client */
 			cp_edns_bad_response(c_out, c_in);
 		return module_finished;
 	}
-	
+
 	/* subnet sent but nothing came back */
-	if (!s_in->subnet_validdata) {
+	if (!s_in->subnet_validdata && !sq->subnet_sent_no_subnet) {
 		/* The authority indicated no support for edns subnet. As a
 		 * consequence the answer ended up in the regular cache. It
 		 * is still useful to put it in the edns subnet cache for
@@ -509,11 +548,23 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 			cp_edns_bad_response(c_out, c_in);
 		return module_finished;
 	}
-	
+
+	/* Purposefully there was no sent subnet, and there is consequently
+	 * no subnet in the answer. If there was, use the subnet in the answer
+	 * anyway. But if there is not, treat it as a prefix 0 answer. */
+	if(sq->subnet_sent_no_subnet && !s_in->subnet_validdata) {
+		/* Fill in 0.0.0.0/0 scope 0, or ::0/0 scope 0, for caching. */
+		s_in->subnet_addr_fam = s_out->subnet_addr_fam;
+		s_in->subnet_source_mask = 0;
+		s_in->subnet_scope_mask = 0;
+		memset(s_in->subnet_addr, 0, INET6_SIZE);
+		s_in->subnet_validdata = 1;
+	}
+
 	/* Being here means we have asked for and got a subnet specific 
 	 * answer. Also, the answer from the authority is not yet cached 
 	 * anywhere. */
-	
+
 	/* can we accept response? */
 	if(s_out->subnet_addr_fam != s_in->subnet_addr_fam ||
 		s_out->subnet_source_mask != s_in->subnet_source_mask ||
@@ -526,6 +577,7 @@ eval_response(struct module_qstate *qstate, int id, struct subnet_qstate *sq)
 		(void)edns_opt_list_remove(&qstate->edns_opts_back_out,
 			qstate->env->cfg->client_subnet_opcode);
 		sq->subnet_sent = 0;
+		sq->subnet_sent_no_subnet = 0;
 		return module_restart_next;
 	}
 
@@ -598,7 +650,7 @@ parse_subnet_option(struct edns_option* ecs_option, struct ecs_data* ecs)
 	return 1;
 }
 
-static void
+void
 subnet_option_from_ss(struct sockaddr_storage *ss, struct ecs_data* ecs,
 	struct config_file* cfg)
 {
@@ -646,6 +698,7 @@ ecs_query_response(struct module_qstate* qstate, struct dns_msg* response,
 		edns_opt_list_remove(&qstate->edns_opts_back_out,
 			qstate->env->cfg->client_subnet_opcode);
 		sq->subnet_sent = 0;
+		sq->subnet_sent_no_subnet = 0;
 		memset(&sq->ecs_server_out, 0, sizeof(sq->ecs_server_out));
 	} else if (!sq->track_max_scope &&
 		FLAGS_GET_RCODE(response->rep->flags) == LDNS_RCODE_NOERROR &&
@@ -664,6 +717,24 @@ ecs_query_response(struct module_qstate* qstate, struct dns_msg* response,
 	return 1;
 }
 
+/** verbose print edns subnet option in pretty print */
+static void
+subnet_log_print(const char* s, struct edns_option* ecs_opt)
+{
+	if(verbosity >= VERB_ALGO) {
+		char buf[256];
+		char* str = buf;
+		size_t str_len = sizeof(buf);
+		if(!ecs_opt) {
+			verbose(VERB_ALGO, "%s (null)", s);
+			return;
+		}
+		(void)sldns_wire2str_edns_subnet_print(&str, &str_len,
+			ecs_opt->opt_data, ecs_opt->opt_len);
+		verbose(VERB_ALGO, "%s %s", s, buf);
+	}
+}
+
 int
 ecs_edns_back_parsed(struct module_qstate* qstate, int id,
 	void* ATTR_UNUSED(cbargs))
@@ -678,6 +749,7 @@ ecs_edns_back_parsed(struct module_qstate* qstate, int id,
 		qstate->env->cfg->client_subnet_opcode)) &&
 		parse_subnet_option(ecs_opt, &sq->ecs_server_in) &&
 		sq->subnet_sent && sq->ecs_server_in.subnet_validdata) {
+			subnet_log_print("answer has edns subnet", ecs_opt);
 			/* Only skip global cache store if we sent an ECS option
 			 * and received one back. Answers from non-whitelisted
 			 * servers will end up in global cache. Answers for
@@ -688,6 +760,9 @@ ecs_edns_back_parsed(struct module_qstate* qstate, int id,
 				sq->ecs_server_in.subnet_scope_mask >
 				sq->max_scope))
 				sq->max_scope = sq->ecs_server_in.subnet_scope_mask;
+	} else if(sq->subnet_sent_no_subnet) {
+		/* The answer can be stored as scope 0, not in global cache. */
+		qstate->no_cache_store = 1;
 	}
 
 	return 1;
@@ -726,11 +801,17 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 				qstate->ext_state[id] = module_finished;
 				return;
 			}
+			subnet_log_print("query has edns subnet", ecs_opt);
 			sq->subnet_downstream = 1;
 		}
 		else if(qstate->mesh_info->reply_list) {
 			subnet_option_from_ss(
-				&qstate->mesh_info->reply_list->query_reply.addr,
+				&qstate->mesh_info->reply_list->query_reply.client_addr,
+				&sq->ecs_client_in, qstate->env->cfg);
+		}
+		else if(qstate->client_addr.ss_family != AF_UNSPEC) {
+			subnet_option_from_ss(
+				&qstate->client_addr,
 				&sq->ecs_client_in, qstate->env->cfg);
 		}
 		
@@ -754,18 +835,30 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 				return;
 		}
 
-		lock_rw_wrlock(&sne->biglock);
-		if (lookup_and_reply(qstate, id, sq)) {
-			sne->num_msg_cache++;
-			lock_rw_unlock(&sne->biglock);
-			verbose(VERB_QUERY, "subnetcache: answered from cache");
-			qstate->ext_state[id] = module_finished;
+		if(!sq->started_no_cache_lookup && !qstate->blacklist) {
+			lock_rw_wrlock(&sne->biglock);
+			if(qstate->mesh_info->reply_list &&
+				lookup_and_reply(qstate, id, sq,
+				qstate->env->cfg->prefetch)) {
+				sne->num_msg_cache++;
+				lock_rw_unlock(&sne->biglock);
+				verbose(VERB_QUERY, "subnetcache: answered from cache");
+				qstate->ext_state[id] = module_finished;
 
-			ecs_opt_list_append(&sq->ecs_client_out,
-				&qstate->edns_opts_front_out, qstate);
-			return;
+				subnet_ecs_opt_list_append(&sq->ecs_client_out,
+					&qstate->edns_opts_front_out, qstate,
+					qstate->region);
+				if(verbosity >= VERB_ALGO) {
+					subnet_log_print("reply has edns subnet",
+						edns_opt_list_find(
+						qstate->edns_opts_front_out,
+						qstate->env->cfg->
+						client_subnet_opcode));
+				}
+				return;
+			}
+			lock_rw_unlock(&sne->biglock);
 		}
-		lock_rw_unlock(&sne->biglock);
 		
 		sq->ecs_server_out.subnet_addr_fam =
 			sq->ecs_client_in.subnet_addr_fam;
@@ -783,7 +876,7 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 			sq->ecs_server_out.subnet_source_mask =
 				qstate->env->cfg->max_client_subnet_ipv6;
 		/* Safe to copy completely, even if the source is limited by the
-		 * configuration. ecs_opt_list_append() will limit the address.
+		 * configuration. subnet_ecs_opt_list_append() will limit the address.
 		 * */
 		memcpy(&sq->ecs_server_out.subnet_addr,
 			sq->ecs_client_in.subnet_addr, INET6_SIZE);
@@ -807,10 +900,19 @@ subnetmod_operate(struct module_qstate *qstate, enum module_ev event,
 		qstate->ext_state[id] = eval_response(qstate, id, sq);
 		if(qstate->ext_state[id] == module_finished &&
 			qstate->return_msg) {
-			ecs_opt_list_append(&sq->ecs_client_out,
-				&qstate->edns_opts_front_out, qstate);
+			subnet_ecs_opt_list_append(&sq->ecs_client_out,
+				&qstate->edns_opts_front_out, qstate,
+				qstate->region);
+			if(verbosity >= VERB_ALGO) {
+				subnet_log_print("reply has edns subnet",
+					edns_opt_list_find(
+					qstate->edns_opts_front_out,
+					qstate->env->cfg->
+					client_subnet_opcode));
+			}
 		}
 		qstate->no_cache_store = sq->started_no_cache_store;
+		qstate->no_cache_lookup = sq->started_no_cache_lookup;
 		return;
 	}
 	if(sq && outbound) {

edns-subnet/subnetmod.h

@@ -76,6 +76,7 @@ struct subnet_msg_cache_data {
 struct subnet_qstate {
 	/** We need the hash for both cache lookup and insert */
 	hashvalue_type qinfo_hash;
+	int qinfo_hash_calculated;
 	/** ecs_data for client communication */
 	struct ecs_data	ecs_client_in;
 	struct ecs_data	ecs_client_out;
@@ -84,6 +85,13 @@ struct subnet_qstate {
 	struct ecs_data	ecs_server_out;
 	int subnet_downstream;
 	int subnet_sent;
+	/**
+	 * If there was no subnet sent because the client used source prefix
+	 * length 0 for omitting the information. Then the answer is cached
+	 * like subnet was a /0 scope. Like the subnet_sent flag, but when
+	 * the EDNS subnet option is omitted because the client asked.
+	 */
+	int subnet_sent_no_subnet;
 	/** keep track of longest received scope, set after receiving CNAME for
 	 * incoming QNAME. */
 	int track_max_scope;
@@ -92,6 +100,8 @@ struct subnet_qstate {
 	uint8_t max_scope;
 	/** has the subnet module been started with no_cache_store? */
 	int started_no_cache_store;
+	/** has the subnet module been started with no_cache_lookup? */
+	int started_no_cache_lookup;
 };
 
 void subnet_data_delete(void* d, void* ATTR_UNUSED(arg));
@@ -143,4 +153,11 @@ int ecs_query_response(struct module_qstate* qstate, struct dns_msg* response,
 /** mark subnet msg to be deleted */
 void subnet_markdel(void* key);
 
+/** Add ecs struct to edns list, after parsing it to wire format. */
+void subnet_ecs_opt_list_append(struct ecs_data* ecs, struct edns_option** list,
+	struct module_qstate *qstate, struct regional *region);
+
+/** Create ecs_data from the sockaddr_storage information. */
+void subnet_option_from_ss(struct sockaddr_storage *ss, struct ecs_data* ecs,
+	struct config_file* cfg);
 #endif /* SUBNETMOD_H */

ipsecmod/ipsecmod.c

@@ -456,7 +456,7 @@ ipsecmod_handle_query(struct module_qstate* qstate,
 	/* Store A/AAAA in cache. */
 	if(!dns_cache_store(qstate->env, &qstate->qinfo,
 		qstate->return_msg->rep, 0, qstate->prefetch_leeway,
-		0, qstate->region, qstate->query_flags)) {
+		0, qstate->region, qstate->query_flags, qstate->qstarttime)) {
 		log_err("ipsecmod: out of memory caching record");
 	}
 	qstate->ext_state[id] = module_finished;

ipset/ipset.c

@@ -138,10 +138,10 @@ ipset_add_rrset_data(struct ipset_env *ie, struct mnl_socket *mnl,
 static int
 ipset_check_zones_for_rrset(struct module_env *env, struct ipset_env *ie,
 	struct mnl_socket *mnl, struct ub_packed_rrset_key *rrset,
-	const char *setname, int af)
+	const char *qname, const int qlen, const char *setname, int af)
 {
 	static char dname[BUFF_LEN];
-	const char *s;
+	const char *ds, *qs;
 	int dlen, plen;
 
 	struct config_strlist *p;
@@ -152,70 +152,73 @@ ipset_check_zones_for_rrset(struct module_env *env, struct ipset_env *ie,
 		log_err("bad domain name");
 		return -1;
 	}
-	if (dname[dlen - 1] == '.') {
-		dlen--;
-	}
 
 	for (p = env->cfg->local_zones_ipset; p; p = p->next) {
+		ds = NULL;
+		qs = NULL;
 		plen = strlen(p->str);
 
-		if (dlen >= plen) {
-			s = dname + (dlen - plen);
-
-			if (strncasecmp(p->str, s, plen) == 0) {
-				d = (struct packed_rrset_data*)rrset->entry.data;
-				ipset_add_rrset_data(ie, mnl, d, setname,
-					af, dname);
-				break;
-			}
+		if (dlen == plen || (dlen > plen && dname[dlen - plen - 1] == '.' )) {
+			ds = dname + (dlen - plen);
+		}
+		if (qlen == plen || (qlen > plen && qname[qlen - plen - 1] == '.' )) {
+			qs = qname + (qlen - plen);
+		}
+		if ((ds && strncasecmp(p->str, ds, plen) == 0)
+			|| (qs && strncasecmp(p->str, qs, plen) == 0)) {
+			d = (struct packed_rrset_data*)rrset->entry.data;
+			ipset_add_rrset_data(ie, mnl, d, setname,
+				af, dname);
+			break;
 		}
 	}
 	return 0;
 }
 
-static int ipset_update(struct module_env *env, struct dns_msg *return_msg, struct ipset_env *ie) {
+static int ipset_update(struct module_env *env, struct dns_msg *return_msg,
+	struct query_info qinfo, struct ipset_env *ie)
+{
 	struct mnl_socket *mnl;
-
 	size_t i;
-
 	const char *setname;
-
 	struct ub_packed_rrset_key *rrset;
-
 	int af;
-
+	static char qname[BUFF_LEN];
+	int qlen;
 
 	mnl = (struct mnl_socket *)ie->mnl;
 	if (!mnl) {
-		// retry to create mnl socket
+		/* retry to create mnl socket */
 		mnl = open_mnl_socket();
 		if (!mnl) {
 			return -1;
 		}
-
 		ie->mnl = mnl;
 	}
 
-	for (i = 0; i < return_msg->rep->rrset_count; ++i) {
+	qlen = sldns_wire2str_dname_buf(qinfo.qname, qinfo.qname_len,
+		qname, BUFF_LEN);
+	if(qlen == 0) {
+		log_err("bad domain name");
+		return -1;
+	}
+
+	for(i = 0; i < return_msg->rep->rrset_count; i++) {
 		setname = NULL;
-
 		rrset = return_msg->rep->rrsets[i];
-
-		if (rrset->rk.type == htons(LDNS_RR_TYPE_A)) {
+		if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_A &&
+			ie->v4_enabled == 1) {
 			af = AF_INET;
-			if ((ie->v4_enabled == 1)) {
-				setname = ie->name_v4;
-			}
-		} else {
+			setname = ie->name_v4;
+		} else if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_AAAA &&
+			ie->v6_enabled == 1) {
 			af = AF_INET6;
-			if ((ie->v6_enabled == 1)) {
-				setname = ie->name_v6;
-			}
+			setname = ie->name_v6;
 		}
 
 		if (setname) {
 			if(ipset_check_zones_for_rrset(env, ie, mnl, rrset,
-				setname, af) == -1)
+				qname, qlen, setname, af) == -1)
 				return -1;
 		}
 	}
@@ -311,7 +314,7 @@ void ipset_operate(struct module_qstate *qstate, enum module_ev event, int id,
 
 	if (iq && (event == module_event_moddone)) {
 		if (qstate->return_msg && qstate->return_msg->rep) {
-			ipset_update(qstate->env, qstate->return_msg, ie);
+			ipset_update(qstate->env, qstate->return_msg, qstate->qinfo, ie);
 		}
 		qstate->ext_state[id] = module_finished;
 		return;

iterator/iter_delegpt.c

@@ -75,8 +75,10 @@ struct delegpt* delegpt_copy(struct delegpt* dp, struct regional* region)
 	copy->ssl_upstream = dp->ssl_upstream;
 	copy->tcp_upstream = dp->tcp_upstream;
 	for(ns = dp->nslist; ns; ns = ns->next) {
-		if(!delegpt_add_ns(copy, region, ns->name, ns->lame))
+		if(!delegpt_add_ns(copy, region, ns->name, ns->lame,
+			ns->tls_auth_name, ns->port))
 			return NULL;
+		copy->nslist->cache_lookup_count = ns->cache_lookup_count;
 		copy->nslist->resolved = ns->resolved;
 		copy->nslist->got4 = ns->got4;
 		copy->nslist->got6 = ns->got6;
@@ -84,8 +86,8 @@ struct delegpt* delegpt_copy(struct delegpt* dp, struct regional* region)
 		copy->nslist->done_pside6 = ns->done_pside6;
 	}
 	for(a = dp->target_list; a; a = a->next_target) {
-		if(!delegpt_add_addr(copy, region, &a->addr, a->addrlen, 
-			a->bogus, a->lame, a->tls_auth_name, NULL))
+		if(!delegpt_add_addr(copy, region, &a->addr, a->addrlen,
+			a->bogus, a->lame, a->tls_auth_name, -1, NULL))
 			return NULL;
 	}
 	return copy;
@@ -102,7 +104,7 @@ delegpt_set_name(struct delegpt* dp, struct regional* region, uint8_t* name)
 
 int 
 delegpt_add_ns(struct delegpt* dp, struct regional* region, uint8_t* name,
-	uint8_t lame)
+	uint8_t lame, char* tls_auth_name, int port)
 {
 	struct delegpt_ns* ns;
 	size_t len;
@@ -120,12 +122,21 @@ delegpt_add_ns(struct delegpt* dp, struct regional* region, uint8_t* name,
 	ns->namelen = len;
 	dp->nslist = ns;
 	ns->name = regional_alloc_init(region, name, ns->namelen);
+	ns->cache_lookup_count = 0;
 	ns->resolved = 0;
 	ns->got4 = 0;
 	ns->got6 = 0;
 	ns->lame = lame;
 	ns->done_pside4 = 0;
 	ns->done_pside6 = 0;
+	ns->port = port;
+	if(tls_auth_name) {
+		ns->tls_auth_name = regional_strdup(region, tls_auth_name);
+		if(!ns->tls_auth_name)
+			return 0;
+	} else {
+		ns->tls_auth_name = NULL;
+	}
 	return ns->name != 0;
 }
 
@@ -159,9 +170,9 @@ delegpt_find_addr(struct delegpt* dp, struct sockaddr_storage* addr,
 	return NULL;
 }
 
-int 
-delegpt_add_target(struct delegpt* dp, struct regional* region, 
-	uint8_t* name, size_t namelen, struct sockaddr_storage* addr, 
+int
+delegpt_add_target(struct delegpt* dp, struct regional* region,
+	uint8_t* name, size_t namelen, struct sockaddr_storage* addr,
 	socklen_t addrlen, uint8_t bogus, uint8_t lame, int* additions)
 {
 	struct delegpt_ns* ns = delegpt_find_ns(dp, name, namelen);
@@ -176,18 +187,27 @@ delegpt_add_target(struct delegpt* dp, struct regional* region,
 		else	ns->got4 = 1;
 		if(ns->got4 && ns->got6)
 			ns->resolved = 1;
+	} else {
+		if(addr_is_ip6(addr, addrlen))
+			ns->done_pside6 = 1;
+		else	ns->done_pside4 = 1;
 	}
-	return delegpt_add_addr(dp, region, addr, addrlen, bogus, lame, NULL,
-		additions);
+	log_assert(ns->port>0);
+	return delegpt_add_addr(dp, region, addr, addrlen, bogus, lame,
+		ns->tls_auth_name, ns->port, additions);
 }
 
-int 
-delegpt_add_addr(struct delegpt* dp, struct regional* region, 
-	struct sockaddr_storage* addr, socklen_t addrlen, uint8_t bogus, 
-	uint8_t lame, char* tls_auth_name, int* additions)
+int
+delegpt_add_addr(struct delegpt* dp, struct regional* region,
+	struct sockaddr_storage* addr, socklen_t addrlen, uint8_t bogus,
+	uint8_t lame, char* tls_auth_name, int port, int* additions)
 {
 	struct delegpt_addr* a;
 	log_assert(!dp->dp_type_mlc);
+	if(port != -1) {
+		log_assert(port>0);
+		sockaddr_store_port(addr, addrlen, port);
+	}
 	/* check for duplicates */
 	if((a = delegpt_find_addr(dp, addr, addrlen))) {
 		if(bogus)
@@ -301,6 +321,45 @@ void delegpt_log(enum verbosity_value v, struct delegpt* dp)
 	}
 }
 
+int
+delegpt_addr_on_result_list(struct delegpt* dp, struct delegpt_addr* find)
+{
+	struct delegpt_addr* a = dp->result_list;
+	while(a) {
+		if(a == find)
+			return 1;
+		a = a->next_result;
+	}
+	return 0;
+}
+
+void
+delegpt_usable_list_remove_addr(struct delegpt* dp, struct delegpt_addr* del)
+{
+	struct delegpt_addr* usa = dp->usable_list, *prev = NULL;
+	while(usa) {
+		if(usa == del) {
+			/* snip off the usable list */
+			if(prev)
+				prev->next_usable = usa->next_usable;
+			else	dp->usable_list = usa->next_usable;
+			return;
+		}
+		prev = usa;
+		usa = usa->next_usable;
+	}
+}
+
+void
+delegpt_add_to_result_list(struct delegpt* dp, struct delegpt_addr* a)
+{
+	if(delegpt_addr_on_result_list(dp, a))
+		return;
+	delegpt_usable_list_remove_addr(dp, a);
+	a->next_result = dp->result_list;
+	dp->result_list = a;
+}
+
 void 
 delegpt_add_unused_targets(struct delegpt* dp)
 {
@@ -324,13 +383,16 @@ delegpt_count_targets(struct delegpt* dp)
 }
 
 size_t 
-delegpt_count_missing_targets(struct delegpt* dp)
+delegpt_count_missing_targets(struct delegpt* dp, int* alllame)
 {
 	struct delegpt_ns* ns;
-	size_t n = 0;
-	for(ns = dp->nslist; ns; ns = ns->next)
-		if(!ns->resolved)
-			n++;
+	size_t n = 0, nlame = 0;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved) continue;
+		n++;
+		if(ns->lame) nlame++;
+	}
+	if(alllame && n == nlame) *alllame = 1;
 	return n;
 }
 
@@ -412,7 +474,8 @@ delegpt_rrset_add_ns(struct delegpt* dp, struct regional* region,
 			(size_t)sldns_read_uint16(nsdata->rr_data[i]))
 			continue; /* bad format */
 		/* add rdata of NS (= wirefmt dname), skip rdatalen bytes */
-		if(!delegpt_add_ns(dp, region, nsdata->rr_data[i]+2, lame))
+		if(!delegpt_add_ns(dp, region, nsdata->rr_data[i]+2, lame,
+			NULL, UNBOUND_DNS_PORT))
 			return 0;
 	}
 	return 1;
@@ -429,7 +492,6 @@ delegpt_add_rrset_A(struct delegpt* dp, struct regional* region,
 	log_assert(!dp->dp_type_mlc);
         memset(&sa, 0, len);
         sa.sin_family = AF_INET;
-        sa.sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
         for(i=0; i<d->count; i++) {
                 if(d->rr_len[i] != 2 + INET_SIZE)
                         continue;
@@ -453,7 +515,6 @@ delegpt_add_rrset_AAAA(struct delegpt* dp, struct regional* region,
 	log_assert(!dp->dp_type_mlc);
         memset(&sa, 0, len);
         sa.sin6_family = AF_INET6;
-        sa.sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
         for(i=0; i<d->count; i++) {
                 if(d->rr_len[i] != 2 + INET6_SIZE) /* rdatalen + len of IP6 */
                         continue;
@@ -555,6 +616,7 @@ void delegpt_free_mlc(struct delegpt* dp)
 	while(n) {
 		nn = n->next;
 		free(n->name);
+		free(n->tls_auth_name);
 		free(n);
 		n = nn;
 	}
@@ -577,7 +639,8 @@ int delegpt_set_name_mlc(struct delegpt* dp, uint8_t* name)
 	return (dp->name != NULL);
 }
 
-int delegpt_add_ns_mlc(struct delegpt* dp, uint8_t* name, uint8_t lame)
+int delegpt_add_ns_mlc(struct delegpt* dp, uint8_t* name, uint8_t lame,
+	char* tls_auth_name, int port)
 {
 	struct delegpt_ns* ns;
 	size_t len;
@@ -598,20 +661,37 @@ int delegpt_add_ns_mlc(struct delegpt* dp, uint8_t* name, uint8_t lame)
 	}
 	ns->next = dp->nslist;
 	dp->nslist = ns;
+	ns->cache_lookup_count = 0;
 	ns->resolved = 0;
 	ns->got4 = 0;
 	ns->got6 = 0;
 	ns->lame = (uint8_t)lame;
 	ns->done_pside4 = 0;
 	ns->done_pside6 = 0;
+	ns->port = port;
+	if(tls_auth_name) {
+		ns->tls_auth_name = strdup(tls_auth_name);
+		if(!ns->tls_auth_name) {
+			free(ns->name);
+			free(ns);
+			return 0;
+		}
+	} else {
+		ns->tls_auth_name = NULL;
+	}
 	return 1;
 }
 
 int delegpt_add_addr_mlc(struct delegpt* dp, struct sockaddr_storage* addr,
-	socklen_t addrlen, uint8_t bogus, uint8_t lame, char* tls_auth_name)
+	socklen_t addrlen, uint8_t bogus, uint8_t lame, char* tls_auth_name,
+	int port)
 {
 	struct delegpt_addr* a;
 	log_assert(dp->dp_type_mlc);
+	if(port != -1) {
+		log_assert(port>0);
+		sockaddr_store_port(addr, addrlen, port);
+	}
 	/* check for duplicates */
 	if((a = delegpt_find_addr(dp, addr, addrlen))) {
 		if(bogus)
@@ -663,8 +743,14 @@ int delegpt_add_target_mlc(struct delegpt* dp, uint8_t* name, size_t namelen,
 		else	ns->got4 = 1;
 		if(ns->got4 && ns->got6)
 			ns->resolved = 1;
+	} else {
+		if(addr_is_ip6(addr, addrlen))
+			ns->done_pside6 = 1;
+		else	ns->done_pside4 = 1;
 	}
-	return delegpt_add_addr_mlc(dp, addr, addrlen, bogus, lame, NULL);
+	log_assert(ns->port>0);
+	return delegpt_add_addr_mlc(dp, addr, addrlen, bogus, lame,
+		ns->tls_auth_name, ns->port);
 }
 
 size_t delegpt_get_mem(struct delegpt* dp)

iterator/iter_delegpt.h

@@ -101,6 +101,8 @@ struct delegpt_ns {
 	uint8_t* name;
 	/** length of name */
 	size_t namelen;
+	/** number of cache lookups for the name */
+	int cache_lookup_count;
 	/** 
 	 * If the name has been resolved. false if not queried for yet.
 	 * true if the A, AAAA queries have been generated.
@@ -126,6 +128,11 @@ struct delegpt_ns {
 	 * Also enabled if a parent-side cache entry exists, or a parent-side
 	 * negative-cache entry exists. */
 	uint8_t done_pside6;
+	/** the TLS authentication name, (if not NULL) to use. */
+	char* tls_auth_name;
+	/** the port to use; it should mostly be the default 53 but configured
+	 *  upstreams can provide nondefault ports. */
+	int port;
 };
 
 /**
@@ -191,10 +198,12 @@ int delegpt_set_name(struct delegpt* dp, struct regional* regional,
  * @param regional: where to allocate the info.
  * @param name: domain name in wire format.
  * @param lame: name is lame, disprefer it.
+ * @param tls_auth_name: TLS authentication name (or NULL).
+ * @param port: port to use for resolved addresses.
  * @return false on error.
  */
-int delegpt_add_ns(struct delegpt* dp, struct regional* regional, 
-	uint8_t* name, uint8_t lame);
+int delegpt_add_ns(struct delegpt* dp, struct regional* regional,
+	uint8_t* name, uint8_t lame, char* tls_auth_name, int port);
 
 /**
  * Add NS rrset; calls add_ns repeatedly.
@@ -271,12 +280,14 @@ int delegpt_add_rrset(struct delegpt* dp, struct regional* regional,
  * @param bogus: if address is bogus.
  * @param lame: if address is lame.
  * @param tls_auth_name: TLS authentication name (or NULL).
+ * @param port: the port to use; if -1 the port is taken from addr.
  * @param additions: will be set to 1 if a new address is added
  * @return false on error.
  */
-int delegpt_add_addr(struct delegpt* dp, struct regional* regional, 
+int delegpt_add_addr(struct delegpt* dp, struct regional* regional,
 	struct sockaddr_storage* addr, socklen_t addrlen,
-	uint8_t bogus, uint8_t lame, char* tls_auth_name, int* additions);
+	uint8_t bogus, uint8_t lame, char* tls_auth_name, int port,
+	int* additions);
 
 /** 
  * Find NS record in name list of delegation point.
@@ -321,9 +332,10 @@ void delegpt_add_unused_targets(struct delegpt* dp);
 /**
  * Count number of missing targets. These are ns names with no resolved flag.
  * @param dp: delegation point.
+ * @param alllame: if set, check if all the missing targets are lame.
  * @return number of missing targets (or 0).
  */
-size_t delegpt_count_missing_targets(struct delegpt* dp);
+size_t delegpt_count_missing_targets(struct delegpt* dp, int* alllame);
 
 /** count total number of targets in dp */
 size_t delegpt_count_targets(struct delegpt* dp);
@@ -404,22 +416,27 @@ int delegpt_set_name_mlc(struct delegpt* dp, uint8_t* name);
  * @param dp: must have been created with delegpt_create_mlc. 
  * @param name: the name to add.
  * @param lame: the name is lame, disprefer.
+ * @param tls_auth_name: TLS authentication name (or NULL).
+ * @param port: port to use for resolved addresses.
  * @return false on error.
  */
-int delegpt_add_ns_mlc(struct delegpt* dp, uint8_t* name, uint8_t lame);
+int delegpt_add_ns_mlc(struct delegpt* dp, uint8_t* name, uint8_t lame,
+	char* tls_auth_name, int port);
 
 /**
  * add an address to a malloced delegation point.
- * @param dp: must have been created with delegpt_create_mlc. 
+ * @param dp: must have been created with delegpt_create_mlc.
  * @param addr: the address.
  * @param addrlen: the length of addr.
  * @param bogus: if address is bogus.
  * @param lame: if address is lame.
  * @param tls_auth_name: TLS authentication name (or NULL).
+ * @param port: the port to use; if -1 the port is taken from addr.
  * @return false on error.
  */
 int delegpt_add_addr_mlc(struct delegpt* dp, struct sockaddr_storage* addr,
-	socklen_t addrlen, uint8_t bogus, uint8_t lame, char* tls_auth_name);
+	socklen_t addrlen, uint8_t bogus, uint8_t lame, char* tls_auth_name,
+	int port);
 
 /**
  * Add target address to the delegation point.
@@ -440,4 +457,29 @@ int delegpt_add_target_mlc(struct delegpt* dp, uint8_t* name, size_t namelen,
 /** get memory in use by dp */
 size_t delegpt_get_mem(struct delegpt* dp);
 
+/**
+ * See if the addr is on the result list.
+ * @param dp: delegation point.
+ * @param find: the pointer is searched for on the result list.
+ * @return 1 if found, 0 if not found.
+ */
+int delegpt_addr_on_result_list(struct delegpt* dp, struct delegpt_addr* find);
+
+/**
+ * Remove the addr from the usable list.
+ * @param dp: the delegation point.
+ * @param del: the addr to remove from the list, the pointer is searched for.
+ */
+void delegpt_usable_list_remove_addr(struct delegpt* dp,
+	struct delegpt_addr* del);
+
+/**
+ * Add the delegpt_addr back to the result list, if it is not already on
+ * the result list. Also removes it from the usable list.
+ * @param dp: delegation point.
+ * @param a: addr to add, nothing happens if it is already on the result list.
+ *	It is removed from the usable list.
+ */
+void delegpt_add_to_result_list(struct delegpt* dp, struct delegpt_addr* a);
+
 #endif /* ITERATOR_ITER_DELEGPT_H */

iterator/iter_fwd.c

@@ -200,21 +200,27 @@ read_fwds_name(struct config_stub* s)
 }
 
 /** set fwd host names */
-static int 
+static int
 read_fwds_host(struct config_stub* s, struct delegpt* dp)
 {
 	struct config_strlist* p;
 	uint8_t* dname;
-	size_t dname_len;
+	char* tls_auth_name;
+	int port;
 	for(p = s->hosts; p; p = p->next) {
 		log_assert(p->str);
-		dname = sldns_str2wire_dname(p->str, &dname_len);
+		dname = authextstrtodname(p->str, &port, &tls_auth_name);
 		if(!dname) {
 			log_err("cannot parse forward %s server name: '%s'", 
 				s->name, p->str);
 			return 0;
 		}
-		if(!delegpt_add_ns_mlc(dp, dname, 0)) {
+#if ! defined(HAVE_SSL_SET1_HOST) && ! defined(HAVE_X509_VERIFY_PARAM_SET1_HOST)
+		if(tls_auth_name)
+			log_err("no name verification functionality in "
+				"ssl library, ignored name for %s", p->str);
+#endif
+		if(!delegpt_add_ns_mlc(dp, dname, 0, tls_auth_name, port)) {
 			free(dname);
 			log_err("out of memory");
 			return 0;
@@ -245,7 +251,7 @@ read_fwds_addr(struct config_stub* s, struct delegpt* dp)
 				"ssl library, ignored name for %s", p->str);
 #endif
 		if(!delegpt_add_addr_mlc(dp, &addr, addrlen, 0, 0,
-			tls_auth_name)) {
+			tls_auth_name, -1)) {
 			log_err("out of memory");
 			return 0;
 		}

iterator/iter_hints.c

@@ -99,8 +99,8 @@ ah(struct delegpt* dp, const char* sv, const char* ip)
 		log_err("could not parse %s", sv);
 		return 0;
 	}
-	if(!delegpt_add_ns_mlc(dp, dname, 0) ||
-	   !extstrtoaddr(ip, &addr, &addrlen) ||
+	if(!delegpt_add_ns_mlc(dp, dname, 0, NULL, UNBOUND_DNS_PORT) ||
+	   !extstrtoaddr(ip, &addr, &addrlen, UNBOUND_DNS_PORT) ||
 	   !delegpt_add_target_mlc(dp, dname, dname_len,
 		&addr, addrlen, 0, 0)) {
 		free(dname);
@@ -213,21 +213,27 @@ read_stubs_name(struct config_stub* s)
 }
 
 /** set stub host names */
-static int 
+static int
 read_stubs_host(struct config_stub* s, struct delegpt* dp)
 {
 	struct config_strlist* p;
-	size_t dname_len;
 	uint8_t* dname;
+	char* tls_auth_name;
+	int port;
 	for(p = s->hosts; p; p = p->next) {
 		log_assert(p->str);
-		dname = sldns_str2wire_dname(p->str, &dname_len);
+		dname = authextstrtodname(p->str, &port, &tls_auth_name);
 		if(!dname) {
 			log_err("cannot parse stub %s nameserver name: '%s'", 
 				s->name, p->str);
 			return 0;
 		}
-		if(!delegpt_add_ns_mlc(dp, dname, 0)) {
+#if ! defined(HAVE_SSL_SET1_HOST) && ! defined(HAVE_X509_VERIFY_PARAM_SET1_HOST)
+		if(tls_auth_name)
+			log_err("no name verification functionality in "
+				"ssl library, ignored name for %s", p->str);
+#endif
+		if(!delegpt_add_ns_mlc(dp, dname, 0, tls_auth_name, port)) {
 			free(dname);
 			log_err("out of memory");
 			return 0;
@@ -258,7 +264,7 @@ read_stubs_addr(struct config_stub* s, struct delegpt* dp)
 				"ssl library, ignored name for %s", p->str);
 #endif
 		if(!delegpt_add_addr_mlc(dp, &addr, addrlen, 0, 0,
-			auth_name)) {
+			auth_name, -1)) {
 			log_err("out of memory");
 			return 0;
 		}
@@ -338,7 +344,7 @@ read_root_hints(struct iter_hints* hints, char* fname)
 		if(sldns_wirerr_get_type(rr, rr_len, dname_len)
 			== LDNS_RR_TYPE_NS) {
 			if(!delegpt_add_ns_mlc(dp, sldns_wirerr_get_rdata(rr,
-				rr_len, dname_len), 0)) {
+				rr_len, dname_len), 0, NULL, UNBOUND_DNS_PORT)) {
 				log_err("out of memory reading root hints");
 				goto stop_read;
 			}

iterator/iter_priv.c

@@ -207,28 +207,6 @@ size_t priv_get_mem(struct iter_priv* priv)
 	return sizeof(*priv) + regional_get_mem(priv->region);
 }
 
-/** remove RR from msgparse RRset, return true if rrset is entirely bad */
-static int
-remove_rr(const char* str, sldns_buffer* pkt, struct rrset_parse* rrset,
-	struct rr_parse* prev, struct rr_parse** rr, struct sockaddr_storage* addr, socklen_t addrlen)
-{
-	if(verbosity >= VERB_QUERY && rrset->dname_len <= LDNS_MAX_DOMAINLEN && str) {
-		uint8_t buf[LDNS_MAX_DOMAINLEN+1];
-		dname_pkt_copy(pkt, buf, rrset->dname);
-		log_name_addr(VERB_QUERY, str, buf, addr, addrlen);
-	}
-	if(prev)
-		prev->next = (*rr)->next;
-	else	rrset->rr_first = (*rr)->next;
-	if(rrset->rr_last == *rr)
-		rrset->rr_last = prev;
-	rrset->rr_count --;
-	rrset->size -= (*rr)->size;
-	/* rr struct still exists, but is unlinked, so that in the for loop
-	 * the rr->next works fine to continue. */
-	return rrset->rr_count == 0;
-}
-
 int priv_rrset_bad(struct iter_priv* priv, sldns_buffer* pkt,
 	struct rrset_parse* rrset)
 {
@@ -261,7 +239,7 @@ int priv_rrset_bad(struct iter_priv* priv, sldns_buffer* pkt,
 					INET_SIZE);
 				memmove(&addr, &sa, len);
 				if(priv_lookup_addr(priv, &addr, len)) {
-					if(remove_rr("sanitize: removing public name with private address", pkt, rrset, prev, &rr, &addr, len))
+					if(msgparse_rrset_remove_rr("sanitize: removing public name with private address", pkt, rrset, prev, rr, &addr, len))
 						return 1;
 					continue;
 				}
@@ -284,7 +262,7 @@ int priv_rrset_bad(struct iter_priv* priv, sldns_buffer* pkt,
 					INET6_SIZE);
 				memmove(&addr, &sa, len);
 				if(priv_lookup_addr(priv, &addr, len)) {
-					if(remove_rr("sanitize: removing public name with private address", pkt, rrset, prev, &rr, &addr, len))
+					if(msgparse_rrset_remove_rr("sanitize: removing public name with private address", pkt, rrset, prev, rr, &addr, len))
 						return 1;
 					continue;
 				}

iterator/iter_resptype.c

@@ -42,6 +42,7 @@
 #include "config.h"
 #include "iterator/iter_resptype.h"
 #include "iterator/iter_delegpt.h"
+#include "iterator/iterator.h"
 #include "services/cache/dns.h"
 #include "util/net_help.h"
 #include "util/data/dname.h"
@@ -105,7 +106,8 @@ response_type_from_cache(struct dns_msg* msg,
 
 enum response_type 
 response_type_from_server(int rdset,
-	struct dns_msg* msg, struct query_info* request, struct delegpt* dp)
+	struct dns_msg* msg, struct query_info* request, struct delegpt* dp,
+	int* empty_nodata_found)
 {
 	uint8_t* origzone = (uint8_t*)"\000"; /* the default */
 	struct ub_packed_rrset_key* s;
@@ -113,7 +115,11 @@ response_type_from_server(int rdset,
 
 	if(!msg || !request)
 		return RESPONSE_TYPE_THROWAWAY;
-	
+	/* If the TC flag is set, the response is incomplete. Too large to
+	 * fit even in TCP or so. Discard it, it cannot be retrieved here. */
+	if((msg->rep->flags & BIT_TC))
+		return RESPONSE_TYPE_THROWAWAY;
+
 	/* If the message is NXDOMAIN, then it answers the question. */
 	if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_NXDOMAIN) {
 		/* make sure its not recursive when we don't want it to */
@@ -280,6 +286,22 @@ response_type_from_server(int rdset,
 
 	/* If we've gotten this far, this is NOERROR/NODATA (which could 
 	 * be an entirely empty message) */
+	/* For entirely empty messages, try again, at first, then accept
+	 * it it happens more. A regular noerror/nodata response has a soa
+	 * negative ttl value in the authority section. This makes it try
+	 * again at another authority. And decides between storing a 5 second
+	 * empty message or a 5 second servfail response. */
+	if(msg->rep->an_numrrsets == 0 && msg->rep->ns_numrrsets == 0 &&
+		msg->rep->ar_numrrsets == 0) {
+		if(empty_nodata_found) {
+			/* detect as throwaway at first, but accept later. */
+			(*empty_nodata_found)++;
+			if(*empty_nodata_found < EMPTY_NODATA_RETRY_COUNT)
+				return RESPONSE_TYPE_THROWAWAY;
+			return RESPONSE_TYPE_ANSWER;
+		}
+		return RESPONSE_TYPE_ANSWER;
+	}
 	/* check if recursive answer; saying it has empty cache */
 	if( (msg->rep->flags&BIT_RA) && !(msg->rep->flags&BIT_AA) && !rdset)
 		return RESPONSE_TYPE_REC_LAME;

iterator/iter_resptype.h

@@ -119,9 +119,11 @@ enum response_type response_type_from_cache(struct dns_msg* msg,
  * @param request: the request that generated the response.
  * @param dp: The delegation point that was being queried
  *          when the response was returned.
+ * @param empty_nodata_found: flag to keep track of empty nodata detection.
  * @return the response type (CNAME or ANSWER).
  */
 enum response_type response_type_from_server(int rdset, 
-	struct dns_msg* msg, struct query_info* request, struct delegpt* dp);
+	struct dns_msg* msg, struct query_info* request, struct delegpt* dp,
+	int* empty_nodata_found);
 
 #endif /* ITERATOR_ITER_RESPTYPE_H */

iterator/iter_scrub.c

@@ -346,6 +346,26 @@ soa_in_auth(struct msg_parse* msg)
 	return 0;
 }
 
+/** Check if type is allowed in the authority section */
+static int
+type_allowed_in_authority_section(uint16_t tp)
+{
+	if(tp == LDNS_RR_TYPE_SOA || tp == LDNS_RR_TYPE_NS ||
+		tp == LDNS_RR_TYPE_DS || tp == LDNS_RR_TYPE_NSEC ||
+		tp == LDNS_RR_TYPE_NSEC3)
+		return 1;
+	return 0;
+}
+
+/** Check if type is allowed in the additional section */
+static int
+type_allowed_in_additional_section(uint16_t tp)
+{
+	if(tp == LDNS_RR_TYPE_A || tp == LDNS_RR_TYPE_AAAA)
+		return 1;
+	return 0;
+}
+
 /**
  * This routine normalizes a response. This includes removing "irrelevant"
  * records from the answer and additional sections and (re)synthesizing
@@ -355,11 +375,13 @@ soa_in_auth(struct msg_parse* msg)
  * @param msg: msg to normalize.
  * @param qinfo: original query.
  * @param region: where to allocate synthesized CNAMEs.
+ * @param env: module env with config options.
  * @return 0 on error.
  */
 static int
 scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg, 
-	struct query_info* qinfo, struct regional* region)
+	struct query_info* qinfo, struct regional* region,
+	struct module_env* env)
 {
 	uint8_t* sname = qinfo->qname;
 	size_t snamelen = qinfo->qname_len;
@@ -511,6 +533,7 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 
 	/* Mark additional names from AUTHORITY */
 	while(rrset && rrset->section == LDNS_SECTION_AUTHORITY) {
+		/* protect internals of recursor by making sure to del these */
 		if(rrset->type==LDNS_RR_TYPE_DNAME ||
 			rrset->type==LDNS_RR_TYPE_CNAME ||
 			rrset->type==LDNS_RR_TYPE_A ||
@@ -519,6 +542,13 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 				"RRset:", pkt, msg, prev, &rrset);
 			continue;
 		}
+		/* Allowed list of types in the authority section */
+		if(env->cfg->harden_unknown_additional &&
+			!type_allowed_in_authority_section(rrset->type)) {
+			remove_rrset("normalize: removing irrelevant "
+				"RRset:", pkt, msg, prev, &rrset);
+			continue;
+		}
 		/* only one NS set allowed in authority section */
 		if(rrset->type==LDNS_RR_TYPE_NS) {
 			/* NS set must be pertinent to the query */
@@ -576,7 +606,6 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 	 * found in ANSWER and AUTHORITY. */
 	/* These records have not been marked OK previously */
 	while(rrset && rrset->section == LDNS_SECTION_ADDITIONAL) {
-		/* FIXME: what about other types? */
 		if(rrset->type==LDNS_RR_TYPE_A || 
 			rrset->type==LDNS_RR_TYPE_AAAA) 
 		{
@@ -589,6 +618,7 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 				continue;
 			}
 		}
+		/* protect internals of recursor by making sure to del these */
 		if(rrset->type==LDNS_RR_TYPE_DNAME || 
 			rrset->type==LDNS_RR_TYPE_CNAME ||
 			rrset->type==LDNS_RR_TYPE_NS) {
@@ -596,6 +626,13 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 				"RRset:", pkt, msg, prev, &rrset);
 			continue;
 		}
+		/* Allowed list of types in the additional section */
+		if(env->cfg->harden_unknown_additional &&
+			!type_allowed_in_additional_section(rrset->type)) {
+			remove_rrset("normalize: removing irrelevant "
+				"RRset:", pkt, msg, prev, &rrset);
+			continue;
+		}
 		prev = rrset;
 		rrset = rrset->rrset_all_next;
 	}
@@ -679,6 +716,56 @@ static int sanitize_nsec_is_overreach(sldns_buffer* pkt,
 	return 0;
 }
 
+/** Remove individual RRs, if the length is wrong. Returns true if the RRset
+ * has been removed. */
+static int
+scrub_sanitize_rr_length(sldns_buffer* pkt, struct msg_parse* msg,
+	struct rrset_parse* prev, struct rrset_parse** rrset, int* added_ede,
+	struct module_qstate* qstate)
+{
+	struct rr_parse* rr, *rr_prev = NULL;
+	for(rr = (*rrset)->rr_first; rr; rr = rr->next) {
+
+		/* Sanity check for length of records
+		 * An A record should be 6 bytes only
+		 * (2 bytes for length and 4 for IPv4 addr)*/
+		if((*rrset)->type == LDNS_RR_TYPE_A && rr->size != 6 ) {
+			if(!*added_ede) {
+				*added_ede = 1;
+				errinf_ede(qstate, "sanitize: records of inappropriate length have been removed.",
+					LDNS_EDE_OTHER);
+			}
+			if(msgparse_rrset_remove_rr("sanitize: removing type A RR of inappropriate length:",
+				pkt, *rrset, rr_prev, rr, NULL, 0)) {
+				remove_rrset("sanitize: removing type A RRset of inappropriate length:",
+					pkt, msg, prev, rrset);
+				return 1;
+			}
+			continue;
+		}
+
+		/* Sanity check for length of records
+		 * An AAAA record should be 18 bytes only
+		 * (2 bytes for length and 16 for IPv6 addr)*/
+		if((*rrset)->type == LDNS_RR_TYPE_AAAA && rr->size != 18 ) {
+			if(!*added_ede) {
+				*added_ede = 1;
+				errinf_ede(qstate, "sanitize: records of inappropriate length have been removed.",
+					LDNS_EDE_OTHER);
+			}
+			if(msgparse_rrset_remove_rr("sanitize: removing type AAAA RR of inappropriate length:",
+				pkt, *rrset, rr_prev, rr, NULL, 0)) {
+				remove_rrset("sanitize: removing type AAAA RRset of inappropriate length:",
+					pkt, msg, prev, rrset);
+				return 1;
+			}
+			continue;
+		}
+		rr_prev = rr;
+	}
+	return 0;
+}
+
 /**
  * Given a response event, remove suspect RRsets from the response.
  * "Suspect" rrsets are potentially poison. Note that this routine expects
@@ -691,15 +778,17 @@ static int sanitize_nsec_is_overreach(sldns_buffer* pkt,
  * @param zonename: name of server zone.
  * @param env: module environment with config and cache.
  * @param ie: iterator environment with private address data.
+ * @param qstate: for setting errinf for EDE error messages.
  * @return 0 on error.
  */
 static int
 scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg, 
 	struct query_info* qinfo, uint8_t* zonename, struct module_env* env,
-	struct iter_env* ie)
+	struct iter_env* ie, struct module_qstate* qstate)
 {
 	int del_addi = 0; /* if additional-holding rrsets are deleted, we
 		do not trust the normalized additional-A-AAAA any more */
+	int added_rrlen_ede = 0;
 	struct rrset_parse* rrset, *prev;
 	prev = NULL;
 	rrset = msg->rrset_first;
@@ -744,6 +833,14 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
 	rrset = msg->rrset_first;
 	while(rrset) {
 
+		/* Sanity check for length of records */
+		if(rrset->type == LDNS_RR_TYPE_A ||
+			rrset->type == LDNS_RR_TYPE_AAAA) {
+			if(scrub_sanitize_rr_length(pkt, msg, prev, &rrset,
+				&added_rrlen_ede, qstate))
+				continue;
+		}
+
 		/* remove private addresses */
 		if( (rrset->type == LDNS_RR_TYPE_A || 
 			rrset->type == LDNS_RR_TYPE_AAAA)) {
@@ -817,7 +914,8 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg,
 int 
 scrub_message(sldns_buffer* pkt, struct msg_parse* msg, 
 	struct query_info* qinfo, uint8_t* zonename, struct regional* region,
-	struct module_env* env, struct iter_env* ie)
+	struct module_env* env, struct module_qstate* qstate,
+	struct iter_env* ie)
 {
 	/* basic sanity checks */
 	log_nametypeclass(VERB_ALGO, "scrub for", zonename, LDNS_RR_TYPE_NS, 
@@ -846,10 +944,10 @@ scrub_message(sldns_buffer* pkt, struct msg_parse* msg,
 	}
 
 	/* normalize the response, this cleans up the additional.  */
-	if(!scrub_normalize(pkt, msg, qinfo, region))
+	if(!scrub_normalize(pkt, msg, qinfo, region, env))
 		return 0;
 	/* delete all out-of-zone information */
-	if(!scrub_sanitize(pkt, msg, qinfo, zonename, env, ie))
+	if(!scrub_sanitize(pkt, msg, qinfo, zonename, env, ie, qstate))
 		return 0;
 	return 1;
 }

iterator/iter_scrub.h

@@ -48,6 +48,7 @@ struct query_info;
 struct regional;
 struct module_env;
 struct iter_env;
+struct module_qstate;
 
 /**
  * Cleanup the passed dns message.
@@ -59,11 +60,13 @@ struct iter_env;
  *	Used to determine out of bailiwick information.
  * @param regional: where to allocate (new) parts of the message.
  * @param env: module environment with config settings and cache. 
+ * @param qstate: for setting errinf for EDE error messages.
  * @param ie: iterator module environment data.
  * @return: false if the message is total waste. true if scrubbed with success.
  */
 int scrub_message(struct sldns_buffer* pkt, struct msg_parse* msg, 
 	struct query_info* qinfo, uint8_t* zonename, struct regional* regional,
-	struct module_env* env, struct iter_env* ie);
+	struct module_env* env, struct module_qstate* qstate,
+	struct iter_env* ie);
 
 #endif /* ITERATOR_ITER_SCRUB_H */

iterator/iter_utils.c

@@ -70,8 +70,11 @@
 
 /** time when nameserver glue is said to be 'recent' */
 #define SUSPICION_RECENT_EXPIRY 86400
-/** penalty to validation failed blacklisted IPs */
-#define BLACKLIST_PENALTY (USEFUL_SERVER_TOP_TIMEOUT*4)
+
+/** if NAT64 is enabled and no NAT64 prefix is configured, first fall back to
+ * DNS64 prefix.  If that is not configured, fall back to this default value.
+ */
+static const char DEFAULT_NAT64_PREFIX[] = "64:ff9b::/96";
 
 /** fillup fetch policy array */
 static void
@@ -144,6 +147,7 @@ caps_white_apply_cfg(rbtree_type* ntree, struct config_file* cfg)
 int
 iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
 {
+	const char *nat64_prefix;
 	int i;
 	/* target fetch policy */
 	if(!read_fetch_policy(iter_env, cfg->target_fetch_policy))
@@ -174,9 +178,35 @@ iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
 		}
 
 	}
+
+	nat64_prefix = cfg->nat64_prefix;
+	if(!nat64_prefix)
+		nat64_prefix = cfg->dns64_prefix;
+	if(!nat64_prefix)
+		nat64_prefix = DEFAULT_NAT64_PREFIX;
+	if(!netblockstrtoaddr(nat64_prefix, 0, &iter_env->nat64_prefix_addr,
+		&iter_env->nat64_prefix_addrlen,
+		&iter_env->nat64_prefix_net)) {
+		log_err("cannot parse nat64-prefix netblock: %s", nat64_prefix);
+		return 0;
+	}
+	if(!addr_is_ip6(&iter_env->nat64_prefix_addr,
+		iter_env->nat64_prefix_addrlen)) {
+		log_err("nat64-prefix is not IPv6: %s", cfg->nat64_prefix);
+		return 0;
+	}
+	if(!prefixnet_is_nat64(iter_env->nat64_prefix_net)) {
+		log_err("nat64-prefix length it not 32, 40, 48, 56, 64 or 96: %s",
+			nat64_prefix);
+		return 0;
+	}
+
 	iter_env->supports_ipv6 = cfg->do_ip6;
 	iter_env->supports_ipv4 = cfg->do_ip4;
+	iter_env->use_nat64 = cfg->do_nat64;
 	iter_env->outbound_msg_retry = cfg->outbound_msg_retry;
+	iter_env->max_sent_count = cfg->max_sent_count;
+	iter_env->max_query_restarts = cfg->max_query_restarts;
 	return 1;
 }
 
@@ -240,7 +270,8 @@ iter_filter_unsuitable(struct iter_env* iter_env, struct module_env* env,
 	if(!iter_env->supports_ipv6 && addr_is_ip6(&a->addr, a->addrlen)) {
 		return -1; /* there is no ip6 available */
 	}
-	if(!iter_env->supports_ipv4 && !addr_is_ip6(&a->addr, a->addrlen)) {
+	if(!iter_env->supports_ipv4 && !iter_env->use_nat64 &&
+	   !addr_is_ip6(&a->addr, a->addrlen)) {
 		return -1; /* there is no ip4 available */
 	}
 	/* check lameness - need zone , class info */
@@ -367,6 +398,7 @@ iter_filter_order(struct iter_env* iter_env, struct module_env* env,
 	struct sock_list* blacklist, time_t prefetch)
 {
 	int got_num = 0, low_rtt = 0, swap_to_front, rtt_band = RTT_BAND, nth;
+	int alllame = 0;
 	size_t num_results;
 	struct delegpt_addr* a, *n, *prev=NULL;
 
@@ -376,7 +408,10 @@ iter_filter_order(struct iter_env* iter_env, struct module_env* env,
 	if(got_num == 0)
 		return 0;
 	if(low_rtt >= USEFUL_SERVER_TOP_TIMEOUT &&
-		(delegpt_count_missing_targets(dp) > 0 || open_target > 0)) {
+		/* If all missing (or not fully resolved) targets are lame,
+		 * then use the remaining lame address. */
+		((delegpt_count_missing_targets(dp, &alllame) > 0 && !alllame) ||
+		open_target > 0)) {
 		verbose(VERB_ALGO, "Bad choices, trying to get more choice");
 		return 0; /* we want more choice. The best choice is a bad one.
 			     return 0 to force the caller to fetch more */
@@ -657,10 +692,10 @@ dns_copy_msg(struct dns_msg* from, struct regional* region)
 void
 iter_dns_store(struct module_env* env, struct query_info* msgqinf,
 	struct reply_info* msgrep, int is_referral, time_t leeway, int pside,
-	struct regional* region, uint16_t flags)
+	struct regional* region, uint16_t flags, time_t qstarttime)
 {
 	if(!dns_cache_store(env, msgqinf, msgrep, is_referral, leeway,
-		pside, region, flags))
+		pside, region, flags, qstarttime))
 		log_err("out of memory: cannot store data in cache");
 }
 
@@ -743,9 +778,15 @@ iter_mark_pside_cycle_targets(struct module_qstate* qstate, struct delegpt* dp)
 
 int
 iter_dp_is_useless(struct query_info* qinfo, uint16_t qflags,
-	struct delegpt* dp)
+	struct delegpt* dp, int supports_ipv4, int supports_ipv6,
+	int use_nat64)
 {
 	struct delegpt_ns* ns;
+	struct delegpt_addr* a;
+
+	if(supports_ipv6 && use_nat64)
+		supports_ipv4 = 1;
+
 	/* check:
 	 *      o RD qflag is on.
 	 *      o no addresses are provided.
@@ -758,13 +799,24 @@ iter_dp_is_useless(struct query_info* qinfo, uint16_t qflags,
 	 */
 	if(!(qflags&BIT_RD))
 		return 0;
-	/* either available or unused targets */
-	if(dp->usable_list || dp->result_list)
-		return 0;
+	/* either available or unused targets,
+	 * if they exist, the dp is not useless. */
+	for(a = dp->usable_list; a; a = a->next_usable) {
+		if(!addr_is_ip6(&a->addr, a->addrlen) && supports_ipv4)
+			return 0;
+		else if(addr_is_ip6(&a->addr, a->addrlen) && supports_ipv6)
+			return 0;
+	}
+	for(a = dp->result_list; a; a = a->next_result) {
+		if(!addr_is_ip6(&a->addr, a->addrlen) && supports_ipv4)
+			return 0;
+		else if(addr_is_ip6(&a->addr, a->addrlen) && supports_ipv6)
+			return 0;
+	}
 
 	/* see if query is for one of the nameservers, which is glue */
-	if( (qinfo->qtype == LDNS_RR_TYPE_A ||
-		qinfo->qtype == LDNS_RR_TYPE_AAAA) &&
+	if( ((qinfo->qtype == LDNS_RR_TYPE_A && supports_ipv4) ||
+		(qinfo->qtype == LDNS_RR_TYPE_AAAA && supports_ipv6)) &&
 		dname_subdomain_c(qinfo->qname, dp->name) &&
 		delegpt_find_ns(dp, qinfo->qname, qinfo->qname_len))
 		return 1;
@@ -1195,6 +1247,9 @@ int iter_lookup_parent_glue_from_cache(struct module_env* env,
 	struct delegpt_ns* ns;
 	size_t num = delegpt_count_targets(dp);
 	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->cache_lookup_count > ITERATOR_NAME_CACHELOOKUP_MAX_PSIDE)
+			continue;
+		ns->cache_lookup_count++;
 		/* get cached parentside A */
 		akey = rrset_cache_lookup(env->rrset_cache, ns->name,
 			ns->namelen, LDNS_RR_TYPE_A, qinfo->qclass,
@@ -1291,8 +1346,7 @@ void iter_dec_attempts(struct delegpt* dp, int d, int outbound_msg_retry)
 	for(a=dp->target_list; a; a = a->next_target) {
 		if(a->attempts >= outbound_msg_retry) {
 			/* add back to result list */
-			a->next_result = dp->result_list;
-			dp->result_list = a;
+			delegpt_add_to_result_list(dp, a);
 		}
 		if(a->attempts > d)
 			a->attempts -= d;

iterator/iter_utils.h

@@ -62,6 +62,15 @@ struct ub_packed_rrset_key;
 struct module_stack;
 struct outside_network;
 
+/* max number of lookups in the cache for target nameserver names.
+ * This stops, for large delegations, N*N lookups in the cache. */
+#define ITERATOR_NAME_CACHELOOKUP_MAX	3
+/* max number of lookups in the cache for parentside glue for nameserver names
+ * This stops, for larger delegations, N*N lookups in the cache.
+ * It is a little larger than the nonpside max, so it allows a couple extra
+ * lookups of parent side glue. */
+#define ITERATOR_NAME_CACHELOOKUP_MAX_PSIDE	5
+
 /**
  * Process config options and set iterator module state.
  * Sets default values if no config is found.
@@ -132,6 +141,7 @@ struct dns_msg* dns_copy_msg(struct dns_msg* from, struct regional* regional);
  * 	can be prefetch-updates.
  * @param region: to copy modified (cache is better) rrs back to.
  * @param flags: with BIT_CD for dns64 AAAA translated queries.
+ * @param qstarttime: time of query start.
  * return void, because we are not interested in alloc errors,
  * 	the iterator and validator can operate on the results in their
  * 	scratch space (the qstate.region) and are not dependent on the cache.
@@ -140,7 +150,7 @@ struct dns_msg* dns_copy_msg(struct dns_msg* from, struct regional* regional);
  */
 void iter_dns_store(struct module_env* env, struct query_info* qinf,
 	struct reply_info* rep, int is_referral, time_t leeway, int pside,
-	struct regional* region, uint16_t flags);
+	struct regional* region, uint16_t flags, time_t qstarttime);
 
 /**
  * Select randomly with n/m probability.
@@ -175,10 +185,17 @@ void iter_mark_pside_cycle_targets(struct module_qstate* qstate,
  * @param qinfo: query name and type
  * @param qflags: query flags with RD flag
  * @param dp: delegpt to check.
+ * @param supports_ipv4: if we support ipv4 for lookups to the target.
+ * 	if not, then the IPv4 addresses are useless.
+ * @param supports_ipv6: if we support ipv6 for lookups to the target.
+ * 	if not, then the IPv6 addresses are useless.
+ * @param use_nat64: if we support NAT64 for lookups to the target.
+ *	if yes, IPv4 addresses are useful even if we don't support IPv4.
  * @return true if dp is useless.
  */
-int iter_dp_is_useless(struct query_info* qinfo, uint16_t qflags, 
-	struct delegpt* dp);
+int iter_dp_is_useless(struct query_info* qinfo, uint16_t qflags,
+	struct delegpt* dp, int supports_ipv4, int supports_ipv6,
+	int use_nat64);
 
 /**
  * See if qname has DNSSEC needs.  This is true if there is a trust anchor above

iterator/iterator.h

@@ -60,12 +60,11 @@ struct rbtree_type;
 /** max number of nxdomains allowed for target lookups for a query and
  * its subqueries */
 #define MAX_TARGET_NX		5
-/** max number of query restarts. Determines max number of CNAME chain. */
-#define MAX_RESTART_COUNT	11
+/** max number of nxdomains allowed for target lookups for a query and
+ * its subqueries when fallback has kicked in */
+#define MAX_TARGET_NX_FALLBACK	(MAX_TARGET_NX*2)
 /** max number of referrals. Makes sure resolver does not run away */
 #define MAX_REFERRAL_COUNT	130
-/** max number of queries-sent-out.  Make sure large NS set does not loop */
-#define MAX_SENT_COUNT		32
 /** max number of queries for which to perform dnsseclameness detection,
  * (rrsigs missing detection) after that, just pick up that response */
 #define DNSSEC_LAME_DETECT_COUNT 4
@@ -80,7 +79,7 @@ struct rbtree_type;
 /**
  * number of labels from QNAME that are always send individually when using
  * QNAME minimisation, even when the number of labels of the QNAME is bigger
- * tham MAX_MINIMISE_COUNT */
+ * than MAX_MINIMISE_COUNT */
 #define MINIMISE_ONE_LAB	4
 #define MINIMISE_MULTIPLE_LABS	(MAX_MINIMISE_COUNT - MINIMISE_ONE_LAB)
 /** at what query-sent-count to stop target fetch policy */
@@ -91,18 +90,22 @@ struct rbtree_type;
 extern int UNKNOWN_SERVER_NICENESS;
 /** maximum timeout before a host is deemed unsuitable, in msec. 
  * After host_ttl this will be timed out and the host will be tried again. 
- * Equals RTT_MAX_TIMEOUT
- */
-#define USEFUL_SERVER_TOP_TIMEOUT	120000
+ * Equals RTT_MAX_TIMEOUT, and thus when RTT_MAX_TIMEOUT is overwritten by
+ * config infra_cache_max_rtt, it will be overwritten as well. */
+extern int USEFUL_SERVER_TOP_TIMEOUT;
+/** penalty to validation failed blacklisted IPs
+ * Equals USEFUL_SERVER_TOP_TIMEOUT*4, and thus when RTT_MAX_TIMEOUT is
+ * overwritten by config infra_cache_max_rtt, it will be overwritten as well. */
+extern int BLACKLIST_PENALTY;
 /** RTT band, within this amount from the best, servers are chosen randomly.
  * Chosen so that the UNKNOWN_SERVER_NICENESS falls within the band of a 
  * fast server, this causes server exploration as a side benefit. msec. */
 #define RTT_BAND 400
-/** Start value for blacklisting a host, 2*USEFUL_SERVER_TOP_TIMEOUT in sec */
-#define INFRA_BACKOFF_INITIAL 240
+/** Number of retries for empty nodata packets before it is accepted. */
+#define EMPTY_NODATA_RETRY_COUNT 2
 
 /**
- * Global state for the iterator. 
+ * Global state for the iterator.
  */
 struct iter_env {
 	/** A flag to indicate whether or not we have an IPv6 route */
@@ -111,6 +114,18 @@ struct iter_env {
 	/** A flag to indicate whether or not we have an IPv4 route */
 	int supports_ipv4;
 
+	/** A flag to locally apply NAT64 to make IPv4 addrs into IPv6 */
+	int use_nat64;
+
+	/** NAT64 prefix address, cf. dns64_env->prefix_addr */
+	struct sockaddr_storage nat64_prefix_addr;
+
+	/** sizeof(sockaddr_in6) */
+	socklen_t nat64_prefix_addrlen;
+
+	/** CIDR mask length of NAT64 prefix */
+	int nat64_prefix_net;
+
 	/** A set of inetaddrs that should never be queried. */
 	struct iter_donotq* donotq;
 
@@ -140,6 +155,12 @@ struct iter_env {
 
 	/** number of retries on outgoing queries */
 	int outbound_msg_retry;
+
+	/** number of queries_sent */
+	int max_sent_count;
+
+	/** max number of query restarts to limit length of CNAME chain */
+	int max_query_restarts;
 };
 
 /**
@@ -217,6 +238,21 @@ enum iter_state {
 	FINISHED_STATE
 };
 
+/**
+ * Shared counters for queries.
+ */
+enum target_count_variables {
+	/** Reference count for the shared iter_qstate->target_count. */
+	TARGET_COUNT_REF = 0,
+	/** Number of target queries spawned for the query and subqueries. */
+	TARGET_COUNT_QUERIES,
+	/** Number of nxdomain responses encountered. */
+	TARGET_COUNT_NX,
+
+	/** This should stay last here, it is used for the allocation */
+	TARGET_COUNT_MAX,
+};
+
 /**
  * Per query state for the iterator module.
  */
@@ -304,21 +340,26 @@ struct iter_qstate {
 	/** the number of times this query has been restarted. */
 	int query_restart_count;
 
-	/** the number of times this query as followed a referral. */
+	/** the number of times this query has followed a referral. */
 	int referral_count;
 
 	/** number of queries fired off */
 	int sent_count;
 	
-	/** number of target queries spawned in [1], for this query and its
-	 * subqueries, the malloced-array is shared, [0] refcount.
-	 * in [2] the number of nxdomains is counted. */
+	/** malloced-array shared with this query and its subqueries. It keeps
+	 * track of the defined enum target_count_variables counters. */
 	int* target_count;
 
 	/** number of target lookups per delegation point. Reset to 0 after
 	 * receiving referral answer. Not shared with subqueries. */
 	int dp_target_count;
 
+	/** Delegation point that triggered the NXNS fallback; shared with
+	 * this query and its subqueries, count-referenced by the reference
+	 * counter in target_count.
+	 * This also marks the fallback activation. */
+	uint8_t** nxns_dp;
+
 	/** if true, already tested for ratelimiting and passed the test */
 	int ratelimit_ok;
 
@@ -376,10 +417,15 @@ struct iter_qstate {
 	 */
 	int refetch_glue;
 
+	/**
+	 * This flag detects that a completely empty nodata was received,
+	 * already so that it is accepted later. */
+	int empty_nodata_found;
+
 	/** list of pending queries to authoritative servers. */
 	struct outbound_list outlist;
 
-	/** QNAME minimisation state, RFC7816 */
+	/** QNAME minimisation state, RFC9156 */
 	enum minimisation_state minimisation_state;
 
 	/** State for capsfail: QNAME minimisation state for comparisons. */
@@ -412,7 +458,14 @@ struct iter_qstate {
 	/** true if there have been parse failures of reply packets */
 	int parse_failures;
 	/** a failure printout address for last received answer */
-	struct comm_reply* fail_reply;
+	union {
+		struct in_addr in;
+#ifdef AF_INET6
+		struct in6_addr in6;
+#endif
+	} fail_addr;
+	/** which fail_addr, 0 is nothing, 4 or 6 */
+	int fail_addr_type;
 };
 
 /**

libunbound/context.c

@@ -70,6 +70,7 @@ context_finalize(struct ub_ctx* ctx)
 	} else {
 		log_init(cfg->logfile, cfg->use_syslog, NULL);
 	}
+	ctx->pipe_pid = getpid();
 	cfg_apply_local_port_policy(cfg, 65536);
 	config_apply(cfg);
 	if(!modstack_setup(&ctx->mods, cfg->module_conf, ctx->env))

libunbound/context.h

@@ -89,6 +89,12 @@ struct ub_ctx {
 	pid_t bg_pid;
 	/** tid of bg worker thread */
 	ub_thread_type bg_tid;
+	/** pid when pipes are created. This was the process when the
+	 * setup was called. Helps with clean up, so we can tell after a fork
+	 * which side of the fork the delete is on. */
+	pid_t pipe_pid;
+	/** when threaded, the worker that exists in the created thread. */
+	struct libworker* thread_worker;
 
 	/** do threading (instead of forking) for async resolution */
 	int dothread;

libunbound/libunbound.c

@@ -305,11 +305,31 @@ ub_ctx_delete(struct ub_ctx* ctx)
 	int do_stop = 1;
 	if(!ctx) return;
 
+	/* if the delete is called but it has forked, and before the fork
+	 * the context was finalized, then the bg worker is not stopped
+	 * from here. There is one worker, but two contexts that refer to
+	 * it and only one should clean up, the one with getpid == pipe_pid.*/
+	if(ctx->created_bg && ctx->pipe_pid != getpid()) {
+		do_stop = 0;
+#ifndef USE_WINSOCK
+		/* Stop events from getting deregistered, if the backend is
+		 * epoll, the epoll fd is the same as the other process.
+		 * That process should deregister them. */
+		if(ctx->qq_pipe->listen_com)
+			ctx->qq_pipe->listen_com->event_added = 0;
+		if(ctx->qq_pipe->res_com)
+			ctx->qq_pipe->res_com->event_added = 0;
+		if(ctx->rr_pipe->listen_com)
+			ctx->rr_pipe->listen_com->event_added = 0;
+		if(ctx->rr_pipe->res_com)
+			ctx->rr_pipe->res_com->event_added = 0;
+#endif
+	}
 	/* see if bg thread is created and if threads have been killed */
 	/* no locks, because those may be held by terminated threads */
 	/* for processes the read pipe is closed and we see that on read */
 #ifdef HAVE_PTHREAD
-	if(ctx->created_bg && ctx->dothread) {
+	if(ctx->created_bg && ctx->dothread && do_stop) {
 		if(pthread_kill(ctx->bg_tid, 0) == ESRCH) {
 			/* thread has been killed */
 			do_stop = 0;
@@ -318,6 +338,23 @@ ub_ctx_delete(struct ub_ctx* ctx)
 #endif /* HAVE_PTHREAD */
 	if(do_stop)
 		ub_stop_bg(ctx);
+	if(ctx->created_bg && ctx->pipe_pid != getpid() && ctx->thread_worker) {
+		/* This delete is happening from a different process. Delete
+		 * the thread worker from this process memory space. The
+		 * thread is not there to do so, so it is freed here. */
+		struct ub_event_base* evbase = comm_base_internal(
+			ctx->thread_worker->base);
+		libworker_delete_event(ctx->thread_worker);
+		ctx->thread_worker = NULL;
+#ifdef USE_MINI_EVENT
+		ub_event_base_free(evbase);
+#else
+		/* cannot event_base_free, because the epoll_fd cleanup
+		 * in libevent could stop the original event_base in the
+		 * other process from working. */
+		free(evbase);
+#endif
+	}
 	libworker_delete_event(ctx->event_worker);
 
 	modstack_desetup(&ctx->mods, ctx->env);
@@ -951,7 +988,7 @@ ub_ctx_set_fwd(struct ub_ctx* ctx, const char* addr)
 	lock_basic_unlock(&ctx->cfglock);
 
 	/* check syntax for addr */
-	if(!extstrtoaddr(addr, &storage, &stlen)) {
+	if(!extstrtoaddr(addr, &storage, &stlen, UNBOUND_DNS_PORT)) {
 		errno=EINVAL;
 		return UB_SYNTAX;
 	}
@@ -1031,7 +1068,7 @@ int ub_ctx_set_stub(struct ub_ctx* ctx, const char* zone, const char* addr,
 	if(addr) {
 		struct sockaddr_storage storage;
 		socklen_t stlen;
-		if(!extstrtoaddr(addr, &storage, &stlen)) {
+		if(!extstrtoaddr(addr, &storage, &stlen, UNBOUND_DNS_PORT)) {
 			errno=EINVAL;
 			return UB_SYNTAX;
 		}

libunbound/libworker.c

@@ -62,6 +62,7 @@
 #include "util/random.h"
 #include "util/config_file.h"
 #include "util/netevent.h"
+#include "util/proxy_protocol.h"
 #include "util/storage/lookup3.h"
 #include "util/storage/slabhash.h"
 #include "util/net_help.h"
@@ -168,15 +169,15 @@ libworker_setup(struct ub_ctx* ctx, int is_bg, struct ub_event_base* eb)
 		hints_delete(w->env->hints);
 		w->env->hints = NULL;
 	}
-	if(cfg->ssl_upstream || (cfg->tls_cert_bundle && cfg->tls_cert_bundle[0]) || cfg->tls_win_cert) {
-		w->sslctx = connect_sslctx_create(NULL, NULL,
-			cfg->tls_cert_bundle, cfg->tls_win_cert);
-		if(!w->sslctx) {
-			/* to make the setup fail after unlock */
-			hints_delete(w->env->hints);
-			w->env->hints = NULL;
-		}
+#ifdef HAVE_SSL
+	w->sslctx = connect_sslctx_create(NULL, NULL,
+		cfg->tls_cert_bundle, cfg->tls_win_cert);
+	if(!w->sslctx) {
+		/* to make the setup fail after unlock */
+		hints_delete(w->env->hints);
+		w->env->hints = NULL;
 	}
+#endif
 	if(!w->is_bg || w->is_bg_thread) {
 		lock_basic_unlock(&ctx->cfglock);
 	}
@@ -265,6 +266,7 @@ libworker_setup(struct ub_ctx* ctx, int is_bg, struct ub_event_base* eb)
 	w->env->kill_sub = &mesh_state_delete;
 	w->env->detect_cycle = &mesh_detect_cycle;
 	comm_base_timept(w->base, &w->env->now, &w->env->now_tv);
+	pp_init(&sldns_write_uint16, &sldns_write_uint32);
 	return w;
 }
 
@@ -395,6 +397,7 @@ int libworker_bg(struct ub_ctx* ctx)
 		w = libworker_setup(ctx, 1, NULL);
 		if(!w) return UB_NOMEM;
 		w->is_bg_thread = 1;
+		ctx->thread_worker = w;
 #ifdef ENABLE_LOCK_CHECKS
 		w->thread_num = 1; /* for nicer DEBUG checklocks */
 #endif
@@ -604,6 +607,8 @@ setup_qinfo_edns(struct libworker* w, struct ctx_query* q,
 	edns->opt_list_out = NULL;
 	edns->opt_list_inplace_cb_out = NULL;
 	edns->padding_block_size = 0;
+	edns->cookie_present = 0;
+	edns->cookie_valid = 0;
 	if(sldns_buffer_capacity(w->back->udp_buff) < 65535)
 		edns->udp_size = (uint16_t)sldns_buffer_capacity(
 			w->back->udp_buff);
@@ -650,7 +655,7 @@ int libworker_fg(struct ub_ctx* ctx, struct ctx_query* q)
 	}
 	/* process new query */
 	if(!mesh_new_callback(w->env->mesh, &qinfo, qflags, &edns, 
-		w->back->udp_buff, qid, libworker_fg_done_cb, q)) {
+		w->back->udp_buff, qid, libworker_fg_done_cb, q, 0)) {
 		free(qinfo.qname);
 		return UB_NOMEM;
 	}
@@ -730,7 +735,7 @@ int libworker_attach_mesh(struct ub_ctx* ctx, struct ctx_query* q,
 	if(async_id)
 		*async_id = q->querynum;
 	if(!mesh_new_callback(w->env->mesh, &qinfo, qflags, &edns, 
-		w->back->udp_buff, qid, libworker_event_done_cb, q)) {
+		w->back->udp_buff, qid, libworker_event_done_cb, q, 0)) {
 		free(qinfo.qname);
 		return UB_NOMEM;
 	}
@@ -867,7 +872,7 @@ handle_newq(struct libworker* w, uint8_t* buf, uint32_t len)
 	q->w = w;
 	/* process new query */
 	if(!mesh_new_callback(w->env->mesh, &qinfo, qflags, &edns, 
-		w->back->udp_buff, qid, libworker_bg_done_cb, q)) {
+		w->back->udp_buff, qid, libworker_bg_done_cb, q, 0)) {
 		add_bg_result(w, q, NULL, UB_NOMEM, NULL, 0);
 	}
 	free(qinfo.qname);
@@ -882,9 +887,10 @@ void libworker_alloc_cleanup(void* arg)
 
 struct outbound_entry* libworker_send_query(struct query_info* qinfo,
 	uint16_t flags, int dnssec, int want_dnssec, int nocaps,
+	int check_ratelimit,
 	struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
 	size_t zonelen, int tcp_upstream, int ssl_upstream, char* tls_auth_name,
-	struct module_qstate* q)
+	struct module_qstate* q, int* was_ratelimited)
 {
 	struct libworker* w = (struct libworker*)q->env->worker;
 	struct outbound_entry* e = (struct outbound_entry*)regional_alloc(
@@ -893,9 +899,10 @@ struct outbound_entry* libworker_send_query(struct query_info* qinfo,
 		return NULL;
 	e->qstate = q;
 	e->qsent = outnet_serviced_query(w->back, qinfo, flags, dnssec,
-		want_dnssec, nocaps, tcp_upstream, ssl_upstream,
+		want_dnssec, nocaps, check_ratelimit, tcp_upstream, ssl_upstream,
 		tls_auth_name, addr, addrlen, zone, zonelen, q,
-		libworker_handle_service_reply, e, w->back->udp_buff, q->env);
+		libworker_handle_service_reply, e, w->back->udp_buff, q->env,
+		was_ratelimited);
 	if(!e->qsent) {
 		return NULL;
 	}
@@ -976,10 +983,11 @@ void worker_sighandler(int ATTR_UNUSED(sig), void* ATTR_UNUSED(arg))
 struct outbound_entry* worker_send_query(struct query_info* ATTR_UNUSED(qinfo),
 	uint16_t ATTR_UNUSED(flags), int ATTR_UNUSED(dnssec),
 	int ATTR_UNUSED(want_dnssec), int ATTR_UNUSED(nocaps),
+	int ATTR_UNUSED(check_ratelimit),
 	struct sockaddr_storage* ATTR_UNUSED(addr), socklen_t ATTR_UNUSED(addrlen),
 	uint8_t* ATTR_UNUSED(zone), size_t ATTR_UNUSED(zonelen), int ATTR_UNUSED(tcp_upstream),
 	int ATTR_UNUSED(ssl_upstream), char* ATTR_UNUSED(tls_auth_name),
-	struct module_qstate* ATTR_UNUSED(q))
+	struct module_qstate* ATTR_UNUSED(q), int* ATTR_UNUSED(was_ratelimited))
 {
 	log_assert(0);
 	return 0;

libunbound/python/libunbound.i

@@ -36,6 +36,9 @@
 %begin %{
 /* store state of warning output, restored at later pop */
 #pragma GCC diagnostic push
+/* ignore warnings for pragma below, where for older GCC it can produce a
+   warning if the cast-function-type warning is absent. */
+#pragma GCC diagnostic ignored "-Wpragmas"
 /* ignore gcc8 METH_NOARGS function cast warnings for swig function pointers */
 #pragma GCC diagnostic ignored "-Wcast-function-type"
 %}
@@ -860,6 +863,9 @@ Result: ['74.125.43.147', '74.125.43.99', '74.125.43.103', '74.125.43.104']
 %inline %{
   //SWIG will see the ub_ctx as a class
   struct ub_ctx {
+          /* Dummy member, so the struct is not empty, MSVC complains about
+           * that. */
+          int dummy;
   };
 %}
 

libunbound/unbound-event.h

@@ -52,8 +52,8 @@
  * unbound was compiled with, otherwise it wouldn't work, the event and
  * event_base structures would be different.
  */
-#ifndef _UB_UNBOUND_EVENT_H
-#define _UB_UNBOUND_EVENT_H
+#ifndef UB_UNBOUND_EVENT_H
+#define UB_UNBOUND_EVENT_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -230,7 +230,7 @@ int ub_ctx_set_event(struct ub_ctx* ctx, struct event_base* base);
  * @param callback: this is called on completion of the resolution.
  * 	It is called as:
  * 	void callback(void* mydata, int rcode, void* packet, int packet_len,
- * 		int sec, char* why_bogus)
+ * 		int sec, char* why_bogus, int was_ratelimited)
  * 	with mydata: the same as passed here, you may pass NULL,
  * 	with rcode: 0 on no error, nonzero for mostly SERVFAIL situations,
  *		this is a DNS rcode.
@@ -241,6 +241,7 @@ int ub_ctx_set_event(struct ub_ctx* ctx, struct event_base* base);
  *	with packet_len: length in bytes of the packet buffer.
  *	with sec: 0 if insecure, 1 if bogus, 2 if DNSSEC secure.
  *	with why_bogus: text string explaining why it is bogus (or NULL).
+ *	with was_ratelimited: if the query was ratelimited.
  *	These point to buffers inside unbound; do not deallocate the packet or
  *	error string.
  *
@@ -261,4 +262,4 @@ int ub_resolve_event(struct ub_ctx* ctx, const char* name, int rrtype,
 }
 #endif
 
-#endif /* _UB_UNBOUND_H */
+#endif /* UB_UNBOUND_EVENT_H */

libunbound/unbound.h

@@ -4,22 +4,22 @@
  * Copyright (c) 2007, NLnet Labs. All rights reserved.
  *
  * This software is open source.
- * 
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
- * 
+ *
  * Redistributions of source code must retain the above copyright notice,
  * this list of conditions and the following disclaimer.
- * 
+ *
  * Redistributions in binary form must reproduce the above copyright notice,
  * this list of conditions and the following disclaimer in the documentation
  * and/or other materials provided with the distribution.
- * 
+ *
  * Neither the name of the NLNET LABS nor the names of its contributors may
  * be used to endorse or promote products derived from this software without
  * specific prior written permission.
- * 
+ *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -36,7 +36,7 @@
 /**
  * \file
  *
- * This file contains functions to resolve DNS queries and 
+ * This file contains functions to resolve DNS queries and
  * validate the answers. Synchronously and asynchronously.
  *
  * Several ways to use this interface from an application wishing
@@ -65,7 +65,7 @@
  *	... or process() calls my_callback() with results.
  *
  *      ... if the application has nothing more to do, wait for answer
- *      ub_wait(ctx); 
+ *      ub_wait(ctx);
  *
  * Application threaded. Blocking.
  *	Blocking, same as above. The current thread does the work.
@@ -83,7 +83,7 @@
  * CRYPTO_set_id_callback and CRYPTO_set_locking_callback.
  *
  * If no threading is compiled in, the above async example uses fork(2) to
- * create a process to perform the work. The forked process exits when the 
+ * create a process to perform the work. The forked process exits when the
  * calling process exits, or ctx_delete() is called.
  * Otherwise, for asynchronous with threading, a worker thread is created.
  *
@@ -94,8 +94,8 @@
  * The second calls another worker thread (or process) to perform the work.
  * And no buffers need to be set up, but a context-switch happens.
  */
-#ifndef _UB_UNBOUND_H
-#define _UB_UNBOUND_H
+#ifndef UB_UNBOUND_H
+#define UB_UNBOUND_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -128,10 +128,10 @@ struct ub_result {
 	/** the class asked for */
 	int qclass;
 
-	/** 
-	 * a list of network order DNS rdata items, terminated with a 
+	/**
+	 * a list of network order DNS rdata items, terminated with a
 	 * NULL pointer, so that data[0] is the first result entry,
-	 * data[1] the second, and the last entry is NULL. 
+	 * data[1] the second, and the last entry is NULL.
 	 * If there was no data, data[0] is NULL.
 	 */
 	char** data;
@@ -139,8 +139,8 @@ struct ub_result {
 	/** the length in bytes of the data items, len[i] for data[i] */
 	int* len;
 
-	/** 
-	 * canonical name for the result (the final cname). 
+	/**
+	 * canonical name for the result (the final cname).
 	 * zero terminated string.
 	 * May be NULL if no canonical name exists.
 	 */
@@ -165,9 +165,9 @@ struct ub_result {
 	 */
 	int havedata;
 
-	/** 
+	/**
 	 * If there was no data, and the domain did not exist, this is true.
-	 * If it is false, and there was no data, then the domain name 
+	 * If it is false, and there was no data, then the domain name
 	 * is purported to exist, but the requested data type is not available.
 	 */
 	int nxdomain;
@@ -182,19 +182,19 @@ struct ub_result {
 	 */
 	int secure;
 
-	/** 
-	 * If the result was not secure (secure==0), and this result is due 
+	/**
+	 * If the result was not secure (secure==0), and this result is due
 	 * to a security failure, bogus is true.
 	 * This means the data has been actively tampered with, signatures
-	 * failed, expected signatures were not present, timestamps on 
+	 * failed, expected signatures were not present, timestamps on
 	 * signatures were out of date and so on.
 	 *
-	 * If !secure and !bogus, this can happen if the data is not secure 
-	 * because security is disabled for that domain name. 
+	 * If !secure and !bogus, this can happen if the data is not secure
+	 * because security is disabled for that domain name.
 	 * This means the data is from a domain where data is not signed.
 	 */
 	int bogus;
-	
+
 	/**
 	 * If the result is bogus this contains a string (zero terminated)
 	 * that describes the failure.  There may be other errors as well
@@ -222,7 +222,7 @@ struct ub_result {
  * The readable function definition looks like:
  * void my_callback(void* my_arg, int err, struct ub_result* result);
  * It is called with
- *	void* my_arg: your pointer to a (struct of) data of your choice, 
+ *	void* my_arg: your pointer to a (struct of) data of your choice,
  *		or NULL.
  *	int err: if 0 all is OK, otherwise an error occurred and no results
  *	     are forthcoming.
@@ -301,8 +301,8 @@ int ub_ctx_set_option(struct ub_ctx* ctx, const char* opt, const char* val);
  * 	This is a power-users interface that lets you specify all sorts
  * 	of options.
  * @param str: the string is malloced and returned here. NULL on error.
- * 	The caller must free() the string.  In cases with multiple 
- * 	entries (auto-trust-anchor-file), a newline delimited list is 
+ * 	The caller must free() the string.  In cases with multiple
+ * 	entries (auto-trust-anchor-file), a newline delimited list is
  * 	returned in the string.
  * @return 0 if OK else an error code (malloc failure, syntax error).
  */
@@ -321,10 +321,10 @@ int ub_ctx_get_option(struct ub_ctx* ctx, const char* opt, char** str);
 int ub_ctx_config(struct ub_ctx* ctx, const char* fname);
 
 /**
- * Set machine to forward DNS queries to, the caching resolver to use. 
- * IP4 or IP6 address. Forwards all DNS requests to that machine, which 
- * is expected to run a recursive resolver. If the proxy is not 
- * DNSSEC-capable, validation may fail. Can be called several times, in 
+ * Set machine to forward DNS queries to, the caching resolver to use.
+ * IP4 or IP6 address. Forwards all DNS requests to that machine, which
+ * is expected to run a recursive resolver. If the proxy is not
+ * DNSSEC-capable, validation may fail. Can be called several times, in
  * that case the addresses are used as backup servers.
  *
  * To read the list of nameservers from /etc/resolv.conf (from DHCP or so),
@@ -389,7 +389,7 @@ int ub_ctx_resolvconf(struct ub_ctx* ctx, const char* fname);
 
 /**
  * Read list of hosts from the filename given.
- * Usually "/etc/hosts". 
+ * Usually "/etc/hosts".
  * These addresses are not flagged as DNSSEC secure when queried for.
  *
  * @param ctx: context.
@@ -403,7 +403,7 @@ int ub_ctx_hosts(struct ub_ctx* ctx, const char* fname);
 /**
  * Add a trust anchor to the given context.
  * The trust anchor is a string, on one line, that holds a valid DNSKEY or
- * DS RR. 
+ * DS RR.
  * @param ctx: context.
  *	At this time it is only possible to add trusted keys before the
  *	first resolve is done.
@@ -465,7 +465,7 @@ int ub_ctx_debugout(struct ub_ctx* ctx, void* out);
  * Set debug verbosity for the context
  * Output is directed to stderr.
  * @param ctx: context.
- * @param d: debug level, 0 is off, 1 is very minimal, 2 is detailed, 
+ * @param d: debug level, 0 is off, 1 is very minimal, 2 is detailed,
  *	and 3 is lots.
  * @return 0 if OK, else error.
  */
@@ -474,10 +474,10 @@ int ub_ctx_debuglevel(struct ub_ctx* ctx, int d);
 /**
  * Set a context behaviour for asynchronous action.
  * @param ctx: context.
- * @param dothread: if true, enables threading and a call to resolve_async() 
+ * @param dothread: if true, enables threading and a call to resolve_async()
  *	creates a thread to handle work in the background.
  *	If false, a process is forked to handle work in the background.
- *	Changes to this setting after async() calls have been made have 
+ *	Changes to this setting after async() calls have been made have
  *	no effect (delete and re-create the context to change).
  * @return 0 if OK, else error.
  */
@@ -495,7 +495,7 @@ int ub_poll(struct ub_ctx* ctx);
 
 /**
  * Wait for a context to finish with results. Calls ub_process() after
- * the wait for you. After the wait, there are no more outstanding 
+ * the wait for you. After the wait, there are no more outstanding
  * asynchronous queries.
  * @param ctx: context.
  * @return: 0 if OK, else error.
@@ -530,11 +530,11 @@ int ub_process(struct ub_ctx* ctx);
  * @param rrtype: type of RR in host order, 1 is A (address).
  * @param rrclass: class of RR in host order, 1 is IN (for internet).
  * @param result: the result data is returned in a newly allocated result
- * 	structure. May be NULL on return, return value is set to an error 
+ * 	structure. May be NULL on return, return value is set to an error
  * 	in that case (out of memory).
  * @return 0 if OK, else error.
  */
-int ub_resolve(struct ub_ctx* ctx, const char* name, int rrtype, 
+int ub_resolve(struct ub_ctx* ctx, const char* name, int rrtype,
 	int rrclass, struct ub_result** result);
 
 /**
@@ -561,11 +561,11 @@ int ub_resolve(struct ub_ctx* ctx, const char* name, int rrtype,
  * 	If an error happens during processing, your callback will be called
  * 	with error set to a nonzero value (and result==NULL).
  * @param async_id: if you pass a non-NULL value, an identifier number is
- *	returned for the query as it is in progress. It can be used to 
+ *	returned for the query as it is in progress. It can be used to
  *	cancel the query.
  * @return 0 if OK, else error.
  */
-int ub_resolve_async(struct ub_ctx* ctx, const char* name, int rrtype, 
+int ub_resolve_async(struct ub_ctx* ctx, const char* name, int rrtype,
 	int rrclass, void* mydata, ub_callback_type callback, int* async_id);
 
 /**
@@ -589,7 +589,7 @@ int ub_cancel(struct ub_ctx* ctx, int async_id);
  */
 void ub_resolve_free(struct ub_result* result);
 
-/** 
+/**
  * Convert error value to a human readable string.
  * @param err: error code from one of the libunbound functions.
  * 	The error codes are from the type enum ub_ctx_err.
@@ -605,7 +605,7 @@ const char* ub_strerror(int err);
 int ub_ctx_print_local_zones(struct ub_ctx* ctx);
 
 /**
- * Add a new zone with the zonetype to the local authority info of the 
+ * Add a new zone with the zonetype to the local authority info of the
  * library.
  * @param ctx: context.  Is finalized by the routine.
  * @param zone_name: name of the zone in text, "example.com"
@@ -613,7 +613,7 @@ int ub_ctx_print_local_zones(struct ub_ctx* ctx);
  * @param zone_type: type of the zone (like for unbound.conf) in text.
  * @return 0 if OK, else error.
  */
-int ub_ctx_zone_add(struct ub_ctx* ctx, const char *zone_name, 
+int ub_ctx_zone_add(struct ub_ctx* ctx, const char *zone_name,
 	const char *zone_type);
 
 /**
@@ -649,7 +649,7 @@ int ub_ctx_data_remove(struct ub_ctx* ctx, const char *data);
  */
 const char* ub_version(void);
 
-/** 
+/**
  * Some global statistics that are not in struct stats_info,
  * this struct is shared on a shm segment (shm-key in unbound.conf)
  */
@@ -695,13 +695,22 @@ struct ub_server_stats {
 	long long num_queries;
 	/** number of queries that have been dropped/ratelimited by ip. */
 	long long num_queries_ip_ratelimited;
+	/** number of queries with a valid DNS Cookie. */
+	long long num_queries_cookie_valid;
+	/** number of queries with only the client part of the DNS Cookie. */
+	long long num_queries_cookie_client;
+	/** number of queries with invalid DNS Cookie. */
+	long long num_queries_cookie_invalid;
 	/** number of queries that had a cache-miss. */
 	long long num_queries_missed_cache;
 	/** number of prefetch queries - cachehits with prefetch */
 	long long num_queries_prefetch;
-
+	/** number of queries which are too late to process */
+	long long num_queries_timed_out;
+	/** the longest wait time in the queue */
+	long long max_query_time_us;
 	/**
-	 * Sum of the querylistsize of the worker for 
+	 * Sum of the querylistsize of the worker for
 	 * every query that missed cache. To calculate average.
 	 */
 	long long sum_query_list_size;
@@ -725,6 +734,8 @@ struct ub_server_stats {
 	long long qtcp;
 	/** number of outgoing queries over TCP */
 	long long qtcp_outgoing;
+	/** number of outgoing queries over UDP */
+	long long qudp_outgoing;
 	/** number of queries over (DNS over) TLS */
 	long long qtls;
 	/** number of queries over (DNS over) HTTPS */
@@ -771,12 +782,12 @@ struct ub_server_stats {
 	long long tcp_accept_usage;
 	/** expired answers served from cache */
 	long long ans_expired;
-	/** histogram data exported to array 
+	/** histogram data exported to array
 	 * if the array is the same size, no data is lost, and
 	 * if all histograms are same size (is so by default) then
 	 * adding up works well. */
 	long long hist[UB_STATS_BUCKET_NUM];
-	
+
 	/** number of message cache entries */
 	long long msg_cache_count;
 	/** number of rrset cache entries */
@@ -786,6 +797,11 @@ struct ub_server_stats {
 	/** number of key cache entries */
 	long long key_cache_count;
 
+	/** maximum number of collisions in the msg cache */
+	long long msg_cache_max_collisions;
+	/** maximum number of collisions in the rrset cache */
+	long long rrset_cache_max_collisions;
+
 	/** number of queries that used dnscrypt */
 	long long num_query_dnscrypt_crypted;
 	/** number of queries that queried dnscrypt certificates */
@@ -817,6 +833,8 @@ struct ub_server_stats {
 	/** number of queries answered from edns-subnet specific data, and
 	 * the answer was from the edns-subnet cache. */
 	long long num_query_subnet_cache;
+	/** number of queries served from cachedb */
+	long long num_query_cachedb;
 	/** number of bytes in the stream wait buffers */
 	long long mem_stream_wait;
 	/** number of bytes in the HTTP2 query buffers */
@@ -829,7 +847,7 @@ struct ub_server_stats {
 	long long rpz_action[UB_STATS_RPZ_ACTION_NUM];
 };
 
-/** 
+/**
  * Statistics to send over the control pipe when asked
  * This struct is made to be memcopied, sent in binary.
  * shm mapped with (number+1) at num_threads+1, with first as total
@@ -858,4 +876,4 @@ struct ub_stats_info {
 }
 #endif
 
-#endif /* _UB_UNBOUND_H */
+#endif /* UB_UNBOUND_H */

libunbound/worker.h

@@ -58,6 +58,7 @@ struct query_info;
  * @param dnssec: if set, EDNS record will have DO bit set.
  * @param want_dnssec: signatures needed.
  * @param nocaps: ignore capsforid(if in config), do not perturb qname.
+ * @param check_ratelimit: if set, will check ratelimit before sending out.
  * @param addr: where to.
  * @param addrlen: length of addr.
  * @param zone: delegation point name.
@@ -67,14 +68,17 @@ struct query_info;
  * @param tls_auth_name: if ssl_upstream, use this name with TLS
  * 	authentication.
  * @param q: which query state to reactivate upon return.
+ * @param was_ratelimited: it will signal back if the query failed to pass the
+ *	ratelimit check.
  * @return: false on failure (memory or socket related). no query was
  *      sent.
  */
 struct outbound_entry* libworker_send_query(struct query_info* qinfo,
 	uint16_t flags, int dnssec, int want_dnssec, int nocaps,
+	int check_ratelimit,
 	struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
 	size_t zonelen, int tcp_upstream, int ssl_upstream, char* tls_auth_name,
-	struct module_qstate* q);
+	struct module_qstate* q, int* was_ratelimited);
 
 /** process incoming serviced query replies from the network */
 int libworker_handle_service_reply(struct comm_point* c, void* arg, int error,
@@ -110,6 +114,7 @@ void worker_sighandler(int sig, void* arg);
  * @param dnssec: if set, EDNS record will have DO bit set.
  * @param want_dnssec: signatures needed.
  * @param nocaps: ignore capsforid(if in config), do not perturb qname.
+ * @param check_ratelimit: if set, will check ratelimit before sending out.
  * @param addr: where to.
  * @param addrlen: length of addr.
  * @param zone: wireformat dname of the zone.
@@ -119,14 +124,17 @@ void worker_sighandler(int sig, void* arg);
  * @param tls_auth_name: if ssl_upstream, use this name with TLS
  * 	authentication.
  * @param q: which query state to reactivate upon return.
+ * @param was_ratelimited: it will signal back if the query failed to pass the
+ *	ratelimit check.
  * @return: false on failure (memory or socket related). no query was
  *      sent.
  */
 struct outbound_entry* worker_send_query(struct query_info* qinfo,
 	uint16_t flags, int dnssec, int want_dnssec, int nocaps,
+	int check_ratelimit,
 	struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
 	size_t zonelen, int tcp_upstream, int ssl_upstream, char* tls_auth_name,
-	struct module_qstate* q);
+	struct module_qstate* q, int* was_ratelimited);
 
 /** 
  * process control messages from the main thread. Frees the control 

makedist.sh

@@ -268,10 +268,12 @@ if [ "$DOWIN" = "yes" ]; then
 		# before 1.0.1i need --cross-compile-prefix=i686-w64-mingw32-
 		if test "$mw64" = "mingw64"; then
 			sslflags="no-asm -DOPENSSL_NO_CAPIENG mingw64"
+			sspdll="/usr/x86_64-w64-mingw32/sys-root/mingw/bin/libssp-0.dll"
 		else
 			sslflags="no-asm -DOPENSSL_NO_CAPIENG mingw"
+			sspdll="/usr/i686-w64-mingw32/sys-root/mingw/bin/libssp-0.dll"
 		fi
-		if test -f /usr/x86_64-w64-mingw32/sys-root/mingw/bin/libssp-0.dll; then
+		if test -f "$sspdll"; then
 			# stack protector lib needs to link in to make
 			# -lws2_32 work in openssl link stage
 			SSPLIB="-l:libssp.a"
@@ -356,8 +358,8 @@ if [ "$DOWIN" = "yes" ]; then
 	replace_version "configure.ac" "$version" "$version2"
     	version="$version2"
     	info "Rebuilding configure script (autoconf) snapshot."
-    	autoconf || error_cleanup "Autoconf failed."
-    	autoheader || error_cleanup "Autoheader failed."
+	autoconf -f || error_cleanup "Autoconf failed."
+	autoheader -f || error_cleanup "Autoheader failed."
     	rm -r autom4te* || echo "ignored"
     fi
 
@@ -439,8 +441,8 @@ if [ "$DOWIN" = "yes" ]; then
 	cp ../../sslsharedinstall/lib64/libssl.dll.a libunbound/.
     fi
     cp ../../unbound_shared/unbound.h ../../unbound_shared/.libs/libunbound*.dll ../../unbound_shared/.libs/libunbound.dll.a ../../unbound_shared/.libs/libunbound.a ../../unbound_shared/.libs/libunbound*.def ../../sslsharedinstall/bin/libcrypto*.dll ../../sslsharedinstall/bin/libssl*.dll ../../wxpinstall/bin/libexpat*.dll ../../wxpinstall/lib/libexpat.dll.a libunbound/.
-    if test -f /usr/x86_64-w64-mingw32/sys-root/mingw/bin/libssp-0.dll; then
-	    cp /usr/x86_64-w64-mingw32/sys-root/mingw/bin/libssp-0.dll libunbound/.
+    if test -f "$sspdll"; then
+	    cp "$sspdll" libunbound/.
     fi
     # zipfile
     zip -r ../$file LICENSE README.txt unbound.exe unbound-anchor.exe unbound-host.exe unbound-control.exe unbound-checkconf.exe unbound-service-install.exe unbound-service-remove.exe unbound-control-setup.cmd example.conf service.conf root.key unbound-website.url create_unbound_ad_servers.cmd warmup.cmd unbound_cache.cmd Changelog libunbound
@@ -509,7 +511,7 @@ if [ `uname -s | grep -i -c darwin` -ne 0 ]; then
 fi
 
 info "Building configure script (autoreconf)."
-autoreconf || error_cleanup "Autoconf failed."
+autoreconf -f || error_cleanup "Autoconf failed."
 
 rm -r autom4te* || error_cleanup "Failed to remove autoconf cache directory."
 
@@ -554,7 +556,7 @@ fi
 
 if [ "$RECONFIGURE" = "yes" ]; then
     info "Rebuilding configure script (autoconf) snapshot."
-    autoreconf || error_cleanup "Autoconf failed."
+    autoreconf -f || error_cleanup "Autoconf failed."
     rm -r autom4te* || error_cleanup "Failed to remove autoconf cache directory."
 fi
 

pythonmod/doc/examples/example0-1.py

@@ -1,9 +1,9 @@
 def init(id, cfg):
-   log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, cfg.python_script))
+   log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, mod_env['script']))
    return True
 
 def init_standard(id, env):
-   log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, env.cfg.port, env.cfg.python_script))
+   log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, env.cfg.port, mod_env['script']))
    return True
 
 def deinit(id):

pythonmod/doc/examples/example0.rst

@@ -50,7 +50,7 @@ Script file must contain four compulsory functions:
 ::
 
    def init(id, cfg):
-      log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, cfg.python_script))
+      log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, mod_env['script']))
       return True
 
 
@@ -69,7 +69,7 @@ Script file must contain four compulsory functions:
 ::
 
     def init_standard(id, env):
-       log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, env.cfg.port, env.cfg.python_script))
+       log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, env.cfg.port, mod_env['script']))
        return True
 
 

pythonmod/doc/modules/config.rst

@@ -129,7 +129,7 @@ config_file
    
    .. attribute:: ifs
    
-      Interface description strings (IP addresses).
+      List of interface description strings (IP addresses).
 
    .. attribute:: num_out_ifs
    
@@ -138,7 +138,7 @@ config_file
 
    .. attribute:: out_ifs
    
-      Outgoing interface description strings (IP addresses).
+      List of outgoing interface description strings (IP addresses).
       
    .. attribute:: root_hints
    
@@ -339,4 +339,5 @@ config_file
 
    .. attribute:: python_script
    
-      Python script file.
+      Linked list of Python script files.
+      Deprecated; `mod_env['script']` should be used instead.

pythonmod/doc/modules/env.rst

@@ -6,8 +6,11 @@ Global variables
 
 .. envvar:: mod_env
 
-   Module environment, contains data pointer for module-specific data.
-   See :class:`pythonmod_env`.
+   Module environment, it is the 'data' pointer for module-specific data
+   in :class:`pythonmod_env`.
+   It is initialized as a dictionary with the 'script' key pointing to the
+   module's python script.
+   It can be further populated during runtime for module-specific data.
 
 
 Predefined constants

pythonmod/doc/modules/functions.rst

@@ -60,7 +60,7 @@ EDNS options
 
 .. function:: edns_opt_list_remove(list, code);
 
-    Remove an ENDS option code from the list.
+    Remove an EDNS option code from the list.
     .. note:: All :class:`edns_option` with the code will be removed
 
     :param list: linked list of :class:`edns_option`

pythonmod/doc/modules/struct.rst

@@ -77,7 +77,7 @@ module_qstate
 
     .. attribute:: edns_opts_back_in_iter
 
-        Iterator for `ends_opts_back_in`.
+        Iterator for `edns_opts_back_in`.
 
     .. attribute:: edns_opts_front_out
 

pythonmod/examples/edns.py

@@ -80,7 +80,7 @@ def init_standard(id, env):
     ..note:: The previously accessible configuration options can now be found in
              env.cfg.
     """
-    log_info("python: inited script {}".format(env.cfg.python_script))
+    log_info("python: inited script {}".format(mod_env['script']))
 
     # Register EDNS option 65001 as a known EDNS option.
     if not register_edns_option(env, 65001, bypass_cache_stage=True,

pythonmod/examples/inplace_callbacks.py

@@ -34,6 +34,9 @@
  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  POSSIBILITY OF SUCH DAMAGE.
 '''
+
+import os
+
 #Try:
 # - dig @localhost nlnetlabs.nl +ednsopt=65002:
 #       This query *could* be answered from cache. If so, unbound will reply
@@ -207,7 +210,7 @@ def inplace_servfail_callback(qinfo, qstate, rep, rcode, edns, opt_list_out,
 
     """
     log_info("python: called back while servfail.")
-    # Append the example ENDS option
+    # Append the example EDNS option
     b = bytearray.fromhex("")
     edns_opt_list_append(opt_list_out, 65003, b, region)
 
@@ -242,6 +245,36 @@ def inplace_query_callback(qinfo, flags, qstate, addr, zone, region, **kwargs):
     return True
 
 
+def inplace_query_response_callback(qstate, response, **kwargs):
+    """
+    Function that will be registered as an inplace callback function.
+    It will be called after receiving a reply from a backend server.
+
+    :param qstate: module qstate. opt_lists are available here;
+    :param response: struct dns_msg. The reply received from the backend server;
+    :param **kwargs: Dictionary that may contain parameters added in a future
+                     release.
+    """
+    log_dns_msg(
+        "python: incoming reply from {}{}".format(qstate.reply.addr, os.linesep),
+        response.qinfo, response.rep
+    )
+    return True
+
+
+def inplace_edns_back_parsed_call(qstate, **kwargs):
+    """
+    Function that will be registered as an inplace callback function.
+    It will be called after EDNS is parsed on a reply from a backend server..
+
+    :param qstate: module qstate. opt_lists are available here;
+    :param **kwargs: Dictionary that may contain parameters added in a future
+                     release.
+    """
+    log_info("python: edns parsed")
+    return True
+
+
 def init_standard(id, env):
     """
     New version of the init function.
@@ -254,7 +287,7 @@ def init_standard(id, env):
              env.cfg.
 
     """
-    log_info("python: inited script {}".format(env.cfg.python_script))
+    log_info("python: inited script {}".format(mod_env['script']))
 
     # Register the inplace_reply_callback function as an inplace callback
     # function when answering a resolved query.
@@ -281,6 +314,16 @@ def init_standard(id, env):
     if not register_inplace_cb_query(inplace_query_callback, env, id):
         return False
 
+    # Register the inplace_edns_back_parsed_call function as an inplace callback
+    # for when a reply is received from a backend server.
+    if not register_inplace_cb_query_response(inplace_query_response_callback, env, id):
+        return False
+
+    # Register the inplace_edns_back_parsed_call function as an inplace callback
+    # for when EDNS is parsed on a reply from a backend server.
+    if not register_inplace_cb_edns_back_parsed_call(inplace_edns_back_parsed_call, env, id):
+        return False
+
     return True
 
 

pythonmod/examples/log.py

@@ -87,7 +87,7 @@ def logDnsMsg(qstate):
     print "-"*100
 
 def init(id, cfg):
-   log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, cfg.python_script))
+   log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, mod_env['script']))
    return True
 
 def deinit(id):

pythonmod/interface.i

@@ -86,6 +86,20 @@
      }
      return list;
    }
+
+   /* converts an array of strings (char**) to a List of strings */
+   PyObject* CharArrayAsStringList(char** array, int len) {
+     PyObject* list;
+     int i;
+
+     if(!array||len==0) return PyList_New(0);
+
+     list = PyList_New(len);
+     for (i=0; i < len; i++) {
+            PyList_SET_ITEM(list, i, PyString_FromString(array[i]));
+     }
+     return list;
+   }
 %}
 
 /* ************************************************************************************ *
@@ -609,9 +623,9 @@ struct mesh_reply {
    struct comm_reply query_reply;
 };
 
-%rename(_addr) comm_reply::addr;
+%rename(_addr) comm_reply::client_addr;
 struct comm_reply {
-   struct sockaddr_storage addr;
+   struct sockaddr_storage client_addr;
 };
 
 %extend comm_reply {
@@ -712,9 +726,10 @@ struct module_env {
     /* --- services --- */
     struct outbound_entry* (*send_query)(struct query_info* qinfo,
         uint16_t flags, int dnssec, int want_dnssec, int nocaps,
+        int check_ratelimit,
         struct sockaddr_storage* addr, socklen_t addrlen,
         uint8_t* zone, size_t zonelen, int tcp_upstream, int ssl_upstream,
-        char* tls_auth_name, struct module_qstate* q);
+        char* tls_auth_name, struct module_qstate* q, int* was_ratelimited);
     void (*detach_subs)(struct module_qstate* qstate);
     int (*attach_sub)(struct module_qstate* qstate,
         struct query_info* qinfo, uint16_t qflags, int prime,
@@ -951,6 +966,9 @@ struct config_str2list {
 /* ************************************************************************************ *
    Structure config_file
  * ************************************************************************************ */
+%ignore config_file::ifs;
+%ignore config_file::out_ifs;
+%ignore config_file::python_script;
 struct config_file {
    int verbosity;
    int stat_interval;
@@ -1034,6 +1052,25 @@ struct config_file {
    struct config_strlist* python_script;
 };
 
+%inline %{
+   PyObject* _get_ifs_tuple(struct config_file* cfg) {
+      return CharArrayAsStringList(cfg->ifs, cfg->num_ifs);
+   }
+   PyObject* _get_ifs_out_tuple(struct config_file* cfg) {
+      return CharArrayAsStringList(cfg->out_ifs, cfg->num_out_ifs);
+   }
+%}
+
+%extend config_file {
+   %pythoncode %{
+        ifs = property(_unboundmodule._get_ifs_tuple)
+        out_ifs = property(_unboundmodule._get_ifs_out_tuple)
+
+        def _deprecated_python_script(self): return "cfg.python_script is deprecated, you can use `mod_env['script']` instead."
+        python_script = property(_deprecated_python_script)
+   %}
+}
+
 /* ************************************************************************************ *
    ASN: Adding structures related to forwards_lookup and dns_cache_find_delegation
  * ************************************************************************************ */
@@ -1374,9 +1411,10 @@ int set_return_msg(struct module_qstate* qstate,
 /* Functions which we will need to lookup delegations */
 struct delegpt* dns_cache_find_delegation(struct module_env* env,
         uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
-        struct regional* region, struct dns_msg** msg, uint32_t timenow);
+        struct regional* region, struct dns_msg** msg, uint32_t timenow,
+        int noexpiredabove, uint8_t* expiretop, size_t expiretoplen);
 int iter_dp_is_useless(struct query_info* qinfo, uint16_t qflags,
-        struct delegpt* dp);
+        struct delegpt* dp, int supports_ipv4, int supports_ipv6, int use_nat64);
 struct iter_hints_stub* hints_lookup_stub(struct iter_hints* hints,
         uint8_t* qname, uint16_t qclass, struct delegpt* dp);
 
@@ -1403,10 +1441,12 @@ struct delegpt* find_delegation(struct module_qstate* qstate, char *nm, size_t n
     qinfo.qclass = LDNS_RR_CLASS_IN;
 
     while(1) {
-        dp = dns_cache_find_delegation(qstate->env, (uint8_t*)nm, nmlen, qinfo.qtype, qinfo.qclass, region, &msg, timenow);
+        dp = dns_cache_find_delegation(qstate->env, (uint8_t*)nm, nmlen, qinfo.qtype, qinfo.qclass, region, &msg, timenow, 0, NULL, 0);
         if(!dp)
             return NULL;
-        if(iter_dp_is_useless(&qinfo, BIT_RD, dp)) {
+        if(iter_dp_is_useless(&qinfo, BIT_RD, dp,
+                qstate->env->cfg->do_ip4, qstate->env->cfg->do_ip6,
+                qstate->env->cfg->do_nat64)) {
             if (dname_is_root((uint8_t*)nm))
                 return NULL;
             nm = (char*)dp->name;
@@ -1547,13 +1587,15 @@ int edns_opt_list_append(struct edns_option** list, uint16_t code, size_t len,
         struct comm_reply* repinfo, struct regional* region,
         struct timeval* start_time, int id, void* python_callback)
     {
-        PyObject *func, *py_edns, *py_qstate, *py_opt_list_out, *py_qinfo;
-        PyObject *py_rep, *py_repinfo, *py_region;
+        PyObject *func = NULL, *py_edns = NULL, *py_qstate = NULL;
+        PyObject *py_opt_list_out = NULL, *py_qinfo = NULL;
+        PyObject *py_rep = NULL, *py_repinfo = NULL, *py_region = NULL;
         PyObject *py_args = NULL, *py_kwargs = NULL, *result = NULL;
         int res = 0;
         double py_start_time = ((double)start_time->tv_sec) + ((double)start_time->tv_usec) / 1.0e6;
 
         PyGILState_STATE gstate = PyGILState_Ensure();
+
         func = (PyObject *) python_callback;
         py_edns = SWIG_NewPointerObj((void*) edns, SWIGTYPE_p_edns_data, 0);
         py_qstate = SWIG_NewPointerObj((void*) qstate,
@@ -1564,20 +1606,24 @@ int edns_opt_list_append(struct edns_option** list, uint16_t code, size_t len,
         py_rep = SWIG_NewPointerObj((void*) rep, SWIGTYPE_p_reply_info, 0);
         py_repinfo = SWIG_NewPointerObj((void*) repinfo, SWIGTYPE_p_comm_reply, 0);
         py_region = SWIG_NewPointerObj((void*) region, SWIGTYPE_p_regional, 0);
-        if(py_qinfo && py_qstate && py_rep && py_edns && py_opt_list_out
-                && py_region && py_repinfo) {
-                py_args = Py_BuildValue("(OOOiOOO)", py_qinfo, py_qstate, py_rep,
-                        rcode, py_edns, py_opt_list_out, py_region);
-                py_kwargs = Py_BuildValue("{s:O,s:d}", "repinfo", py_repinfo, "start_time",
-                          py_start_time);
-                if(py_args && py_kwargs) {
-                        result = PyObject_Call(func, py_args, py_kwargs);
-                } else {
-                        log_err("pythonmod: malloc failure in python_inplace_cb_reply_generic");
-                }
-        } else {
-                log_err("pythonmod: malloc failure in python_inplace_cb_reply_generic");
+        if(!(py_qinfo && py_qstate && py_rep && py_edns && py_opt_list_out
+                && py_region && py_repinfo)) {
+                log_err("pythonmod: swig pointer failure in python_inplace_cb_reply_generic");
+                goto out;
         }
+        py_args = Py_BuildValue("(OOOiOOO)", py_qinfo, py_qstate, py_rep,
+                rcode, py_edns, py_opt_list_out, py_region);
+        py_kwargs = Py_BuildValue("{s:O,s:d}", "repinfo", py_repinfo, "start_time",
+                py_start_time);
+        if(!(py_args && py_kwargs)) {
+                log_err("pythonmod: BuildValue failure in python_inplace_cb_reply_generic");
+                goto out;
+        }
+        result = PyObject_Call(func, py_args, py_kwargs);
+        if (result) {
+            res = PyInt_AsLong(result);
+        }
+out:
         Py_XDECREF(py_edns);
         Py_XDECREF(py_qstate);
         Py_XDECREF(py_opt_list_out);
@@ -1587,9 +1633,6 @@ int edns_opt_list_append(struct edns_option** list, uint16_t code, size_t len,
         Py_XDECREF(py_region);
         Py_XDECREF(py_args);
         Py_XDECREF(py_kwargs);
-        if (result) {
-            res = PyInt_AsLong(result);
-        }
         Py_XDECREF(result);
         PyGILState_Release(gstate);
         return res;
@@ -1637,29 +1680,34 @@ int edns_opt_list_append(struct edns_option** list, uint16_t code, size_t len,
         int res = 0;
         PyObject *func = python_callback;
         PyObject *py_args = NULL, *py_kwargs = NULL, *result = NULL;
+        PyObject *py_qinfo = NULL;
+        PyObject *py_qstate = NULL;
+        PyObject *py_addr = NULL;
+        PyObject *py_zone = NULL;
+        PyObject *py_region = NULL;
 
         PyGILState_STATE gstate = PyGILState_Ensure();
 
-        PyObject *py_qinfo = SWIG_NewPointerObj((void*) qinfo, SWIGTYPE_p_query_info, 0);
-        PyObject *py_qstate = SWIG_NewPointerObj((void*) qstate, SWIGTYPE_p_module_qstate, 0);
-        PyObject *py_addr = SWIG_NewPointerObj((void *) addr, SWIGTYPE_p_sockaddr_storage, 0);
-        PyObject *py_zone = PyBytes_FromStringAndSize((const char *)zone, zonelen);
-        PyObject *py_region = SWIG_NewPointerObj((void*) region, SWIGTYPE_p_regional, 0);
-        if(py_qinfo && py_qstate && py_addr && py_zone && py_region) {
-                py_args = Py_BuildValue("(OiOOOO)", py_qinfo, flags, py_qstate, py_addr, py_zone, py_region);
-                py_kwargs = Py_BuildValue("{}");
-                if(py_args && py_kwargs) {
-                        result = PyObject_Call(func, py_args, py_kwargs);
-                        if (result) {
-                            res = PyInt_AsLong(result);
-                        }
-                } else {
-                        log_err("pythonmod: malloc failure in python_inplace_cb_query_generic");
-                }
-        } else {
-                log_err("pythonmod: malloc failure in python_inplace_cb_query_generic");
+        py_qinfo = SWIG_NewPointerObj((void*) qinfo, SWIGTYPE_p_query_info, 0);
+        py_qstate = SWIG_NewPointerObj((void*) qstate, SWIGTYPE_p_module_qstate, 0);
+        py_addr = SWIG_NewPointerObj((void *) addr, SWIGTYPE_p_sockaddr_storage, 0);
+        py_zone = PyBytes_FromStringAndSize((const char *)zone, zonelen);
+        py_region = SWIG_NewPointerObj((void*) region, SWIGTYPE_p_regional, 0);
+        if(!(py_qinfo && py_qstate && py_addr && py_zone && py_region)) {
+                log_err("pythonmod: swig pointer failure in python_inplace_cb_query_generic");
+                goto out;
         }
-
+        py_args = Py_BuildValue("(OiOOOO)", py_qinfo, flags, py_qstate, py_addr, py_zone, py_region);
+        py_kwargs = Py_BuildValue("{}");
+        if(!(py_args && py_kwargs)) {
+                log_err("pythonmod: BuildValue failure in python_inplace_cb_query_generic");
+                goto out;
+        }
+        result = PyObject_Call(func, py_args, py_kwargs);
+        if (result) {
+            res = PyInt_AsLong(result);
+        }
+out:
         Py_XDECREF(py_qinfo);
         Py_XDECREF(py_qstate);
         Py_XDECREF(py_addr);
@@ -1683,6 +1731,105 @@ int edns_opt_list_append(struct edns_option** list, uint16_t code, size_t len,
         if (ret) Py_INCREF(py_cb);
         return ret;
     }
+
+    int python_inplace_cb_query_response(struct module_qstate* qstate,
+        struct dns_msg* response, int id, void* python_callback)
+    {
+        int res = 0;
+        PyObject *func = python_callback;
+        PyObject *py_qstate = NULL;
+        PyObject *py_response = NULL;
+        PyObject *py_args = NULL;
+        PyObject *py_kwargs = NULL;
+        PyObject *result = NULL;
+
+        PyGILState_STATE gstate = PyGILState_Ensure();
+
+        py_qstate = SWIG_NewPointerObj((void*) qstate, SWIGTYPE_p_module_qstate, 0);
+        py_response = SWIG_NewPointerObj((void*) response, SWIGTYPE_p_dns_msg, 0);
+        if(!(py_qstate && py_response)) {
+                log_err("pythonmod: swig pointer failure in python_inplace_cb_query_response");
+                goto out;
+        }
+        py_args = Py_BuildValue("(OO)", py_qstate, py_response);
+        py_kwargs = Py_BuildValue("{}");
+        if(!(py_args && py_kwargs)) {
+                log_err("pythonmod: BuildValue failure in python_inplace_cb_query_response");
+                goto out;
+        }
+        result = PyObject_Call(func, py_args, py_kwargs);
+        if (result) {
+            res = PyInt_AsLong(result);
+        }
+out:
+        Py_XDECREF(py_qstate);
+        Py_XDECREF(py_response);
+
+        Py_XDECREF(py_args);
+        Py_XDECREF(py_kwargs);
+        Py_XDECREF(result);
+
+        PyGILState_Release(gstate);
+
+        return res;
+    }
+
+    static int register_inplace_cb_query_response(PyObject* py_cb,
+        struct module_env* env, int id)
+    {
+        int ret = inplace_cb_register(python_inplace_cb_query_response,
+            inplace_cb_query_response, (void*) py_cb, env, id);
+        if (ret) Py_INCREF(py_cb);
+        return ret;
+    }
+
+    int python_inplace_cb_edns_back_parsed_call(struct module_qstate* qstate,
+        int id, void* python_callback)
+    {
+        int res = 0;
+        PyObject *func = python_callback;
+        PyObject *py_qstate = NULL;
+        PyObject *py_args = NULL;
+        PyObject *py_kwargs = NULL;
+        PyObject *result = NULL;
+
+        PyGILState_STATE gstate = PyGILState_Ensure();
+
+        py_qstate = SWIG_NewPointerObj((void*) qstate, SWIGTYPE_p_module_qstate, 0);
+        if(!py_qstate) {
+                log_err("pythonmod: swig pointer failure in python_inplace_cb_edns_back_parsed_call");
+                goto out;
+        }
+        py_args = Py_BuildValue("(O)", py_qstate);
+        py_kwargs = Py_BuildValue("{}");
+        if(!(py_args && py_kwargs)) {
+                log_err("pythonmod: BuildValue failure in python_inplace_cb_edns_back_parsed_call");
+                goto out;
+        }
+        result = PyObject_Call(func, py_args, py_kwargs);
+        if (result) {
+            res = PyInt_AsLong(result);
+        }
+out:
+        Py_XDECREF(py_qstate);
+
+        Py_XDECREF(py_args);
+        Py_XDECREF(py_kwargs);
+        Py_XDECREF(result);
+
+        PyGILState_Release(gstate);
+
+        return res;
+    }
+
+    static int register_inplace_cb_edns_back_parsed_call(PyObject* py_cb,
+        struct module_env* env, int id)
+    {
+        int ret = inplace_cb_register(python_inplace_cb_edns_back_parsed_call,
+            inplace_cb_edns_back_parsed, (void*) py_cb, env, id);
+        if (ret) Py_INCREF(py_cb);
+        return ret;
+    }
 %}
 /* C declarations */
 int inplace_cb_register(void* cb, enum inplace_cb_list_type type, void* cbarg,
@@ -1699,3 +1846,7 @@ static int register_inplace_cb_reply_servfail(PyObject* py_cb,
     struct module_env* env, int id);
 static int register_inplace_cb_query(PyObject *py_cb,
     struct module_env* env, int id);
+static int register_inplace_cb_query_response(PyObject *py_cb,
+    struct module_env* env, int id);
+static int register_inplace_cb_edns_back_parsed_call(PyObject *py_cb,
+    struct module_env* env, int id);

pythonmod/pythonmod.c

@@ -112,6 +112,34 @@ struct pythonmod_qstate {
 	PyObject* data;
 };
 
+/* The dict from __main__ could have remnants from a previous script
+ * invocation, in a multi python module setup. Usually this is fine since newer
+ * scripts will update their values. The obvious erroneous case is when mixing
+ * python scripts that make use of both 'init' and 'init_standard'. This
+ * results in 'init_standard' to persist on following scripts that don't use it
+ * (thus not replacing it). This is also problematic in case where a script
+ * does not define a required function but a previously loaded script did. The
+ * current solution is to make sure to clean offensive remnants that influence
+ * further parsing of the individual scripts.
+ */
+static void
+clean_python_function_objects(PyObject* dict) {
+	const char* function_names[] = {
+		"init",
+		"init_standard",
+		"deinit",
+		"operate",
+		"inform_super"
+	};
+	size_t i;
+
+	for(i=0; i<sizeof(function_names)/sizeof(function_names[0]); i++) {
+		if(PyDict_GetItemString(dict, function_names[i]) != NULL) {
+			PyDict_DelItemString(dict, function_names[i]);
+		}
+	}
+};
+
 /* Generated */
 #ifndef S_SPLINT_S
 #include "pythonmod/interface.h"
@@ -252,13 +280,24 @@ cleanup:
 	Py_XDECREF(exc_tb);
 }
 
+/* we only want to unwind Python once at exit */
+static void
+pythonmod_atexit(void)
+{
+   log_assert(py_mod_count == 0);
+   log_assert(mainthr != NULL);
+ 
+   PyEval_RestoreThread(mainthr);
+   Py_Finalize();
+}
+
 int pythonmod_init(struct module_env* env, int id)
 {
    int py_mod_idx = py_mod_count++;
-    
+
    /* Initialize module */
    FILE* script_py = NULL;
-   PyObject* py_init_arg, *res;
+   PyObject* py_init_arg = NULL, *res = NULL, *fname = NULL;
    PyGILState_STATE gil;
    int init_standard = 1, i = 0;
 #if PY_MAJOR_VERSION < 3
@@ -292,47 +331,98 @@ int pythonmod_init(struct module_env* env, int id)
    /* Initialize Python libraries */
    if (py_mod_count==1 && !Py_IsInitialized()) 
    {
+#if PY_VERSION_HEX >= 0x03080000
+      PyStatus status;
+      PyPreConfig preconfig;
+      PyConfig config;
+#endif
 #if PY_MAJOR_VERSION >= 3
       wchar_t progname[8];
       mbstowcs(progname, "unbound", 8);
 #else
       char *progname = "unbound";
 #endif
+#if PY_VERSION_HEX < 0x03080000
       Py_SetProgramName(progname);
+#else
+      /* Python must be preinitialized, before the PyImport_AppendInittab
+       * call. */
+      PyPreConfig_InitPythonConfig(&preconfig);
+      status = Py_PreInitialize(&preconfig);
+      if(PyStatus_Exception(status)) {
+	log_err("python exception in Py_PreInitialize: %s%s%s",
+		(status.func?status.func:""), (status.func?": ":""),
+		(status.err_msg?status.err_msg:""));
+	return 0;
+      }
+#endif
       Py_NoSiteFlag = 1;
 #if PY_MAJOR_VERSION >= 3
       PyImport_AppendInittab(SWIG_name, (void*)SWIG_init);
 #endif
+#if PY_VERSION_HEX < 0x03080000
       Py_Initialize();
+#else
+      PyConfig_InitPythonConfig(&config);
+      status = PyConfig_SetString(&config, &config.program_name, progname);
+      if(PyStatus_Exception(status)) {
+	log_err("python exception in PyConfig_SetString(.. program_name ..): %s%s%s",
+		(status.func?status.func:""), (status.func?": ":""),
+		(status.err_msg?status.err_msg:""));
+	PyConfig_Clear(&config);
+	return 0;
+      }
+      config.site_import = 0;
+      status = Py_InitializeFromConfig(&config);
+      if(PyStatus_Exception(status)) {
+	log_err("python exception in Py_InitializeFromConfig: %s%s%s",
+		(status.func?status.func:""), (status.func?": ":""),
+		(status.err_msg?status.err_msg:""));
+	PyConfig_Clear(&config);
+	return 0;
+      }
+      PyConfig_Clear(&config);
+#endif
 #if PY_MAJOR_VERSION <= 2 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION <= 6)
       /* initthreads only for python 3.6 and older */
       PyEval_InitThreads();
 #endif
       SWIG_init();
       mainthr = PyEval_SaveThread();
+
+      /* register callback to unwind Python at exit */
+      atexit(pythonmod_atexit);
    }
 
    gil = PyGILState_Ensure();
 
    if (py_mod_count==1) {
       /* Initialize Python */
-      PyRun_SimpleString("import sys \n");
+      if(PyRun_SimpleString("import sys \n") < 0 ) {
+         log_err("pythonmod: cannot initialize core module: unboundmodule.py");
+         goto python_init_fail;
+      }
       PyRun_SimpleString("sys.path.append('.') \n");
+      PyRun_SimpleString("sys.path.append('"RUN_DIR"') \n");
+      PyRun_SimpleString("sys.path.append('"SHARE_DIR"') \n");
       if(env->cfg->directory && env->cfg->directory[0]) {
          char wdir[1524];
          snprintf(wdir, sizeof(wdir), "sys.path.append('%s') \n",
          env->cfg->directory);
          PyRun_SimpleString(wdir);
       }
-      PyRun_SimpleString("sys.path.append('"RUN_DIR"') \n");
-      PyRun_SimpleString("sys.path.append('"SHARE_DIR"') \n");
-      PyRun_SimpleString("import distutils.sysconfig \n");
-      PyRun_SimpleString("sys.path.append(distutils.sysconfig.get_python_lib(1,0)) \n");
-      if (PyRun_SimpleString("from unboundmodule import *\n") < 0)
+      if(PyRun_SimpleString("import site\n") < 0) {
+         log_err("pythonmod: cannot initialize core module: unboundmodule.py");
+         goto python_init_fail;
+      }
+      if(PyRun_SimpleString("sys.path.extend(site.getsitepackages())\n") < 0) {
+         log_err("pythonmod: cannot initialize core module: unboundmodule.py");
+         goto python_init_fail;
+      }
+      if(PyRun_SimpleString("from unboundmodule import *\n") < 0)
       {
          log_err("pythonmod: cannot initialize core module: unboundmodule.py");
-         PyGILState_Release(gil);
-         return 0;
+         goto python_init_fail;
       }
    }
 
@@ -348,18 +438,32 @@ int pythonmod_init(struct module_env* env, int id)
    if (script_py == NULL)
    {
       log_err("pythonmod: can't open file %s for reading", pe->fname);
-      PyGILState_Release(gil);
-      return 0;
+      goto python_init_fail;
    }
 
    /* Load file */
    pe->module = PyImport_AddModule("__main__");
+   Py_XINCREF(pe->module);
    pe->dict = PyModule_GetDict(pe->module);
-   pe->data = PyDict_New();
-   Py_XINCREF(pe->data);
-   PyModule_AddObject(pe->module, "mod_env", pe->data);
+   Py_XINCREF(pe->dict);
+   clean_python_function_objects(pe->dict);
 
-   /* TODO: deallocation of pe->... if an error occurs */
+   pe->data = PyDict_New();
+   /* add the script filename to the global "mod_env" for trivial access */
+   fname = PyString_FromString(pe->fname);
+   if(PyDict_SetItemString(pe->data, "script", fname) < 0) {
+	log_err("pythonmod: could not add item to dictionary");
+	Py_XDECREF(fname);
+	goto python_init_fail;
+   }
+   Py_XDECREF(fname);
+   Py_XINCREF(pe->data);  /* reference will be stolen below */
+   if(PyModule_AddObject(pe->module, "mod_env", pe->data) < 0) {
+	log_err("pythonmod: could not add mod_env object");
+	Py_XDECREF(pe->data);  /* 2 times, here and on python_init_fail; */
+	                       /* on failure the reference is not stolen */
+	goto python_init_fail;
+   }
 
    if (PyRun_SimpleFile(script_py, pe->fname) < 0) {
 #if PY_MAJOR_VERSION <= 2 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 9)
@@ -390,18 +494,30 @@ int pythonmod_init(struct module_env* env, int id)
       fstr = malloc(flen+1);
       if(!fstr) {
 	      log_err("malloc failure to print parse error");
-	      PyGILState_Release(gil);
+
+/* close the file */
+#if PY_MAJOR_VERSION < 3
+	      Py_XDECREF(PyFileObject);
+#else
 	      fclose(script_py);
-	      return 0;
+#endif
+
+	      goto python_init_fail;
       }
       fseek(script_py, 0, SEEK_SET);
       if(fread(fstr, flen, 1, script_py) < 1) {
 	      log_err("file read failed to print parse error: %s: %s",
 		pe->fname, strerror(errno));
-	      PyGILState_Release(gil);
-	      fclose(script_py);
 	      free(fstr);
-	      return 0;
+
+/* close the file */
+#if PY_MAJOR_VERSION < 3
+	      Py_XDECREF(PyFileObject);
+#else
+	      fclose(script_py);
+#endif
+
+	      goto python_init_fail;
       }
       fstr[flen] = 0;
       /* we compile the string, but do not run it, to stop side-effects */
@@ -409,17 +525,26 @@ int pythonmod_init(struct module_env* env, int id)
        * that we are expecting */
       (void)Py_CompileString(fstr, pe->fname, Py_file_input);
 #endif
+
       log_py_err();
-      PyGILState_Release(gil);
+
+/* close the file */
+#if PY_MAJOR_VERSION < 3
+      Py_XDECREF(PyFileObject);
+#else
       fclose(script_py);
+#endif
+
 #if PY_MAJOR_VERSION <= 2 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 9)
       /* no cleanup needed for python before 3.9 */
 #else
       /* cleanup for python 3.9 and newer */
       free(fstr);
 #endif
-      return 0;
+      goto python_init_fail;
    }
+
+/* close the file */
 #if PY_MAJOR_VERSION < 3
    Py_XDECREF(PyFileObject);
 #else
@@ -432,28 +557,28 @@ int pythonmod_init(struct module_env* env, int id)
       if ((pe->func_init = PyDict_GetItemString(pe->dict, "init")) == NULL)
       {
          log_err("pythonmod: function init is missing in %s", pe->fname);
-         PyGILState_Release(gil);
-         return 0;
+         goto python_init_fail;
       }
    }
+   Py_XINCREF(pe->func_init);
    if ((pe->func_deinit = PyDict_GetItemString(pe->dict, "deinit")) == NULL)
    {
       log_err("pythonmod: function deinit is missing in %s", pe->fname);
-      PyGILState_Release(gil);
-      return 0;
+      goto python_init_fail;
    }
+   Py_XINCREF(pe->func_deinit);
    if ((pe->func_operate = PyDict_GetItemString(pe->dict, "operate")) == NULL)
    {
       log_err("pythonmod: function operate is missing in %s", pe->fname);
-      PyGILState_Release(gil);
-      return 0;
+      goto python_init_fail;
    }
+   Py_XINCREF(pe->func_operate);
    if ((pe->func_inform = PyDict_GetItemString(pe->dict, "inform_super")) == NULL)
    {
       log_err("pythonmod: function inform_super is missing in %s", pe->fname);
-      PyGILState_Release(gil);
-      return 0;
+      goto python_init_fail;
    }
+   Py_XINCREF(pe->func_inform);
 
    if (init_standard)
    {
@@ -469,21 +594,31 @@ int pythonmod_init(struct module_env* env, int id)
    {
       log_err("pythonmod: Exception occurred in function init");
       log_py_err();
-      Py_XDECREF(res);
-      Py_XDECREF(py_init_arg);
-      PyGILState_Release(gil);
-      return 0;
+      goto python_init_fail;
    }
 
    Py_XDECREF(res);
    Py_XDECREF(py_init_arg);
    PyGILState_Release(gil);
-
    return 1;
+
+python_init_fail:
+   Py_XDECREF(pe->module);
+   Py_XDECREF(pe->dict);
+   Py_XDECREF(pe->data);
+   Py_XDECREF(pe->func_init);
+   Py_XDECREF(pe->func_deinit);
+   Py_XDECREF(pe->func_operate);
+   Py_XDECREF(pe->func_inform);
+   Py_XDECREF(res);
+   Py_XDECREF(py_init_arg);
+   PyGILState_Release(gil);
+   return 0;
 }
 
 void pythonmod_deinit(struct module_env* env, int id)
 {
+   int cbtype;
    struct pythonmod_env* pe = env->modinfo[id];
    if(pe == NULL)
       return;
@@ -503,18 +638,24 @@ void pythonmod_deinit(struct module_env* env, int id)
       /* Free result if any */
       Py_XDECREF(res);
       /* Free shared data if any */
+      Py_XDECREF(pe->module);
+      Py_XDECREF(pe->dict);
       Py_XDECREF(pe->data);
+      Py_XDECREF(pe->func_init);
+      Py_XDECREF(pe->func_deinit);
+      Py_XDECREF(pe->func_inform);
+      Py_XDECREF(pe->func_operate);
       PyGILState_Release(gil);
 
-      if(--py_mod_count==0) {
-         PyEval_RestoreThread(mainthr);
-         Py_Finalize();
-         mainthr = NULL;
-      }
+      py_mod_count--;
    }
    pe->fname = NULL;
    free(pe);
 
+   /* iterate over all possible callback types and clean up each in turn */
+   for (cbtype = 0; cbtype < inplace_cb_types_total; cbtype++)
+      inplace_cb_delete(env, cbtype, id);
+
    /* Module is deallocated in Python */
    env->modinfo[id] = NULL;
 }

pythonmod/pythonmod.h

@@ -82,4 +82,12 @@ int python_inplace_cb_query_generic(
         uint8_t* zone, size_t zonelen, struct regional* region, int id,
         void* python_callback);
 
+/** Declared here for fptr_wlist access. The definition is in interface.i. */
+int python_inplace_cb_query_response(struct module_qstate* qstate,
+    struct dns_msg* response, int id, void* python_callback);
+
+/** Declared here for fptr_wlist access. The definition is in interface.i. */
+int python_inplace_cb_edns_back_parsed_call(struct module_qstate* qstate,
+    int id, void* python_callback);
+
 #endif /* PYTHONMOD_H */

pythonmod/pythonmod_utils.c

@@ -72,7 +72,8 @@ int storeQueryInCache(struct module_qstate* qstate, struct query_info* qinfo,
 	}
 
 	return dns_cache_store(qstate->env, qinfo, msgrep, is_referral,
-		qstate->prefetch_leeway, 0, NULL, qstate->query_flags);
+		qstate->prefetch_leeway, 0, NULL, qstate->query_flags,
+		qstate->qstarttime);
 }
 
 /* Invalidate the message associated with query_info stored in message cache */
@@ -171,14 +172,14 @@ int createResponse(struct module_qstate* qstate, sldns_buffer* pkt)
 }
 
 
-/* Convert reply->addr to string */
+/* Convert reply->client_addr to string */
 void reply_addr2str(struct comm_reply* reply, char* dest, int maxlen)
 {
-	int af = (int)((struct sockaddr_in*) &(reply->addr))->sin_family;
-	void* sinaddr = &((struct sockaddr_in*) &(reply->addr))->sin_addr;
+	int af = (int)((struct sockaddr_in*) &(reply->client_addr))->sin_family;
+	void* sinaddr = &((struct sockaddr_in*) &(reply->client_addr))->sin_addr;
 
 	if(af == AF_INET6)
-		sinaddr = &((struct sockaddr_in6*)&(reply->addr))->sin6_addr;
+		sinaddr = &((struct sockaddr_in6*)&(reply->client_addr))->sin6_addr;
 	dest[0] = 0;
 	if (inet_ntop(af, sinaddr, dest, (socklen_t)maxlen) == 0)
 		return;

pythonmod/ubmodule-msg.py

@@ -35,7 +35,7 @@
 import os
 
 def init(id, cfg):
-    log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, cfg.python_script))
+    log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, mod_env['script']))
     return True
 
 def deinit(id):

pythonmod/ubmodule-tst.py

@@ -33,7 +33,7 @@
  POSSIBILITY OF SUCH DAMAGE.
 '''
 def init(id, cfg):
-    log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, cfg.python_script))
+    log_info("pythonmod: init called, module id is %d port: %d script: %s" % (id, cfg.port, mod_env['script']))
     return True
 
 def deinit(id):
@@ -55,12 +55,15 @@ def setTTL(qstate, ttl):
 
 def dataHex(data, prefix=""):
     res = ""
-    for i in range(0, (len(data)+15)/16):
+    for i in range(0, int((len(data)+15)/16)):
         res += "%s0x%02X | " % (prefix, i*16)
-        d = map(lambda x:ord(x), data[i*16:i*16+17])
+        if type(data[0]) == type(1):
+            d = map(lambda x:int(x), data[i*16:i*16+17])
+        else:
+            d = map(lambda x:ord(x), data[i*16:i*16+17])
         for ch in d:
             res += "%02X " % ch
-        for i in range(0,17-len(d)):
+        for i in range(0,17-len(data[i*16:i*16+17])):
             res += "   "
         res += "| "
         for ch in d:
@@ -72,31 +75,31 @@ def dataHex(data, prefix=""):
     return res
 
 def printReturnMsg(qstate):
-    print "Return MSG rep   :: flags: %04X, QDcount: %d, Security:%d, TTL=%d" % (qstate.return_msg.rep.flags, qstate.return_msg.rep.qdcount,qstate.return_msg.rep.security, qstate.return_msg.rep.ttl)
-    print "           qinfo :: qname:",qstate.return_msg.qinfo.qname_list, qstate.return_msg.qinfo.qname_str, "type:",qstate.return_msg.qinfo.qtype_str, "class:",qstate.return_msg.qinfo.qclass_str
+    print("Return MSG rep   :: flags: %04X, QDcount: %d, Security:%d, TTL=%d" % (qstate.return_msg.rep.flags, qstate.return_msg.rep.qdcount,qstate.return_msg.rep.security, qstate.return_msg.rep.ttl))
+    print("           qinfo :: qname:",qstate.return_msg.qinfo.qname_list, qstate.return_msg.qinfo.qname_str, "type:",qstate.return_msg.qinfo.qtype_str, "class:",qstate.return_msg.qinfo.qclass_str)
     if (qstate.return_msg.rep):
-        print "RRSets:",qstate.return_msg.rep.rrset_count
+        print("RRSets:",qstate.return_msg.rep.rrset_count)
         prevkey = None
         for i in range(0,qstate.return_msg.rep.rrset_count):
             r = qstate.return_msg.rep.rrsets[i]
             rk = r.rk
-            print i,":",rk.dname_list, rk.dname_str, "flags: %04X" % rk.flags,
-            print "type:",rk.type_str,"(%d)" % ntohs(rk.type), "class:",rk.rrset_class_str,"(%d)" % ntohs(rk.rrset_class)
+            print(i,":",rk.dname_list, rk.dname_str, "flags: %04X" % rk.flags)
+            print("type:",rk.type_str,"(%d)" % ntohs(rk.type), "class:",rk.rrset_class_str,"(%d)" % ntohs(rk.rrset_class))
 
             d = r.entry.data
-            print "    RRDatas:",d.count+d.rrsig_count
+            print("    RRDatas:",d.count+d.rrsig_count)
             for j in range(0,d.count+d.rrsig_count):
-                print "    ",j,":","TTL=",d.rr_ttl[j],"RR data:"
-                print dataHex(d.rr_data[j],"         ")
+                print("    ",j,":","TTL=",d.rr_ttl[j],"RR data:")
+                print(dataHex(d.rr_data[j],"         "))
 
 def operate(id, event, qstate, qdata):
     log_info("pythonmod: operate called, id: %d, event:%s" % (id, strmodulevent(event)))
-    #print "pythonmod: per query data", qdata
+    #print("pythonmod: per query data", qdata)
 
-    print "Query:", ''.join(map(lambda x:chr(max(32,ord(x))),qstate.qinfo.qname)), qstate.qinfo.qname_list, 
-    print "Type:",qstate.qinfo.qtype_str,"(%d)" % qstate.qinfo.qtype,
-    print "Class:",qstate.qinfo.qclass_str,"(%d)" % qstate.qinfo.qclass
-    print
+    print("Query:", qstate.qinfo.qname, qstate.qinfo.qname_list, qstate.qinfo.qname_str)
+    print("Type:",qstate.qinfo.qtype_str,"(%d)" % qstate.qinfo.qtype)
+    print("Class:",qstate.qinfo.qclass_str,"(%d)" % qstate.qinfo.qclass)
+    print("")
 
     # TEST:
     #   > dig @127.0.0.1 www.seznam.cz A
@@ -118,7 +121,7 @@ def operate(id, event, qstate, qdata):
             invalidateQueryInCache(qstate, qstate.return_msg.qinfo)
 
             if (qstate.return_msg.rep.authoritative):
-                print "X"*300
+                print("X"*300)
 
             setTTL(qstate, 10) #do cache nastavime TTL na 10
             if not storeQueryInCache(qstate, qstate.return_msg.qinfo, qstate.return_msg.rep, 0):

respip/respip.c

@@ -833,8 +833,11 @@ static int
 respip_use_rpz(struct resp_addr* raddr, struct rpz* r,
 	enum respip_action* action,
 	struct ub_packed_rrset_key** data, int* rpz_log, char** log_name,
-	int* rpz_cname_override, struct regional* region, int* is_rpz)
+	int* rpz_cname_override, struct regional* region, int* is_rpz,
+	int* rpz_passthru)
 {
+	if(rpz_passthru && *rpz_passthru)
+		return 0;
 	if(r->action_override == RPZ_DISABLED_ACTION) {
 		*is_rpz = 0;
 		return 1;
@@ -848,6 +851,9 @@ respip_use_rpz(struct resp_addr* raddr, struct rpz* r,
 		*data = r->cname_override;
 		*rpz_cname_override = 1;
 	}
+	if(*action == respip_always_transparent /* RPZ_PASSTHRU_ACTION */
+		&& rpz_passthru)
+		*rpz_passthru = 1;
 	*rpz_log = r->log;
 	if(r->log_name)
 		if(!(*log_name = regional_strdup(region, r->log_name)))
@@ -861,7 +867,7 @@ respip_rewrite_reply(const struct query_info* qinfo,
 	const struct respip_client_info* cinfo, const struct reply_info* rep,
 	struct reply_info** new_repp, struct respip_action_info* actinfo,
 	struct ub_packed_rrset_key** alias_rrset, int search_only,
-	struct regional* region, struct auth_zones* az)
+	struct regional* region, struct auth_zones* az, int* rpz_passthru)
 {
 	const uint8_t* ctaglist;
 	size_t ctaglen;
@@ -934,7 +940,7 @@ respip_rewrite_reply(const struct query_info* qinfo,
 			ipset->tagname, ipset->num_tags);
 	}
 	lock_rw_rdlock(&az->rpz_lock);
-	for(a = az->rpz_first; a && !raddr; a = a->rpz_az_next) {
+	for(a = az->rpz_first; a && !raddr && !(rpz_passthru && *rpz_passthru); a = a->rpz_az_next) {
 		lock_rw_rdlock(&a->lock);
 		r = a->rpz;
 		if(!r->taglist || taglist_intersect(r->taglist, 
@@ -943,7 +949,7 @@ respip_rewrite_reply(const struct query_info* qinfo,
 				r->respip_set, &rrset_id, &rr_id))) {
 				if(!respip_use_rpz(raddr, r, &action, &data,
 					&rpz_log, &log_name, &rpz_cname_override,
-					region, &rpz_used)) {
+					region, &rpz_used, rpz_passthru)) {
 					log_err("out of memory");
 					lock_rw_unlock(&raddr->lock);
 					lock_rw_unlock(&a->lock);
@@ -964,7 +970,7 @@ respip_rewrite_reply(const struct query_info* qinfo,
 						addr_to_str(&raddr->node.addr,
 							raddr->node.addrlen,
 							nm, sizeof(nm));
-						verbose(VERB_ALGO, "respip: rpz response-ip trigger %s/%d on %s %s with action %s", nm, raddr->node.net, qn, ip, rpz_action_to_string(respip_action_to_rpz_action(action)));
+						verbose(VERB_ALGO, "respip: rpz: response-ip trigger %s/%d on %s %s with action %s", nm, raddr->node.net, qn, ip, rpz_action_to_string(respip_action_to_rpz_action(action)));
 					}
 					/* break to make sure 'a' stays pointed
 					 * to used auth_zone, and keeps lock */
@@ -1094,7 +1100,8 @@ respip_operate(struct module_qstate* qstate, enum module_ev event, int id,
 			if(!respip_rewrite_reply(&qstate->qinfo,
 				qstate->client_info, qstate->return_msg->rep,
 				&new_rep, &actinfo, &alias_rrset, 0,
-				qstate->region, qstate->env->auth_zones)) {
+				qstate->region, qstate->env->auth_zones,
+				&qstate->rpz_passthru)) {
 				goto servfail;
 			}
 			if(actinfo.action != respip_none) {
@@ -1169,7 +1176,7 @@ respip_merge_cname(struct reply_info* base_rep,
 
 	/* see if the target reply would be subject to a response-ip action. */
 	if(!respip_rewrite_reply(qinfo, cinfo, tgt_rep, &tmp_rep, &actinfo,
-		&alias_rrset, 1, region, az))
+		&alias_rrset, 1, region, az, NULL))
 		return 0;
 	if(actinfo.action != respip_none) {
 		log_info("CNAME target of redirect response-ip action would "
@@ -1283,7 +1290,7 @@ respip_set_is_empty(const struct respip_set* set)
 void
 respip_inform_print(struct respip_action_info* respip_actinfo, uint8_t* qname,
 	uint16_t qtype, uint16_t qclass, struct local_rrset* local_alias,
-	struct comm_reply* repinfo)
+	struct sockaddr_storage* addr, socklen_t addrlen)
 {
 	char srcip[128], respip[128], txt[512];
 	unsigned port;
@@ -1293,15 +1300,15 @@ respip_inform_print(struct respip_action_info* respip_actinfo, uint8_t* qname,
 
 	if(local_alias)
 		qname = local_alias->rrset->rk.dname;
-	port = (unsigned)((repinfo->addr.ss_family == AF_INET) ?
-		ntohs(((struct sockaddr_in*)&repinfo->addr)->sin_port) :
-		ntohs(((struct sockaddr_in6*)&repinfo->addr)->sin6_port));
-	addr_to_str(&repinfo->addr, repinfo->addrlen, srcip, sizeof(srcip));
+	port = (unsigned)((addr->ss_family == AF_INET) ?
+		ntohs(((struct sockaddr_in*)addr)->sin_port) :
+		ntohs(((struct sockaddr_in6*)addr)->sin6_port));
+	addr_to_str(addr, addrlen, srcip, sizeof(srcip));
 	addr_to_str(&respip_addr->addr, respip_addr->addrlen,
 		respip, sizeof(respip));
 	if(respip_actinfo->rpz_log) {
 		txtlen += snprintf(txt+txtlen, sizeof(txt)-txtlen, "%s",
-			"RPZ applied ");
+			"rpz: applied ");
 		if(respip_actinfo->rpz_cname_override)
 			actionstr = rpz_action_to_string(
 				RPZ_CNAME_OVERRIDE_ACTION);

respip/respip.h

@@ -176,6 +176,8 @@ int respip_merge_cname(struct reply_info* base_rep,
  *   will be set (or intact) accordingly but the modified reply won't be built.
  * @param az: auth zones containing RPZ information.
  * @param region: allocator to build *new_repp.
+ * @param rpz_passthru: keeps track of query state can have passthru that
+ *   stops further rpz processing. Or NULL for cached answer processing.
  * @return 1 on success, 0 on error.
  */
 int respip_rewrite_reply(const struct query_info* qinfo,
@@ -183,7 +185,8 @@ int respip_rewrite_reply(const struct query_info* qinfo,
 	const struct reply_info *rep, struct reply_info** new_repp,
 	struct respip_action_info* actinfo,
 	struct ub_packed_rrset_key** alias_rrset,
-	int search_only, struct regional* region, struct auth_zones* az);
+	int search_only, struct regional* region, struct auth_zones* az,
+	int* rpz_passthru);
 
 /**
  * Get the response-ip function block.
@@ -248,11 +251,13 @@ int respip_set_is_empty(const struct respip_set* set);
  * @param local_alias: set to a local alias if the query matches an alias in
  *  a local zone.  In this case its owner name will be considered the actual
  *  query name.
- * @param repinfo: reply info containing the client's source address and port.
+ * @param addr: the client's source address and port.
+ * @param addrlen: the client's source address length.
  */
 void respip_inform_print(struct respip_action_info* respip_actinfo,
 	uint8_t* qname, uint16_t qtype, uint16_t qclass,
-	struct local_rrset* local_alias, struct comm_reply* repinfo);
+	struct local_rrset* local_alias, struct sockaddr_storage* addr,
+	socklen_t addrlen);
 
 /**
  * Find resp_addr in tree, create and add to tree if it does not exist.

services/authzone.c

@@ -132,6 +132,7 @@ msg_create(struct regional* region, struct query_info* qinfo)
 		return NULL;
 	msg->rep->flags = (uint16_t)(BIT_QR | BIT_AA);
 	msg->rep->authoritative = 1;
+	msg->rep->reason_bogus = LDNS_EDE_NONE;
 	msg->rep->qdcount = 1;
 	/* rrsets is NULL, no rrsets yet */
 	return msg;
@@ -1305,8 +1306,8 @@ az_remove_rr(struct auth_zone* z, uint8_t* rr, size_t rr_len,
 		auth_data_delete(node);
 	}
 	if(z->rpz) {
-		rpz_remove_rr(z->rpz, z->namelen, dname, dname_len, rr_type,
-			rr_class, rdata, rdatalen);
+		rpz_remove_rr(z->rpz, z->name, z->namelen, dname, dname_len,
+			rr_type, rr_class, rdata, rdatalen);
 	}
 	return 1;
 }
@@ -1882,6 +1883,8 @@ static int auth_zone_zonemd_check_hash(struct auth_zone* z,
 	struct regional* region = NULL;
 	struct sldns_buffer* buf = NULL;
 	uint32_t soa_serial = 0;
+	char* unsupported_reason = NULL;
+	int only_unsupported = 1;
 	region = env->scratch;
 	regional_free_all(region);
 	buf = env->scratch_buffer;
@@ -1911,6 +1914,7 @@ static int auth_zone_zonemd_check_hash(struct auth_zone* z,
 			&hashalgo, &hash, &hashlen)) {
 			/* malformed RR */
 			*reason = "ZONEMD rdata malformed";
+			only_unsupported = 0;
 			continue;
 		}
 		/* check for duplicates */
@@ -1920,25 +1924,51 @@ static int auth_zone_zonemd_check_hash(struct auth_zone* z,
 			 * is not allowed. */
 			*reason = "ZONEMD RRSet contains more than one RR "
 				"with the same scheme and hash algorithm";
+			only_unsupported = 0;
 			continue;
 		}
 		regional_free_all(region);
 		if(serial != soa_serial) {
 			*reason = "ZONEMD serial is wrong";
+			only_unsupported = 0;
 			continue;
 		}
+		*reason = NULL;
 		if(auth_zone_generate_zonemd_check(z, scheme, hashalgo,
 			hash, hashlen, region, buf, reason)) {
 			/* success */
+			if(*reason) {
+				if(!unsupported_reason)
+					unsupported_reason = *reason;
+				/* continue to check for valid ZONEMD */
+				if(verbosity >= VERB_ALGO) {
+					char zstr[255+1];
+					dname_str(z->name, zstr);
+					verbose(VERB_ALGO, "auth-zone %s ZONEMD %d %d is unsupported: %s", zstr, (int)scheme, (int)hashalgo, *reason);
+				}
+				*reason = NULL;
+				continue;
+			}
 			if(verbosity >= VERB_ALGO) {
 				char zstr[255+1];
 				dname_str(z->name, zstr);
-				verbose(VERB_ALGO, "auth-zone %s ZONEMD hash is correct", zstr);
+				if(!*reason)
+					verbose(VERB_ALGO, "auth-zone %s ZONEMD hash is correct", zstr);
 			}
 			return 1;
 		}
+		only_unsupported = 0;
 		/* try next one */
 	}
+	/* have we seen no failures but only unsupported algo,
+	 * and one unsupported algorithm, or more. */
+	if(only_unsupported && unsupported_reason) {
+		/* only unsupported algorithms, with valid serial, not
+		 * malformed. Did not see supported algorithms, failed or
+		 * successful ones. */
+		*reason = unsupported_reason;
+		return 1;
+	}
 	/* fail, we may have reason */
 	if(!*reason)
 		*reason = "no ZONEMD records found";
@@ -2445,6 +2475,7 @@ az_find_ce(struct auth_zone* z, struct query_info* qinfo,
 	struct auth_rrset** rrset)
 {
 	struct auth_data* n = node;
+	struct auth_rrset* lookrrset;
 	*ce = NULL;
 	*rrset = NULL;
 	if(!node_exact) {
@@ -2467,21 +2498,23 @@ az_find_ce(struct auth_zone* z, struct query_info* qinfo,
 		/* see if the current candidate has issues */
 		/* not zone apex and has type NS */
 		if(n->namelen != z->namelen &&
-			(*rrset=az_domain_rrset(n, LDNS_RR_TYPE_NS)) &&
+			(lookrrset=az_domain_rrset(n, LDNS_RR_TYPE_NS)) &&
 			/* delegate here, but DS at exact the dp has notype */
 			(qinfo->qtype != LDNS_RR_TYPE_DS || 
 			n->namelen != qinfo->qname_len)) {
 			/* referral */
 			/* this is ce and the lowernode is nonexisting */
 			*ce = n;
-			return 0;
+			*rrset = lookrrset;
+			node_exact = 0;
 		}
 		/* not equal to qname and has type DNAME */
 		if(n->namelen != qinfo->qname_len &&
-			(*rrset=az_domain_rrset(n, LDNS_RR_TYPE_DNAME))) {
+			(lookrrset=az_domain_rrset(n, LDNS_RR_TYPE_DNAME))) {
 			/* this is ce and the lowernode is nonexisting */
 			*ce = n;
-			return 0;
+			*rrset = lookrrset;
+			node_exact = 0;
 		}
 
 		if(*ce == NULL && !domain_has_only_nsec3(n)) {
@@ -2726,6 +2759,7 @@ az_change_dnames(struct dns_msg* msg, uint8_t* oldname, uint8_t* newname,
 			== 0) {
 			msg->rep->rrsets[i]->rk.dname = newname;
 			msg->rep->rrsets[i]->rk.dname_len = newlen;
+			msg->rep->rrsets[i]->entry.hash = rrset_key_hash(&msg->rep->rrsets[i]->rk);
 		}
 	}
 }
@@ -3669,7 +3703,7 @@ addr_matches_master(struct auth_master* master, struct sockaddr_storage* addr,
 	/* compare address (but not port number, that is the destination
 	 * port of the master, the port number of the received notify is
 	 * allowed to by any port on that master) */
-	if(extstrtoaddr(master->host, &a, &alen) &&
+	if(extstrtoaddr(master->host, &a, &alen, UNBOUND_DNS_PORT) &&
 		sockaddr_cmp_addr(addr, addrlen, &a, alen)==0) {
 		*fromhost = master;
 		return 1;
@@ -3937,7 +3971,7 @@ probe_copy_masters_for_allow_notify(struct auth_xfer* xfr)
 	struct auth_master* list = NULL, *last = NULL;
 	struct auth_master* p;
 	/* build up new list with copies */
-	for(p = xfr->task_probe->masters; p; p=p->next) {
+	for(p = xfr->task_transfer->masters; p; p=p->next) {
 		struct auth_master* m = auth_master_copy(p);
 		if(!m) {
 			auth_free_masters(list);
@@ -4456,7 +4490,7 @@ chunkline_get_line_collated(struct auth_chunk** chunk, size_t* chunk_pos,
 	return 1;
 }
 
-/** process $ORIGIN for http */
+/** process $ORIGIN for http, 0 nothing, 1 done, 2 error */
 static int
 http_parse_origin(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
 {
@@ -4467,13 +4501,16 @@ http_parse_origin(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
 		pstate->origin_len = sizeof(pstate->origin);
 		s = sldns_str2wire_dname_buf(sldns_strip_ws(line+8),
 			pstate->origin, &pstate->origin_len);
-		if(s) pstate->origin_len = 0;
+		if(s) {
+			pstate->origin_len = 0;
+			return 2;
+		}
 		return 1;
 	}
 	return 0;
 }
 
-/** process $TTL for http */
+/** process $TTL for http, 0 nothing, 1 done, 2 error */
 static int
 http_parse_ttl(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
 {
@@ -4481,8 +4518,12 @@ http_parse_ttl(sldns_buffer* buf, struct sldns_file_parse_state* pstate)
 	if(strncmp(line, "$TTL", 4) == 0 &&
 		isspace((unsigned char)line[4])) {
 		const char* end = NULL;
+		int overflow = 0;
 		pstate->default_ttl = sldns_str2period(
-			sldns_strip_ws(line+5), &end);
+			sldns_strip_ws(line+5), &end, &overflow);
+		if(overflow) {
+			return 2;
+		}
 		return 1;
 	}
 	return 0;
@@ -4493,15 +4534,20 @@ static int
 chunkline_non_comment_RR(struct auth_chunk** chunk, size_t* chunk_pos,
 	sldns_buffer* buf, struct sldns_file_parse_state* pstate)
 {
+	int ret;
 	while(chunkline_get_line_collated(chunk, chunk_pos, buf)) {
 		if(chunkline_is_comment_line_or_empty(buf)) {
 			/* a comment, go to next line */
 			continue;
 		}
-		if(http_parse_origin(buf, pstate)) {
+		if((ret=http_parse_origin(buf, pstate))!=0) {
+			if(ret == 2)
+				return 0;
 			continue; /* $ORIGIN has been handled */
 		}
-		if(http_parse_ttl(buf, pstate)) {
+		if((ret=http_parse_ttl(buf, pstate))!=0) {
+			if(ret == 2)
+				return 0;
 			continue; /* $TTL has been handled */
 		}
 		return 1;
@@ -5007,6 +5053,7 @@ apply_http(struct auth_xfer* xfr, struct auth_zone* z,
 	struct sldns_file_parse_state pstate;
 	struct auth_chunk* chunk;
 	size_t chunk_pos;
+	int ret;
 	memset(&pstate, 0, sizeof(pstate));
 	pstate.default_ttl = 3600;
 	if(xfr->namelen < sizeof(pstate.origin)) {
@@ -5063,10 +5110,24 @@ apply_http(struct auth_xfer* xfr, struct auth_zone* z,
 			continue;
 		}
 		/* parse line and add RR */
-		if(http_parse_origin(scratch_buffer, &pstate)) {
+		if((ret=http_parse_origin(scratch_buffer, &pstate))!=0) {
+			if(ret == 2) {
+				verbose(VERB_ALGO, "error parsing ORIGIN on line [%s:%d] %s",
+					xfr->task_transfer->master->file,
+					pstate.lineno,
+					sldns_buffer_begin(scratch_buffer));
+				return 0;
+			}
 			continue; /* $ORIGIN has been handled */
 		}
-		if(http_parse_ttl(scratch_buffer, &pstate)) {
+		if((ret=http_parse_ttl(scratch_buffer, &pstate))!=0) {
+			if(ret == 2) {
+				verbose(VERB_ALGO, "error parsing TTL on line [%s:%d] %s",
+					xfr->task_transfer->master->file,
+					pstate.lineno,
+					sldns_buffer_begin(scratch_buffer));
+				return 0;
+			}
 			continue; /* $TTL has been handled */
 		}
 		if(!http_parse_add_rr(xfr, z, scratch_buffer, &pstate)) {
@@ -5324,7 +5385,7 @@ xfr_transfer_lookup_host(struct auth_xfer* xfr, struct module_env* env)
 	struct edns_data edns;
 	sldns_buffer* buf = env->scratch_buffer;
 	if(!master) return 0;
-	if(extstrtoaddr(master->host, &addr, &addrlen)) {
+	if(extstrtoaddr(master->host, &addr, &addrlen, UNBOUND_DNS_PORT)) {
 		/* not needed, host is in IP addr format */
 		return 0;
 	}
@@ -5362,6 +5423,8 @@ xfr_transfer_lookup_host(struct auth_xfer* xfr, struct module_env* env)
 	edns.opt_list_out = NULL;
 	edns.opt_list_inplace_cb_out = NULL;
 	edns.padding_block_size = 0;
+	edns.cookie_present = 0;
+	edns.cookie_valid = 0;
 	if(sldns_buffer_capacity(buf) < 65535)
 		edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
 	else	edns.udp_size = 65535;
@@ -5370,7 +5433,7 @@ xfr_transfer_lookup_host(struct auth_xfer* xfr, struct module_env* env)
 	 * called straight away */
 	lock_basic_unlock(&xfr->lock);
 	if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
-		&auth_xfer_transfer_lookup_callback, xfr)) {
+		&auth_xfer_transfer_lookup_callback, xfr, 0)) {
 		lock_basic_lock(&xfr->lock);
 		log_err("out of memory lookup up master %s", master->host);
 		return 0;
@@ -5455,6 +5518,8 @@ xfr_transfer_init_fetch(struct auth_xfer* xfr, struct module_env* env)
 			addr_to_str(&addr, addrlen, as, sizeof(as));
 			verbose(VERB_ALGO, "auth zone %s transfer next HTTP fetch from %s started", zname, as);
 		}
+		/* Create or refresh the list of allow_notify addrs */
+		probe_copy_masters_for_allow_notify(xfr);
 		return 1;
 	}
 
@@ -6280,6 +6345,16 @@ xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
 
 	/* get master addr */
 	if(xfr->task_probe->scan_addr) {
+		/* parse the host string to get the auth name for scan_addr */
+		if(xfr->task_probe->scan_specific &&
+			xfr->task_probe->scan_specific->host &&
+			!authextstrtoaddr(xfr->task_probe->scan_specific->host, &addr, &addrlen, &auth_name)) {
+			char zname[255+1];
+			dname_str(xfr->name, zname);
+			log_err("%s: failed lookup, cannot probe to master %s",
+				zname, master->host);
+			return 0;
+		}
 		addrlen = xfr->task_probe->scan_addr->addrlen;
 		memmove(&addr, &xfr->task_probe->scan_addr->addr, addrlen);
 	} else {
@@ -6293,18 +6368,6 @@ xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
 				zname, master->host);
 			return 0;
 		}
-		if (auth_name != NULL) {
-			if (addr.ss_family == AF_INET
-			&&  (int)ntohs(((struct sockaddr_in *)&addr)->sin_port)
-		            == env->cfg->ssl_port)
-				((struct sockaddr_in *)&addr)->sin_port
-					= htons((uint16_t)env->cfg->port);
-			else if (addr.ss_family == AF_INET6
-			&&  (int)ntohs(((struct sockaddr_in6 *)&addr)->sin6_port)
-		            == env->cfg->ssl_port)
-                        	((struct sockaddr_in6 *)&addr)->sin6_port
-					= htons((uint16_t)env->cfg->port);
-		}
 	}
 
 	/* create packet */
@@ -6314,11 +6377,12 @@ xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
 		xfr->task_probe->id = GET_RANDOM_ID(env->rnd);
 	xfr_create_soa_probe_packet(xfr, env->scratch_buffer, 
 		xfr->task_probe->id);
-	/* we need to remove the cp if we have a different ip4/ip6 type now */
+	/* we need to remove the cp if we have a different ip4/ip6 type now.
+	 * And also for tls probes to get a new tcp commpoint. */
 	if(xfr->task_probe->cp &&
 		((xfr->task_probe->cp_is_ip6 && !addr_is_ip6(&addr, addrlen)) ||
-		(!xfr->task_probe->cp_is_ip6 && addr_is_ip6(&addr, addrlen)))
-		) {
+		(!xfr->task_probe->cp_is_ip6 && addr_is_ip6(&addr, addrlen)) ||
+		auth_name != NULL)) {
 		comm_point_delete(xfr->task_probe->cp);
 		xfr->task_probe->cp = NULL;
 	}
@@ -6326,14 +6390,19 @@ xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
 		if(addr_is_ip6(&addr, addrlen))
 			xfr->task_probe->cp_is_ip6 = 1;
 		else 	xfr->task_probe->cp_is_ip6 = 0;
-		xfr->task_probe->cp = outnet_comm_point_for_udp(env->outnet,
-			auth_xfer_probe_udp_callback, xfr, &addr, addrlen);
+		if(auth_name != NULL)
+			xfr->task_probe->cp = outnet_comm_point_for_tcp(env->outnet,
+				auth_xfer_probe_tcp_callback, xfr, &addr, addrlen, env->scratch_buffer, -1, auth_name != NULL, auth_name);
+		else
+			xfr->task_probe->cp = outnet_comm_point_for_udp(env->outnet,
+				auth_xfer_probe_udp_callback, xfr, &addr, addrlen);
 		if(!xfr->task_probe->cp) {
 			char zname[255+1], as[256];
 			dname_str(xfr->name, zname);
 			addr_to_str(&addr, addrlen, as, sizeof(as));
-			verbose(VERB_ALGO, "cannot create udp cp for "
-				"probe %s to %s", zname, as);
+			verbose(VERB_ALGO, "cannot create %s cp for "
+				"probe %s to %s", (auth_name?"tcp":"udp"),
+				zname, as);
 			return 0;
 		}
 	}
@@ -6346,8 +6415,9 @@ xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
 		}
 	}
 
-	/* send udp packet */
-	if(!comm_point_send_udp_msg(xfr->task_probe->cp, env->scratch_buffer,
+	/* send udp packet, for an UDP commpoint. */
+	if(auth_name == NULL &&
+	   !comm_point_send_udp_msg(xfr->task_probe->cp, env->scratch_buffer,
 		(struct sockaddr*)&addr, addrlen, 0)) {
 		char zname[255+1], as[256];
 		dname_str(xfr->name, zname);
@@ -6356,13 +6426,18 @@ xfr_probe_send_probe(struct auth_xfer* xfr, struct module_env* env,
 			zname, as);
 		return 0;
 	}
+	/* for a TCP commpoint, the packet is waiting to be sent in the
+	 * TCP commpoint buffer. The callback returns when there is a reply. */
 	if(verbosity >= VERB_ALGO) {
 		char zname[255+1], as[256];
 		dname_str(xfr->name, zname);
 		addr_to_str(&addr, addrlen, as, sizeof(as));
-		verbose(VERB_ALGO, "auth zone %s soa probe sent to %s", zname,
-			as);
+		verbose(VERB_ALGO, "auth zone %s soa probe sent to %s%s%s",
+			zname, as, (auth_name?"#":""),
+			(auth_name?auth_name:""));
 	}
+	if(auth_name != NULL)
+		timeout *= 10; /* TCP and TLS get a longer timeout */
 	xfr->task_probe->timeout = timeout;
 #ifndef S_SPLINT_S
 	t.tv_sec = timeout/1000;
@@ -6500,6 +6575,94 @@ auth_xfer_probe_udp_callback(struct comm_point* c, void* arg, int err,
 	return 0;
 }
 
+/** callback for task_probe tcp replies */
+int
+auth_xfer_probe_tcp_callback(struct comm_point* c, void* arg, int err,
+	struct comm_reply* ATTR_UNUSED(repinfo))
+{
+	struct auth_xfer* xfr = (struct auth_xfer*)arg;
+	struct module_env* env;
+	log_assert(xfr->task_probe);
+	lock_basic_lock(&xfr->lock);
+	env = xfr->task_probe->env;
+	if(!env || env->outnet->want_to_quit) {
+		lock_basic_unlock(&xfr->lock);
+		return 0; /* stop on quit */
+	}
+	/* stop the timer */
+	comm_timer_disable(xfr->task_probe->timer);
+
+	/* see if we got a reply and what that means */
+	if(err == NETEVENT_NOERROR) {
+		uint32_t serial = 0;
+		if(check_packet_ok(c->buffer, LDNS_RR_TYPE_SOA, xfr,
+			&serial)) {
+			/* successful lookup */
+			if(verbosity >= VERB_ALGO) {
+				char buf[256];
+				dname_str(xfr->name, buf);
+				verbose(VERB_ALGO, "auth zone %s: soa probe "
+					"serial is %u", buf, (unsigned)serial);
+			}
+			/* see if this serial indicates that the zone has
+			 * to be updated */
+			if(xfr_serial_means_update(xfr, serial)) {
+				/* if updated, start the transfer task, if needed */
+				verbose(VERB_ALGO, "auth_zone updated, start transfer");
+				if(xfr->task_transfer->worker == NULL) {
+					struct auth_master* master =
+						xfr_probe_current_master(xfr);
+					/* if we have download URLs use them
+					 * in preference to this master we
+					 * just probed the SOA from */
+					if(xfr->task_transfer->masters &&
+						xfr->task_transfer->masters->http)
+						master = NULL;
+					xfr_probe_disown(xfr);
+					xfr_start_transfer(xfr, env, master);
+					return 0;
+
+				}
+				/* other tasks are running, we don't do this anymore */
+				xfr_probe_disown(xfr);
+				lock_basic_unlock(&xfr->lock);
+				/* return, we don't sent a reply to this udp packet,
+				 * and we setup the tasks to do next */
+				return 0;
+			} else {
+				verbose(VERB_ALGO, "auth_zone master reports unchanged soa serial");
+				/* we if cannot find updates amongst the
+				 * masters, this means we then have a new lease
+				 * on the zone */
+				xfr->task_probe->have_new_lease = 1;
+			}
+		} else {
+			if(verbosity >= VERB_ALGO) {
+				char buf[256];
+				dname_str(xfr->name, buf);
+				verbose(VERB_ALGO, "auth zone %s: bad reply to soa probe", buf);
+			}
+		}
+	} else {
+		if(verbosity >= VERB_ALGO) {
+			char buf[256];
+			dname_str(xfr->name, buf);
+			verbose(VERB_ALGO, "auth zone %s: soa probe failed", buf);
+		}
+	}
+
+	/* failed lookup or not an update */
+	/* delete commpoint so a new one is created, with a fresh port nr */
+	comm_point_delete(xfr->task_probe->cp);
+	xfr->task_probe->cp = NULL;
+
+	/* if the result was not a successful probe, we need
+	 * to send the next one */
+	xfr_probe_nextmaster(xfr);
+	xfr_probe_send_or_end(xfr, env);
+	return 0;
+}
+
 /** lookup a host name for its addresses, if needed */
 static int
 xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
@@ -6513,7 +6676,7 @@ xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
 	struct edns_data edns;
 	sldns_buffer* buf = env->scratch_buffer;
 	if(!master) return 0;
-	if(extstrtoaddr(master->host, &addr, &addrlen)) {
+	if(extstrtoaddr(master->host, &addr, &addrlen, UNBOUND_DNS_PORT)) {
 		/* not needed, host is in IP addr format */
 		return 0;
 	}
@@ -6553,6 +6716,8 @@ xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
 	edns.opt_list_out = NULL;
 	edns.opt_list_inplace_cb_out = NULL;
 	edns.padding_block_size = 0;
+	edns.cookie_present = 0;
+	edns.cookie_valid = 0;
 	if(sldns_buffer_capacity(buf) < 65535)
 		edns.udp_size = (uint16_t)sldns_buffer_capacity(buf);
 	else	edns.udp_size = 65535;
@@ -6561,7 +6726,7 @@ xfr_probe_lookup_host(struct auth_xfer* xfr, struct module_env* env)
 	 * called straight away */
 	lock_basic_unlock(&xfr->lock);
 	if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
-		&auth_xfer_probe_lookup_callback, xfr)) {
+		&auth_xfer_probe_lookup_callback, xfr, 0)) {
 		lock_basic_lock(&xfr->lock);
 		log_err("out of memory lookup up master %s", master->host);
 		return 0;
@@ -7450,7 +7615,7 @@ static void add_rrlist_rrsigs_into_data(struct packed_rrset_data* data,
 		size_t j;
 		if(!rrlist[i])
 			continue;
-		if(rrlist[i] && rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
+		if(rrlist[i]->type == LDNS_RR_TYPE_ZONEMD &&
 			query_dname_compare(z->name, node->name)==0) {
 			/* omit RRSIGs over type ZONEMD at apex */
 			continue;
@@ -7632,13 +7797,16 @@ int auth_zone_generate_zonemd_check(struct auth_zone* z, int scheme,
 {
 	uint8_t gen[512];
 	size_t genlen = 0;
+	*reason = NULL;
 	if(!zonemd_hashalgo_supported(hashalgo)) {
+		/* allow it */
 		*reason = "unsupported algorithm";
-		return 0;
+		return 1;
 	}
 	if(!zonemd_scheme_supported(scheme)) {
+		/* allow it */
 		*reason = "unsupported scheme";
-		return 0;
+		return 1;
 	}
 	if(hashlen < 12) {
 		/* the ZONEMD draft requires digests to fail if too small */
@@ -7726,7 +7894,7 @@ static int zonemd_dnssec_verify_rrset(struct auth_zone* z,
 		auth_zone_log(z->name, VERB_ALGO,
 			"zonemd: verify %s RRset with DNSKEY", typestr);
 	}
-	sec = dnskeyset_verify_rrset(env, ve, &pk, dnskey, sigalg, why_bogus,
+	sec = dnskeyset_verify_rrset(env, ve, &pk, dnskey, sigalg, why_bogus, NULL,
 		LDNS_SECTION_ANSWER, NULL);
 	if(sec == sec_status_secure) {
 		return 1;
@@ -8003,9 +8171,13 @@ auth_zone_verify_zonemd_with_key(struct auth_zone* z, struct module_env* env,
 	}
 
 	/* success! log the success */
-	auth_zone_log(z->name, VERB_ALGO, "ZONEMD verification successful");
+	if(reason)
+		auth_zone_log(z->name, VERB_ALGO, "ZONEMD %s", reason);
+	else	auth_zone_log(z->name, VERB_ALGO, "ZONEMD verification successful");
 	if(result) {
-		*result = strdup("ZONEMD verification successful");
+		if(reason)
+			*result = strdup(reason);
+		else	*result = strdup("ZONEMD verification successful");
 		if(!*result) log_err("out of memory");
 	}
 }
@@ -8065,7 +8237,7 @@ zonemd_get_dnskey_from_anchor(struct auth_zone* z, struct module_env* env,
 	auth_zone_log(z->name, VERB_QUERY,
 		"zonemd: verify DNSKEY RRset with trust anchor");
 	sec = val_verify_DNSKEY_with_TA(env, ve, keystorage, anchor->ds_rrset,
-		anchor->dnskey_rrset, NULL, why_bogus, NULL);
+		anchor->dnskey_rrset, NULL, why_bogus, NULL, NULL);
 	regional_free_all(env->scratch);
 	if(sec == sec_status_secure) {
 		/* success */
@@ -8124,7 +8296,7 @@ auth_zone_verify_zonemd_key_with_ds(struct auth_zone* z,
 	keystorage->rk.rrset_class = htons(z->dclass);
 	auth_zone_log(z->name, VERB_QUERY, "zonemd: verify zone DNSKEY with DS");
 	sec = val_verify_DNSKEY_with_DS(env, ve, keystorage, ds, sigalg,
-		why_bogus, NULL);
+		why_bogus, NULL, NULL);
 	regional_free_all(env->scratch);
 	if(sec == sec_status_secure) {
 		/* success */
@@ -8340,7 +8512,7 @@ zonemd_lookup_dnskey(struct auth_zone* z, struct module_env* env)
 	/* the callback can be called straight away */
 	lock_rw_unlock(&z->lock);
 	if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0,
-		&auth_zonemd_dnskey_lookup_callback, z)) {
+		&auth_zonemd_dnskey_lookup_callback, z, 0)) {
 		lock_rw_wrlock(&z->lock);
 		log_err("out of memory lookup of %s for zonemd",
 			(fetch_ds?"DS":"DNSKEY"));

services/authzone.h

@@ -678,6 +678,9 @@ void auth_xfer_timer(void* arg);
 /** callback for commpoint udp replies to task_probe */
 int auth_xfer_probe_udp_callback(struct comm_point* c, void* arg, int err,
         struct comm_reply* repinfo);
+/** callback for commpoint tcp replies to task_probe */
+int auth_xfer_probe_tcp_callback(struct comm_point* c, void* arg, int err,
+        struct comm_reply* repinfo);
 /** callback for task_transfer tcp connections */
 int auth_xfer_transfer_tcp_callback(struct comm_point* c, void* arg, int err,
         struct comm_reply* repinfo);
@@ -747,6 +750,9 @@ int zonemd_scheme_supported(int scheme);
  * @param region: temp region for allocs during canonicalisation.
  * @param buf: temp buffer during canonicalisation.
  * @param reason: string returned with failure reason.
+ * 	If the hash cannot be checked, but it is allowed, for unknown
+ * 	algorithms, the routine returns success, and the reason is nonNULL,
+ * 	with the allowance reason.
  * @return false on failure.
  */
 int auth_zone_generate_zonemd_check(struct auth_zone* z, int scheme,

services/cache/dns.c

@@ -68,11 +68,16 @@
  * 	in a prefetch situation to be updated (without becoming sticky).
  * @param qrep: update rrsets here if cache is better
  * @param region: for qrep allocs.
+ * @param qstarttime: time when delegations were looked up, this is perhaps
+ *	earlier than the time in now. The time is used to determine if RRsets
+ *	of type NS have expired, so that they can only be updated using
+ *	lookups of delegation points that did not use them, since they had
+ *	expired then.
  */
 static void
 store_rrsets(struct module_env* env, struct reply_info* rep, time_t now,
 	time_t leeway, int pside, struct reply_info* qrep,
-	struct regional* region)
+	struct regional* region, time_t qstarttime)
 {
 	size_t i;
 	/* see if rrset already exists in cache, if not insert it. */
@@ -81,8 +86,8 @@ store_rrsets(struct module_env* env, struct reply_info* rep, time_t now,
 		rep->ref[i].id = rep->rrsets[i]->id;
 		/* update ref if it was in the cache */
 		switch(rrset_cache_update(env->rrset_cache, &rep->ref[i],
-				env->alloc, now + ((ntohs(rep->ref[i].key->rk.type)==
-				LDNS_RR_TYPE_NS && !pside)?0:leeway))) {
+				env->alloc, ((ntohs(rep->ref[i].key->rk.type)==
+				LDNS_RR_TYPE_NS && !pside)?qstarttime:now + leeway))) {
 		case 0: /* ref unchanged, item inserted */
 			break;
 		case 2: /* ref updated, cache is superior */
@@ -127,35 +132,11 @@ msg_cache_remove(struct module_env* env, uint8_t* qname, size_t qnamelen,
 	slabhash_remove(env->msg_cache, h, &k);
 }
 
-/** remove servfail msg cache entry */
-static void
-msg_del_servfail(struct module_env* env, struct query_info* qinfo,
-	uint32_t flags)
-{
-	struct msgreply_entry* e;
-	/* see if the entry is servfail, and then remove it, so that
-	 * lookups move from the cacheresponse stage to the recursionresponse
-	 * stage */
-	e = msg_cache_lookup(env, qinfo->qname, qinfo->qname_len,
-		qinfo->qtype, qinfo->qclass, flags, 0, 0);
-	if(!e) return;
-	/* we don't check for the ttl here, also expired servfail entries
-	 * are removed.  If the user uses serve-expired, they would still be
-	 * used to answer from cache */
-	if(FLAGS_GET_RCODE(((struct reply_info*)e->entry.data)->flags)
-		!= LDNS_RCODE_SERVFAIL) {
-		lock_rw_unlock(&e->entry.lock);
-		return;
-	}
-	lock_rw_unlock(&e->entry.lock);
-	msg_cache_remove(env, qinfo->qname, qinfo->qname_len, qinfo->qtype,
-		qinfo->qclass, flags);
-}
-
 void 
 dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
 	hashvalue_type hash, struct reply_info* rep, time_t leeway, int pside,
-	struct reply_info* qrep, uint32_t flags, struct regional* region)
+	struct reply_info* qrep, uint32_t flags, struct regional* region,
+	time_t qstarttime)
 {
 	struct msgreply_entry* e;
 	time_t ttl = rep->ttl;
@@ -170,18 +151,26 @@ dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
 	/* there was a reply_info_sortref(rep) here but it seems to be
 	 * unnecessary, because the cache gets locked per rrset. */
 	reply_info_set_ttls(rep, *env->now);
-	store_rrsets(env, rep, *env->now, leeway, pside, qrep, region);
+	store_rrsets(env, rep, *env->now, leeway, pside, qrep, region,
+		qstarttime);
 	if(ttl == 0 && !(flags & DNSCACHE_STORE_ZEROTTL)) {
 		/* we do not store the message, but we did store the RRs,
 		 * which could be useful for delegation information */
 		verbose(VERB_ALGO, "TTL 0: dropped msg from cache");
-		free(rep);
-		/* if the message is SERVFAIL in cache, remove that SERVFAIL,
+		reply_info_delete(rep, NULL);
+		/* if the message is in the cache, remove that msg,
 		 * so that the TTL 0 response can be returned for future
-		 * responses (i.e. don't get answered by the servfail from
+		 * responses (i.e. don't get answered from
 		 * cache, but instead go to recursion to get this TTL0
-		 * response). */
-		msg_del_servfail(env, qinfo, flags);
+		 * response).
+		 * Possible messages that could be in the cache:
+		 * - SERVFAIL
+		 * - NXDOMAIN
+		 * - NODATA
+		 * - an older record that is expired
+		 * - an older record that did not yet expire */
+		msg_cache_remove(env, qinfo->qname, qinfo->qname_len,
+			qinfo->qtype, qinfo->qclass, flags);
 		return;
 	}
 
@@ -194,10 +183,51 @@ dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
 	slabhash_insert(env->msg_cache, hash, &e->entry, rep, env->alloc);
 }
 
+/** see if an rrset is expired above the qname, return upper qname. */
+static int
+rrset_expired_above(struct module_env* env, uint8_t** qname, size_t* qnamelen,
+	uint16_t searchtype, uint16_t qclass, time_t now, uint8_t* expiretop,
+	size_t expiretoplen)
+{
+	struct ub_packed_rrset_key *rrset;
+	uint8_t lablen;
+
+	while(*qnamelen > 0) {
+		/* look one label higher */
+		lablen = **qname;
+		*qname += lablen + 1;
+		*qnamelen -= lablen + 1;
+		if(*qnamelen <= 0)
+			break;
+
+		/* looks up with a time of 0, to see expired entries */
+		if((rrset = rrset_cache_lookup(env->rrset_cache, *qname,
+			*qnamelen, searchtype, qclass, 0, 0, 0))) {
+			struct packed_rrset_data* data =
+				(struct packed_rrset_data*)rrset->entry.data;
+			if(now > data->ttl) {
+				/* it is expired, this is not wanted */
+				lock_rw_unlock(&rrset->entry.lock);
+				log_nametypeclass(VERB_ALGO, "this rrset is expired", *qname, searchtype, qclass);
+				return 1;
+			}
+			/* it is not expired, continue looking */
+			lock_rw_unlock(&rrset->entry.lock);
+		}
+
+		/* do not look above the expiretop. */
+		if(expiretop && *qnamelen == expiretoplen &&
+			query_dname_compare(*qname, expiretop)==0)
+			break;
+	}
+	return 0;
+}
+
 /** find closest NS or DNAME and returns the rrset (locked) */
 static struct ub_packed_rrset_key*
 find_closest_of_type(struct module_env* env, uint8_t* qname, size_t qnamelen, 
-	uint16_t qclass, time_t now, uint16_t searchtype, int stripfront)
+	uint16_t qclass, time_t now, uint16_t searchtype, int stripfront,
+	int noexpiredabove, uint8_t* expiretop, size_t expiretoplen)
 {
 	struct ub_packed_rrset_key *rrset;
 	uint8_t lablen;
@@ -212,8 +242,40 @@ find_closest_of_type(struct module_env* env, uint8_t* qname, size_t qnamelen,
 	/* snip off front part of qname until the type is found */
 	while(qnamelen > 0) {
 		if((rrset = rrset_cache_lookup(env->rrset_cache, qname, 
-			qnamelen, searchtype, qclass, 0, now, 0)))
-			return rrset;
+			qnamelen, searchtype, qclass, 0, now, 0))) {
+			uint8_t* origqname = qname;
+			size_t origqnamelen = qnamelen;
+			if(!noexpiredabove)
+				return rrset;
+			/* if expiretop set, do not look above it, but
+			 * qname is equal, so the just found result is also
+			 * the nonexpired above part. */
+			if(expiretop && qnamelen == expiretoplen &&
+				query_dname_compare(qname, expiretop)==0)
+				return rrset;
+			/* check for expiry, but we have to let go of the rrset
+			 * for the lock ordering */
+			lock_rw_unlock(&rrset->entry.lock);
+			/* the expired_above function always takes off one
+			 * label (if qnamelen>0) and returns the final qname
+			 * where it searched, so we can continue from there
+			 * turning the O N*N search into O N. */
+			if(!rrset_expired_above(env, &qname, &qnamelen,
+				searchtype, qclass, now, expiretop,
+				expiretoplen)) {
+				/* we want to return rrset, but it may be
+				 * gone from cache, if so, just loop like
+				 * it was not in the cache in the first place.
+				 */
+				if((rrset = rrset_cache_lookup(env->
+					rrset_cache, origqname, origqnamelen,
+					searchtype, qclass, 0, now, 0))) {
+					return rrset;
+				}
+			}
+			log_nametypeclass(VERB_ALGO, "ignoring rrset because expired rrsets exist above it", origqname, searchtype, qclass);
+			continue;
+		}
 
 		/* snip off front label */
 		lablen = *qname;
@@ -324,6 +386,9 @@ cache_fill_missing(struct module_env* env, uint16_t qclass,
 	struct ub_packed_rrset_key* akey;
 	time_t now = *env->now;
 	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->cache_lookup_count > ITERATOR_NAME_CACHELOOKUP_MAX)
+			continue;
+		ns->cache_lookup_count++;
 		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
 			ns->namelen, LDNS_RR_TYPE_A, qclass, 0, now, 0);
 		if(akey) {
@@ -428,6 +493,7 @@ dns_msg_create(uint8_t* qname, size_t qnamelen, uint16_t qtype,
 		return NULL; /* integer overflow protection */
 	msg->rep->flags = BIT_QR; /* with QR, no AA */
 	msg->rep->qdcount = 1;
+	msg->rep->reason_bogus = LDNS_EDE_NONE;
 	msg->rep->rrsets = (struct ub_packed_rrset_key**)
 		regional_alloc(region, 
 		capacity*sizeof(struct ub_packed_rrset_key*));
@@ -461,7 +527,8 @@ dns_msg_ansadd(struct dns_msg* msg, struct regional* region,
 struct delegpt* 
 dns_cache_find_delegation(struct module_env* env, uint8_t* qname, 
 	size_t qnamelen, uint16_t qtype, uint16_t qclass, 
-	struct regional* region, struct dns_msg** msg, time_t now)
+	struct regional* region, struct dns_msg** msg, time_t now,
+	int noexpiredabove, uint8_t* expiretop, size_t expiretoplen)
 {
 	/* try to find closest NS rrset */
 	struct ub_packed_rrset_key* nskey;
@@ -469,7 +536,7 @@ dns_cache_find_delegation(struct module_env* env, uint8_t* qname,
 	struct delegpt* dp;
 
 	nskey = find_closest_of_type(env, qname, qnamelen, qclass, now,
-		LDNS_RR_TYPE_NS, 0);
+		LDNS_RR_TYPE_NS, 0, noexpiredabove, expiretop, expiretoplen);
 	if(!nskey) /* hope the caller has hints to prime or something */
 		return NULL;
 	nsdata = (struct packed_rrset_data*)nskey->entry.data;
@@ -524,6 +591,8 @@ gen_dns_msg(struct regional* region, struct query_info* q, size_t num)
 		sizeof(struct reply_info) - sizeof(struct rrset_ref));
 	if(!msg->rep)
 		return NULL;
+	msg->rep->reason_bogus = LDNS_EDE_NONE;
+	msg->rep->reason_bogus_str = NULL;
 	if(num > RR_COUNT_MAX)
 		return NULL; /* integer overflow protection */
 	msg->rep->rrsets = (struct ub_packed_rrset_key**)
@@ -550,6 +619,14 @@ tomsg(struct module_env* env, struct query_info* q, struct reply_info* r,
 				r->serve_expired_ttl < now) {
 				return NULL;
 			}
+			/* Ignore expired failure answers */
+			if(FLAGS_GET_RCODE(r->flags) !=
+				LDNS_RCODE_NOERROR &&
+				FLAGS_GET_RCODE(r->flags) !=
+				LDNS_RCODE_NXDOMAIN &&
+				FLAGS_GET_RCODE(r->flags) !=
+				LDNS_RCODE_YXDOMAIN)
+				return 0;
 		} else {
 			return NULL;
 		}
@@ -577,6 +654,11 @@ tomsg(struct module_env* env, struct query_info* q, struct reply_info* r,
 	msg->rep->ar_numrrsets = r->ar_numrrsets;
 	msg->rep->rrset_count = r->rrset_count;
 	msg->rep->authoritative = r->authoritative;
+	msg->rep->reason_bogus = r->reason_bogus;
+	if(r->reason_bogus_str) {
+		msg->rep->reason_bogus_str = regional_strdup(region, r->reason_bogus_str);
+	}
+
 	if(!rrset_array_lock(r->ref, r->rrset_count, now_control)) {
 		return NULL;
 	}
@@ -632,6 +714,7 @@ rrset_msg(struct ub_packed_rrset_key* rrset, struct regional* region,
 	msg->rep->ns_numrrsets = 0;
 	msg->rep->ar_numrrsets = 0;
 	msg->rep->rrset_count = 1;
+	msg->rep->reason_bogus = LDNS_EDE_NONE;
 	msg->rep->rrsets[0] = packed_rrset_copy_region(rrset, region, now);
 	if(!msg->rep->rrsets[0]) /* copy CNAME */
 		return NULL;
@@ -670,6 +753,7 @@ synth_dname_msg(struct ub_packed_rrset_key* rrset, struct regional* region,
 	msg->rep->ns_numrrsets = 0;
 	msg->rep->ar_numrrsets = 0;
 	msg->rep->rrset_count = 1;
+	msg->rep->reason_bogus = LDNS_EDE_NONE;
 	msg->rep->rrsets[0] = packed_rrset_copy_region(rrset, region, now);
 	if(!msg->rep->rrsets[0]) /* copy DNAME */
 		return NULL;
@@ -835,7 +919,7 @@ dns_cache_lookup(struct module_env* env,
 	 * consistent with the DNAME */
 	if(!no_partial &&
 		(rrset=find_closest_of_type(env, qname, qnamelen, qclass, now,
-		LDNS_RR_TYPE_DNAME, 1))) {
+		LDNS_RR_TYPE_DNAME, 1, 0, NULL, 0))) {
 		/* synthesize a DNAME+CNAME message based on this */
 		enum sec_status sec_status = sec_status_unchecked;
 		struct dns_msg* msg = synth_dname_msg(rrset, region, now, &k,
@@ -968,7 +1052,7 @@ dns_cache_lookup(struct module_env* env,
 int
 dns_cache_store(struct module_env* env, struct query_info* msgqinf,
         struct reply_info* msgrep, int is_referral, time_t leeway, int pside,
-	struct regional* region, uint32_t flags)
+	struct regional* region, uint32_t flags, time_t qstarttime)
 {
 	struct reply_info* rep = NULL;
 	/* alloc, malloc properly (not in region, like msg is) */
@@ -978,7 +1062,6 @@ dns_cache_store(struct module_env* env, struct query_info* msgqinf,
 	/* ttl must be relative ;i.e. 0..86400 not  time(0)+86400.
 	 * the env->now is added to message and RRsets in this routine. */
 	/* the leeway is used to invalidate other rrsets earlier */
-
 	if(is_referral) {
 		/* store rrsets */
 		struct rrset_ref ref;
@@ -991,11 +1074,11 @@ dns_cache_store(struct module_env* env, struct query_info* msgqinf,
 			/*ignore ret: it was in the cache, ref updated */
 			/* no leeway for typeNS */
 			(void)rrset_cache_update(env->rrset_cache, &ref, 
-				env->alloc, *env->now + 
+				env->alloc,
 				((ntohs(ref.key->rk.type)==LDNS_RR_TYPE_NS
-				 && !pside) ? 0:leeway));
+				 && !pside) ? qstarttime:*env->now + leeway));
 		}
-		free(rep);
+		reply_info_delete(rep, NULL);
 		return 1;
 	} else {
 		/* store msg, and rrsets */
@@ -1015,7 +1098,7 @@ dns_cache_store(struct module_env* env, struct query_info* msgqinf,
 		rep->flags &= ~(BIT_AA | BIT_CD);
 		h = query_info_hash(&qinf, (uint16_t)flags);
 		dns_cache_store_msg(env, &qinf, h, rep, leeway, pside, msgrep,
-			flags, region);
+			flags, region, qstarttime);
 		/* qname is used inside query_info_entrysetup, and set to 
 		 * NULL. If it has not been used, free it. free(0) is safe. */
 		free(qinf.qname);

services/cache/dns.h

@@ -88,11 +88,13 @@ struct dns_msg {
  * @param flags: flags with BIT_CD for AAAA queries in dns64 translation.
  *   The higher 16 bits are used internally to customize the cache policy.
  *   (See DNSCACHE_STORE_xxx flags).
+ * @param qstarttime: time when the query was started, and thus when the
+ * 	delegations were looked up.
  * @return 0 on alloc error (out of memory).
  */
 int dns_cache_store(struct module_env* env, struct query_info* qinf,
         struct reply_info* rep, int is_referral, time_t leeway, int pside,
-	struct regional* region, uint32_t flags);
+	struct regional* region, uint32_t flags, time_t qstarttime);
 
 /**
  * Store message in the cache. Stores in message cache and rrset cache.
@@ -112,11 +114,14 @@ int dns_cache_store(struct module_env* env, struct query_info* qinf,
  * 	can be updated to full TTL even in prefetch situations.
  * @param qrep: message that can be altered with better rrs from cache.
  * @param flags: customization flags for the cache policy.
+ * @param qstarttime: time when the query was started, and thus when the
+ * 	delegations were looked up.
  * @param region: to allocate into for qmsg.
  */
 void dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
 	hashvalue_type hash, struct reply_info* rep, time_t leeway, int pside,
-	struct reply_info* qrep, uint32_t flags, struct regional* region);
+	struct reply_info* qrep, uint32_t flags, struct regional* region,
+	time_t qstarttime);
 
 /**
  * Find a delegation from the cache.
@@ -129,11 +134,18 @@ void dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
  * @param msg: if not NULL, delegation message is returned here, synthesized
  *	from the cache.
  * @param timenow: the time now, for checking if TTL on cache entries is OK.
+ * @param noexpiredabove: if set, no expired NS rrsets above the one found
+ * 	are tolerated. It only returns delegations where the delegations above
+ * 	it are valid.
+ * @param expiretop: if not NULL, name where check for expiry ends for
+ * 	noexpiredabove.
+ * @param expiretoplen: length of expiretop dname.
  * @return new delegation or NULL on error or if not found in cache.
  */
 struct delegpt* dns_cache_find_delegation(struct module_env* env, 
 	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass, 
-	struct regional* region, struct dns_msg** msg, time_t timenow);
+	struct regional* region, struct dns_msg** msg, time_t timenow,
+	int noexpiredabove, uint8_t* expiretop, size_t expiretoplen);
 
 /**
  * generate dns_msg from cached message

services/cache/infra.c

@@ -67,6 +67,11 @@ int infra_dp_ratelimit = 0;
  *  in queries per second. */
 int infra_ip_ratelimit = 0;
 
+/** ratelimit value for client ip addresses,
+ *  in queries per second.
+ *  For clients with a valid DNS Cookie. */
+int infra_ip_ratelimit_cookie = 0;
+
 size_t 
 infra_sizefunc(void* k, void* ATTR_UNUSED(d))
 {
@@ -721,13 +726,13 @@ infra_get_lame_rtt(struct infra_cache* infra,
 			else	*rtt = USEFUL_SERVER_TOP_TIMEOUT-1000;
 		}
 	}
+	/* expired entry */
 	if(timenow > host->ttl) {
-		/* expired entry */
+
 		/* see if this can be a re-probe of an unresponsive server */
 		/* minus 1000 because that is outside of the RTTBAND, so
 		 * blacklisted servers stay blacklisted if this is chosen */
-		if(host->rtt.rto >= USEFUL_SERVER_TOP_TIMEOUT ||
-			infra->infra_keep_probing) {
+		if(host->rtt.rto >= USEFUL_SERVER_TOP_TIMEOUT) {
 			lock_rw_unlock(&e->lock);
 			*rtt = USEFUL_SERVER_TOP_TIMEOUT-1000;
 			*lame = 0;
@@ -834,14 +839,13 @@ static struct lruhash_entry* infra_find_ratedata(struct infra_cache* infra,
 
 /** find data item in array for ip addresses */
 static struct lruhash_entry* infra_find_ip_ratedata(struct infra_cache* infra,
-	struct comm_reply* repinfo, int wr)
+	struct sockaddr_storage* addr, socklen_t addrlen, int wr)
 {
 	struct ip_rate_key key;
-	hashvalue_type h = hash_addr(&(repinfo->addr),
-		repinfo->addrlen, 0);
+	hashvalue_type h = hash_addr(addr, addrlen, 0);
 	memset(&key, 0, sizeof(key));
-	key.addr = repinfo->addr;
-	key.addrlen = repinfo->addrlen;
+	key.addr = *addr;
+	key.addrlen = addrlen;
 	key.entry.hash = h;
 	return slabhash_lookup(infra->client_ip_rates, h, &key, wr);
 }
@@ -876,10 +880,9 @@ static void infra_create_ratedata(struct infra_cache* infra,
 
 /** create rate data item for ip address */
 static void infra_ip_create_ratedata(struct infra_cache* infra,
-	struct comm_reply* repinfo, time_t timenow)
+	struct sockaddr_storage* addr, socklen_t addrlen, time_t timenow)
 {
-	hashvalue_type h = hash_addr(&(repinfo->addr),
-	repinfo->addrlen, 0);
+	hashvalue_type h = hash_addr(addr, addrlen, 0);
 	struct ip_rate_key* k = (struct ip_rate_key*)calloc(1, sizeof(*k));
 	struct ip_rate_data* d = (struct ip_rate_data*)calloc(1, sizeof(*d));
 	if(!k || !d) {
@@ -887,8 +890,8 @@ static void infra_ip_create_ratedata(struct infra_cache* infra,
 		free(d);
 		return; /* alloc failure */
 	}
-	k->addr = repinfo->addr;
-	k->addrlen = repinfo->addrlen;
+	k->addr = *addr;
+	k->addrlen = addrlen;
 	lock_rw_init(&k->entry.lock);
 	k->entry.hash = h;
 	k->entry.key = k;
@@ -898,8 +901,9 @@ static void infra_ip_create_ratedata(struct infra_cache* infra,
 	slabhash_insert(infra->client_ip_rates, h, &k->entry, d, NULL);
 }
 
-/** find the second and return its rate counter, if none, remove oldest */
-static int* infra_rate_find_second(void* data, time_t t)
+/** Find the second and return its rate counter. If none and should_add, remove
+ *  oldest to accommodate. Else return none. */
+static int* infra_rate_find_second_or_none(void* data, time_t t, int should_add)
 {
 	struct rate_data* d = (struct rate_data*)data;
 	int i, oldest;
@@ -907,6 +911,7 @@ static int* infra_rate_find_second(void* data, time_t t)
 		if(d->timestamp[i] == t)
 			return &(d->qps[i]);
 	}
+	if(!should_add) return NULL;
 	/* remove oldest timestamp, and insert it at t with 0 qps */
 	oldest = 0;
 	for(i=0; i<RATE_WINDOW; i++) {
@@ -918,21 +923,41 @@ static int* infra_rate_find_second(void* data, time_t t)
 	return &(d->qps[oldest]);
 }
 
-int infra_rate_max(void* data, time_t now)
+/** find the second and return its rate counter, if none, remove oldest to
+ *  accommodate */
+static int* infra_rate_give_second(void* data, time_t t)
+{
+    return infra_rate_find_second_or_none(data, t, 1);
+}
+
+/** find the second and return its rate counter only if it exists. Caller
+ *  should check for NULL return value */
+static int* infra_rate_get_second(void* data, time_t t)
+{
+    return infra_rate_find_second_or_none(data, t, 0);
+}
+
+int infra_rate_max(void* data, time_t now, int backoff)
 {
 	struct rate_data* d = (struct rate_data*)data;
 	int i, max = 0;
 	for(i=0; i<RATE_WINDOW; i++) {
-		if(now-d->timestamp[i] <= RATE_WINDOW) {
-			if(d->qps[i] > max)
+		if(backoff) {
+			if(now-d->timestamp[i] <= RATE_WINDOW &&
+				d->qps[i] > max) {
 				max = d->qps[i];
+			}
+		} else {
+			if(now == d->timestamp[i]) {
+				return d->qps[i];
+			}
 		}
 	}
 	return max;
 }
 
 int infra_ratelimit_inc(struct infra_cache* infra, uint8_t* name,
-	size_t namelen, time_t timenow, struct query_info* qinfo,
+	size_t namelen, time_t timenow, int backoff, struct query_info* qinfo,
 	struct comm_reply* replylist)
 {
 	int lim, max;
@@ -949,13 +974,13 @@ int infra_ratelimit_inc(struct infra_cache* infra, uint8_t* name,
 	/* find or insert ratedata */
 	entry = infra_find_ratedata(infra, name, namelen, 1);
 	if(entry) {
-		int premax = infra_rate_max(entry->data, timenow);
-		int* cur = infra_rate_find_second(entry->data, timenow);
+		int premax = infra_rate_max(entry->data, timenow, backoff);
+		int* cur = infra_rate_give_second(entry->data, timenow);
 		(*cur)++;
-		max = infra_rate_max(entry->data, timenow);
+		max = infra_rate_max(entry->data, timenow, backoff);
 		lock_rw_unlock(&entry->lock);
 
-		if(premax < lim && max >= lim) {
+		if(premax <= lim && max > lim) {
 			char buf[257], qnm[257], ts[12], cs[12], ip[128];
 			dname_str(name, buf);
 			dname_str(qinfo->qname, qnm);
@@ -963,19 +988,19 @@ int infra_ratelimit_inc(struct infra_cache* infra, uint8_t* name,
 			sldns_wire2str_class_buf(qinfo->qclass, cs, sizeof(cs));
 			ip[0]=0;
 			if(replylist) {
-				addr_to_str((struct sockaddr_storage *)&replylist->addr,
-					replylist->addrlen, ip, sizeof(ip));
+				addr_to_str((struct sockaddr_storage *)&replylist->remote_addr,
+					replylist->remote_addrlen, ip, sizeof(ip));
 				verbose(VERB_OPS, "ratelimit exceeded %s %d query %s %s %s from %s", buf, lim, qnm, cs, ts, ip);
 			} else {
 				verbose(VERB_OPS, "ratelimit exceeded %s %d query %s %s %s", buf, lim, qnm, cs, ts);
 			}
 		}
-		return (max < lim);
+		return (max <= lim);
 	}
 
 	/* create */
 	infra_create_ratedata(infra, name, namelen, timenow);
-	return (1 < lim);
+	return (1 <= lim);
 }
 
 void infra_ratelimit_dec(struct infra_cache* infra, uint8_t* name,
@@ -987,14 +1012,19 @@ void infra_ratelimit_dec(struct infra_cache* infra, uint8_t* name,
 		return; /* not enabled */
 	entry = infra_find_ratedata(infra, name, namelen, 1);
 	if(!entry) return; /* not cached */
-	cur = infra_rate_find_second(entry->data, timenow);
+	cur = infra_rate_get_second(entry->data, timenow);
+	if(cur == NULL) {
+		/* our timenow is not available anymore; nothing to decrease */
+		lock_rw_unlock(&entry->lock);
+		return;
+	}
 	if((*cur) > 0)
 		(*cur)--;
 	lock_rw_unlock(&entry->lock);
 }
 
 int infra_ratelimit_exceeded(struct infra_cache* infra, uint8_t* name,
-	size_t namelen, time_t timenow)
+	size_t namelen, time_t timenow, int backoff)
 {
 	struct lruhash_entry* entry;
 	int lim, max;
@@ -1010,10 +1040,10 @@ int infra_ratelimit_exceeded(struct infra_cache* infra, uint8_t* name,
 	entry = infra_find_ratedata(infra, name, namelen, 0);
 	if(!entry)
 		return 0; /* not cached */
-	max = infra_rate_max(entry->data, timenow);
+	max = infra_rate_max(entry->data, timenow, backoff);
 	lock_rw_unlock(&entry->lock);
 
-	return (max >= lim);
+	return (max > lim);
 }
 
 size_t 
@@ -1026,8 +1056,50 @@ infra_get_mem(struct infra_cache* infra)
 	return s;
 }
 
+/* Returns 1 if the limit has not been exceeded, 0 otherwise. */
+static int
+check_ip_ratelimit(struct sockaddr_storage* addr, socklen_t addrlen,
+	struct sldns_buffer* buffer, int premax, int max, int has_cookie)
+{
+	int limit;
+
+	if(has_cookie) limit = infra_ip_ratelimit_cookie;
+	else           limit = infra_ip_ratelimit;
+
+	/* Disabled */
+	if(limit == 0) return 1;
+
+	if(premax <= limit && max > limit) {
+		char client_ip[128], qnm[LDNS_MAX_DOMAINLEN+1+12+12];
+		addr_to_str(addr, addrlen, client_ip, sizeof(client_ip));
+		qnm[0]=0;
+		if(sldns_buffer_limit(buffer)>LDNS_HEADER_SIZE &&
+			LDNS_QDCOUNT(sldns_buffer_begin(buffer))!=0) {
+			(void)sldns_wire2str_rrquestion_buf(
+				sldns_buffer_at(buffer, LDNS_HEADER_SIZE),
+				sldns_buffer_limit(buffer)-LDNS_HEADER_SIZE,
+				qnm, sizeof(qnm));
+			if(strlen(qnm)>0 && qnm[strlen(qnm)-1]=='\n')
+				qnm[strlen(qnm)-1] = 0; /*remove newline*/
+			if(strchr(qnm, '\t'))
+				*strchr(qnm, '\t') = ' ';
+			if(strchr(qnm, '\t'))
+				*strchr(qnm, '\t') = ' ';
+			verbose(VERB_OPS, "ip_ratelimit exceeded %s %d%s %s",
+				client_ip, limit,
+				has_cookie?"(cookie)":"", qnm);
+		} else {
+			verbose(VERB_OPS, "ip_ratelimit exceeded %s %d%s (no query name)",
+				client_ip, limit,
+				has_cookie?"(cookie)":"");
+		}
+	}
+	return (max <= limit);
+}
+
 int infra_ip_ratelimit_inc(struct infra_cache* infra,
-  struct comm_reply* repinfo, time_t timenow, struct sldns_buffer* buffer)
+	struct sockaddr_storage* addr, socklen_t addrlen, time_t timenow,
+	int has_cookie, int backoff, struct sldns_buffer* buffer)
 {
 	int max;
 	struct lruhash_entry* entry;
@@ -1037,42 +1109,18 @@ int infra_ip_ratelimit_inc(struct infra_cache* infra,
 		return 1;
 	}
 	/* find or insert ratedata */
-	entry = infra_find_ip_ratedata(infra, repinfo, 1);
+	entry = infra_find_ip_ratedata(infra, addr, addrlen, 1);
 	if(entry) {
-		int premax = infra_rate_max(entry->data, timenow);
-		int* cur = infra_rate_find_second(entry->data, timenow);
+		int premax = infra_rate_max(entry->data, timenow, backoff);
+		int* cur = infra_rate_give_second(entry->data, timenow);
 		(*cur)++;
-		max = infra_rate_max(entry->data, timenow);
+		max = infra_rate_max(entry->data, timenow, backoff);
 		lock_rw_unlock(&entry->lock);
-
-		if(premax < infra_ip_ratelimit && max >= infra_ip_ratelimit) {
-			char client_ip[128], qnm[LDNS_MAX_DOMAINLEN+1+12+12];
-			addr_to_str((struct sockaddr_storage *)&repinfo->addr,
-				repinfo->addrlen, client_ip, sizeof(client_ip));
-			qnm[0]=0;
-			if(sldns_buffer_limit(buffer)>LDNS_HEADER_SIZE &&
-				LDNS_QDCOUNT(sldns_buffer_begin(buffer))!=0) {
-				(void)sldns_wire2str_rrquestion_buf(
-					sldns_buffer_at(buffer, LDNS_HEADER_SIZE),
-					sldns_buffer_limit(buffer)-LDNS_HEADER_SIZE,
-					qnm, sizeof(qnm));
-				if(strlen(qnm)>0 && qnm[strlen(qnm)-1]=='\n')
-					qnm[strlen(qnm)-1] = 0; /*remove newline*/
-				if(strchr(qnm, '\t'))
-					*strchr(qnm, '\t') = ' ';
-				if(strchr(qnm, '\t'))
-					*strchr(qnm, '\t') = ' ';
-				verbose(VERB_OPS, "ip_ratelimit exceeded %s %d %s",
-					client_ip, infra_ip_ratelimit, qnm);
-			} else {
-				verbose(VERB_OPS, "ip_ratelimit exceeded %s %d (no query name)",
-					client_ip, infra_ip_ratelimit);
-			}
-		}
-		return (max <= infra_ip_ratelimit);
+		return check_ip_ratelimit(addr, addrlen, buffer, premax, max,
+			has_cookie);
 	}
 
 	/* create */
-	infra_ip_create_ratedata(infra, repinfo, timenow);
+	infra_ip_create_ratedata(infra, addr, addrlen, timenow);
 	return 1;
 }

services/cache/infra.h

@@ -153,6 +153,8 @@ struct rate_key {
 
 /** ip ratelimit, 0 is off */
 extern int infra_ip_ratelimit;
+/** ip ratelimit for DNS Cookie clients, 0 is off */
+extern int infra_ip_ratelimit_cookie;
 
 /**
  * key for ip_ratelimit lookups, a source IP.
@@ -368,6 +370,7 @@ long long infra_get_host_rto(struct infra_cache* infra,
  * @param name: zone name
  * @param namelen: zone name length
  * @param timenow: what time it is now.
+ * @param backoff: if backoff is enabled.
  * @param qinfo: for logging, query name.
  * @param replylist: for logging, querier's address (if any).
  * @return 1 if it could be incremented. 0 if the increment overshot the
@@ -375,7 +378,7 @@ long long infra_get_host_rto(struct infra_cache* infra,
  * Failures like alloc failures are not returned (probably as 1).
  */
 int infra_ratelimit_inc(struct infra_cache* infra, uint8_t* name,
-	size_t namelen, time_t timenow, struct query_info* qinfo,
+	size_t namelen, time_t timenow, int backoff, struct query_info* qinfo,
 	struct comm_reply* replylist);
 
 /**
@@ -398,13 +401,15 @@ void infra_ratelimit_dec(struct infra_cache* infra, uint8_t* name,
  * @param name: zone name
  * @param namelen: zone name length
  * @param timenow: what time it is now.
+ * @param backoff: if backoff is enabled.
  * @return true if exceeded.
  */
 int infra_ratelimit_exceeded(struct infra_cache* infra, uint8_t* name,
-	size_t namelen, time_t timenow);
+	size_t namelen, time_t timenow, int backoff);
 
-/** find the maximum rate stored, not too old. 0 if no information. */
-int infra_rate_max(void* data, time_t now);
+/** find the maximum rate stored. 0 if no information.
+ *  When backoff is enabled look for the maximum in the whole RATE_WINDOW. */
+int infra_rate_max(void* data, time_t now, int backoff);
 
 /** find the ratelimit in qps for a domain. 0 if no limit for domain. */
 int infra_find_ratelimit(struct infra_cache* infra, uint8_t* name,
@@ -413,14 +418,17 @@ int infra_find_ratelimit(struct infra_cache* infra, uint8_t* name,
 /** Update query ratelimit hash and decide
  *  whether or not a query should be dropped.
  *  @param infra: infra cache
- *  @param repinfo: information about client
+ *  @param addr: client address
+ *  @param addrlen: client address length
  *  @param timenow: what time it is now.
+ *  @param has_cookie: if the request came with a DNS Cookie.
+ *  @param backoff: if backoff is enabled.
  *  @param buffer: with query for logging.
  *  @return 1 if it could be incremented. 0 if the increment overshot the
  *  ratelimit and the query should be dropped. */
 int infra_ip_ratelimit_inc(struct infra_cache* infra,
-	struct comm_reply* repinfo, time_t timenow,
-	struct sldns_buffer* buffer);
+	struct sockaddr_storage* addr, socklen_t addrlen, time_t timenow,
+	int has_cookie, int backoff, struct sldns_buffer* buffer);
 
 /**
  * Get memory used by the infra cache.

services/listen_dnsport.c

@@ -4,22 +4,22 @@
  * Copyright (c) 2007, NLnet Labs. All rights reserved.
  *
  * This software is open source.
- * 
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
- * 
+ *
  * Redistributions of source code must retain the above copyright notice,
  * this list of conditions and the following disclaimer.
- * 
+ *
  * Redistributions in binary form must reproduce the above copyright notice,
  * this list of conditions and the following disclaimer in the documentation
  * and/or other materials provided with the distribution.
- * 
+ *
  * Neither the name of the NLNET LABS nor the names of its contributors may
  * be used to endorse or promote products derived from this software without
  * specific prior written permission.
- * 
+ *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -47,6 +47,7 @@
 #ifdef USE_TCP_FASTOPEN
 #include <netinet/tcp.h>
 #endif
+#include <ctype.h>
 #include "services/listen_dnsport.h"
 #include "services/outside_network.h"
 #include "util/netevent.h"
@@ -78,9 +79,11 @@
 #ifdef HAVE_NET_IF_H
 #include <net/if.h>
 #endif
-
+#ifdef HAVE_LINUX_NET_TSTAMP_H
+#include <linux/net_tstamp.h>
+#endif
 /** number of queued TCP connections for listen() */
-#define TCP_BACKLOG 256 
+#define TCP_BACKLOG 256
 
 #ifndef THREADS_DISABLED
 /** lock on the counter of stream buffer memory */
@@ -123,12 +126,12 @@ verbose_print_addr(struct addrinfo *addr)
 			(void)strlcpy(buf, "(null)", sizeof(buf));
 		}
 		buf[sizeof(buf)-1] = 0;
-		verbose(VERB_ALGO, "creating %s%s socket %s %d", 
+		verbose(VERB_ALGO, "creating %s%s socket %s %d",
 			addr->ai_socktype==SOCK_DGRAM?"udp":
 			addr->ai_socktype==SOCK_STREAM?"tcp":"otherproto",
 			addr->ai_family==AF_INET?"4":
 			addr->ai_family==AF_INET6?"6":
-			"_otherfam", buf, 
+			"_otherfam", buf,
 			ntohs(((struct sockaddr_in*)addr->ai_addr)->sin_port));
 	}
 }
@@ -139,7 +142,9 @@ verbose_print_unbound_socket(struct unbound_socket* ub_sock)
 	if(verbosity >= VERB_ALGO) {
 		log_info("listing of unbound_socket structure:");
 		verbose_print_addr(ub_sock->addr);
-		log_info("s is: %d, fam is: %s", ub_sock->s, ub_sock->fam == AF_INET?"AF_INET":"AF_INET6");
+		log_info("s is: %d, fam is: %s, acl: %s", ub_sock->s,
+			ub_sock->fam == AF_INET?"AF_INET":"AF_INET6",
+			ub_sock->acl?"yes":"no");
 	}
 }
 
@@ -184,7 +189,7 @@ systemd_get_activated(int family, int socktype, int listen,
 			log_err("systemd sd_listen_fds(): %s", strerror(-r));
 		return -1;
 	}
-	
+
 	for(i = 0; i < r; i++) {
 		if(sd_is_socket(SD_LISTEN_FDS_START + i, family, socktype, listen)) {
 			s = SD_LISTEN_FDS_START + i;
@@ -250,7 +255,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 			return -1;
 		}
 #else
-		if(WSAGetLastError() == WSAEAFNOSUPPORT || 
+		if(WSAGetLastError() == WSAEAFNOSUPPORT ||
 			WSAGetLastError() == WSAEPROTONOSUPPORT) {
 			*noproto = 1;
 			return -1;
@@ -267,7 +272,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 #endif
 	if(listen) {
 #ifdef SO_REUSEADDR
-		if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void*)&on, 
+		if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void*)&on,
 			(socklen_t)sizeof(on)) < 0) {
 			log_err("setsockopt(.. SO_REUSEADDR ..) failed: %s",
 				sock_strerror(errno));
@@ -365,9 +370,9 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		socklen_t slen = (socklen_t)sizeof(got);
 #  ifdef SO_RCVBUFFORCE
 		/* Linux specific: try to use root permission to override
-		 * system limits on rcvbuf. The limit is stored in 
+		 * system limits on rcvbuf. The limit is stored in
 		 * /proc/sys/net/core/rmem_max or sysctl net.core.rmem_max */
-		if(setsockopt(s, SOL_SOCKET, SO_RCVBUFFORCE, (void*)&rcv, 
+		if(setsockopt(s, SOL_SOCKET, SO_RCVBUFFORCE, (void*)&rcv,
 			(socklen_t)sizeof(rcv)) < 0) {
 			if(errno != EPERM) {
 				log_err("setsockopt(..., SO_RCVBUFFORCE, "
@@ -378,7 +383,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 				return -1;
 			}
 #  endif /* SO_RCVBUFFORCE */
-			if(setsockopt(s, SOL_SOCKET, SO_RCVBUF, (void*)&rcv, 
+			if(setsockopt(s, SOL_SOCKET, SO_RCVBUF, (void*)&rcv,
 				(socklen_t)sizeof(rcv)) < 0) {
 				log_err("setsockopt(..., SO_RCVBUF, "
 					"...) failed: %s", sock_strerror(errno));
@@ -389,7 +394,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 			}
 			/* check if we got the right thing or if system
 			 * reduced to some system max.  Warn if so */
-			if(getsockopt(s, SOL_SOCKET, SO_RCVBUF, (void*)&got, 
+			if(getsockopt(s, SOL_SOCKET, SO_RCVBUF, (void*)&got,
 				&slen) >= 0 && got < rcv/2) {
 				log_warn("so-rcvbuf %u was not granted. "
 					"Got %u. To fix: start with "
@@ -410,9 +415,9 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		socklen_t slen = (socklen_t)sizeof(got);
 #  ifdef SO_SNDBUFFORCE
 		/* Linux specific: try to use root permission to override
-		 * system limits on sndbuf. The limit is stored in 
+		 * system limits on sndbuf. The limit is stored in
 		 * /proc/sys/net/core/wmem_max or sysctl net.core.wmem_max */
-		if(setsockopt(s, SOL_SOCKET, SO_SNDBUFFORCE, (void*)&snd, 
+		if(setsockopt(s, SOL_SOCKET, SO_SNDBUFFORCE, (void*)&snd,
 			(socklen_t)sizeof(snd)) < 0) {
 			if(errno != EPERM) {
 				log_err("setsockopt(..., SO_SNDBUFFORCE, "
@@ -423,7 +428,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 				return -1;
 			}
 #  endif /* SO_SNDBUFFORCE */
-			if(setsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&snd, 
+			if(setsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&snd,
 				(socklen_t)sizeof(snd)) < 0) {
 				log_err("setsockopt(..., SO_SNDBUF, "
 					"...) failed: %s", sock_strerror(errno));
@@ -434,7 +439,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 			}
 			/* check if we got the right thing or if system
 			 * reduced to some system max.  Warn if so */
-			if(getsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&got, 
+			if(getsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&got,
 				&slen) >= 0 && got < snd/2) {
 				log_warn("so-sndbuf %u was not granted. "
 					"Got %u. To fix: start with "
@@ -457,9 +462,16 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		int action;
 # endif
 # if defined(IPV6_V6ONLY)
-		if(v6only) {
+		if(v6only
+#   ifdef HAVE_SYSTEMD
+			/* Systemd wants to control if the socket is v6 only
+			 * or both, with BindIPv6Only=default, ipv6-only or
+			 * both in systemd.socket, so it is not set here. */
+			&& !got_fd_from_systemd
+#   endif
+			) {
 			int val=(v6only==2)?0:1;
-			if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, 
+			if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY,
 				(void*)&val, (socklen_t)sizeof(val)) < 0) {
 				log_err("setsockopt(..., IPV6_V6ONLY"
 					", ...) failed: %s", sock_strerror(errno));
@@ -489,6 +501,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 			return -1;
 		}
 # elif defined(IPV6_MTU)
+#   ifndef USE_WINSOCK
 		/*
 		 * On Linux, to send no larger than 1280, the PMTUD is
 		 * disabled by default for datagrams anyway, so we set
@@ -496,13 +509,29 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		 */
 		if (setsockopt(s, IPPROTO_IPV6, IPV6_MTU,
 			(void*)&mtu, (socklen_t)sizeof(mtu)) < 0) {
-			log_err("setsockopt(..., IPV6_MTU, ...) failed: %s", 
+			log_err("setsockopt(..., IPV6_MTU, ...) failed: %s",
 				sock_strerror(errno));
 			sock_close(s);
 			*noproto = 0;
 			*inuse = 0;
 			return -1;
 		}
+#   elif defined(IPV6_USER_MTU)
+		/* As later versions of the mingw crosscompiler define
+		 * IPV6_MTU, do the same for windows but use IPV6_USER_MTU
+		 * instead which is writable; IPV6_MTU is readonly there. */
+		if (setsockopt(s, IPPROTO_IPV6, IPV6_USER_MTU,
+			(void*)&mtu, (socklen_t)sizeof(mtu)) < 0) {
+			if (WSAGetLastError() != WSAENOPROTOOPT) {
+				log_err("setsockopt(..., IPV6_USER_MTU, ...) failed: %s",
+					wsa_strerror(WSAGetLastError()));
+				sock_close(s);
+				*noproto = 0;
+				*inuse = 0;
+				return -1;
+			}
+		}
+#   endif /* USE_WINSOCK */
 # endif /* IPv6 MTU */
 # if defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT)
 #  if defined(IP_PMTUDISC_OMIT)
@@ -549,7 +578,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		int action;
 #   if defined(IP_PMTUDISC_OMIT)
 		action = IP_PMTUDISC_OMIT;
-		if (setsockopt(s, IPPROTO_IP, IP_MTU_DISCOVER, 
+		if (setsockopt(s, IPPROTO_IP, IP_MTU_DISCOVER,
 			&action, (socklen_t)sizeof(action)) < 0) {
 
 			if (errno != EINVAL) {
@@ -582,7 +611,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		/* the IP_DONTFRAG option if defined in the 11.0 OSX headers,
 		 * but does not work on that version, so we exclude it */
 		int off = 0;
-		if (setsockopt(s, IPPROTO_IP, IP_DONTFRAG, 
+		if (setsockopt(s, IPPROTO_IP, IP_DONTFRAG,
 			&off, (socklen_t)sizeof(off)) < 0) {
 			log_err("setsockopt(..., IP_DONTFRAG, ...) failed: %s",
 				strerror(errno));
@@ -620,7 +649,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		if(WSAGetLastError() != WSAEADDRINUSE &&
 			WSAGetLastError() != WSAEADDRNOTAVAIL &&
 			!(WSAGetLastError() == WSAEACCES && verbosity < 4 && !listen)) {
-			log_err_addr("can't bind socket", 
+			log_err_addr("can't bind socket",
 				wsa_strerror(WSAGetLastError()),
 				(struct sockaddr_storage*)addr, addrlen);
 		}
@@ -722,7 +751,7 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
     }
 #endif
 #ifdef SO_REUSEADDR
-	if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void*)&on, 
+	if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void*)&on,
 		(socklen_t)sizeof(on)) < 0) {
 		log_err("setsockopt(.. SO_REUSEADDR ..) failed: %s",
 			sock_strerror(errno));
@@ -758,8 +787,15 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
 	(void)reuseport;
 #endif /* defined(SO_REUSEPORT) */
 #if defined(IPV6_V6ONLY)
-	if(addr->ai_family == AF_INET6 && v6only) {
-		if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, 
+	if(addr->ai_family == AF_INET6 && v6only
+#  ifdef HAVE_SYSTEMD
+		/* Systemd wants to control if the socket is v6 only
+		 * or both, with BindIPv6Only=default, ipv6-only or
+		 * both in systemd.socket, so it is not set here. */
+		&& !got_fd_from_systemd
+#  endif
+		) {
+		if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY,
 			(void*)&on, (socklen_t)sizeof(on)) < 0) {
 			log_err("setsockopt(..., IPV6_V6ONLY, ...) failed: %s",
 				sock_strerror(errno));
@@ -811,7 +847,7 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
 				addr->ai_addrlen);
 		}
 #else
-		log_err_addr("can't bind socket", 
+		log_err_addr("can't bind socket",
 			wsa_strerror(WSAGetLastError()),
 			(struct sockaddr_storage*)addr->ai_addr,
 			addr->ai_addrlen);
@@ -839,7 +875,7 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
 	/* 5 is recommended on linux */
 	qlen = 5;
 #endif
-	if ((setsockopt(s, IPPROTO_TCP, TCP_FASTOPEN, &qlen, 
+	if ((setsockopt(s, IPPROTO_TCP, TCP_FASTOPEN, &qlen,
 		  sizeof(qlen))) == -1 ) {
 #ifdef ENOPROTOOPT
 		/* squelch ENOPROTOOPT: freebsd server mode with kernel support
@@ -965,7 +1001,7 @@ err:
  * Create socket from getaddrinfo results
  */
 static int
-make_sock(int stype, const char* ifname, const char* port, 
+make_sock(int stype, const char* ifname, const char* port,
 	struct addrinfo *hints, int v6only, int* noip6, size_t rcv, size_t snd,
 	int* reuseport, int transparent, int tcp_mss, int nodelay, int freebind,
 	int use_systemd, int dscp, struct unbound_socket* ub_sock)
@@ -981,10 +1017,10 @@ make_sock(int stype, const char* ifname, const char* port,
 			return -1;
 		}
 #endif
-		log_err("node %s:%s getaddrinfo: %s %s", 
+		log_err("node %s:%s getaddrinfo: %s %s",
 			ifname?ifname:"default", port, gai_strerror(r),
 #ifdef EAI_SYSTEM
-			r==EAI_SYSTEM?(char*)strerror(errno):""
+			(r==EAI_SYSTEM?(char*)strerror(errno):"")
 #else
 			""
 #endif
@@ -1014,13 +1050,14 @@ make_sock(int stype, const char* ifname, const char* port,
 	ub_sock->addr = res;
 	ub_sock->s = s;
 	ub_sock->fam = hints->ai_family;
+	ub_sock->acl = NULL;
 
 	return s;
 }
 
 /** make socket and first see if ifname contains port override info */
 static int
-make_sock_port(int stype, const char* ifname, const char* port, 
+make_sock_port(int stype, const char* ifname, const char* port,
 	struct addrinfo *hints, int v6only, int* noip6, size_t rcv, size_t snd,
 	int* reuseport, int transparent, int tcp_mss, int nodelay, int freebind,
 	int use_systemd, int dscp, struct unbound_socket* ub_sock)
@@ -1058,11 +1095,13 @@ make_sock_port(int stype, const char* ifname, const char* port,
  * @param list: list head. changed.
  * @param s: fd.
  * @param ftype: if fd is UDP.
+ * @param pp2_enabled: if PROXYv2 is enabled for this port.
  * @param ub_sock: socket with address.
  * @return false on failure. list in unchanged then.
  */
 static int
-port_insert(struct listen_port** list, int s, enum listen_type ftype, struct unbound_socket* ub_sock)
+port_insert(struct listen_port** list, int s, enum listen_type ftype,
+	int pp2_enabled, struct unbound_socket* ub_sock)
 {
 	struct listen_port* item = (struct listen_port*)malloc(
 		sizeof(struct listen_port));
@@ -1071,14 +1110,34 @@ port_insert(struct listen_port** list, int s, enum listen_type ftype, struct unb
 	item->next = *list;
 	item->fd = s;
 	item->ftype = ftype;
+	item->pp2_enabled = pp2_enabled;
 	item->socket = ub_sock;
 	*list = item;
 	return 1;
 }
 
+/** set fd to receive software timestamps */
+static int
+set_recvtimestamp(int s)
+{
+#ifdef HAVE_LINUX_NET_TSTAMP_H
+	int opt = SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_SOFTWARE;
+	if (setsockopt(s, SOL_SOCKET, SO_TIMESTAMPNS, (void*)&opt, (socklen_t)sizeof(opt)) < 0) {
+		log_err("setsockopt(..., SO_TIMESTAMPNS, ...) failed: %s",
+			strerror(errno));
+		return 0;
+	}
+	return 1;
+#else
+	log_err("packets timestamping is not supported on this platform");
+	(void)s;
+	return 0;
+#endif
+}
+
 /** set fd to receive source address packet info */
 static int
-set_recvpktinfo(int s, int family) 
+set_recvpktinfo(int s, int family)
 {
 #if defined(IPV6_RECVPKTINFO) || defined(IPV6_PKTINFO) || (defined(IP_RECVDSTADDR) && defined(IP_SENDSRCADDR)) || defined(IP_PKTINFO)
 	int on = 1;
@@ -1157,7 +1216,7 @@ if_is_ssl(const char* ifname, const char* port, int ssl_port,
  * @param do_auto: use automatic interface detection.
  * 	If enabled, then ifname must be the wildcard name.
  * @param do_udp: if udp should be used.
- * @param do_tcp: if udp should be used.
+ * @param do_tcp: if tcp should be used.
  * @param hints: for getaddrinfo. family and flags have to be set by caller.
  * @param port: Port number to use (as string).
  * @param list: list of open ports, appended to, changed to point to list head.
@@ -1166,6 +1225,7 @@ if_is_ssl(const char* ifname, const char* port, int ssl_port,
  * @param ssl_port: ssl service port number
  * @param tls_additional_port: list of additional ssl service port numbers.
  * @param https_port: DoH service port number
+ * @param proxy_protocol_port: list of PROXYv2 port numbers.
  * @param reuseport: try to set SO_REUSEPORT if nonNULL and true.
  * 	set to false on exit if reuseport failed due to no kernel support.
  * @param transparent: set IP_TRANSPARENT socket option.
@@ -1175,40 +1235,50 @@ if_is_ssl(const char* ifname, const char* port, int ssl_port,
  * @param use_systemd: if true, fetch sockets from systemd.
  * @param dnscrypt_port: dnscrypt service port number
  * @param dscp: DSCP to use.
+ * @param sock_queue_timeout: the sock_queue_timeout from config. Seconds to
+ * 	wait to discard if UDP packets have waited for long in the socket
+ * 	buffer.
  * @return: returns false on error.
  */
 static int
-ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp, 
+ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 	struct addrinfo *hints, const char* port, struct listen_port** list,
 	size_t rcv, size_t snd, int ssl_port,
 	struct config_strlist* tls_additional_port, int https_port,
+	struct config_strlist* proxy_protocol_port,
 	int* reuseport, int transparent, int tcp_mss, int freebind,
-	int http2_nodelay, int use_systemd, int dnscrypt_port, int dscp)
+	int http2_nodelay, int use_systemd, int dnscrypt_port, int dscp,
+	int sock_queue_timeout)
 {
 	int s, noip6=0;
 	int is_https = if_is_https(ifname, port, https_port);
+	int is_dnscrypt = if_is_dnscrypt(ifname, port, dnscrypt_port);
+	int is_pp2 = if_is_pp2(ifname, port, proxy_protocol_port);
 	int nodelay = is_https && http2_nodelay;
 	struct unbound_socket* ub_sock;
-#ifdef USE_DNSCRYPT
-	int is_dnscrypt = ((strchr(ifname, '@') && 
-			atoi(strchr(ifname, '@')+1) == dnscrypt_port) ||
-			(!strchr(ifname, '@') && atoi(port) == dnscrypt_port));
-#else
-	int is_dnscrypt = 0;
-	(void)dnscrypt_port;
-#endif
 
 	if(!do_udp && !do_tcp)
 		return 0;
 
+	if(is_pp2) {
+		if(is_dnscrypt) {
+			fatal_exit("PROXYv2 and DNSCrypt combination not "
+				"supported!");
+		} else if(is_https) {
+			fatal_exit("PROXYv2 and DoH combination not "
+				"supported!");
+		}
+	}
+
 	if(do_auto) {
 		ub_sock = calloc(1, sizeof(struct unbound_socket));
 		if(!ub_sock)
 			return 0;
-		if((s = make_sock_port(SOCK_DGRAM, ifname, port, hints, 1, 
+		if((s = make_sock_port(SOCK_DGRAM, ifname, port, hints, 1,
 			&noip6, rcv, snd, reuseport, transparent,
 			tcp_mss, nodelay, freebind, use_systemd, dscp, ub_sock)) == -1) {
-			freeaddrinfo(ub_sock->addr);
+			if(ub_sock->addr)
+				freeaddrinfo(ub_sock->addr);
 			free(ub_sock);
 			if(noip6) {
 				log_warn("IPv6 protocol not available");
@@ -1219,14 +1289,20 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 		/* getting source addr packet info is highly non-portable */
 		if(!set_recvpktinfo(s, hints->ai_family)) {
 			sock_close(s);
-			freeaddrinfo(ub_sock->addr);
+			if(ub_sock->addr)
+				freeaddrinfo(ub_sock->addr);
 			free(ub_sock);
 			return 0;
 		}
-		if(!port_insert(list, s,
-		   is_dnscrypt?listen_type_udpancil_dnscrypt:listen_type_udpancil, ub_sock)) {
+		if (sock_queue_timeout && !set_recvtimestamp(s)) {
+			log_warn("socket timestamping is not available");
+		}
+		if(!port_insert(list, s, is_dnscrypt
+			?listen_type_udpancil_dnscrypt:listen_type_udpancil,
+			is_pp2, ub_sock)) {
 			sock_close(s);
-			freeaddrinfo(ub_sock->addr);
+			if(ub_sock->addr)
+				freeaddrinfo(ub_sock->addr);
 			free(ub_sock);
 			return 0;
 		}
@@ -1235,10 +1311,11 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 		if(!ub_sock)
 			return 0;
 		/* regular udp socket */
-		if((s = make_sock_port(SOCK_DGRAM, ifname, port, hints, 1, 
+		if((s = make_sock_port(SOCK_DGRAM, ifname, port, hints, 1,
 			&noip6, rcv, snd, reuseport, transparent,
 			tcp_mss, nodelay, freebind, use_systemd, dscp, ub_sock)) == -1) {
-			freeaddrinfo(ub_sock->addr);
+			if(ub_sock->addr)
+				freeaddrinfo(ub_sock->addr);
 			free(ub_sock);
 			if(noip6) {
 				log_warn("IPv6 protocol not available");
@@ -1246,10 +1323,17 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 			}
 			return 0;
 		}
-		if(!port_insert(list, s,
-		   is_dnscrypt?listen_type_udp_dnscrypt:listen_type_udp, ub_sock)) {
+		if (sock_queue_timeout && !set_recvtimestamp(s)) {
+			log_warn("socket timestamping is not available");
+		}
+		if(!port_insert(list, s, is_dnscrypt
+			?listen_type_udp_dnscrypt :
+			(sock_queue_timeout ?
+				listen_type_udpancil:listen_type_udp),
+			is_pp2, ub_sock)) {
 			sock_close(s);
-			freeaddrinfo(ub_sock->addr);
+			if(ub_sock->addr)
+				freeaddrinfo(ub_sock->addr);
 			free(ub_sock);
 			return 0;
 		}
@@ -1269,10 +1353,11 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 			port_type = listen_type_tcp_dnscrypt;
 		else
 			port_type = listen_type_tcp;
-		if((s = make_sock_port(SOCK_STREAM, ifname, port, hints, 1, 
+		if((s = make_sock_port(SOCK_STREAM, ifname, port, hints, 1,
 			&noip6, 0, 0, reuseport, transparent, tcp_mss, nodelay,
 			freebind, use_systemd, dscp, ub_sock)) == -1) {
-			freeaddrinfo(ub_sock->addr);
+			if(ub_sock->addr)
+				freeaddrinfo(ub_sock->addr);
 			free(ub_sock);
 			if(noip6) {
 				/*log_warn("IPv6 protocol not available");*/
@@ -1282,9 +1367,10 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 		}
 		if(is_ssl)
 			verbose(VERB_ALGO, "setup TCP for SSL service");
-		if(!port_insert(list, s, port_type, ub_sock)) {
+		if(!port_insert(list, s, port_type, is_pp2, ub_sock)) {
 			sock_close(s);
-			freeaddrinfo(ub_sock->addr);
+			if(ub_sock->addr)
+				freeaddrinfo(ub_sock->addr);
 			free(ub_sock);
 			return 0;
 		}
@@ -1292,7 +1378,7 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 	return 1;
 }
 
-/** 
+/**
  * Add items to commpoint list in front.
  * @param c: commpoint to add.
  * @param front: listen struct.
@@ -1343,7 +1429,7 @@ void listen_desetup_locks(void)
 	}
 }
 
-struct listen_dnsport* 
+struct listen_dnsport*
 listen_create(struct comm_base* base, struct listen_port* ports,
 	size_t bufsize, int tcp_accept_count, int tcp_idle_timeout,
 	int harden_large_queries, uint32_t http_max_streams,
@@ -1369,40 +1455,42 @@ listen_create(struct comm_base* base, struct listen_port* ports,
 	while(ports) {
 		struct comm_point* cp = NULL;
 		if(ports->ftype == listen_type_udp ||
-		   ports->ftype == listen_type_udp_dnscrypt)
-			cp = comm_point_create_udp(base, ports->fd, 
-				front->udp_buff, cb, cb_arg, ports->socket);
-		else if(ports->ftype == listen_type_tcp ||
-				ports->ftype == listen_type_tcp_dnscrypt)
-			cp = comm_point_create_tcp(base, ports->fd, 
+		   ports->ftype == listen_type_udp_dnscrypt) {
+			cp = comm_point_create_udp(base, ports->fd,
+				front->udp_buff, ports->pp2_enabled, cb,
+				cb_arg, ports->socket);
+		} else if(ports->ftype == listen_type_tcp ||
+				ports->ftype == listen_type_tcp_dnscrypt) {
+			cp = comm_point_create_tcp(base, ports->fd,
 				tcp_accept_count, tcp_idle_timeout,
 				harden_large_queries, 0, NULL,
 				tcp_conn_limit, bufsize, front->udp_buff,
-				ports->ftype, cb, cb_arg, ports->socket);
-		else if(ports->ftype == listen_type_ssl ||
+				ports->ftype, ports->pp2_enabled, cb, cb_arg,
+				ports->socket);
+		} else if(ports->ftype == listen_type_ssl ||
 			ports->ftype == listen_type_http) {
-			cp = comm_point_create_tcp(base, ports->fd, 
+			cp = comm_point_create_tcp(base, ports->fd,
 				tcp_accept_count, tcp_idle_timeout,
 				harden_large_queries,
 				http_max_streams, http_endpoint,
 				tcp_conn_limit, bufsize, front->udp_buff,
-				ports->ftype, cb, cb_arg, ports->socket);
-			if(http_notls && ports->ftype == listen_type_http)
-				cp->ssl = NULL;
-			else
-				cp->ssl = sslctx;
+				ports->ftype, ports->pp2_enabled, cb, cb_arg,
+				ports->socket);
 			if(ports->ftype == listen_type_http) {
 				if(!sslctx && !http_notls) {
-				  log_warn("HTTPS port configured, but no TLS "
-					"tls-service-key or tls-service-pem "
-					"set");
+					log_warn("HTTPS port configured, but "
+						"no TLS tls-service-key or "
+						"tls-service-pem set");
 				}
 #ifndef HAVE_SSL_CTX_SET_ALPN_SELECT_CB
-				if(!http_notls)
-				  log_warn("Unbound is not compiled with an "
-					"OpenSSL version supporting ALPN "
-					" (OpenSSL >= 1.0.2). This is required "
-					"to use DNS-over-HTTPS");
+				if(!http_notls) {
+					log_warn("Unbound is not compiled "
+						"with an OpenSSL version "
+						"supporting ALPN "
+						"(OpenSSL >= 1.0.2). This "
+						"is required to use "
+						"DNS-over-HTTPS");
+				}
 #endif
 #ifndef HAVE_NGHTTP2_NGHTTP2_H
 				log_warn("Unbound is not compiled with "
@@ -1411,14 +1499,30 @@ listen_create(struct comm_base* base, struct listen_port* ports,
 #endif
 			}
 		} else if(ports->ftype == listen_type_udpancil ||
-				  ports->ftype == listen_type_udpancil_dnscrypt)
-			cp = comm_point_create_udp_ancil(base, ports->fd, 
-				front->udp_buff, cb, cb_arg, ports->socket);
+				  ports->ftype == listen_type_udpancil_dnscrypt) {
+#if defined(AF_INET6) && defined(IPV6_PKTINFO) && defined(HAVE_RECVMSG)
+			cp = comm_point_create_udp_ancil(base, ports->fd,
+				front->udp_buff, ports->pp2_enabled, cb,
+				cb_arg, ports->socket);
+#else
+			log_warn("This system does not support UDP ancilliary data.");
+#endif
+		}
 		if(!cp) {
-			log_err("can't create commpoint");	
+			log_err("can't create commpoint");
 			listen_delete(front);
 			return NULL;
 		}
+		if((http_notls && ports->ftype == listen_type_http) ||
+			(ports->ftype == listen_type_tcp) ||
+			(ports->ftype == listen_type_udp) ||
+			(ports->ftype == listen_type_udpancil) ||
+			(ports->ftype == listen_type_tcp_dnscrypt) ||
+			(ports->ftype == listen_type_udp_dnscrypt) ||
+			(ports->ftype == listen_type_udpancil_dnscrypt))
+			cp->ssl = NULL;
+		else
+			cp->ssl = sslctx;
 		cp->dtenv = dtenv;
 		cp->do_not_close = 1;
 #ifdef USE_DNSCRYPT
@@ -1465,10 +1569,10 @@ listen_list_delete(struct listen_list* list)
 	}
 }
 
-void 
+void
 listen_delete(struct listen_dnsport* front)
 {
-	if(!front) 
+	if(!front)
 		return;
 	listen_list_delete(front->cps);
 #ifdef USE_DNSCRYPT
@@ -1674,7 +1778,7 @@ int resolve_interface_names(char** ifs, int num_ifs,
 #endif /* HAVE_GETIFADDRS */
 }
 
-struct listen_port* 
+struct listen_port*
 listening_ports_open(struct config_file* cfg, char** ifs, int num_ifs,
 	int* reuseport)
 {
@@ -1706,32 +1810,95 @@ listening_ports_open(struct config_file* cfg, char** ifs, int num_ifs,
 	}
 	/* create ip4 and ip6 ports so that return addresses are nice. */
 	if(do_auto || num_ifs == 0) {
+		if(do_auto && cfg->if_automatic_ports &&
+			cfg->if_automatic_ports[0]!=0) {
+			char* now = cfg->if_automatic_ports;
+			while(now && *now) {
+				char* after;
+				int extraport;
+				while(isspace((unsigned char)*now))
+					now++;
+				if(!*now)
+					break;
+				after = now;
+				extraport = (int)strtol(now, &after, 10);
+				if(extraport < 0 || extraport > 65535) {
+					log_err("interface-automatic-ports port number out of range, at position %d of '%s'", (int)(now-cfg->if_automatic_ports)+1, cfg->if_automatic_ports);
+					listening_ports_free(list);
+					return NULL;
+				}
+				if(extraport == 0 && now == after) {
+					log_err("interface-automatic-ports could not be parsed, at position %d of '%s'", (int)(now-cfg->if_automatic_ports)+1, cfg->if_automatic_ports);
+					listening_ports_free(list);
+					return NULL;
+				}
+				now = after;
+				snprintf(portbuf, sizeof(portbuf), "%d", extraport);
+				if(do_ip6) {
+					hints.ai_family = AF_INET6;
+					if(!ports_create_if("::0",
+						do_auto, cfg->do_udp, do_tcp,
+						&hints, portbuf, &list,
+						cfg->so_rcvbuf, cfg->so_sndbuf,
+						cfg->ssl_port, cfg->tls_additional_port,
+						cfg->https_port,
+						cfg->proxy_protocol_port,
+						reuseport, cfg->ip_transparent,
+						cfg->tcp_mss, cfg->ip_freebind,
+						cfg->http_nodelay, cfg->use_systemd,
+						cfg->dnscrypt_port, cfg->ip_dscp, cfg->sock_queue_timeout)) {
+						listening_ports_free(list);
+						return NULL;
+					}
+				}
+				if(do_ip4) {
+					hints.ai_family = AF_INET;
+					if(!ports_create_if("0.0.0.0",
+						do_auto, cfg->do_udp, do_tcp,
+						&hints, portbuf, &list,
+						cfg->so_rcvbuf, cfg->so_sndbuf,
+						cfg->ssl_port, cfg->tls_additional_port,
+						cfg->https_port,
+						cfg->proxy_protocol_port,
+						reuseport, cfg->ip_transparent,
+						cfg->tcp_mss, cfg->ip_freebind,
+						cfg->http_nodelay, cfg->use_systemd,
+						cfg->dnscrypt_port, cfg->ip_dscp, cfg->sock_queue_timeout)) {
+						listening_ports_free(list);
+						return NULL;
+					}
+				}
+			}
+			return list;
+		}
 		if(do_ip6) {
 			hints.ai_family = AF_INET6;
-			if(!ports_create_if(do_auto?"::0":"::1", 
-				do_auto, cfg->do_udp, do_tcp, 
+			if(!ports_create_if(do_auto?"::0":"::1",
+				do_auto, cfg->do_udp, do_tcp,
 				&hints, portbuf, &list,
 				cfg->so_rcvbuf, cfg->so_sndbuf,
 				cfg->ssl_port, cfg->tls_additional_port,
-				cfg->https_port, reuseport, cfg->ip_transparent,
+				cfg->https_port, cfg->proxy_protocol_port,
+				reuseport, cfg->ip_transparent,
 				cfg->tcp_mss, cfg->ip_freebind,
 				cfg->http_nodelay, cfg->use_systemd,
-				cfg->dnscrypt_port, cfg->ip_dscp)) {
+				cfg->dnscrypt_port, cfg->ip_dscp, cfg->sock_queue_timeout)) {
 				listening_ports_free(list);
 				return NULL;
 			}
 		}
 		if(do_ip4) {
 			hints.ai_family = AF_INET;
-			if(!ports_create_if(do_auto?"0.0.0.0":"127.0.0.1", 
-				do_auto, cfg->do_udp, do_tcp, 
+			if(!ports_create_if(do_auto?"0.0.0.0":"127.0.0.1",
+				do_auto, cfg->do_udp, do_tcp,
 				&hints, portbuf, &list,
 				cfg->so_rcvbuf, cfg->so_sndbuf,
 				cfg->ssl_port, cfg->tls_additional_port,
-				cfg->https_port, reuseport, cfg->ip_transparent,
+				cfg->https_port, cfg->proxy_protocol_port,
+				reuseport, cfg->ip_transparent,
 				cfg->tcp_mss, cfg->ip_freebind,
 				cfg->http_nodelay, cfg->use_systemd,
-				cfg->dnscrypt_port, cfg->ip_dscp)) {
+				cfg->dnscrypt_port, cfg->ip_dscp, cfg->sock_queue_timeout)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -1742,13 +1909,14 @@ listening_ports_open(struct config_file* cfg, char** ifs, int num_ifs,
 				continue;
 			hints.ai_family = AF_INET6;
 			if(!ports_create_if(ifs[i], 0, cfg->do_udp,
-				do_tcp, &hints, portbuf, &list, 
+				do_tcp, &hints, portbuf, &list,
 				cfg->so_rcvbuf, cfg->so_sndbuf,
 				cfg->ssl_port, cfg->tls_additional_port,
-				cfg->https_port, reuseport, cfg->ip_transparent,
+				cfg->https_port, cfg->proxy_protocol_port,
+				reuseport, cfg->ip_transparent,
 				cfg->tcp_mss, cfg->ip_freebind,
 				cfg->http_nodelay, cfg->use_systemd,
-				cfg->dnscrypt_port, cfg->ip_dscp)) {
+				cfg->dnscrypt_port, cfg->ip_dscp, cfg->sock_queue_timeout)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -1757,13 +1925,14 @@ listening_ports_open(struct config_file* cfg, char** ifs, int num_ifs,
 				continue;
 			hints.ai_family = AF_INET;
 			if(!ports_create_if(ifs[i], 0, cfg->do_udp,
-				do_tcp, &hints, portbuf, &list, 
+				do_tcp, &hints, portbuf, &list,
 				cfg->so_rcvbuf, cfg->so_sndbuf,
 				cfg->ssl_port, cfg->tls_additional_port,
-				cfg->https_port, reuseport, cfg->ip_transparent,
+				cfg->https_port, cfg->proxy_protocol_port,
+				reuseport, cfg->ip_transparent,
 				cfg->tcp_mss, cfg->ip_freebind,
 				cfg->http_nodelay, cfg->use_systemd,
-				cfg->dnscrypt_port, cfg->ip_dscp)) {
+				cfg->dnscrypt_port, cfg->ip_dscp, cfg->sock_queue_timeout)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -1783,7 +1952,8 @@ void listening_ports_free(struct listen_port* list)
 		}
 		/* rc_ports don't have ub_socket */
 		if(list->socket) {
-			freeaddrinfo(list->socket->addr);
+			if(list->socket->addr)
+				freeaddrinfo(list->socket->addr);
 			free(list->socket);
 		}
 		free(list);
@@ -1794,8 +1964,8 @@ void listening_ports_free(struct listen_port* list)
 size_t listen_get_mem(struct listen_dnsport* listen)
 {
 	struct listen_list* p;
-	size_t s = sizeof(*listen) + sizeof(*listen->base) + 
-		sizeof(*listen->udp_buff) + 
+	size_t s = sizeof(*listen) + sizeof(*listen->base) +
+		sizeof(*listen->udp_buff) +
 		sldns_buffer_capacity(listen->udp_buff);
 #ifdef USE_DNSCRYPT
 	s += sizeof(*listen->dnscrypt_udp_buff);
@@ -1876,7 +2046,7 @@ void tcp_req_info_clear(struct tcp_req_info* req)
 	}
 	req->open_req_list = NULL;
 	req->num_open_req = 0;
-	
+
 	/* free pending writable result packets */
 	item = req->done_req_list;
 	while(item) {
@@ -1935,7 +2105,7 @@ tcp_req_info_setup_listen(struct tcp_req_info* req)
 		wr = 1;
 	if(!req->read_is_closed)
 		rd = 1;
-	
+
 	if(wr) {
 		req->cp->tcp_is_reading = 0;
 		comm_point_stop_listening(req->cp);
@@ -2071,7 +2241,7 @@ tcp_req_info_handle_readdone(struct tcp_req_info* req)
 	}
 	req->in_worker_handle = 0;
 	/* it should be waiting in the mesh for recursion.
-	 * If mesh failed to add a new entry and called commpoint_drop_reply. 
+	 * If mesh failed to add a new entry and called commpoint_drop_reply.
 	 * Then the mesh state has been cleared. */
 	if(req->is_drop) {
 		/* the reply has been dropped, stream has been closed. */
@@ -2131,7 +2301,7 @@ tcp_req_info_add_result(struct tcp_req_info* req, uint8_t* buf, size_t len)
 	last = req->done_req_list;
 	while(last && last->next)
 		last = last->next;
-	
+
 	/* create new element */
 	item = (struct tcp_req_done_item*)malloc(sizeof(*item));
 	if(!item) {
@@ -2490,7 +2660,7 @@ static int http2_query_read_done(struct http2_session* h2_session,
 			"buffer already assigned to stream");
 		return -1;
 	}
-    
+
     /* the c->buffer might be used by mesh_send_reply and no be cleard
 	 * need to be cleared before use */
 	sldns_buffer_clear(h2_session->c->buffer);