for 0.7.2: fixup building in subdir.

git-svn-id: file:///svn/unbound/branches/beta-0.7@797 be551aaa-1e26-0410-a405-d3ace91eadb9

Makefile.in

@@ -20,16 +20,21 @@ exec_prefix=@exec_prefix@
 bindir=@bindir@
 mandir=@mandir@
 libdir=@libdir@
+# datarootdir is here to please some checkers, use datadir.
 datarootdir=@datarootdir@
 datadir=@datadir@
 includedir=@includedir@
 doxygen=@doxygen@
 libtool=@libtool@
 ldnsdir=@ldnsdir@
+staticexe=@staticexe@
+configfile=@ub_conf_file@
 
+YACC=@YACC@
+LEX=@LEX@
 CC=@CC@
-CPPFLAGS=-I. @CPPFLAGS@ @DEFS@
-CFLAGS=-I. @CFLAGS@
+CPPFLAGS=-I$(srcdir) -I. @CPPFLAGS@ @DEFS@
+CFLAGS=@CFLAGS@
 LDFLAGS=@LDFLAGS@
 LIBS=@LIBS@
 LIBOBJS=@LIBOBJS@
@@ -42,53 +47,130 @@ endif
 BUILD=build/
 
 LINT=splint
-LINTFLAGS=+quiet -weak -warnposix -unrecog -Din_addr_t=uint32_t -Du_int=unsigned -Du_char=uint8_t -preproc
+LINTFLAGS=+quiet -weak -warnposix -unrecog -Din_addr_t=uint32_t -Du_int=unsigned -Du_char=uint8_t -preproc -Drlimit=rlimit64 -D__gnuc_va_list=va_list
 # compat with openssl linux edition.
-LINTFLAGS+="-DBN_ULONG=unsigned long" -Dkrb5_int32=int "-Dkrb5_ui_4=unsigned int" -DPQ_64BIT=uint64_t
+LINTFLAGS+="-DBN_ULONG=unsigned long" -Dkrb5_int32=int "-Dkrb5_ui_4=unsigned int" -DPQ_64BIT=uint64_t -DRC4_INT=unsigned
 
-INSTALL=$(srcdir)/install-sh 
+INSTALL=$(srcdir)/install-sh
 
-COMMON_SRC=$(wildcard services/*.c util/*.c)
+COMMON_SRC=$(patsubst $(srcdir)/%,%, $(wildcard $(srcdir)/services/*.c \
+	$(srcdir)/services/cache/*.c $(srcdir)/util/*.c \
+	$(srcdir)/util/data/*.c $(srcdir)/util/storage/*.c \
+	$(srcdir)/iterator/*.c $(srcdir)/validator/*.c)) \
+	util/configparser.c util/configlexer.c testcode/checklocks.c
 COMMON_OBJ=$(addprefix $(BUILD),$(COMMON_SRC:.c=.o))
 COMPAT_OBJ=$(addprefix $(BUILD)compat/,$(LIBOBJS))
-UNITTEST_SRC=testcode/unitmain.c $(COMMON_SRC)
+UNITTEST_SRC=$(patsubst $(srcdir)/%,%, \
+	$(wildcard $(srcdir)/testcode/unit*.c)) \
+	testcode/readhex.c testcode/ldns-testpkts.c checkconf/worker_cb.c \
+	$(COMMON_SRC)
 UNITTEST_OBJ=$(addprefix $(BUILD),$(UNITTEST_SRC:.c=.o)) $(COMPAT_OBJ)
-DAEMON_SRC=$(wildcard daemon/*.c) $(COMMON_SRC)
+DAEMON_SRC=$(patsubst $(srcdir)/%,%, $(wildcard $(srcdir)/daemon/*.c)) \
+	$(COMMON_SRC)
 DAEMON_OBJ=$(addprefix $(BUILD),$(DAEMON_SRC:.c=.o)) $(COMPAT_OBJ)
-TESTBOUND_SRC=testcode/testbound.c testcode/ldns-testpkts.c daemon/worker.c testcode/replay.c testcode/fake_event.c $(filter-out util/netevent.c services/listen_dnsport.c services/outside_network.c, $(COMMON_SRC))
+CHECKCONF_SRC=checkconf/unbound-checkconf.c checkconf/worker_cb.c $(COMMON_SRC)
+CHECKCONF_OBJ=$(addprefix $(BUILD),$(CHECKCONF_SRC:.c=.o)) $(COMPAT_OBJ)
+TESTBOUND_SRC=testcode/testbound.c testcode/ldns-testpkts.c \
+	daemon/worker.c daemon/daemon.c daemon/stats.c testcode/replay.c \
+	testcode/fake_event.c $(filter-out util/netevent.c \
+	services/listen_dnsport.c services/outside_network.c, $(COMMON_SRC))
 TESTBOUND_OBJ=$(addprefix $(BUILD),$(TESTBOUND_SRC:.c=.o)) $(COMPAT_OBJ)
-ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) $(TESTBOUND_SRC)
-ALL_OBJ=$(addprefix $(BUILD),$(ALL_SRC:.c=.o) $(addprefix compat/,$(LIBOBJS))) $(COMPAT_OBJ)
+LOCKVERIFY_SRC=testcode/lock_verify.c checkconf/worker_cb.c $(COMMON_SRC)
+LOCKVERIFY_OBJ=$(addprefix $(BUILD),$(LOCKVERIFY_SRC:.c=.o)) $(COMPAT_OBJ)
+PKTVIEW_SRC=testcode/pktview.c testcode/readhex.c checkconf/worker_cb.c \
+	$(COMMON_SRC)
+PKTVIEW_OBJ=$(addprefix $(BUILD),$(PKTVIEW_SRC:.c=.o)) $(COMPAT_OBJ)
+SIGNIT_SRC=testcode/signit.c checkconf/worker_cb.c $(COMMON_SRC)
+SIGNIT_OBJ=$(addprefix $(BUILD),$(SIGNIT_SRC:.c=.o)) $(COMPAT_OBJ)
+MEMSTATS_SRC=testcode/memstats.c checkconf/worker_cb.c $(COMMON_SRC)
+MEMSTATS_OBJ=$(addprefix $(BUILD),$(MEMSTATS_SRC:.c=.o)) $(COMPAT_OBJ)
+ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) \
+	$(TESTBOUND_SRC) $(LOCKVERIFY_SRC) $(PKTVIEW_SRC) $(SIGNIT_SRC) \
+	$(MEMSTATS_SRC) $(CHECKCONF_SRC)
+ALL_OBJ=$(addprefix $(BUILD),$(ALL_SRC:.c=.o) \
+	$(addprefix compat/,$(LIBOBJS))) $(COMPAT_OBJ)
 
 COMPILE=$(LIBTOOL) --tag=CC --mode=compile $(CC) $(CPPFLAGS) $(CFLAGS)
-LINK=$(LIBTOOL) --tag=CC --mode=link $(CC) $(CFLAGS) $(LDFLAGS)
+LINK=$(LIBTOOL) --tag=CC --mode=link $(CC) $(staticexe) $(RUNTIME_PATH) $(CFLAGS) $(LDFLAGS)
 LINK_LIB=$(LIBTOOL) --tag=CC --mode=link $(CC) $(CFLAGS) $(LDFLAGS) -release $(VERSION)
 
+.PHONY:	clean realclean doc lint all install uninstall tests test download_ldns strip
+
 $(BUILD)%.o:    $(srcdir)/%.c 
 	$(INFO) Build $<
+	@if test ! -z "$(ldnsdir)" -a ! -e $(ldnsdir)/include/ldns/ldns.h; \
+		then (cd $(ldnsdir); $(MAKE) copy-headers); fi
 	@if test ! -d $(dir $@); then $(INSTALL) -d $(patsubst %/,%,$(dir $@)); fi
 	$Q$(COMPILE) -c $< -o $@
 
-.PHONY:	clean realclean doc lint all 
+all:	$(COMMON_OBJ) unbound unbound-checkconf
 
-all:	$(COMMON_OBJ) unbound unittest testbound
+tests:	all unittest testbound lock-verify pktview signit memstats
 
-unbound:	$(DAEMON_OBJ)
+test:	tests
+	bash testcode/do-tests.sh
+
+ifeq ($(patsubst ldns-src%,ldns-src,$(ldnsdir)),ldns-src)
+ldnslib=$(ldnsdir)/lib/libldns.a
+$(ldnslib):
+	@if test ! -z "$(ldnsdir)"; \
+	then (cd $(ldnsdir) && $(MAKE)); fi
+else
+ldnslib=
+endif
+
+unbound:	$(DAEMON_OBJ) $(ldnslib)
 	$(INFO) Link $@
-	$Q$(LINK) -o $@ $^ $(LIBS)
+	$Q$(LINK) -o $@ $(sort $(DAEMON_OBJ)) $(LIBS)
 
-unittest:	$(UNITTEST_OBJ)
+unbound-checkconf:	$(CHECKCONF_OBJ) $(ldnslib)
 	$(INFO) Link $@
-	$Q$(LINK) -o $@ $^ $(LIBS)
+	$Q$(LINK) -o $@ $(sort $(CHECKCONF_OBJ)) $(LIBS)
 
-testbound:	$(TESTBOUND_OBJ)
+unittest:	$(UNITTEST_OBJ) $(ldnslib)
 	$(INFO) Link $@
-	$Q$(LINK) -o $@ $^ $(LIBS)
+	$Q$(LINK) -o $@ $(sort $(UNITTEST_OBJ)) $(LIBS)
 
-testcode/ldns-testpkts.c:	$(ldnsdir)/examples/ldns-testpkts.c \
-			$(ldnsdir)/examples/ldns-testpkts.h
-	cp $(ldnsdir)/examples/ldns-testpkts.c testcode/ldns-testpkts.c
-	cp $(ldnsdir)/examples/ldns-testpkts.h testcode/ldns-testpkts.h
+testbound:	$(TESTBOUND_OBJ) $(ldnslib)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $(sort $(TESTBOUND_OBJ)) $(LIBS)
+
+lock-verify:	$(LOCKVERIFY_OBJ) $(ldnslib)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $(sort $(LOCKVERIFY_OBJ)) $(LIBS)
+
+pktview:	$(PKTVIEW_OBJ) $(ldnslib)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $(sort $(PKTVIEW_OBJ)) $(LIBS)
+
+signit:	$(SIGNIT_OBJ) $(ldnslib)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $(sort $(SIGNIT_OBJ)) $(LIBS)
+
+memstats:	$(MEMSTATS_OBJ) $(ldnslib)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $(sort $(MEMSTATS_OBJ)) $(LIBS)
+
+#testcode/ldns-testpkts.c:	$(ldnsdir)/examples/ldns-testpkts.c \
+#			$(ldnsdir)/examples/ldns-testpkts.h
+#	cp $(ldnsdir)/examples/ldns-testpkts.c testcode/ldns-testpkts.c
+#	cp $(ldnsdir)/examples/ldns-testpkts.h testcode/ldns-testpkts.h
+
+util/config_file.c:	util/configparser.h
+util/configlexer.c:  $(srcdir)/util/configlexer.lex util/configparser.h
+	$(INFO) Lex $<
+ifeq "$(strip $(LEX))" ":"
+	$Qecho "rebuild lexer, but no lex program, skipped"
+else
+	@if test ! -d util; then $(INSTALL) -d util; fi
+	$Qecho "#include \"util/configyyrename.h\"" > $@
+	$Q$(LEX) -t $< >> $@
+endif
+
+util/configparser.c util/configparser.h:  $(srcdir)/util/configparser.y
+	$(INFO) Yacc $<
+	@if test ! -d util; then $(INSTALL) -d util; fi
+	$Q$(YACC) -d -o util/configparser.c $<
 
 clean:
 	rm -f *.o *.d *.lo *~ tags
@@ -97,14 +179,17 @@ clean:
 realclean: clean
 	rm -f config.status config.log config.h.in config.h
 	rm -f configure config.sub config.guess ltmain.sh aclocal.m4 libtool
+	rm -f util/configlexer.c util/configparser.c util/configparser.h
+	if test -d ldns-src; then rm -rf ldns-src; fi
 	rm -f Makefile 
 
-lint:
-	$Qfor i in $(sort $(ALL_SRC)); do \
-		echo lint $$i; \
-		$(LINT) $(LINTFLAGS) -I. -I$(srcdir) -I$(ldnsdir)/include $(srcdir)/$$i ; \
-		if [ $$? -ne 0 ] ; then exit 1 ; fi ; \
-	done
+$(BUILD)%.lint:    $(srcdir)/%.c 
+	$(INFO) Lint $<
+	@if test ! -d $(dir $@); then $(INSTALL) -d $(patsubst %/,%,$(dir $@)); fi
+	$Q$(LINT) $(LINTFLAGS) -I. -I$(srcdir) -I$(ldnsdir)/include $<
+	$Qtouch $@
+
+lint:	$(addprefix $(BUILD),$(filter-out util/configparser.lint,$(filter-out util/configlexer.lint,$(sort $(ALL_SRC:.c=.lint)))))
 
 tags:	$(srcdir)/*.[ch] $(srcdir)/*/*.[ch]
 	ctags  -f $(srcdir)/tags $(srcdir)/*.[ch] $(srcdir)/*/*.[ch]
@@ -114,12 +199,46 @@ ifdef doxygen
 	$(doxygen) $(srcdir)/doc/unbound.doxygen
 endif
 
+strip:
+	strip unbound
+	strip unbound-checkconf
+
+install:
+	$(INSTALL) -d $(bindir)
+	$(INSTALL) -d $(mandir)
+	$(INSTALL) -d $(mandir)/man8
+	$(INSTALL) -d $(mandir)/man5
+	$(LIBTOOL) --mode=install cp unbound $(bindir)/unbound
+	$(LIBTOOL) --mode=install cp unbound-checkconf $(bindir)/unbound-checkconf
+	$(INSTALL) -c -m 644 $(srcdir)/doc/unbound.8 $(mandir)/man8
+	$(INSTALL) -c -m 644 $(srcdir)/doc/unbound-checkconf.8 $(mandir)/man8
+	$(INSTALL) -c -m 644 $(srcdir)/doc/unbound.conf.5 $(mandir)/man5
+	if test ! -e $(configfile); then $(INSTALL) -d `dirname $(configfile)`; $(INSTALL) -c -m 644 $(srcdir)/doc/example.conf $(configfile); fi
+
+uninstall:
+	rm -f -- $(bindir)/unbound $(bindir)/unbound-checkconf
+	rm -f -- $(mandir)/man8/unbound.8 $(mandir)/man8/unbound-checkconf.8 $(mandir)/man5/unbound.conf.5
+	@echo
+	@echo "You still need to remove `dirname $(configfile)` , $(configfile) by hand"
+
+download_ldns:
+	svn export https://www.nlnetlabs.nl/ldns/svn/trunk/makedist.sh ldns_makedist.sh
+	./ldns_makedist.sh -s -d https://www.nlnetlabs.nl/ldns/svn/trunk
+	mv ldns-*_pre_*.tar.gz ldns-src.tar.gz
+	rm ldns-*_pre_*.tar.gz.sha1 ldns_makedist.sh
+
 # Automatic dependencies.
 $(BUILD)%.d: $(srcdir)/%.c
 	$(INFO) Depend $<
+	@if test ! -z "$(ldnsdir)" -a ! -e $(ldnsdir)/include/ldns/ldns.h; \
+		then (cd $(ldnsdir); $(MAKE) copy-headers); fi
 	@if test ! -d $(dir $@); then $(INSTALL) -d $(patsubst %/,%,$(dir $@)); fi
 	$Q$(SHELL) -ec '$(CC) -MM $(CPPFLAGS) $(CFLAGS) $< \
 	              | sed '\''s!\(.*\)\.o[ :]*!$(dir $@)\1.o $@ : !g'\'' > $@; \
 	              [ -s $@ ] || rm -f $@'
 
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(MAKECMDGOALS),realclean)
 -include $(addprefix $(BUILD),$(ALL_SRC:.c=.d))
+endif
+endif

README

@@ -0,0 +1,10 @@
+Unbound README
+* ./configure && make && make install
+* on BSDs and Solaris use gmake (GNU make).
+* If it complains that no libevent can be found; libevent is useful when
+  using many (10000) outgoing ports. By default 16 ports are used and the
+  builtin alternative is equally capable.
+* More detailed README, README.svn, README.tests in doc directory
+* manual pages can be found in doc directory, and are installed, unbound(8).
+* example configuration file doc/example.conf
+

aclocal.m4

@@ -1,7 +1,7 @@
-# generated automatically by aclocal 1.9.6 -*- Autoconf -*-
+# generated automatically by aclocal 1.10 -*- Autoconf -*-
 
 # Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-# 2005  Free Software Foundation, Inc.
+# 2005, 2006  Free Software Foundation, Inc.
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
 # with or without modifications, as long as this notice is preserved.
@@ -1597,7 +1597,7 @@ linux*)
 
   # Append ld.so.conf contents to the search path
   if test -f /etc/ld.so.conf; then
-    lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;s/[:,	]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;/^$/d' | tr '\n' ' '`
+    lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/^[	 ]*//;s/#.*//;/^[^\/]/d;s/[:,	]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;/^$/d' | tr '\n' ' '`
     sys_lib_dlsearch_path_spec="/lib${libsuff} /usr/lib${libsuff} $lt_ld_extra"
   fi
 
@@ -6370,6 +6370,7 @@ do
     done
   done
 done
+IFS=$as_save_IFS
 lt_ac_max=0
 lt_ac_count=0
 # Add /usr/xpg4/bin/sed as it is typically found on Solaris
@@ -6402,6 +6403,7 @@ for lt_ac_sed in $lt_ac_sed_list /usr/xpg4/bin/sed; do
 done
 ])
 SED=$lt_cv_path_SED
+AC_SUBST([SED])
 AC_MSG_RESULT([$SED])
 ])
 

acx_pthread.m4

@@ -0,0 +1,280 @@
+##### http://autoconf-archive.cryp.to/acx_pthread.html
+#
+# SYNOPSIS
+#
+#   ACX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]])
+#
+# DESCRIPTION
+#
+#   This macro figures out how to build C programs using POSIX threads.
+#   It sets the PTHREAD_LIBS output variable to the threads library and
+#   linker flags, and the PTHREAD_CFLAGS output variable to any special
+#   C compiler flags that are needed. (The user can also force certain
+#   compiler flags/libs to be tested by setting these environment
+#   variables.)
+#
+#   Also sets PTHREAD_CC to any special C compiler that is needed for
+#   multi-threaded programs (defaults to the value of CC otherwise).
+#   (This is necessary on AIX to use the special cc_r compiler alias.)
+#
+#   NOTE: You are assumed to not only compile your program with these
+#   flags, but also link it with them as well. e.g. you should link
+#   with $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS
+#   $LIBS
+#
+#   If you are only building threads programs, you may wish to use
+#   these variables in your default LIBS, CFLAGS, and CC:
+#
+#          LIBS="$PTHREAD_LIBS $LIBS"
+#          CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+#          CC="$PTHREAD_CC"
+#
+#   In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute
+#   constant has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to
+#   that name (e.g. PTHREAD_CREATE_UNDETACHED on AIX).
+#
+#   ACTION-IF-FOUND is a list of shell commands to run if a threads
+#   library is found, and ACTION-IF-NOT-FOUND is a list of commands to
+#   run it if it is not found. If ACTION-IF-FOUND is not specified, the
+#   default action will define HAVE_PTHREAD.
+#
+#   Please let the authors know if this macro fails on any platform, or
+#   if you have any other suggestions or comments. This macro was based
+#   on work by SGJ on autoconf scripts for FFTW (http://www.fftw.org/)
+#   (with help from M. Frigo), as well as ac_pthread and hb_pthread
+#   macros posted by Alejandro Forero Cuervo to the autoconf macro
+#   repository. We are also grateful for the helpful feedback of
+#   numerous users.
+#
+# LAST MODIFICATION
+#
+#   2006-05-29
+#
+# COPYLEFT
+#
+#   Copyright (c) 2006 Steven G. Johnson <stevenj@alum.mit.edu>
+#
+#   This program is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU General Public License as
+#   published by the Free Software Foundation; either version 2 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+#   General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program; if not, write to the Free Software
+#   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+#   02111-1307, USA.
+#
+#   As a special exception, the respective Autoconf Macro's copyright
+#   owner gives unlimited permission to copy, distribute and modify the
+#   configure scripts that are the output of Autoconf when processing
+#   the Macro. You need not follow the terms of the GNU General Public
+#   License when using or distributing such scripts, even though
+#   portions of the text of the Macro appear in them. The GNU General
+#   Public License (GPL) does govern all other use of the material that
+#   constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the
+#   Autoconf Macro released by the Autoconf Macro Archive. When you
+#   make and distribute a modified version of the Autoconf Macro, you
+#   may extend this special exception to the GPL to apply to your
+#   modified version as well.
+
+AC_DEFUN([ACX_PTHREAD], [
+AC_REQUIRE([AC_CANONICAL_HOST])
+AC_LANG_SAVE
+AC_LANG_C
+acx_pthread_ok=no
+
+# We used to check for pthread.h first, but this fails if pthread.h
+# requires special compiler flags (e.g. on True64 or Sequent).
+# It gets checked for in the link test anyway.
+
+# First of all, check if the user has set any of the PTHREAD_LIBS,
+# etcetera environment variables, and if threads linking works using
+# them:
+if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then
+        save_CFLAGS="$CFLAGS"
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+        save_LIBS="$LIBS"
+        LIBS="$PTHREAD_LIBS $LIBS"
+        AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS])
+        AC_TRY_LINK_FUNC(pthread_join, acx_pthread_ok=yes)
+        AC_MSG_RESULT($acx_pthread_ok)
+        if test x"$acx_pthread_ok" = xno; then
+                PTHREAD_LIBS=""
+                PTHREAD_CFLAGS=""
+        fi
+        LIBS="$save_LIBS"
+        CFLAGS="$save_CFLAGS"
+fi
+
+# We must check for the threads library under a number of different
+# names; the ordering is very important because some systems
+# (e.g. DEC) have both -lpthread and -lpthreads, where one of the
+# libraries is broken (non-POSIX).
+
+# Create a list of thread flags to try.  Items starting with a "-" are
+# C compiler flags, and other items are library names, except for "none"
+# which indicates that we try without any flags at all, and "pthread-config"
+# which is a program returning the flags for the Pth emulation library.
+
+acx_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
+
+# The ordering *is* (sometimes) important.  Some notes on the
+# individual items follow:
+
+# pthreads: AIX (must check this before -lpthread)
+# none: in case threads are in libc; should be tried before -Kthread and
+#       other compiler flags to prevent continual compiler warnings
+# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h)
+# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able)
+# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread)
+# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads)
+# -pthreads: Solaris/gcc
+# -mthreads: Mingw32/gcc, Lynx/gcc
+# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it
+#      doesn't hurt to check since this sometimes defines pthreads too;
+#      also defines -D_REENTRANT)
+#      ... -mt is also the pthreads flag for HP/aCC
+# pthread: Linux, etcetera
+# --thread-safe: KAI C++
+# pthread-config: use pthread-config program (for GNU Pth library)
+
+case "${host_cpu}-${host_os}" in
+        *solaris*)
+
+        # On Solaris (at least, for some versions), libc contains stubbed
+        # (non-functional) versions of the pthreads routines, so link-based
+        # tests will erroneously succeed.  (We need to link with -pthreads/-mt/
+        # -lpthread.)  (The stubs are missing pthread_cleanup_push, or rather
+        # a function called by this macro, so we could check for that, but
+        # who knows whether they'll stub that too in a future libc.)  So,
+        # we'll just look for -pthreads and -lpthread first:
+
+        acx_pthread_flags="-pthreads pthread -mt -pthread $acx_pthread_flags"
+        ;;
+esac
+
+if test x"$acx_pthread_ok" = xno; then
+for flag in $acx_pthread_flags; do
+
+        case $flag in
+                none)
+                AC_MSG_CHECKING([whether pthreads work without any flags])
+                ;;
+
+                -*)
+                AC_MSG_CHECKING([whether pthreads work with $flag])
+                PTHREAD_CFLAGS="$flag"
+                ;;
+
+		pthread-config)
+		AC_CHECK_PROG(acx_pthread_config, pthread-config, yes, no)
+		if test x"$acx_pthread_config" = xno; then continue; fi
+		PTHREAD_CFLAGS="`pthread-config --cflags`"
+		PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
+		;;
+
+                *)
+                AC_MSG_CHECKING([for the pthreads library -l$flag])
+                PTHREAD_LIBS="-l$flag"
+                ;;
+        esac
+
+        save_LIBS="$LIBS"
+        save_CFLAGS="$CFLAGS"
+        LIBS="$PTHREAD_LIBS $LIBS"
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+
+        # Check for various functions.  We must include pthread.h,
+        # since some functions may be macros.  (On the Sequent, we
+        # need a special flag -Kthread to make this header compile.)
+        # We check for pthread_join because it is in -lpthread on IRIX
+        # while pthread_create is in libc.  We check for pthread_attr_init
+        # due to DEC craziness with -lpthreads.  We check for
+        # pthread_cleanup_push because it is one of the few pthread
+        # functions on Solaris that doesn't have a non-functional libc stub.
+        # We try pthread_create on general principles.
+        AC_TRY_LINK([#include <pthread.h>],
+                    [pthread_t th; pthread_join(th, 0);
+                     pthread_attr_init(0); pthread_cleanup_push(0, 0);
+                     pthread_create(0,0,0,0); pthread_cleanup_pop(0); ],
+                    [acx_pthread_ok=yes])
+
+        LIBS="$save_LIBS"
+        CFLAGS="$save_CFLAGS"
+
+        AC_MSG_RESULT($acx_pthread_ok)
+        if test "x$acx_pthread_ok" = xyes; then
+                break;
+        fi
+
+        PTHREAD_LIBS=""
+        PTHREAD_CFLAGS=""
+done
+fi
+
+# Various other checks:
+if test "x$acx_pthread_ok" = xyes; then
+        save_LIBS="$LIBS"
+        LIBS="$PTHREAD_LIBS $LIBS"
+        save_CFLAGS="$CFLAGS"
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+
+        # Detect AIX lossage: JOINABLE attribute is called UNDETACHED.
+	AC_MSG_CHECKING([for joinable pthread attribute])
+	attr_name=unknown
+	for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do
+	    AC_TRY_LINK([#include <pthread.h>], [int attr=$attr; return attr;],
+                        [attr_name=$attr; break])
+	done
+        AC_MSG_RESULT($attr_name)
+        if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then
+            AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name,
+                               [Define to necessary symbol if this constant
+                                uses a non-standard name on your system.])
+        fi
+
+        AC_MSG_CHECKING([if more special flags are required for pthreads])
+        flag=no
+        case "${host_cpu}-${host_os}" in
+            *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";;
+            *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";;
+        esac
+        AC_MSG_RESULT(${flag})
+        if test "x$flag" != xno; then
+            PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS"
+        fi
+
+        LIBS="$save_LIBS"
+        CFLAGS="$save_CFLAGS"
+
+        # More AIX lossage: must compile with xlc_r or cc_r
+	if test x"$GCC" != xyes; then
+          AC_CHECK_PROGS(PTHREAD_CC, xlc_r cc_r, ${CC})
+        else
+          PTHREAD_CC=$CC
+	fi
+else
+        PTHREAD_CC="$CC"
+fi
+
+AC_SUBST(PTHREAD_LIBS)
+AC_SUBST(PTHREAD_CFLAGS)
+AC_SUBST(PTHREAD_CC)
+
+# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
+if test x"$acx_pthread_ok" = xyes; then
+        ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1])
+        :
+else
+        acx_pthread_ok=no
+        $2
+fi
+AC_LANG_RESTORE
+])dnl ACX_PTHREAD

checkconf/unbound-checkconf.c

@@ -0,0 +1,198 @@
+/*
+ * checkconf/unbound-checkconf.c - config file checker for unbound.conf file.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * The config checker checks for syntax and other errors in the unbound.conf
+ * file, and can be used to check for errors before the server is started
+ * or sigHUPped.
+ * Exit status 1 means an error.
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/module.h"
+#include "util/net_help.h"
+#include "util/regional.h"
+#include "iterator/iterator.h"
+#include "validator/validator.h"
+#include <pwd.h>
+
+/** Give checkconf usage, and exit (1). */
+static void
+usage()
+{
+	printf("Usage:	unbound-checkconf [file]\n");
+	printf("	Checks unbound configuration file for errors.\n");
+	printf("file	if omitted %s is used.\n", CONFIGFILE);
+	printf("-h	show this usage help.\n");
+	printf("Version %s\n", PACKAGE_VERSION);
+	printf("BSD licensed, see LICENSE in source package for details.\n");
+	printf("Report bugs to %s\n", PACKAGE_BUGREPORT);
+	exit(1);
+}
+
+/** check if module works with config */
+static void
+check_mod(struct config_file* cfg, struct module_func_block* fb)
+{
+	struct module_env env;
+	memset(&env, 0, sizeof(env));
+	env.cfg = cfg;
+	env.scratch = regional_create();
+	env.scratch_buffer = ldns_buffer_new(BUFSIZ);
+	if(!env.scratch || !env.scratch_buffer)
+		fatal_exit("out of memory");
+	if(!(*fb->init)(&env, 0)) {
+		fatal_exit("bad config for %s module", fb->name);
+	}
+	(*fb->deinit)(&env, 0);
+	ldns_buffer_free(env.scratch_buffer);
+	regional_destroy(env.scratch);
+}
+
+/** check configuration for errors */
+static void
+morechecks(struct config_file* cfg)
+{
+	int i;
+	struct sockaddr_storage a;
+	socklen_t alen;
+	for(i=0; i<cfg->num_ifs; i++) {
+		if(!ipstrtoaddr(cfg->ifs[i], UNBOUND_DNS_PORT, &a, &alen)) {
+			fatal_exit("cannot parse interface specified as '%s'",
+				cfg->ifs[i]);
+		}
+	}
+	for(i=0; i<cfg->num_out_ifs; i++) {
+		if(!ipstrtoaddr(cfg->out_ifs[i], UNBOUND_DNS_PORT, 
+			&a, &alen)) {
+			fatal_exit("cannot parse outgoing-interface "
+				"specified as '%s'", cfg->out_ifs[i]);
+		}
+	}
+
+	if(cfg->verbosity < 0)
+		fatal_exit("verbosity value < 0");
+	if(cfg->num_threads < 0 || cfg->num_threads > 10000)
+		fatal_exit("num_threads value weird");
+	if(!cfg->do_ip4 && !cfg->do_ip6)
+		fatal_exit("ip4 and ip6 are both disabled, pointless");
+	if(!cfg->do_udp && !cfg->do_tcp)
+		fatal_exit("udp and tcp are both disabled, pointless");
+
+	if(cfg->chrootdir && strncmp(cfg->chrootdir, cfg->directory,
+		strlen(cfg->chrootdir)) != 0)
+		fatal_exit("working directory %s not in chrootdir %s",
+			cfg->directory, cfg->chrootdir);
+	if(cfg->chrootdir && cfg->pidfile && cfg->pidfile[0] &&
+		strncmp(cfg->chrootdir, cfg->pidfile,
+			strlen(cfg->chrootdir)) != 0)
+		fatal_exit("pid file %s not in chrootdir %s",
+			cfg->pidfile, cfg->chrootdir);
+	if(cfg->chrootdir && cfg->logfile && cfg->logfile[0] &&
+		strncmp(cfg->chrootdir, cfg->logfile,
+			strlen(cfg->chrootdir)) != 0)
+		fatal_exit("log file %s not in chrootdir %s",
+			cfg->logfile, cfg->chrootdir);
+	
+	if(strcmp(cfg->module_conf, "iterator") != 0 &&
+		strcmp(cfg->module_conf, "validator iterator") != 0) {
+		fatal_exit("module conf %s is not known to work",
+			cfg->module_conf);
+	}
+
+	if(cfg->username && cfg->username[0]) {
+		struct passwd *pwd;
+		if((pwd = getpwnam(cfg->username)) == NULL)
+			fatal_exit("user '%s' does not exist.", cfg->username);
+		endpwent();
+	}
+}
+
+/** check config file */
+static void
+checkconf(char* cfgfile)
+{
+	struct config_file* cfg = config_create();
+	if(!cfg)
+		fatal_exit("out of memory");
+	if(!config_read(cfg, cfgfile)) {
+		/* config_read prints messages to stderr */
+		config_delete(cfg);
+		exit(1);
+	}
+	morechecks(cfg);
+	check_mod(cfg, iter_get_funcblock());
+	check_mod(cfg, val_get_funcblock());
+	config_delete(cfg);
+	printf("unbound-checkconf: no errors in %s\n", cfgfile);
+}
+
+/** getopt global, in case header files fail to declare it. */
+extern int optind;
+/** getopt global, in case header files fail to declare it. */
+extern char* optarg;
+
+/** Main routine for checkconf */
+int main(int argc, char* argv[])
+{
+	int c;
+	char* f;
+	log_ident_set("unbound-checkconf");
+	log_init(NULL, 0, NULL);
+	checklock_start();
+	/* parse the options */
+	while( (c=getopt(argc, argv, "h")) != -1) {
+		switch(c) {
+		case '?':
+		case 'h':
+		default:
+			usage();
+		}
+	}
+	argc -= optind;
+	argv += optind;
+	if(argc != 0 && argc != 1)
+		usage();
+	if(argc == 1)
+		f = argv[0];
+	else	f = CONFIGFILE;
+	checkconf(f);
+	checklock_stop();
+	return 0;
+}

checkconf/worker_cb.c

@@ -0,0 +1,109 @@
+/*
+ * checkconf/worker_cb.c - fake callback routines to make fptr_wlist work
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains fake callback functions, so that the symbols exist
+ * and the fptr_wlist continues to work even if the daemon/worker is not
+ * linked into the resulting program.
+ */
+#include "config.h"
+#include "util/log.h"
+struct comm_reply;
+struct comm_point;
+struct module_qstate;
+
+int worker_handle_control_cmd(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int worker_handle_request(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(repinfo))
+{
+	log_assert(0);
+	return 0;
+}
+
+int worker_handle_reply(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int worker_handle_service_reply(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+void worker_sighandler(int ATTR_UNUSED(sig), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+int worker_send_packet(ldns_buffer* ATTR_UNUSED(pkt), 
+	struct sockaddr_storage* ATTR_UNUSED(addr), 
+	socklen_t ATTR_UNUSED(addrlen), int ATTR_UNUSED(timeout), 
+	struct module_qstate* ATTR_UNUSED(q), int ATTR_UNUSED(use_tcp))
+{
+	log_assert(0);
+	return 0;
+}
+
+struct outbound_entry* worker_send_query(uint8_t* ATTR_UNUSED(qname), 
+	size_t ATTR_UNUSED(qnamelen), uint16_t ATTR_UNUSED(qtype), 
+	uint16_t ATTR_UNUSED(qclass), uint16_t ATTR_UNUSED(flags), 
+	int ATTR_UNUSED(dnssec), struct sockaddr_storage* ATTR_UNUSED(addr), 
+	socklen_t ATTR_UNUSED(addrlen), struct module_qstate* ATTR_UNUSED(q))
+{
+	log_assert(0);
+	return 0;
+}
+
+void 
+worker_alloc_cleanup(void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}

config.h.in

@@ -0,0 +1,426 @@
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Pathname to the Unbound configuration file */
+#undef CONFIGFILE
+
+/* Define if you want to use debug lock checking (slow). */
+#undef ENABLE_LOCK_CHECKS
+
+/* Define to 1 if you have the <arpa/inet.h> header file. */
+#undef HAVE_ARPA_INET_H
+
+/* Whether the C compiler accepts the "format" attribute */
+#undef HAVE_ATTR_FORMAT
+
+/* Whether the C compiler accepts the "unused" attribute */
+#undef HAVE_ATTR_UNUSED
+
+/* Define to 1 if your system has a working `chown' function. */
+#undef HAVE_CHOWN
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
+
+/* Define to 1 if you have the `event_base_free' function. */
+#undef HAVE_EVENT_BASE_FREE
+
+/* Define to 1 if you have the `event_base_once' function. */
+#undef HAVE_EVENT_BASE_ONCE
+
+/* Define to 1 if you have the <event.h> header file. */
+#undef HAVE_EVENT_H
+
+/* Define to 1 if you have the `fork' function. */
+#undef HAVE_FORK
+
+/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */
+#undef HAVE_FSEEKO
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#undef HAVE_GETADDRINFO
+
+/* Define to 1 if you have the <getopt.h> header file. */
+#undef HAVE_GETOPT_H
+
+/* Define to 1 if you have the `inet_aton' function. */
+#undef HAVE_INET_ATON
+
+/* Define to 1 if you have the `inet_ntop' function. */
+#undef HAVE_INET_NTOP
+
+/* Define to 1 if you have the `inet_pton' function. */
+#undef HAVE_INET_PTON
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#undef HAVE_LIBCRYPTO
+
+/* Define to 1 if you have the `nsl' library (-lnsl). */
+#undef HAVE_LIBNSL
+
+/* Define to 1 if you have the `rt' library (-lrt). */
+#undef HAVE_LIBRT
+
+/* Define to 1 if you have the `socket' library (-lsocket). */
+#undef HAVE_LIBSOCKET
+
+/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
+   to 0 otherwise. */
+#undef HAVE_MALLOC
+
+/* Define to 1 if you have the `memmove' function. */
+#undef HAVE_MEMMOVE
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* Define to 1 if you have the <netinet/in.h> header file. */
+#undef HAVE_NETINET_IN_H
+
+/* Define to 1 if you have the <openssl/err.h> header file. */
+#undef HAVE_OPENSSL_ERR_H
+
+/* Define to 1 if you have the <openssl/ssl.h> header file. */
+#undef HAVE_OPENSSL_SSL_H
+
+/* Define if you have POSIX threads libraries and header files. */
+#undef HAVE_PTHREAD
+
+/* Define to 1 if the system has the type `pthread_rwlock_t'. */
+#undef HAVE_PTHREAD_RWLOCK_T
+
+/* Define to 1 if the system has the type `pthread_spinlock_t'. */
+#undef HAVE_PTHREAD_SPINLOCK_T
+
+/* Define to 1 if you have the `snprintf' function. */
+#undef HAVE_SNPRINTF
+
+/* Using Solaris threads */
+#undef HAVE_SOLARIS_THREADS
+
+/* Define if you have the SSL libraries installed. */
+#undef HAVE_SSL
+
+/* Define to 1 if you have the <stdarg.h> header file. */
+#undef HAVE_STDARG_H
+
+/* Define to 1 if you have the <stdbool.h> header file. */
+#undef HAVE_STDBOOL_H
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the `strlcpy' function. */
+#undef HAVE_STRLCPY
+
+/* Define to 1 if you have the <syslog.h> header file. */
+#undef HAVE_SYSLOG_H
+
+/* Define to 1 if you have the <sys/param.h> header file. */
+#undef HAVE_SYS_PARAM_H
+
+/* Define to 1 if you have the <sys/socket.h> header file. */
+#undef HAVE_SYS_SOCKET_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <sys/uio.h> header file. */
+#undef HAVE_SYS_UIO_H
+
+/* Define to 1 if you have the <time.h> header file. */
+#undef HAVE_TIME_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to 1 if you have the `vfork' function. */
+#undef HAVE_VFORK
+
+/* Define to 1 if you have the <vfork.h> header file. */
+#undef HAVE_VFORK_H
+
+/* Define to 1 if `fork' works. */
+#undef HAVE_WORKING_FORK
+
+/* Define to 1 if `vfork' works. */
+#undef HAVE_WORKING_VFORK
+
+/* Define to the maximum message length to pass to syslog. */
+#undef MAXSYSLOGMSGLEN
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to necessary symbol if this constant uses a non-standard name on
+   your system. */
+#undef PTHREAD_CREATE_JOINABLE
+
+/* Define as the return type of signal handlers (`int' or `void'). */
+#undef RETSIGTYPE
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Define to 'int' or type of struct msghdr.msg_iovlen. */
+#undef TYPE_MSGIOVLEN
+
+/* use statistics for allocs and frees, for debug use */
+#undef UNBOUND_ALLOC_STATS
+
+/* Define if you want to use internal select based events */
+#undef USE_MINI_EVENT
+
+/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
+   `char[]'. */
+#undef YYTEXT_POINTER
+
+/* Define to 1 if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+# undef _ALL_SOURCE
+#endif
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+#undef _FILE_OFFSET_BITS
+
+/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */
+#undef _LARGEFILE_SOURCE
+
+/* Define for large files, on AIX-style hosts. */
+#undef _LARGE_FILES
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const
+
+/* Define to `int' if <sys/types.h> doesn't define. */
+#undef gid_t
+
+/* in_addr_t */
+#undef in_addr_t
+
+/* in_port_t */
+#undef in_port_t
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+   calls it, or to nothing if 'inline' is not supported under any name.  */
+#ifndef __cplusplus
+#undef inline
+#endif
+
+/* Define to `short' if <sys/types.h> does not define. */
+#undef int16_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef int32_t
+
+/* Define to `long long' if <sys/types.h> does not define. */
+#undef int64_t
+
+/* Define to `char' if <sys/types.h> does not define. */
+#undef int8_t
+
+/* Define to rpl_malloc if the replacement function should be used. */
+#undef malloc
+
+/* Define to `long int' if <sys/types.h> does not define. */
+#undef off_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef pid_t
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+#undef size_t
+
+/* Define to 'int' if not defined */
+#undef socklen_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef ssize_t
+
+/* Define to `unsigned char' if <sys/types.h> does not define. */
+#undef u_char
+
+/* Define to `int' if <sys/types.h> doesn't define. */
+#undef uid_t
+
+/* Define to `unsigned short' if <sys/types.h> does not define. */
+#undef uint16_t
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+#undef uint32_t
+
+/* Define to `unsigned long long' if <sys/types.h> does not define. */
+#undef uint64_t
+
+/* Define to `unsigned char' if <sys/types.h> does not define. */
+#undef uint8_t
+
+/* Define as `fork' if `vfork' does not work. */
+#undef vfork
+
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#include <stddef.h>
+#endif
+
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#endif
+
+#include <errno.h>
+
+#if HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+
+#ifdef HAVE_OPENSSL_ERR_H
+#include <openssl/err.h>
+#endif
+
+#ifdef HAVE_ATTR_FORMAT
+#  define ATTR_FORMAT(archetype, string_index, first_to_check) \
+    __attribute__ ((format (archetype, string_index, first_to_check)))
+#else /* !HAVE_ATTR_FORMAT */
+#  define ATTR_FORMAT(archetype, string_index, first_to_check) /* empty */
+#endif /* !HAVE_ATTR_FORMAT */
+#if defined(DOXYGEN)
+#  define ATTR_UNUSED(x)  x
+#elif defined(__cplusplus)
+#  define ATTR_UNUSED(x)
+#elif defined(HAVE_ATTR_UNUSED)
+#  define ATTR_UNUSED(x)  x __attribute__((unused))
+#else /* !HAVE_ATTR_UNUSED */
+#  define ATTR_UNUSED(x)  x
+#endif /* !HAVE_ATTR_UNUSED */
+#ifndef HAVE_FSEEKO
+#define fseeko fseek
+#define ftello ftell
+#endif /* HAVE_FSEEKO */
+#if defined(HAVE_EVENT_H) && !defined(HAVE_EVENT_BASE_ONCE) && (defined(HAVE_PTHREAD) || defined(HAVE_SOLARIS_THREADS))
+   /* using version of libevent that is not threadsafe. */
+#  define LIBEVENT_SIGNAL_PROBLEM 1
+#endif
+
+#ifndef RAND_MAX
+#define RAND_MAX	2147483647
+#endif
+
+#ifndef IPV6_MIN_MTU
+#define IPV6_MIN_MTU 1280
+#endif /* IPV6_MIN_MTU */
+
+#ifndef CHECKED_INET6
+#  define CHECKED_INET6
+#  ifdef AF_INET6
+#    define INET6
+#  else
+#    define AF_INET6        28
+#  endif
+#endif /* CHECKED_INET6 */
+
+/* maximum nesting of included files */
+#define MAXINCLUDES 10
+
+#ifndef HAVE_SNPRINTF
+#include <stdarg.h>
+int snprintf (char *str, size_t count, const char *fmt, ...);
+int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
+#endif /* HAVE_SNPRINTF */
+#ifndef HAVE_INET_PTON
+int inet_pton(int af, const char* src, void* dst);
+#endif /* HAVE_INET_PTON */
+#ifndef HAVE_INET_NTOP
+const char *inet_ntop(int af, const void *src, char *dst, size_t size);
+#endif
+#ifndef HAVE_INET_ATON
+int inet_aton(const char *cp, struct in_addr *addr);
+#endif
+#ifndef HAVE_MEMMOVE
+void *memmove(void *dest, const void *src, size_t n);
+#endif
+#ifndef HAVE_STRLCPY
+size_t strlcpy(char *dst, const char *src, size_t siz);
+#endif
+#ifndef HAVE_GETADDRINFO
+struct sockaddr_storage;
+#include "compat/fake-rfc2553.h"
+#endif
+
+#include "ldns/ldns.h"
+
+#ifdef UNBOUND_ALLOC_STATS
+#  define malloc(s) unbound_stat_malloc_log(s, __FILE__, __LINE__, __func__)
+#  define calloc(n,s) unbound_stat_calloc_log(n, s, __FILE__, __LINE__, __func__)
+#  define free(p) unbound_stat_free_log(p, __FILE__, __LINE__, __func__)
+#  define realloc(p,s) unbound_stat_realloc_log(p, s, __FILE__, __LINE__, __func__)
+void *unbound_stat_malloc(size_t size);
+void *unbound_stat_calloc(size_t nmemb, size_t size);
+void unbound_stat_free(void *ptr);
+void *unbound_stat_realloc(void *ptr, size_t size);
+void *unbound_stat_malloc_log(size_t size, const char* file, int line,
+	const char* func);
+void *unbound_stat_calloc_log(size_t nmemb, size_t size, const char* file,
+	int line, const char* func);
+void unbound_stat_free_log(void *ptr, const char* file, int line,
+	const char* func);
+void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
+	int line, const char* func);
+#endif /* UNBOUND_ALLOC_STATS */
+
+/** default port for DNS traffic. */
+#define UNBOUND_DNS_PORT 53
+

configure.ac

@@ -2,11 +2,40 @@
 # Process this file with autoconf to produce a configure script.
 AC_PREREQ(2.57)
 
-AC_INIT(unbound, 0.0, wouter@nlnetlabs.nl, unbound)
+AC_INIT(unbound, 0.7.2, unbound-bugs@nlnetlabs.nl, unbound)
 
 CFLAGS=
 AC_AIX
 
+dnl
+dnl By default set $sysconfdir to /etc and $localstatedir to /var
+dnl
+case "$prefix" in
+        NONE)
+                case "$sysconfdir" in
+                        '${prefix}/etc')
+                                sysconfdir=/etc
+                        ;;
+                esac
+                case "$localstatedir" in
+                        '${prefix}/var')
+                                localstatedir=/var
+                        ;;
+                esac
+        ;;
+esac
+
+#
+# Determine configuration file
+ub_conf_file=${sysconfdir}/unbound/unbound.conf
+AC_ARG_WITH([conf_file],
+        AC_HELP_STRING([--with-conf-file=path], [Pathname to the Unbound configuration file]),
+	[ub_conf_file=$withval])
+AC_SUBST(ub_conf_file)
+# the eval is to evaluate shell expansion twice, once
+# for $nsd_conf_file and once for the ${prefix} within it.
+AC_DEFINE_UNQUOTED(CONFIGFILE, ["`eval echo $ub_conf_file`"], [Pathname to the Unbound configuration file])
+
 dnl routine to help check for compiler flags.
 AC_DEFUN([CHECK_COMPILER_FLAG],
 [
@@ -90,7 +119,7 @@ CHECK_COMPILER_FLAG(O2, [CFLAGS="$CFLAGS -O2"])
 
 AC_CHECK_HEADERS([getopt.h time.h],,, [AC_INCLUDES_DEFAULT])
 
-CHECK_COMPILER_FLAG_NEEDED(-std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600,
+CHECK_COMPILER_FLAG_NEEDED(-std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600 -D_ALL_SOURCE,
 [
 #include "confdefs.h"
 #include <stdlib.h>
@@ -121,7 +150,7 @@ int test() {
 	str = gai_strerror(0);
 	return a;
 }
-], [CFLAGS="$CFLAGS -std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600"])
+], [CFLAGS="$CFLAGS -std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600 -D_ALL_SOURCE"])
 
 CHECK_COMPILER_FLAG_NEEDED(-std=c99,
 [
@@ -251,6 +280,15 @@ if test "$srcdir" != "."; then
 	CPPFLAGS="$CPPFLAGS -I$srcdir";
 fi
 
+# skip these tests, we do not need them.
+AC_DEFUN([AC_PROG_F77], [:])
+AC_DEFUN([AC_PROG_FC], [:])
+AC_DEFUN([AC_PROG_CXX], [:])
+AC_DEFUN([AC_PROG_CXXCPP], [:])
+AC_DEFUN([AC_PROG_OBJC], [:])
+AC_DEFUN([AC_PROG_OBJCCPP], [:])
+AC_DEFUN([AC_LIBTOOL_CXX], [:])
+AC_DEFUN([AC_LIBTOOL_F77], [:])
 # Use libtool 
 AC_CHECK_PROGS(libtool, [glibtool libtool15 libtool], [./libtool])
 AC_PATH_TOOL(AR, ar, [false])
@@ -263,11 +301,13 @@ AC_CANONICAL_HOST
 if echo "$host_os" | grep "sunos4" >/dev/null; then
 	lt_cv_sys_max_cmd_len=32750;
 fi
+AC_PROG_LEX
+AC_PROG_YACC
 AC_PROG_LIBTOOL
 
 # Checks for header files.
 AC_HEADER_STDC
-AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h arpa/inet.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/uio.h arpa/inet.h syslog.h],,, [AC_INCLUDES_DEFAULT])
 
 # check for types
 AC_CHECK_TYPE(int8_t, char)
@@ -311,17 +351,66 @@ AC_INCLUDES_DEFAULT
 #endif
 ])
 
+# Check the type of struct msghdr msg_iovlen.
+AC_MSG_CHECKING([sign of struct msghdr.msg_iovlen])
+AC_RUN_IFELSE([
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/socket.h>
+int main()
+{
+	struct msghdr h;
+	h.msg_iovlen = -1;
+	if(10/h.msg_iovlen == 0) return 0; /* unsigned */
+	return 1; /* signed */
+}
+], [ 
+# if compiled and exit status 0
+cv_msgiovlen_type="unsigned" ], [ 
+# not compiled or bad exit status
+cv_msgiovlen_type="int" ], 
+# when cross compiling
+[ cv_msgiovlen_type="int" ])
+AC_MSG_RESULT($cv_msgiovlen_type)
+AC_DEFINE_UNQUOTED([TYPE_MSGIOVLEN], $cv_msgiovlen_type, [Define to 'int' or type of struct msghdr.msg_iovlen.])
+
+# Check for yyunput
+AC_DEFUN([CHECK_YYUNPUT],
+[AC_MSG_CHECKING(whether lex can not generate yyunput)
+AC_CACHE_VAL(ac_cv_c_yy_unput,
+[ac_cv_c_yy_unput=no
+if $LEX --nounput -h 2>&1 | grep "do not generate" 2>&1 >/dev/null; then
+   ac_cv_c_yy_unput="yes"
+else
+   ac_cv_c_yy_unput="no"
+fi
+])
+
+AC_MSG_RESULT($ac_cv_c_yy_unput)
+if test $ac_cv_c_yy_unput = yes; then
+	LEX="$LEX --nounput"
+fi
+])dnl
+
+CHECK_YYUNPUT
+
+# add option to disable the evil rpath
+dnl Check whether to use rpath or not
+AC_ARG_ENABLE(rpath,
+        [  --disable-rpath         disable hardcoded rpath (default=enabled)],
+	enable_rpath=$enableval, enable_rpath=yes)
+
 # Checks for libraries.
 AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
                                     [enable SSL (will check /usr/local/ssl
-                            /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr/sfw /usr)]),[
+                            /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /opt/local /usr/sfw /usr)]),[
         ],[
             withval="yes"
         ])
     if test x_$withval != x_no; then
         AC_MSG_CHECKING(for SSL)
         if test x_$withval = x_ -o x_$withval = x_yes; then
-            withval="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr/sfw /usr"
+            withval="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /opt/local /usr/sfw /usr"
         fi
         for dir in $withval; do
             ssldir="$dir"
@@ -338,7 +427,9 @@ AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
             AC_MSG_RESULT(found in $ssldir)
             HAVE_SSL=yes
             LDFLAGS="$LDFLAGS -L$ssldir/lib";
-	    RUNTIME_PATH="$RUNTIME_PATH -R$ssldir/lib"
+	    if test "x$enable_rpath" = xyes; then
+	        RUNTIME_PATH="$RUNTIME_PATH -R$ssldir/lib"
+	    fi
             AC_CHECK_LIB(crypto, HMAC_CTX_init,, [
                     AC_MSG_ERROR([OpenSSL found in $ssldir, but version 0.9.7 or higher is required])
                 ])
@@ -347,17 +438,54 @@ AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
 	AC_SUBST(RUNTIME_PATH)
     fi
 AC_CHECK_HEADERS([openssl/ssl.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([openssl/err.h],,, [AC_INCLUDES_DEFAULT])
 
+# check for thread library.
+AC_ARG_WITH(pthreads, AC_HELP_STRING([--with-pthreads], 
+ [use pthreads library, or --without-pthreads to disable threading support.]), 
+ [ ],[ withval="yes" ])
+ub_have_pthreads=no
+if test x_$withval != x_no; then
+	sinclude(acx_pthread.m4)
+	ACX_PTHREAD([
+		AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.])
+		LIBS="$PTHREAD_LIBS $LIBS"
+		CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+		CC="$PTHREAD_CC"
+		ub_have_pthreads=yes
+		AC_CHECK_TYPES([pthread_spinlock_t, pthread_rwlock_t],,,[#include <pthread.h>])
+		])
+fi
+
+# check solaris thread library 
+AC_ARG_WITH(solaris-threads, AC_HELP_STRING([--with-solaris-threads], 
+	[use solaris native thread library.]), [ ],[ withval="no" ])
+if test x_$withval != x_no; then
+	if test x_$ub_have_pthreads != x_no; then
+	    AC_WARN([Have pthreads already, ignoring --with-solaris-threads])
+	else
+	AC_SEARCH_LIBS(thr_create, [thread],
+	[
+    		AC_DEFINE(HAVE_SOLARIS_THREADS, 1, [Using Solaris threads])
+
+		CHECK_COMPILER_FLAG(mt, [CFLAGS="$CFLAGS -mt"],
+			[CFLAGS="$CFLAGS -D_REENTRANT"])
+	] , [ 
+		AC_ERROR([no solaris threads found.]) 
+	])
+	fi
+fi
+
+# check for library used by libevent after 1.3c
+AC_CHECK_LIB(rt, clock_gettime)
 # check for libevent
 AC_ARG_WITH(libevent, AC_HELP_STRING([--with-libevent=pathname],
-                                    [set path to libevent  (will check /usr/local /usr/lib /usr/pkg /usr/sfw /usr)]),[
-        ],[
-            withval="yes"
-        ])
+    [set path to libevent  (will check /usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr)]),
+    [ ],[ withval="yes" ])
 if test x_$withval != x_no; then
         AC_MSG_CHECKING(for libevent)
         if test x_$withval = x_ -o x_$withval = x_yes; then
-            withval="/usr/local /usr/lib /usr/pkg /usr/sfw /usr";
+            withval="/usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr";
         fi
         for dir in $withval; do
             thedir="$dir"
@@ -374,44 +502,65 @@ if test x_$withval != x_no; then
                 	CPPFLAGS="$CPPFLAGS -I$thedir";
             		LDFLAGS="$thedir/.libs/*.o $LDFLAGS";
 		else
-            		AC_MSG_ERROR(Cannot find the libevent library in $withval)
+            		AC_MSG_ERROR([Cannot find the libevent library in $withval
+You can restart ./configure --with-libevent=no to use a builtin alternative.
+Please note that this alternative is not as capable as libevent when using
+many outgoing ports.  ])
 		fi
         else
             AC_MSG_RESULT(found in $thedir)
             LDFLAGS="$LDFLAGS -L$thedir/lib";
-	    RUNTIME_PATH="$RUNTIME_PATH -R$thedir/lib"
+	    if test "x$enable_rpath" = xyes; then
+	        RUNTIME_PATH="$RUNTIME_PATH -R$thedir/lib"
+	    fi
         fi
 	AC_SUBST(RUNTIME_PATH)
 	AC_SEARCH_LIBS(event_set, [event])
 	AC_CHECK_HEADERS([event.h],,, [AC_INCLUDES_DEFAULT])
 	AC_CHECK_FUNCS([event_base_free]) # only in libevent 1.2 and later
+	AC_CHECK_FUNCS([event_base_once]) # only in libevent 1.4? and later
 else
 	AC_DEFINE(USE_MINI_EVENT, 1, [Define if you want to use internal select based events])
 fi
 
+# set static linking if requested
+AC_SUBST(staticexe)
+staticexe=""
+AC_ARG_ENABLE(staticexe, AC_HELP_STRING([--enable-static-exe],
+	[ enable to compile executables statically against event, ldns libs, for debug purposes ]), 
+	, )
+if test x_$enable_static_exe = x_yes; then
+	staticexe="-static"
+fi
+
+# set lock checking if requested
+AC_ARG_ENABLE(lock_checks, AC_HELP_STRING([--enable-lock-checks],
+	[ enable to check lock and unlock calls, for debug purposes ]), 
+	, )
+if test x_$enable_lock_checks = x_yes; then
+	AC_DEFINE(ENABLE_LOCK_CHECKS, 1, [Define if you want to use debug lock checking (slow).])
+fi
+
+# set memory allocation checking if requested
+AC_ARG_ENABLE(alloc-checks, AC_HELP_STRING([--enable-alloc-checks],
+	[ enable to check memory allocation, for debug purposes ]), 
+	, )
+if test x_$enable_alloc_checks = x_yes; then
+	AC_DEFINE(UNBOUND_ALLOC_STATS, 1, [use statistics for allocs and frees, for debug use])
+else
+	AC_FUNC_MALLOC
+fi
+
 # check to see if libraries are needed for these functions.
 AC_CHECK_LIB(socket, socket)
 AC_CHECK_LIB(nsl, inet_pton)
 
 AC_FUNC_CHOWN
 AC_FUNC_FORK
-AC_FUNC_MALLOC
 AC_TYPE_SIGNAL
 AC_FUNC_FSEEKO
 AC_SYS_LARGEFILE
 
-AC_REPLACE_FUNCS(inet_aton)
-AC_REPLACE_FUNCS(inet_pton)
-AC_REPLACE_FUNCS(inet_ntop)
-AC_REPLACE_FUNCS(snprintf)
-AC_REPLACE_FUNCS(strlcpy)
-AC_REPLACE_FUNCS(memmove)
-
-AC_CHECK_FUNCS([getaddrinfo])
-if test $ac_cv_func_getaddrinfo = no; then
-AC_LIBOBJ([fake-rfc2553])
-fi
-
 # check this after all other compilation checks, since the linking of the lib
 # may break checks after this.
 AC_ARG_WITH(ldns,
@@ -423,9 +572,52 @@ AC_ARG_WITH(ldns,
 	AC_SUBST(ldnsdir)
 ])
 
-AC_CHECK_LIB(ldns, ldns_rr_new,, [AC_MSG_ERROR([Can't find ldns library])])
+AC_CHECK_LIB(ldns, ldns_rr_new, [
+dnl ldns was found, check compat functions
+AC_REPLACE_FUNCS(inet_aton)
+AC_REPLACE_FUNCS(inet_pton)
+AC_REPLACE_FUNCS(inet_ntop)
+AC_REPLACE_FUNCS(snprintf)
+AC_REPLACE_FUNCS(strlcpy)
+AC_REPLACE_FUNCS(memmove)
 
-AC_DEFINE_UNQUOTED([MAXSYSLOGMSGLEN], [512], [Define to the maximum message length to pass to syslog.])
+AC_CHECK_FUNCS([getaddrinfo])
+if test $ac_cv_func_getaddrinfo = no; then
+AC_LIBOBJ([fake-rfc2553])
+fi
+
+] , [
+dnl use the builtin ldns-src.tar.gz file to build ldns.
+if test ! -f $srcdir/ldns-src.tar.gz; then
+	AC_MSG_ERROR([No ldns library found and no ldns-src.tar.gz, use --with-ldns=path.])
+fi
+echo "****************************************************************"
+echo "***                                                          ***"
+echo "*** ldns library not found (--with-ldns= to specify by hand) ***"
+echo "*** Building ldns library from package included in this one. ***"
+echo "***                                                          ***"
+echo "****************************************************************"
+if test -d ldns-src; then rm -rf ldns-src; fi
+mkdir ldns-src; cp $srcdir/ldns-src.tar.gz ldns-src/ldns-src.tar.gz; (cd ldns-src; gzip -cd ldns-src.tar.gz | tar xf -); rm -f ldns-src/ldns-src.tar.gz
+ldnsdir=`ls -d ldns-src/ldns*`
+AC_MSG_NOTICE([Configure $ldnsdir])
+echo "(cd $ldnsdir; ./configure)"
+(cd $ldnsdir; ./configure)
+AC_MSG_NOTICE([Finished $ldnsdir configure])
+CPPFLAGS="$CPPFLAGS -I$ldnsdir/include"
+LDFLAGS="$LDFLAGS $ldnsdir/lib/*.o"
+AC_SUBST(ldnsdir)
+dnl use ldns compat functions
+AC_DEFINE(HAVE_INET_ATON)
+AC_DEFINE(HAVE_INET_PTON)
+AC_DEFINE(HAVE_INET_NTOP)
+AC_DEFINE(HAVE_SNPRINTF)
+AC_DEFINE(HAVE_STRLCPY)
+AC_DEFINE(HAVE_MEMMOVE)
+AC_DEFINE(HAVE_GETADDRINFO)
+])
+
+AC_DEFINE_UNQUOTED([MAXSYSLOGMSGLEN], [5120], [Define to the maximum message length to pass to syslog.])
 
 AH_BOTTOM([
 #include <stdio.h>
@@ -442,6 +634,8 @@ AH_BOTTOM([
 #include <stdint.h>
 #endif
 
+#include <errno.h>
+
 #if HAVE_SYS_PARAM_H
 #include <sys/param.h>
 #endif
@@ -450,6 +644,10 @@ AH_BOTTOM([
 #include <sys/socket.h>
 #endif
 
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+
 #ifdef HAVE_NETINET_IN_H
 #include <netinet/in.h>
 #endif
@@ -462,6 +660,10 @@ AH_BOTTOM([
 #include <getopt.h>
 #endif
 
+#ifdef HAVE_OPENSSL_ERR_H
+#include <openssl/err.h>
+#endif
+
 #ifdef HAVE_ATTR_FORMAT
 #  define ATTR_FORMAT(archetype, string_index, first_to_check) \
     __attribute__ ((format (archetype, string_index, first_to_check)))
@@ -481,6 +683,10 @@ AH_BOTTOM([
 #define fseeko fseek
 #define ftello ftell
 #endif /* HAVE_FSEEKO */
+#if defined(HAVE_EVENT_H) && !defined(HAVE_EVENT_BASE_ONCE) && (defined(HAVE_PTHREAD) || defined(HAVE_SOLARIS_THREADS))
+   /* using version of libevent that is not threadsafe. */
+#  define LIBEVENT_SIGNAL_PROBLEM 1
+#endif
 
 #ifndef RAND_MAX
 #define RAND_MAX	2147483647
@@ -499,6 +705,9 @@ AH_BOTTOM([
 #  endif
 #endif /* CHECKED_INET6 */
 
+/* maximum nesting of included files */
+#define MAXINCLUDES 10
+
 #ifndef HAVE_SNPRINTF
 #include <stdarg.h>
 int snprintf (char *str, size_t count, const char *fmt, ...);
@@ -526,8 +735,27 @@ struct sockaddr_storage;
 
 #include "ldns/ldns.h"
 
-/** default port to listen for queries, passed to getaddrinfo */
-#define UNBOUND_DNS_PORT "53"
+#ifdef UNBOUND_ALLOC_STATS
+#  define malloc(s) unbound_stat_malloc_log(s, __FILE__, __LINE__, __func__)
+#  define calloc(n,s) unbound_stat_calloc_log(n, s, __FILE__, __LINE__, __func__)
+#  define free(p) unbound_stat_free_log(p, __FILE__, __LINE__, __func__)
+#  define realloc(p,s) unbound_stat_realloc_log(p, s, __FILE__, __LINE__, __func__)
+void *unbound_stat_malloc(size_t size);
+void *unbound_stat_calloc(size_t nmemb, size_t size);
+void unbound_stat_free(void *ptr);
+void *unbound_stat_realloc(void *ptr, size_t size);
+void *unbound_stat_malloc_log(size_t size, const char* file, int line,
+	const char* func);
+void *unbound_stat_calloc_log(size_t nmemb, size_t size, const char* file,
+	int line, const char* func);
+void unbound_stat_free_log(void *ptr, const char* file, int line,
+	const char* func);
+void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
+	int line, const char* func);
+#endif /* UNBOUND_ALLOC_STATS */
+
+/** default port for DNS traffic. */
+#define UNBOUND_DNS_PORT 53
 ])
 
 AC_CONFIG_FILES([Makefile])

daemon/daemon.c

@@ -0,0 +1,479 @@
+/*
+ * daemon/daemon.c - collection of workers that handles requests.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * The daemon consists of global settings and a number of workers.
+ */
+
+#include "config.h"
+#include "daemon/daemon.h"
+#include "daemon/worker.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/data/msgreply.h"
+#include "util/storage/slabhash.h"
+#include "services/listen_dnsport.h"
+#include "services/cache/rrset.h"
+#include "services/cache/infra.h"
+#include "util/module.h"
+#include "iterator/iterator.h"
+#include "validator/validator.h"
+#include "util/fptr_wlist.h"
+#include <signal.h>
+
+/** How many quit requests happened. */
+static int sig_record_quit = 0;
+/** How many reload requests happened. */
+static int sig_record_reload = 0;
+
+/** used when no other sighandling happens, so we don't die
+  * when multiple signals in quick succession are sent to us. 
+  * @param sig: signal number.
+  * @return signal handler return type (void or int).
+  */
+static RETSIGTYPE record_sigh(int sig)
+{
+#ifdef LIBEVENT_SIGNAL_PROBLEM
+	verbose(VERB_OPS, "quit on signal, no cleanup and statistics, "
+		"because installed libevent version is not threadsafe");
+	exit(0);
+#endif 
+	switch(sig)
+	{
+		case SIGTERM:
+		case SIGQUIT:
+		case SIGINT:
+			sig_record_quit++;
+			break;
+		case SIGHUP:
+			sig_record_reload++;
+			break;
+		default:
+			log_err("ignoring signal %d", sig);
+	}
+}
+
+/** 
+ * Signal handling during the time when netevent is disabled.
+ * Stores signals to replay later.
+ */
+static void
+signal_handling_record()
+{
+	if( signal(SIGTERM, record_sigh) == SIG_ERR ||
+		signal(SIGQUIT, record_sigh) == SIG_ERR ||
+		signal(SIGINT, record_sigh) == SIG_ERR ||
+		signal(SIGHUP, record_sigh) == SIG_ERR)
+		log_err("install sighandler: %s", strerror(errno));
+}
+
+/**
+ * Replay old signals.
+ * @param wrk: worker that handles signals.
+ */
+static void
+signal_handling_playback(struct worker* wrk)
+{
+	if(sig_record_reload)
+		worker_sighandler(SIGHUP, wrk);
+	if(sig_record_quit)
+		worker_sighandler(SIGTERM, wrk);
+	sig_record_quit = 0;
+	sig_record_reload = 0;
+}
+
+struct daemon* 
+daemon_init()
+{
+	struct daemon* daemon = (struct daemon*)calloc(1, 
+		sizeof(struct daemon));
+	if(!daemon)
+		return NULL;
+	signal_handling_record();
+	checklock_start();
+	ERR_load_crypto_strings();
+	daemon->need_to_exit = 0;
+	daemon->num_modules = 0;
+	if(!(daemon->env = (struct module_env*)calloc(1, 
+		sizeof(*daemon->env)))) {
+		free(daemon);
+		return NULL;
+	}
+	alloc_init(&daemon->superalloc, NULL, 0);
+	return daemon;	
+}
+
+int 
+daemon_open_shared_ports(struct daemon* daemon)
+{
+	log_assert(daemon);
+	if(daemon->cfg->port == daemon->listening_port)
+		return 1;
+	listening_ports_free(daemon->ports);
+	if(!(daemon->ports=listening_ports_open(daemon->cfg)))
+		return 0;
+	daemon->listening_port = daemon->cfg->port;
+	return 1;
+}
+
+/** count number of modules (words) in the string */
+static int
+count_modules(const char* s)
+{
+	int num = 0;
+	if(!s)
+		return 0;
+	while(*s) {
+		/* skip whitespace */
+		while(*s && isspace((int)*s))
+			s++;
+		if(*s && !isspace((int)*s)) {
+			/* skip identifier */
+			num++;
+			while(*s && !isspace((int)*s))
+				s++;
+		}
+	}
+	return num;
+}
+
+/**
+ * Get funcblock for module name
+ * @param str: string with module name. Advanced to next value on success.
+ * @return funcblock or NULL on error.
+ */
+static struct module_func_block*
+daemon_module_factory(const char** str)
+{
+	/* these are the modules available */
+	int num = 2;
+	const char* names[] = {"iterator", "validator", NULL};
+	struct module_func_block* (*fb[])(void) = 
+		{&iter_get_funcblock, &val_get_funcblock, NULL};
+
+	int i;
+	const char* s = *str;
+	while(*s && isspace((int)*s))
+		s++;
+	for(i=0; i<num; i++) {
+		if(strncmp(names[i], s, strlen(names[i])) == 0) {
+			s += strlen(names[i]);
+			*str = s;
+			return (*fb[i])();
+		}
+	}
+	return NULL;
+}
+
+/**
+ * Read config file module settings and set up the modfunc block
+ * @param daemon: the daemon.
+ * @return false on error
+ */
+static int
+daemon_config_modules(struct daemon* daemon)
+{
+	const char* str = daemon->cfg->module_conf;
+	int i;
+	verbose(VERB_DETAIL, "module config: \"%s\"", str);
+	daemon->num_modules = count_modules(str);
+	if(daemon->num_modules == 0) {
+		log_err("error: no modules specified");
+		return 0;
+	}
+	if(daemon->num_modules > MAX_MODULE) {
+		log_err("error: too many modules (%d max %d)",
+			daemon->num_modules, MAX_MODULE);
+		return 0;
+	}
+	daemon->modfunc = (struct module_func_block**)calloc((size_t)
+		daemon->num_modules, sizeof(struct module_func_block*));
+	if(!daemon->modfunc) {
+		log_err("out of memory");
+		return 0;
+	}
+	for(i=0; i<daemon->num_modules; i++) {
+		daemon->modfunc[i] = daemon_module_factory(&str);
+		if(!daemon->modfunc[i]) {
+			log_err("Unknown value for first module in: '%s'",
+				str);
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/**
+ * Desetup the modules, deinit, delete.
+ * @param daemon: the daemon.
+ */
+static void
+daemon_desetup_modules(struct daemon* daemon)
+{
+	int i;
+	for(i=0; i<daemon->num_modules; i++) {
+		log_assert(fptr_whitelist_mod_deinit(
+			daemon->modfunc[i]->deinit));
+		(*daemon->modfunc[i]->deinit)(daemon->env, i);
+	}
+	daemon->num_modules = 0;
+	free(daemon->modfunc);
+	daemon->modfunc = 0;
+}
+
+/**
+ * Setup modules. Assigns ids and calls module_init.
+ * @param daemon: the daemon
+ */
+static void daemon_setup_modules(struct daemon* daemon)
+{
+	int i;
+	if(daemon->num_modules != 0)
+		daemon_desetup_modules(daemon);
+	/* fixed setup of the modules */
+	if(!daemon_config_modules(daemon)) {
+		fatal_exit("failed to setup modules");
+	}
+	daemon->env->cfg = daemon->cfg;
+	daemon->env->alloc = &daemon->superalloc;
+	daemon->env->worker = NULL;
+	daemon->env->need_to_validate = 0; /* set by module init below */
+	for(i=0; i<daemon->num_modules; i++) {
+		log_info("init module %d: %s", i, daemon->modfunc[i]->name);
+		log_assert(fptr_whitelist_mod_init(daemon->modfunc[i]->init));
+		if(!(*daemon->modfunc[i]->init)(daemon->env, i)) {
+			fatal_exit("module init for module %s failed",
+				daemon->modfunc[i]->name);
+		}
+	}
+}
+
+/**
+ * Allocate empty worker structures. With backptr and thread-number,
+ * from 0..numthread initialised. Used as user arguments to new threads.
+ * @param daemon: the daemon with (new) config settings.
+ */
+static void 
+daemon_create_workers(struct daemon* daemon)
+{
+	int i;
+	log_assert(daemon && daemon->cfg);
+	daemon->num = daemon->cfg->num_threads;
+	daemon->workers = (struct worker**)calloc((size_t)daemon->num, 
+		sizeof(struct worker*));
+	for(i=0; i<daemon->num; i++) {
+		if(!(daemon->workers[i] = worker_create(daemon, i)))
+			fatal_exit("could not create worker");
+	}
+}
+
+/**
+ * Close all pipes except for the numbered thread.
+ * @param daemon: daemon to close pipes in.
+ * @param thr: thread number 0..num-1 of thread to skip.
+ */
+void close_other_pipes(struct daemon* daemon, int thr)
+{
+	int i;
+	for(i=0; i<daemon->num; i++)
+		if(i!=thr) {
+			if(daemon->workers[i]->cmd_send_fd != -1) {
+				close(daemon->workers[i]->cmd_send_fd);
+				daemon->workers[i]->cmd_send_fd = -1;
+			}
+			if(daemon->workers[i]->cmd_recv_fd != -1) {
+				close(daemon->workers[i]->cmd_recv_fd);
+				daemon->workers[i]->cmd_recv_fd = -1;
+			}
+		}
+}
+
+/**
+ * Function to start one thread. 
+ * @param arg: user argument.
+ * @return: void* user return value could be used for thread_join results.
+ */
+static void* 
+thread_start(void* arg)
+{
+	struct worker* worker = (struct worker*)arg;
+	log_thread_set(&worker->thread_num);
+	ub_thread_blocksigs();
+#if !defined(HAVE_PTHREAD) && !defined(HAVE_SOLARIS_THREADS)
+	/* close pipe ends used by main */
+	close(worker->cmd_send_fd);
+	worker->cmd_send_fd = -1;
+	close_other_pipes(worker->daemon, worker->thread_num);
+#endif
+	if(!worker_init(worker, worker->daemon->cfg, worker->daemon->ports, 0))
+		fatal_exit("Could not initialize thread");
+
+	worker_work(worker);
+	return NULL;
+}
+
+/**
+ * Fork and init the other threads. Main thread returns for special handling.
+ * @param daemon: the daemon with other threads to fork.
+ */
+static void
+daemon_start_others(struct daemon* daemon)
+{
+	int i;
+	log_assert(daemon);
+	verbose(VERB_ALGO, "start threads");
+	/* skip i=0, is this thread */
+	for(i=1; i<daemon->num; i++) {
+		ub_thread_create(&daemon->workers[i]->thr_id,
+			thread_start, daemon->workers[i]);
+#if !defined(HAVE_PTHREAD) && !defined(HAVE_SOLARIS_THREADS)
+		/* close pipe end of child */
+		close(daemon->workers[i]->cmd_recv_fd);
+		daemon->workers[i]->cmd_recv_fd = -1;
+#endif /* no threads */
+	}
+}
+
+/**
+ * Stop the other threads.
+ * @param daemon: the daemon with other threads.
+ */
+static void
+daemon_stop_others(struct daemon* daemon)
+{
+	int i;
+	log_assert(daemon);
+	verbose(VERB_ALGO, "stop threads");
+	/* skip i=0, is this thread */
+	/* use i=0 buffer for sending cmds; because we are #0 */
+	for(i=1; i<daemon->num; i++) {
+		worker_send_cmd(daemon->workers[i], 
+			daemon->workers[0]->front->udp_buff, worker_cmd_quit);
+	}
+	/* wait for them to quit */
+	for(i=1; i<daemon->num; i++) {
+		/* join it to make sure its dead */
+		verbose(VERB_ALGO, "join %d", i);
+		ub_thread_join(daemon->workers[i]->thr_id);
+		verbose(VERB_ALGO, "join success %d", i);
+	}
+}
+
+void 
+daemon_fork(struct daemon* daemon)
+{
+	log_assert(daemon);
+
+	/* setup modules */
+	daemon_setup_modules(daemon);
+
+	/* first create all the worker structures, so we can pass
+	 * them to the newly created threads. 
+	 */
+	daemon_create_workers(daemon);
+	
+	/* Now create the threads and init the workers.
+	 * By the way, this is thread #0 (the main thread).
+	 */
+	daemon_start_others(daemon);
+
+	/* Special handling for the main thread. This is the thread
+	 * that handles signals.
+	 */
+	if(!worker_init(daemon->workers[0], daemon->cfg, daemon->ports, 1))
+		fatal_exit("Could not initialize main thread");
+	signal_handling_playback(daemon->workers[0]);
+
+	/* Start resolver service on main thread. */
+	log_info("start of service (%s).", PACKAGE_STRING);
+	worker_work(daemon->workers[0]);
+
+	/* we exited! a signal happened! Stop other threads */
+	daemon_stop_others(daemon);
+
+	daemon->need_to_exit = daemon->workers[0]->need_to_exit;
+}
+
+void 
+daemon_cleanup(struct daemon* daemon)
+{
+	int i;
+	log_assert(daemon);
+	/* before stopping main worker, handle signals ourselves, so we
+	   don't die on multiple reload signals for example. */
+	signal_handling_record();
+	log_thread_set(NULL);
+	/* clean up caches because
+	 * a) RRset IDs will be recycled after a reload, causing collisions
+	 * b) validation config can change, thus rrset, msg, keycache clear 
+	 * The infra cache is kept, the timing and edns info is still valid */
+	slabhash_clear(&daemon->env->rrset_cache->table);
+	slabhash_clear(daemon->env->msg_cache);
+	/* key cache is cleared by module desetup during next daemon_init() */
+	for(i=0; i<daemon->num; i++)
+		worker_delete(daemon->workers[i]);
+	free(daemon->workers);
+	daemon->workers = NULL;
+	daemon->num = 0;
+	daemon->cfg = NULL;
+}
+
+void 
+daemon_delete(struct daemon* daemon)
+{
+	if(!daemon)
+		return;
+	daemon_desetup_modules(daemon);
+	listening_ports_free(daemon->ports);
+	if(daemon->env) {
+		slabhash_delete(daemon->env->msg_cache);
+		rrset_cache_delete(daemon->env->rrset_cache);
+		infra_delete(daemon->env->infra_cache);
+	}
+	alloc_clear(&daemon->superalloc);
+	free(daemon->pidfile);
+	free(daemon->env);
+	free(daemon);
+	/* libcrypto cleanup */
+	/* CONF_modules_unload(1); */
+	EVP_cleanup();
+	/* ENGINE_cleanup(); */
+	CRYPTO_cleanup_all_ex_data(); /* safe, no more threads right now */
+	ERR_remove_state(0);
+	ERR_free_strings();
+	checklock_stop();
+}

daemon/daemon.h

@@ -0,0 +1,117 @@
+/*
+ * daemon/daemon.h - collection of workers that handles requests.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * The daemon consists of global settings and a number of workers.
+ */
+
+#ifndef DAEMON_H
+#define DAEMON_H
+
+#include "util/locks.h"
+#include "util/alloc.h"
+struct config_file;
+struct worker;
+struct listen_port;
+struct slabhash;
+struct module_env;
+struct rrset_cache;
+
+/**
+ * Structure holding worker list.
+ * Holds globally visible information.
+ */
+struct daemon {
+	/** The config settings */
+	struct config_file* cfg;
+	/** pidfile that is used */
+	char* pidfile;
+	/** port number that has ports opened. */
+	int listening_port;
+	/** listening ports, opened, to be shared by threads */
+	struct listen_port* ports;
+	/** num threads allocated */
+	int num;
+	/** the worker entries */
+	struct worker** workers;
+	/** do we need to exit unbound (or is it only a reload?) */
+	int need_to_exit;
+	/** master allocation cache */
+	struct alloc_cache superalloc;
+	/** the module environment master value, copied and changed by threads*/
+	struct module_env* env;
+	/** number of modules active, ids from 0 to num-1. */
+	int num_modules;
+	/** the module callbacks, array of num_modules length */
+	struct module_func_block** modfunc;
+};
+
+/**
+ * Initialize daemon structure.
+ * @return: The daemon structure, or NULL on error.
+ */
+struct daemon* daemon_init();
+
+/**
+ * Open shared listening ports (if needed).
+ * The cfg member pointer must have been set for the daemon.
+ * @param daemon: the daemon.
+ * @return: false on error.
+ */
+int daemon_open_shared_ports(struct daemon* daemon);
+
+/**
+ * Fork workers and start service.
+ * When the routine exits, it is no longer forked.
+ * @param daemon: the daemon.
+ */
+void daemon_fork(struct daemon* daemon);
+
+/**
+ * Close off the worker thread information.
+ * Bring the daemon back into state ready for daemon_fork again.
+ * @param daemon: the daemon.
+ */
+void daemon_cleanup(struct daemon* daemon);
+
+/**
+ * Delete workers, close listening ports.
+ * @param daemon: the daemon.
+ */
+void daemon_delete(struct daemon* daemon);
+
+#endif /* DAEMON_H */

daemon/stats.c

@@ -0,0 +1,72 @@
+/*
+ * daemon/stats.c - collect runtime performance indicators.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file describes the data structure used to collect runtime performance
+ * numbers. These 'statistics' may be of interest to the operator.
+ */
+#include "config.h"
+#include "daemon/stats.h"
+#include "daemon/worker.h"
+#include "services/mesh.h"
+
+void server_stats_init(struct server_stats* stats)
+{
+	memset(stats, 0, sizeof(stats));
+}
+
+void server_stats_querymiss(struct server_stats* stats, struct worker* worker)
+{
+	stats->num_queries_missed_cache++;
+	stats->sum_query_list_size += worker->env.mesh->all.count;
+	if(worker->env.mesh->all.count > stats->max_query_list_size)
+		stats->max_query_list_size = worker->env.mesh->all.count;
+}
+
+void server_stats_log(struct server_stats* stats, int threadnum)
+{
+	log_info("server stats for thread %d: %u queries, %u from cache", 
+		threadnum, (unsigned)stats->num_queries, 
+		(unsigned)(stats->num_queries - 
+			stats->num_queries_missed_cache));
+	log_info("server stats for thread %d: requestlist max %u avg %g "
+		"exceeded %u", threadnum, (unsigned)stats->max_query_list_size,
+		stats->num_queries_missed_cache?
+			(double)stats->sum_query_list_size/
+			stats->num_queries_missed_cache : 0.0,
+		(unsigned)stats->num_query_list_exceeded);
+}

daemon/stats.h

@@ -0,0 +1,77 @@
+/*
+ * daemon/stats.h - collect runtime performance indicators.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file describes the data structure used to collect runtime performance
+ * numbers. These 'statistics' may be of interest to the operator.
+ */
+
+#ifndef DAEMON_STATS_H
+#define DAEMON_STATS_H
+struct worker;
+
+/** per worker statistics */
+struct server_stats {
+	/** number of queries from clients received. */
+	size_t num_queries;
+	/** number of queries that had a cache-miss. */
+	size_t num_queries_missed_cache;
+
+	/**
+	 * Sum of the querylistsize of the worker for 
+	 * every query that missed cache. To calculate average.
+	 */
+	size_t sum_query_list_size;
+	/** max value of query list size reached. */
+	size_t max_query_list_size;
+	/** number of times that the query_list_size was insufficient */
+	size_t num_query_list_exceeded;
+};
+
+/** 
+ * Initialize server stats to 0.
+ * @param stats: what to init (this is alloced by the caller).
+ */
+void server_stats_init(struct server_stats* stats);
+
+/** add query if it missed the cache */
+void server_stats_querymiss(struct server_stats* stats, struct worker* worker);
+
+/** display the stats to the log */
+void server_stats_log(struct server_stats* stats, int threadnum);
+
+#endif /* DAEMON_STATS_H */

daemon/unbound.c

@@ -42,26 +42,334 @@
 
 #include "config.h"
 #include "util/log.h"
-#include "daemon/worker.h"
+#include "daemon/daemon.h"
+#include "util/config_file.h"
+#include "util/storage/slabhash.h"
+#include "services/listen_dnsport.h"
+#include "services/cache/rrset.h"
+#include "services/cache/infra.h"
+#include "util/data/msgreply.h"
+#include "util/module.h"
+#include <signal.h>
+#include <fcntl.h>
+#include <pwd.h>
+#include <sys/resource.h>
 
-/** buffer size for network connections */
-#define BUFSZ 65552
+/** global debug value to keep track of heap memory allocation */
+void* unbound_start_brk = 0;
 
 /** print usage. */
 static void usage()
 {
-	printf("usage: unbound [options]\n");
+	printf("usage:  unbound [options]\n");
 	printf("	start unbound daemon DNS resolver.\n");
 	printf("-h	this help\n");
-	printf("-p port	the port to listen on\n");
-	printf("-v	verbose (multiple times increase verbosity)\n");
-	printf("-f ip	set forwarder address\n");
-	printf("-z port	set forwarder port\n");
+	printf("-c file	config file to read instead of %s\n", CONFIGFILE);
+	printf("	file format is described in unbound.conf(5).\n");
+	printf("-d	do not fork into the background.\n");
+	printf("-v	verbose (more times to increase verbosity)\n");
 	printf("Version %s\n", PACKAGE_VERSION);
 	printf("BSD licensed, see LICENSE in source package for details.\n");
 	printf("Report bugs to %s\n", PACKAGE_BUGREPORT);
 }
 
+/** check file descriptor count */
+static void
+checkrlimits(struct config_file* cfg)
+{
+	int list = ((cfg->do_ip4?1:0) + (cfg->do_ip6?1:0)) * 
+		((cfg->do_udp?1:0) + (cfg->do_tcp?1 + 
+			(int)cfg->incoming_num_tcp:0));
+	size_t ifs = (size_t)(cfg->num_ifs==0?1:cfg->num_ifs);
+	size_t listen_num = list*ifs;
+	size_t out_ifs = (size_t)(cfg->num_out_ifs==0?1:cfg->num_out_ifs);
+	size_t outnum = cfg->outgoing_num_ports*out_ifs + cfg->outgoing_num_tcp;
+	size_t misc = 4; /* logfile, pidfile, stdout... */
+	size_t perthread = listen_num + outnum + 2/*cmdpipe*/ + 2/*libevent*/
+		+ misc; 
+#if !defined(HAVE_PTHREAD) && !defined(HAVE_SOLARIS_THREADS)
+	int numthread = 1; /* it forks */
+#else
+	int numthread = cfg->num_threads;
+#endif
+	size_t total = numthread * perthread + misc;
+	struct rlimit rlim;
+	if(getrlimit(RLIMIT_NOFILE, &rlim) < 0) {
+		log_warn("getrlimit: %s", strerror(errno));
+		return;
+	}
+	if(rlim.rlim_cur == (rlim_t)RLIM_INFINITY)
+		return;
+	if((size_t)rlim.rlim_cur < total) {
+		log_err("Not enough sockets available. Increase "
+			"ulimit(open files).");
+		log_err("or decrease number of threads, outgoing num ports, "
+			"outgoing num tcp or number of interfaces");
+		log_err("estimate %u fds high mark, %u available", 
+			(unsigned)total, (unsigned)rlim.rlim_cur);
+		fatal_exit("Not enough file descriptors available");
+	}
+}
+
+/** set verbosity, check rlimits, cache settings */
+static void
+apply_settings(struct daemon* daemon, struct config_file* cfg, 
+	int cmdline_verbose)
+{
+	/* apply if they have changed */
+	daemon->cfg = cfg;
+	verbosity = cmdline_verbose + cfg->verbosity;
+	config_apply(cfg);
+	if(!daemon->env->msg_cache ||
+	   cfg->msg_cache_size != slabhash_get_size(daemon->env->msg_cache) ||
+	   cfg->msg_cache_slabs != daemon->env->msg_cache->size) {
+		slabhash_delete(daemon->env->msg_cache);
+		daemon->env->msg_cache = slabhash_create(cfg->msg_cache_slabs, 
+			HASH_DEFAULT_STARTARRAY, cfg->msg_cache_size, 
+			msgreply_sizefunc, query_info_compare,
+			query_entry_delete, reply_info_delete, NULL);
+		if(!daemon->env->msg_cache) {
+			fatal_exit("malloc failure updating config settings");
+		}
+	}
+	if((daemon->env->rrset_cache = rrset_cache_adjust(
+		daemon->env->rrset_cache, cfg, &daemon->superalloc)) == 0)
+		fatal_exit("malloc failure updating config settings");
+	if((daemon->env->infra_cache = infra_adjust(daemon->env->infra_cache, 
+		cfg))==0)
+		fatal_exit("malloc failure updating config settings");
+	checkrlimits(cfg);
+}
+
+/** Read existing pid from pidfile. 
+ * @param file: file name of pid file.
+ * @return: the pid from the file or -1 if none.
+ */
+static pid_t
+readpid (const char* file)
+{
+	int fd;
+	pid_t pid;
+	char pidbuf[32];
+	char* t;
+	ssize_t l;
+
+	if ((fd = open(file, O_RDONLY)) == -1) {
+		if(errno != ENOENT)
+			log_err("Could not read pidfile %s: %s",
+				file, strerror(errno));
+		return -1;
+	}
+
+	if (((l = read(fd, pidbuf, sizeof(pidbuf)))) == -1) {
+		if(errno != ENOENT)
+			log_err("Could not read pidfile %s: %s",
+				file, strerror(errno));
+		close(fd);
+		return -1;
+	}
+
+	close(fd);
+
+	/* Empty pidfile means no pidfile... */
+	if (l == 0) {
+		return -1;
+	}
+
+	pidbuf[sizeof(pidbuf)-1] = 0;
+	pid = (pid_t)strtol(pidbuf, &t, 10);
+	
+	if (*t && *t != '\n') {
+		return -1;
+	}
+	return pid;
+}
+
+/** write pid to file. 
+ * @param pidfile: file name of pid file.
+ * @param pid: pid to write to file.
+ */
+static void
+writepid (const char* pidfile, pid_t pid)
+{
+	FILE* f;
+
+	if ((f = fopen(pidfile, "w")) ==  NULL ) {
+		log_err("cannot open pidfile %s: %s", 
+			pidfile, strerror(errno));
+		return;
+	}
+	if(fprintf(f, "%lu\n", (unsigned long)pid) < 0) {
+		log_err("cannot write to pidfile %s: %s", 
+			pidfile, strerror(errno));
+	}
+	fclose(f);
+}
+
+/**
+ * check old pid file.
+ * @param cfg: the config settings
+ */
+static void
+checkoldpid(struct config_file* cfg)
+{
+	pid_t old;
+	char* file = cfg->pidfile;
+	if(cfg->chrootdir && cfg->chrootdir[0] &&
+		strncmp(file, cfg->chrootdir, strlen(cfg->chrootdir))==0) {
+		file += strlen(cfg->chrootdir);
+	}
+	if((old = readpid(file)) != -1) {
+		/* see if it is still alive */
+		if(kill(old, 0) == 0 || errno == EPERM)
+			log_warn("unbound is already running as pid %u.", 
+				(unsigned)old);
+		else	log_warn("did not exit gracefully last time (%u)", 
+				(unsigned)old);
+	}
+}
+
+/** detach from command line */
+static void
+detach(struct config_file* cfg)
+{
+	int fd;
+	/* Take off... */
+	switch (fork()) {
+		case 0:
+			break;
+		case -1:
+			unlink(cfg->pidfile);
+			fatal_exit("fork failed: %s", strerror(errno));
+		default:
+			/* exit interactive session */
+			exit(0);
+	}
+	/* detach */
+	if(setsid() == -1)
+		fatal_exit("setsid() failed: %s", strerror(errno));
+	if ((fd = open("/dev/null", O_RDWR, 0)) != -1) {
+		(void)dup2(fd, STDIN_FILENO);
+		(void)dup2(fd, STDOUT_FILENO);
+		(void)dup2(fd, STDERR_FILENO);
+		if (fd > 2)
+			(void)close(fd);
+	}
+}
+
+/** daemonize, drop user priviliges and chroot if needed */
+static void
+do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
+{
+	uid_t uid;
+	gid_t gid;
+	/* initialize, but not to 0 (root) */
+	memset(&uid, -12, sizeof(uid));
+	memset(&gid, -12, sizeof(gid));
+	log_assert(cfg);
+
+	/* daemonize last to be able to print error to user */
+	if(cfg->directory && cfg->directory[0]) {
+		if(chdir(cfg->directory)) {
+			fatal_exit("Could not chdir to %s: %s",
+				cfg->directory, strerror(errno));
+		}
+		verbose(VERB_DETAIL, "chdir to %s", cfg->directory);
+	}
+	if(cfg->username && cfg->username[0]) {
+		struct passwd *pwd;
+		if((pwd = getpwnam(cfg->username)) == NULL)
+			fatal_exit("user '%s' does not exist.", cfg->username);
+		uid = pwd->pw_uid;
+		gid = pwd->pw_gid;
+		endpwent();
+	}
+	if(cfg->chrootdir && cfg->chrootdir[0]) {
+		if(chroot(cfg->chrootdir))
+			fatal_exit("unable to chroot to %s: %s", 
+				cfg->chrootdir, strerror(errno));
+		verbose(VERB_DETAIL, "chroot to %s", cfg->chrootdir);
+	}
+	if(cfg->username && cfg->username[0]) {
+		if(setgid(gid) != 0)
+			fatal_exit("unable to set group id of %s: %s", 
+				cfg->username, strerror(errno));
+		if(setuid(uid) != 0)
+			fatal_exit("unable to set user id of %s: %s", 
+				cfg->username, strerror(errno));
+		verbose(VERB_DETAIL, "drop user privileges, run as %s", 
+			cfg->username);
+	}
+	/* check old pid file before forking */
+	if(cfg->pidfile && cfg->pidfile[0]) {
+		checkoldpid(cfg);
+	}
+
+	/* init logfile just before fork */
+	log_init(cfg->logfile, cfg->use_syslog, cfg->chrootdir);
+	if(!debug_mode && cfg->do_daemonize) {
+		detach(cfg);
+	}
+	if(cfg->pidfile && cfg->pidfile[0]) {
+		char* pf = cfg->pidfile;
+		if(cfg->chrootdir && cfg->chrootdir[0] &&
+			strncmp(pf, cfg->chrootdir, strlen(cfg->chrootdir))==0)
+			pf += strlen(cfg->chrootdir);
+		writepid(pf, getpid());
+		if(!(daemon->pidfile = strdup(pf)))
+			log_err("pidf: malloc failed");
+	}
+}
+
+/**
+ * Run the daemon. 
+ * @param cfgfile: the config file name.
+ * @param cmdline_verbose: verbosity resulting from commandline -v.
+ *    These increase verbosity as specified in the config file.
+ * @param debug_mode: if set, do not daemonize.
+ */
+static void 
+run_daemon(char* cfgfile, int cmdline_verbose, int debug_mode)
+{
+	struct config_file* cfg = NULL;
+	struct daemon* daemon = NULL;
+	int done_chroot = 0;
+
+	if(!(daemon = daemon_init()))
+		fatal_exit("alloc failure");
+	while(!daemon->need_to_exit) {
+		if(done_chroot)
+			log_info("Restart of %s.", PACKAGE_STRING);
+		else	log_info("Start of %s.", PACKAGE_STRING);
+
+		/* config stuff */
+		if(!(cfg = config_create()))
+			fatal_exit("Could not alloc config defaults");
+		if(!config_read(cfg, cfgfile))
+			fatal_exit("Could not read config file: %s", cfgfile);
+		apply_settings(daemon, cfg, cmdline_verbose);
+	
+		/* prepare */
+		if(!daemon_open_shared_ports(daemon))
+			fatal_exit("could not open ports");
+		if(!done_chroot) { 
+			do_chroot(daemon, cfg, debug_mode); 
+			done_chroot = 1; 
+		} else log_init(cfg->logfile, cfg->use_syslog, cfg->chrootdir);
+		/* work */
+		daemon_fork(daemon);
+
+		/* clean up for restart */
+		verbose(VERB_ALGO, "cleanup.");
+		daemon_cleanup(daemon);
+		config_delete(cfg);
+	}
+	verbose(VERB_ALGO, "Exit cleanup.");
+	if(daemon->pidfile)
+		unlink(daemon->pidfile);
+	daemon_delete(daemon);
+}
+
 /** getopt global, in case header files fail to declare it. */
 extern int optind;
 /** getopt global, in case header files fail to declare it. */
@@ -76,35 +384,27 @@ extern char* optarg;
 int 
 main(int argc, char* argv[])
 {
-	struct worker* worker = NULL;
-	int do_ip4=1, do_ip6=1, do_udp=1, do_tcp=1;
-	size_t numports=3;
-	int baseport=10000;
-	const char* port = UNBOUND_DNS_PORT;
 	int c;
-	const char* fwd = "127.0.0.1";
-	const char* fwdport = UNBOUND_DNS_PORT;
+	char* cfgfile = CONFIGFILE;
+	int cmdline_verbose = 0;
+	int debug_mode = 0;
 
-	log_init();
-	log_info("Start of %s.", PACKAGE_STRING);
+	/* take debug snapshot of heap */
+	unbound_start_brk = sbrk(0);
+
+	log_init(NULL, 0, NULL);
 	/* parse the options */
-	while( (c=getopt(argc, argv, "f:hvp:z:")) != -1) {
+	while( (c=getopt(argc, argv, "c:dhv")) != -1) {
 		switch(c) {
-		case 'f':
-			fwd = optarg;
-			break;
-		case 'z':
-			fwdport = optarg;
-			break;
-		case 'p':
-			if(!atoi(optarg))
-				fatal_exit("invalid port '%s'", optarg);
-			port = optarg;
-			baseport = atoi(optarg)+2000;
-			verbose(VERB_ALGO, "using port: %s", port);
+		case 'c':
+			cfgfile = optarg;
 			break;
 		case 'v':
-			verbosity ++;
+			cmdline_verbose ++;
+			verbosity++;
+			break;
+		case 'd':
+			debug_mode = 1;
 			break;
 		case '?':
 		case 'h':
@@ -121,23 +421,7 @@ main(int argc, char* argv[])
 		return 1;
 	}
 
-	/* setup */
-	worker = worker_init(port, do_ip4, do_ip6, do_udp, do_tcp, BUFSZ,
-		numports, baseport);
-	if(!worker) {
-		fatal_exit("could not initialize");
-	}
-	if(!worker_set_fwd(worker, fwd, fwdport)) {
-		worker_delete(worker);
-		fatal_exit("could not set forwarder address");
-	}
-	
-	/* drop user priviliges and chroot if needed */
-	log_info("start of service (%s).", PACKAGE_STRING);
-	worker_work(worker);
-
-	/* cleanup */
-	verbose(VERB_ALGO, "Exit cleanup.");
-	worker_delete(worker);
+	run_daemon(cfgfile, cmdline_verbose, debug_mode);
+	log_init(NULL, 0, NULL); /* close logfile */
 	return 0;
 }

daemon/worker.h

@@ -45,17 +45,44 @@
 
 #include "config.h"
 #include "util/netevent.h"
+#include "util/locks.h"
+#include "util/alloc.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgparse.h"
+#include "daemon/stats.h"
+#include "util/module.h"
 struct listen_dnsport;
 struct outside_network;
+struct config_file;
+struct daemon;
+struct listen_port;
+struct ub_randstate;
+struct regional;
 
 /** size of table used for random numbers. large to be more secure. */
 #define RND_STATE_SIZE 256
 
+/** worker commands */
+enum worker_commands {
+	/** make the worker quit */
+	worker_cmd_quit
+};
+
 /**
  * Structure holding working information for unbound.
  * Holds globally visible information.
  */
 struct worker {
+	/** the thread number (in daemon array). First in struct for debug. */
+	int thread_num;
+	/** global shared daemon structure */
+	struct daemon* daemon;
+	/** thread id */
+	ub_thread_t thr_id;
+	/** fd 0 of socketpair, write commands for worker to this one */
+	int cmd_send_fd;
+	/** fd 1 of socketpair, worker listens on this one */
+	int cmd_recv_fd;
 	/** the event base this worker works with */
 	struct comm_base* base;
 	/** the frontside listening interface where request events come in */
@@ -63,40 +90,48 @@ struct worker {
 	/** the backside outside network interface to the auth servers */
 	struct outside_network* back;
 	/** the signal handler */
-	struct comm_signal *comsig;
+	struct comm_signal* comsig;
+	/** commpoint to listen to commands. */
+	struct comm_point* cmd_com;
 
-	/** number of requests currently active */
-	int num_requests;
-	/** our one and only query, packet buffer and where to send. */
-	struct comm_reply query_reply;
-	/** id of query */
-	uint16_t query_id;
-
-	/** address to forward to */
-	struct sockaddr_storage fwd_addr;
-	/** length of fwd_addr */
-	socklen_t fwd_addrlen;
+	/** number of requests that can be handled by this worker */
+	size_t request_size;
 
 	/** random() table for this worker. */
-	char* rndstate;
+	struct ub_randstate* rndstate;
+	/** do we need to restart or quit (on signal) */
+	int need_to_exit;
+	/** allocation cache for this thread */
+	struct alloc_cache alloc;
+	/** per thread statistics */
+	struct server_stats stats;
+	/** thread scratch regional */
+	struct regional* scratchpad;
+
+	/** module environment passed to modules, changed for this thread */
+	struct module_env env;
 };
 
+/**
+ * Create the worker structure. Bare bones version, zeroed struct,
+ * with backpointers only. Use worker_init on it later.
+ * @param daemon: the daemon that this worker thread is part of.
+ * @param id: the thread number from 0.. numthreads-1.
+ * @return: the new worker or NULL on alloc failure.
+ */
+struct worker* worker_create(struct daemon* daemon, int id);
+
 /**
  * Initialize worker.
  * Allocates event base, listens to ports
- * @param port: the port number to bind to.
- * @param do_ip4: listen to ip4 queries.
- * @param do_ip6: listen to ip6 queries.
- * @param do_udp: listen to udp queries.
- * @param do_tcp: listen to tcp queries.
- * @param buffer_size: size of datagram buffer.
- * @param numports: number of outgoing ports.
- * @param base_port: -1 or specify base of outgoing port range.
- * @return: The worker, or NULL on error.
+ * @param worker: worker to initialize, created with worker_create.
+ * @param cfg: configuration settings.
+ * @param ports: list of shared query ports.
+ * @param do_sigs: if true, worker installs signal handlers.
+ * @return: false on error.
  */
-struct worker* worker_init(const char* port, int do_ip4, int do_ip6,
-	int do_udp, int do_tcp, size_t buffer_size, size_t numports,
-	int base_port);
+int worker_init(struct worker* worker, struct config_file *cfg, 
+	struct listen_port* ports, int do_sigs);
 
 /**
  * Make worker work.
@@ -109,12 +144,77 @@ void worker_work(struct worker* worker);
 void worker_delete(struct worker* worker);
 
 /**
- * Set forwarder
- * @param worker: the worker to modify.
- * @param ip: the server name.
- * @param port: port on server or NULL for default 53.
- * @return: false on error.
+ * Send a command to a worker. Uses blocking writes.
+ * @param worker: worker to send command to.
+ * @param buffer: an empty buffer to use.
+ * @param cmd: command to send.
  */
-int worker_set_fwd(struct worker* worker, const char* ip, const char* port);
+void worker_send_cmd(struct worker* worker, ldns_buffer* buffer,
+        enum worker_commands cmd);
+
+/**
+ * Worker signal handler function. User argument is the worker itself.
+ * @param sig: signal number.
+ * @param arg: the worker (main worker) that handles signals.
+ */
+void worker_sighandler(int sig, void* arg);
+
+/**
+ * Worker service routine to send udp messages for modules.
+ * @param pkt: packet to send.
+ * @param addr: where to.
+ * @param addrlen: length of addr.
+ * @param timeout: seconds to wait until timeout.
+ * @param q: wich query state to reactivate upon return.
+ * @param use_tcp: true to use TCP, false for UDP.
+ * @return: false on failure (memory or socket related). no query was
+ *      sent.
+ */
+int worker_send_packet(ldns_buffer* pkt, struct sockaddr_storage* addr,
+	socklen_t addrlen, int timeout, struct module_qstate* q, int use_tcp);
+
+/**
+ * Worker service routine to send serviced queries to authoritative servers.
+ * @param qname: query name. (host order)
+ * @param qnamelen: length in bytes of qname, including trailing 0.
+ * @param qtype: query type. (host order)
+ * @param qclass: query class. (host order)
+ * @param flags: host order flags word, with opcode and CD bit.
+ * @param dnssec: if set, EDNS record will have DO bit set.
+ * @param addr: where to.
+ * @param addrlen: length of addr.
+ * @param q: wich query state to reactivate upon return.
+ * @return: false on failure (memory or socket related). no query was
+ *      sent.
+ */
+struct outbound_entry* worker_send_query(uint8_t* qname, size_t qnamelen, 
+	uint16_t qtype, uint16_t qclass, uint16_t flags, int dnssec, 
+	struct sockaddr_storage* addr, socklen_t addrlen,
+	struct module_qstate* q);
+
+/** 
+ * process control messages from the main thread. 
+ * @param c: comm point to read from.
+ * @param arg: worker.
+ * @param error: error status of comm point.
+ * @param reply_info: not used.
+ */
+int worker_handle_control_cmd(struct comm_point* c, void* arg, int error, 
+	struct comm_reply* reply_info);
+
+/** handles callbacks from listening event interface */
+int worker_handle_request(struct comm_point* c, void* arg, int error,
+	struct comm_reply* repinfo);
+
+/** process incoming replies from the network */
+int worker_handle_reply(struct comm_point* c, void* arg, int error, 
+	struct comm_reply* reply_info);
+
+/** process incoming serviced query replies from the network */
+int worker_handle_service_reply(struct comm_point* c, void* arg, int error, 
+	struct comm_reply* reply_info);
+
+/** cleanup the cache to remove all rrset IDs from it, arg is worker */
+void worker_alloc_cleanup(void* arg);
 
 #endif /* DAEMON_WORKER_H */

doc/CREDITS

@@ -0,0 +1,9 @@
+Unbound was developed at NLnet Labs by Wouter Wijngaards.
+
+Unbound was devised by Bill Manning, David Blacka, and Matt Larson 
+from the University of California and from Verisign. The Java prototype
+was made in further cooperation with Geoff Sisson and Roy Arends from Nominet.
+
+At NLnet Labs, Jelte Jansen and Mark Santcroos reviewed the unbound C sources.
+
+Jakob Schlyter - for advice on secure settings, random numbers and blacklists.

doc/README

@@ -4,18 +4,63 @@ http://unbound.net
 
 This software is under BSD license, see LICENSE for details.
 
-* Needs the following libraries
+* Download the latest version of this software from 
+  	http://unbound.net 
+  or get a beta version from the svn repository at 
+  	http://unbound.net/svn/
+
+* Uses the following libraries; 
   * ldns	http://www.nlnetlabs.nl/ldns/			(BSD license)
   * libevent	http://www.monkey.org/~provos/libevent/		(BSD license)
 
-* Make and install: ./configure; make
-  * --with-ldns=/path/to/ldns
-  * --with-libevent=/path/to/libevent
-  	Can be set to either the system install or the build directory.
-	--with-libevent=no gives an alternative to libevent (based on select).
-
 * Create build environment 
-  * run libtoolize -c if config.sub is missing.
+  * run libtoolize -c if config.sub is missing, or run glibtoolize.
   * autoreconf (autoheader && autoconf), if ./configure is missing.
 
-* mailto:wouter@nlnetlabs.nl
+* Make and install: ./configure; make; make install
+  * Use GNU make; default on linux, often called 'gmake' on BSD and Solaris.
+  * --with-ldns=/path/to/ldns
+  	If ldns is not detected on the system, a prepackaged tarball
+	of the ldns library is used to compile and statically link against.
+	If detected on the system, it will dynamically link against it.
+  * --with-libevent=/path/to/libevent
+  	Can be set to either the system install or the build directory.
+	--with-libevent=no gives a builtin alternative implementation.
+	libevent is useful when having many (thousands) of outgoing ports.
+	This improves randomization and spoof resistance. For the default 
+	of 16 ports the builtin alternative works well.
+  * --without-pthreads 
+	This disables pthreads, and uses Solaris thr library or no threading. 
+	Without this option the pthreads library is detected automatically.
+  * --enable-static-exe
+	This enables a debug option to statically link, against ldns and 
+	libevent libraries.
+  * --enable-lock-checks
+  	This enables a debug option to check lock and unlock calls. It needs
+	a recent pthreads library to work.
+  * --enable-alloc-checks
+	This enables a debug option to check malloc (calloc, realloc, free).
+	The server periodically checks if the amount of memory used fits with
+	the amount of memory it thinks it should be using, and reports 
+	memory usage in detail.
+
+* 'make test' attempts to run a series of tests, depending on the support
+  programs that are installed.
+
+Known issues
+------------
+o If libevent is older (before 1.3c), unbound will exit instead of reload
+  on sighup. On a restart 'did not exit gracefully last time' warning is 
+  printed. Perform ./configure --with-libevent=no or update libevent, rerun 
+  configure and recompile unbound to make sighup work correctly.
+
+Acknowledgements
+----------------
+o Thanks to David Blacka and Matt Larson (Verisign) for the unbound-java
+  prototype. Design and code from that prototype has been used to create
+  this program. Such as the iterator state machine and the cache design.
+o Other code origins are from the NSD (NLnet Labs) and LDNS (NLnet Labs)
+  projects. Such as buffer, region-allocator and red-black tree code.
+o See Credits file for contributors.
+
+* mailto:unbound-bugs@nlnetlabs.nl

doc/README.tests

@@ -1,6 +1,9 @@
 README unbound tests
 
-There is a test setup for unbound.
+There is a test setup for unbound. Use
+	make test
+To make and run the tests. The results are summarized at the end.
+You can also run ./unittest by hand, if the system lacks other debug tools.
 
 testdata/ contains the data for tests. 
 testcode/ contains scripts and c code for the tests.
@@ -8,4 +11,4 @@ testcode/ contains scripts and c code for the tests.
 do-tests.sh : runs all the tests in the testdata directory.
 testbed.sh : compiles on a set of (user specific) hosts and runs do-tests.
 
-Tests are run using tpkg.
+Tests are run using testcode/mini_tpkg.sh.

doc/TODO

@@ -0,0 +1,58 @@
+TODO items.
+o use real entropy to make random (ID, port) numbers more random.
+o in production mode, do not free memory on exit. In debug mode, test leaks.
+o profile memory allocation, and if performance issues, use special memory
+  allocator. For example, with caches per thread.
+o #define BIT_... different on bigendian and smallendian systems so that
+  the htons on flags is not needed to send a message from the cache.
+o speed up pkt domain name decompression loop detection using counter perhaps.
+o detect OS/400 pthreads implementation that allows upgrading to writelock
+  on pthreads rwlocks and use it to examine-rd before storing-wr rrset cache.
+o understand synthesized DNAMEs, so those TTL=0 packets are cached properly.
+o understand NSEC/NSEC3, aggressive negative caching, so that updates to
+  NSEC/NSEC3 will result in proper negative responses.
+o fallback without EDNS if result is NOTIMPL, now only on FORMERR like in java.
+o scrubber has slow pkt_subdomain and pkt_strict_subdomain functions.
+o get serverselection algorithm out of local optimum.
+  make subtargets to get rtt info for a couple of targets, like fetch-policy.
+  or send out multiple queries to multiple servers.
+o donotqueryaddresses allow specification of subnets, use trie to store.
+  extend default list to contain illegal ip4 and ip6 addresses.
+o memory profile; and if needed put serviced queries under qstate->region
+  and special purpose region code, reuse blocks, shrink if too big.
+o check query, option to enforce presence of qdsection in noerror answers.
+o configuration option where port 53 is used for send and receive, no other
+  ports are used.
+o (option) to not send replies to clients after a timeout of (say 5 secs) has
+  passed, but keep task active for later retries by client.
+o private TTL feature
+o pretend-dnssec-unaware, and pretend-edns-unaware modes for debug/workshops.
+o delegpt use rbtree for ns-list, to avoid slowdown for very large NS sets.
+o be able to have different listen and query-to addresses to bind to,
+  so you can listen to localhost and query-to to the internet.
+o reprime and refresh oft used data before timeout.
+o retain prime results in a overlaid roothints file.
+o store primed key data in a overlaid keyhints file (sort of like drafttimers).
+o windows version, auto update feature, a query to check for the version.
+o autoreport of problems
+o logrotation, syslog
+o command the server with TSIG inband. get-config, clearcache, 
+	get stats, get memstats, get ..., reload, clear one zone from cache
+o watch for spoof nearmisses.
+o improve compression of DNS packets by first puttig uncompressible rrs, then
+  compress to their rdata.
+o if one server is not responsive do not spend 75 secs on that server, but
+  try other servers with lower rtt.
+o NSID rfc 5001 support.
+o timers rfc 5011 support.
+o Treat YXDOMAIN from a DNAME properly, in iterator (not throwaway), validator.
+o grab ports nonconsequtive and change the set after a while (change within
+  a given range). Could be bad for OS if wrong port. unsure if it helps secure.
+o make timeout backoffs randomized (a couple percent random) to spread traffic.
+o inspect date on executable, then warn user in log if its more than 1 year.
+o proactively prime root, stubs and trust anchors, feature.
+  early failure, faster on first query, but more traffic.
+o use privilege separation, to change privilege options during reload securely
+  not needed.
+o check if for PowerDNS(2.9.21) CNAME in Answer section & rcode=NXDOMAIN needs
+  to be fixed up to be rcode=NOERROR?

doc/example.conf

@@ -0,0 +1,272 @@
+#
+# Example configuration file.
+#
+# See unbound.conf(5) man page.
+#
+# this is a comment.
+
+#Use this to include other text into the file.
+#include: "otherfile.conf"
+
+# The server clause sets the main parameters. 
+server:
+	# whitespace is not necessary, but looks cleaner.
+
+	# verbosity number, 0 is least verbose. 1 is default.
+	verbosity: 1
+
+	# number of threads to create. 1 disables threading.
+	# num-threads: 1
+
+	# specify the interfaces to answer queries from by ip-address.
+	# The default is to listen to localhost (127.0.0.1 and ::1).
+	# specify 0.0.0.0 and ::0 to bind to all available interfaces.
+	# specify every interface on a new 'interface:' labelled line.
+	# The listen interfaces are not changed on reload, only on restart.
+	# interface: 192.0.2.153
+	# interface: 192.0.2.154
+	# interface: 2001:DB8::5
+
+	# port to answer queries from
+	# port: 53
+
+	# specify the interfaces to send outgoing queries to authoritative
+	# server from by ip-address. If none, the default (all) interface
+	# is used. Specify every interface on a 'outgoing-interface:' line.
+	# outgoing-interface: 192.0.2.153
+	# outgoing-interface: 2001:DB8::5
+	# outgoing-interface: 2001:DB8::6
+
+	# unbound needs to send packets to authoritative nameservers.
+	# it uses a range of ports for that.
+	# the start number of the port range
+	# outgoing-port: 1053
+
+	# number of port to allocate per thread, determines the size of the
+	# port range. A larger port range gives more resistance to certain
+	# spoof attacks, as it gets harder to guess which port is used. 
+	# But also takes more system resources (for open sockets).
+	# outgoing-range: 16
+
+	# number of outgoing simultaneous tcp buffers to hold per thread.
+	# outgoing-num-tcp: 10
+
+	# number of incoming simultaneous tcp buffers to hold per thread.
+	# incoming-num-tcp: 10
+	
+	# buffer size for handling DNS data. No messages larger than this
+	# size can be sent or received, by UDP or TCP. In bytes.
+	# msg-buffer-size: 65552
+
+	# the amount of memory to use for the message cache.
+	# in bytes. default is 4 Mb
+	# msg-cache-size: 4194304
+
+	# the number of slabs to use for the message cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# msg-cache-slabs: 4
+
+	# the number of queries that a thread gets to service.
+	# num-queries-per-thread: 1024
+
+	# the amount of memory to use for the RRset cache.
+	# in bytes. default is 4 Mb
+	# rrset-cache-size: 4194304
+
+	# the number of slabs to use for the RRset cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# rrset-cache-slabs: 4
+	
+	# the time to live (TTL) value cap for RRsets and messages in the
+	# cache. Items are not cached for longer. In seconds.
+	# cache-max-ttl: 864000
+
+	# the time to live (TTL) value for cached roundtrip times and
+	# EDNS version information for hosts. In seconds.
+	# infra-host-ttl: 900
+
+	# the time to live (TTL) value for cached lame delegations. In sec.
+	# infra-lame-ttl: 900
+
+	# the number of slabs to use for the Infrastructure cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# infra-cache-slabs: 4
+
+	# the maximum number of hosts that are cached (roundtrip times, EDNS).
+	# infra-cache-numhosts: 10000
+
+	# the maximum size of the lame zones cached per host. in bytes.
+	# infra-cache-lame-size: 10240
+
+	# Enable IPv4, "yes" or "no".
+	# do-ip4: yes
+
+	# Enable IPv6, "yes" or "no".
+	# do-ip6: yes
+
+	# Enable UDP, "yes" or "no".
+	# do-udp: yes
+
+	# Enable TCP, "yes" or "no".
+	# do-tcp: yes
+
+	# if given, a chroot(2) is done to the given directory.
+	# i.e. you can chroot to the working directory, for example,
+	# for extra security, but make sure all files are in that directory.
+	# If you give "" no chroot is performed.
+	# chroot: "/etc/unbound"
+
+	# if given, user privileges are dropped (after binding port),
+	# and the given username is assumed. Default is user "unbound".
+	# If you give "" no priviliges are dropped.
+	# username: "unbound"
+
+	# the working directory.
+	# directory: "/etc/unbound"
+
+	# the log file, "" means log to stderr. 
+	# Use of this option sets use-syslog to "no".
+	# logfile: ""
+	
+	# Log to syslog(3) if yes. The log facility LOG_DAEMON is used to 
+	# log to, with identity "unbound". If yes, it overrides the logfile.
+	# use-syslog: yes 
+
+	# the pid file.
+	# pidfile: "/etc/unbound/unbound.pid"
+	
+	# file to read root hints from.
+	# get one from ftp://FTP.INTERNIC.NET/domain/named.cache
+	# root-hints: ""
+	
+	# enable to not answer id.server and hostname.bind queries.
+	# hide-identity: no
+	
+	# enable to not answer version.server and version.bind queries.
+	# hide-version: no
+	
+	# the identity to report. Leave "" or default to return hostname.
+	# identity: ""
+	
+	# the version to report. Leave "" or default to return package version.
+	# version: ""
+	
+	# the target fetch policy.
+	# series of integers describing the policy per dependency depth. 
+	# The number of values in the list determines the maximum dependency 
+	# depth the recursor will pursue before giving up. Each integer means:
+	# 	-1 : fetch all targets opportunistically,
+	# 	0: fetch on demand,
+	#	positive value: fetch that many targets opportunistically.
+	# Enclose the list of numbers between quotes ("").
+	# target-fetch-policy: "3 2 1 0 0"
+	
+	# Harden against very small EDNS buffer sizes. 
+	# harden-short-bufsize: no
+	
+	# Harden against unseemly large queries.
+	# harden-large-queries: no
+	
+	# Harden against out of zone rrsets, to avoid spoofing attempts. 
+	# harden-glue: yes
+	
+	# Harden against receiving dnssec-stripped data. If you turn it
+	# off, failing to validate dnskey data for a trustanchor will 
+	# trigger insecure mode for that zone (like without a trustanchor).
+	# Default on, which insists on dnssec data for trust-anchored zones.
+	# harden-dnssec-stripped: yes
+	
+	# Do not query the following addresses. No DNS queries are sent there.
+	# List one address per entry. List classless netblocks with /size,
+	# do-not-query-address: 127.0.0.1/8
+	# do-not-query-address: ::1
+	
+	# if yes, the above default do-not-query-address entries are present.
+	# if no, localhost can be queried (for testing and debugging).
+	# do-not-query-localhost: yes
+	
+	# module configuration of the server. A string with identifiers
+	# separated by spaces. "iterator" or "validator iterator"
+	# module-config: "validator iterator"
+	
+	# File with trusted keys for validation. Specify more than one file
+	# with several entries, one file per entry.
+	# Zone file format, with DS and DNSKEY entries.
+	# trust-anchor-file: ""
+	
+	# Trusted key for validation. DS or DNSKEY. specify the RR on a
+	# single line, surrounded by "". TTL is ignored. class is IN default.
+	# (These examples are from August 2007 and may not be valid anymore).
+	# trust-anchor: "nlnetlabs.nl. DNSKEY 257 3 5 AQPzzTWMz8qSWIQlfRnPckx2BiVmkVN6LPupO3mbz7FhLSnm26n6iG9N Lby97Ji453aWZY3M5/xJBSOS2vWtco2t8C0+xeO1bc/d6ZTy32DHchpW 6rDH1vp86Ll+ha0tmwyy9QP7y2bVw5zSbFCrefk8qCUBgfHm9bHzMG1U BYtEIQ=="
+	# trust-anchor: "jelte.nlnetlabs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"
+
+	# File with trusted keys for validation. Specify more than one file
+	# with several entries, one file per entry. Like trust-anchor-file
+	# but has a different file format. Format is BIND-9 style format, 
+	# the trusted-keys { name flag proto algo "key"; }; clauses are read.
+	# trusted-keys-file: ""
+
+	# Override the date for validation with a specific fixed date.
+	# Do not set this unless you are debugging signature inception
+	# and expiration. "" or "0" turns the feature off. 
+	# val-override-date: ""
+	
+	# The time to live for bogus data, rrsets and messages. This avoids
+	# some of the revalidation, until the time interval expires. in secs.
+	# val-bogus-ttl: 900
+	
+	# Should additional section of secure message also be kept clean of
+	# unsecure data. Useful to shield the users of this validator from
+	# potential bogus data in the additional section. All unsigned data 
+	# in the additional section is removed from secure messages.
+	# val-clean-additional: yes
+	
+	# Turn permissive mode on to permit bogus messages. Thus, messages
+	# for which security checks failed will be returned to clients,
+	# instead of SERVFAIL. It still performs the security checks, which
+	# result in interesting log files and possibly the AD bit in
+	# replies if the message is found secure. The default is off.
+	# val-permissive-mode: no
+	
+	# It is possible to configure NSEC3 maximum iteration counts per
+	# keysize. Keep this table very short, as linear search is done.
+	# A message with an NSEC3 with larger count is marked insecure.
+	# List in ascending order the keysize and count values.
+	# val-nsec3-keysize-iterations: "1024 150 2048 500 4096 2500"
+
+	# the amount of memory to use for the key cache.
+	# in bytes. default is 4 Mb
+	# key-cache-size: 4194304
+
+	# the number of slabs to use for the key cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# key-cache-slabs: 4
+
+
+# Stub zones.
+# Create entries like below, to make all queries for 'example.com' and 
+# 'example.org' go to the given list of nameservers. list zero or more 
+# nameservers by hostname or by ipaddress.
+# stub-zone:
+#	name: "example.com"
+#	stub-addr: 192.0.2.68
+# stub-zone:
+#	name: "example.org"
+#	stub-host: ns.example.com.
+
+# Forward zones
+# Create entries like below, to make all queries for 'example.com' and
+# 'example.org' go to the given list of servers. These servers have to handle
+# recursion to other nameservers. List zero or more nameservers by hostname
+# or by ipaddress. Use an entry with name "." to forward all queries.
+# forward-zone:
+# 	name: "example.com"
+# 	forward-addr: 192.0.2.68
+# 	forward-addr: 192.0.2.73@5355  # forward to port 5355.
+# forward-zone:
+# 	name: "example.org"
+# 	forward-host: fwd.example.com

doc/plan

@@ -29,14 +29,26 @@ Roughly the boxes are as follows:
     This stage takes longer, due to complexity in the iterator module.
     Twice as long; one box for module layout, one box for iterator module.
 0.5 Validator - validator module.
-0.6 Bigger and better - good syslog, compressed packets, port ranges,
-    config file format with config checker. 
-0.7 Local zones feature - localzones stubzones fwdzones, no leak rfc1918.
-0.8 Library use - resolver validator lib (and test apps)
-0.9 Corner cases - be able to resolve in the wild. Run fuzzers.
+0.6 Bigger and better - Operational useful features (config, log, memory)
+0.7 Put to a limited audience.
+    gamma/alpha core functionality test release, to a small audience.
+    partial functionality. For more extensive use and testing.
+0.8 Local zones feature - localzones stubzones fwdzones, no leak rfc1918.
+    views support; for selective recursive service.
+0.9 Library use - resolver validator lib (and test apps)
+0.10 Corner cases - be able to resolve in the wild. Run fuzzers.
     Run as many tests as we can think of.
-0.10 Beta release. Run shadow for a resolver in production for several
+    Go through logs and check for long, unresolved cases
+    Use profiler.
+0.11 Beta release. Run shadow for a resolver in production for several
     weeks.
+0.12 Features features
+    aggressive negative caching for NSEC, NSEC3.
+    multiple queries per question, server exploration, server selection.
+    option to use real entropy for randomness (mix it in once in a while).
+    check query, option to enforce qdsection checking (forgery-resilience).
+    NSID support.
+    Be able to prime roots using several queries (only NS on first).
 
 For boxes 0.5-1.0 the planning is to be revised, at the 0.5 stage external
 coders are welcome.  Since the project is bigger, there is room for them.
@@ -115,7 +127,7 @@ Styleguide:
 * Speed test of caching forwarder.
 
 *** Basic Resolver
-* Create module interfacce and module caller algorithm.
+* Create module interface and module caller algorithm.
 * Daemon config to use modules. Test the module caller.
 * Create basic iterator and scrubber modules.
 	* Test every state of the iterator by passing test data into
@@ -133,20 +145,63 @@ Styleguide:
 	* Test resolver.
 	* Speed test.
 
+*** Put to a limited audience
+* The alpha/gamma core functionality, svn access to limited audience.
+* Support features and requests as they arise.
+* Provide real-world experiences.
+
 *** Bigger and Better
-* DNS packet compression of output.
-	* test packet compression (that it compresses.)
-	* config option to compress yes/no
-	* speed test with and without compression.
 * Config file syntax checker program. Tests on checker.
-* Use a huge number of random outgoing ports.
 * Logging first class feature with config options.
+	X with logfile turnover to avoid Gbs of logs.
+	* use syslog optional.
+* donotqueryaddresses with trie for blocking entire netblocks.
+* Memory overhaul, special allocators for hashtable caches, and mesh qstates.
+	* keep a preallocated list of region-chunks per worker thread.
+	* allocate region struct and cleanup list in region itself; use
+	  linked list cleanup list. unit test on this. do not call region
+	  to avoid name-collision with nsd regions, 'regional'.
+* read root hints from file.
+* failover to next server in 1 second, instead of 100 seconds on one server.
+X failure to return answer, w. reason (donotq, noanswer servers, cannot
+  find servers, validationfail w.classification, error), 
+  with threadno, starttime and endtime and qname/type/class, prime/qflags, 
+  from-clients, from-internal, has-subrequests, a nice error report,
+  so that an excerpt from those times can be made from the logs.
+  logfileparsing tool that makes these excerpts and emails them.
+  Not done; user can change verbosity and kill -HUP.
+* clear cache as a callback from the new-rrset-id routine.
+X make overload mode work; phase 0 all ok, phase 1 some threads close ports,
+  to let other threads pick up work. phase 2, all threads closed, so all open
+  the ports again and drop all non-cache-reply queries.
+  Keep mutexed num-overloaded-threads counter. thread incs it when it hits
+  max number of user queries serviced in mesh. threads decs it when it
+  falls below 90% of the max. if incs, and not all threads closed, phase 1,
+  else, phase 2 start is broadcast over command pipes. if decs, open ports
+  if phase 1, start servicing, phase is 0 again. Make robust against delays.
+  readme: max about 1 second worth of incoming queries, 10k perhaps,
+	or 1/number of seconds it takes start up of 10k.
+  Not done. Implement drop when full. 
+* the source includes a copy of the ldns lib for ease of building by 
+  new users. Detect system installed ldns, if installed ldns is OK; use 
+  dynamic linking against it, otherwise static linking against packaged ldns.
+* no greedy TTL algo (and test). 
+* maximum TTL, cap incoming values, and config option.
 
 *** Local zones feature.
 * Build in local zone features. First the total stop for1912.
 * Then 'local content' for minimal serving of localhost.localdomain,
   and so on. 
-* Remember jakob's diagram.
+* Remember jakob's diagram. views support, selective recursive service:
+	* acl for allowed recursion (RD=1), then drop or refused query.
+	  like 10.0.0.0/8 allow, 0.0.0.0/0 refuse, ... in-order.
+	  perhaps also, same list to disallow RD=0 access, like;
+	  allow_recursion, drop_recursion, refuse_recursion, drop_all
+	* static answers for queries, fixed RRs from cfg, option
+	  query for that RR returns answer with that RR.
+	* blacklist (return fixed nxdomain for domain and below), option
+	  can be used to block AS112 traffic, option to unblock a zone.
+	* after checking acl, do iter: static, blacklist, forwards, recurse.
 * Forward-local-zone to NSD.
 	- include in package, autoforkexec on localhost to do so.
 * forward local zone to remote server.
@@ -169,6 +224,31 @@ Styleguide:
 * Try to setup corner cases of (mis)configured DNS service/websites.
 * Resolve msoft, google, yahoo, etc weird websites.
 * Try to resolve many many different queries, perhaps compared with bind.
+* create module testers, specific for the modules
+	* read a file with cache contents and settings, provide fake
+	  environment for module-handle-state-X functions, then check
+	  resulting module state structure to correct answer.
+* speed test cache responses.
+* using two servers, compare answer differences between bind and unbound.
+  this gives false differences due to changes in the rest of internet.
+
+*** Beta release. 
+* Run shadow for a resolver in production for several weeks.
+* Check logs for errors, long queries.
+* Run in valgrind, speed profiling (as production shadow).
+
+*** Features features
+* aggressive negative caching for NSEC, NSEC3.
+* multiple queries per question, server exploration, server selection.
+* option to use real entropy for randomness (mix it in once in a while).
+* check query, enforce qdsection, checking for forgery-resilience.
+* NSID support.
+* support TSIG on queries, for validating resolver deployment.
+* Nicer statistics
+* private TTL, dTLS features.
+* retry-mode, where a bogus result triggers a retry-mode query, where a list
+  of responses over a time interval is collected, and each is validated.
+* draft-timers, DLV features.
 
 treeshrew/
 	validator/ *.c *.h
@@ -225,10 +305,10 @@ treeshrew/
 		unbound.c for validating caching recursive dns server.
 		scheduler.c for the modules.
 
-	libunbound-all/
+	libunbound/
 		app linkable. Can be configged to do whatever,
 		validator, iterator, validating iterator, forwarding stub.
-	libunbound-fwd/
+	libforwardbound/
 		app linkable forwarding stub. Small lib.
 
 	ask_cachor/ *.c *.h

doc/requirements.txt

@@ -79,3 +79,99 @@ o An authoritative name server.
 o Too many Features.
 
 
+5. Choices
+----------
+o rfc2181 decourages duplicates RRs in RRsets. unbound does not create
+  duplicates, but when presented with duplicates on the wire from the
+  authoritative servers, does not perform duplicate removal.
+  It does do some rrsig duplicate removal, in the msgparser, for dnssec qtype
+  rrsig and any, because of special rrsig processing in the msgparser.
+o The harden-glue feature, when yes all out of zone glue is deleted, when
+  no out of zone glue is used for further resolving, is more complicated 
+  than that, see below.
+  Main points:
+  	* rfc2182 trust handling is used. 
+	* data is let through only in very specific cases
+	* spoofability remains possible.
+  Not all glue is let through (despite the name of the option). Only glue 
+  which is present in a delegation, of type A and AAAA, where the name is
+  present in the NS record in the authority section is let through.
+  The glue that is let through is stored in the cache (marked as 'from the
+  additional section'). And will then be used for sending queries to. It
+  will not be present in the reply to the client (if RD is off).
+  A direct query for that name will attempt to get a msg into the message
+  cache. Since A and AAAA queries are not synthesized by the unbound cache,
+  this query will be (eventually) sent to the authoritative server and its
+  answer will be put in the cache, marked as 'from the answer section' and
+  thus remove the 'from the additional section' data, and this record is 
+  returned to the client.
+  The message has a TTL smaller or equal to the TTL of the answer RR.
+  If the cache memory is low; the answer RR may be dropped, and a glue
+  RR may be inserted, within the message TTL time, and thus return the
+  spoofed glue to a client. When the message expires, it is refetched and
+  the cached RR is updated with the correct content.
+  The server can be spoofed by getting it to visit a especially prepared 
+  domain. This domain then inserts an address for another authoritative 
+  server into the cache, when visiting that other domain, this address may
+  then be used to send queries to. And fake answers may be returned.
+  If the other domain is signed by DNSSEC, the fakes will be detected.
+
+  In summary, the harden glue feature presents a security risk if
+  disabled. Disabling the feature leads to possible better performance
+  as more glue is present for the recursive service to use. The feature
+  is implemented so as to minimise the security risk, while trying to 
+  keep this performance gain.
+o The method by which dnssec-lameness is detected is not secure. DNSSEC lame
+  is when a server has the zone in question, but lacks dnssec data, such as
+  signatures. The method to detect dnssec lameness looks at nonvalidated 
+  data from the parent of a zone. This can be used, by spoofing the parent,
+  to create a false sense of dnssec-lameness in the child, or a false sense
+  or dnssec-non-lameness in the child. The first results in the server marked
+  lame, and not used for 900 seconds, and the second will result in a 
+  validator failure (SERVFAIL again), when the query is validated later on.
+
+  Concluding, a spoof of the parent delegation can be used for many cases
+  of denial of service. I.e. a completely different NS set could be returned,
+  or the information withheld. All of these alterations can be caught by
+  the validator if the parent is signed, and result in 900 seconds bogus. 
+  The dnssec-lameness detection is used to detect operator failures, 
+  before the validator will properly verify the messages.
+
+  Also for zones for which no chain of trust exists, but a DS is given by the
+  parent, dnssec-lameness detection enables. This delivers dnnsec to our 
+  clients when possible (for client validators).
+
+  The following issue needs to be resolved:
+	a server that serves both a parent and child zone, where
+	parent is signed, but child is not. The server must not be marked 
+	lame for the parent zone, because the child answer is not signed. 
+  Instead of a false positive, we want false negatives; failure to 
+  detect dnssec-lameness is less of a problem than marking honest 
+  servers lame. dnssec-lameness is a config error and deserves the trouble.
+  So, only messages that identify the zone are used to mark the zone
+  lame. The zone is identified by SOA or NS RRsets in the answer/auth.
+  That includes almost all negative responses and also A, AAAA qtypes.
+  That would be most responses from servers.
+  For referrals, delegations that add a single label can be checked to be
+  from their zone, this covers most delegation-centric zones.
+
+  So possibly, for complicated setups, with multiple (parent-child) zones 
+  on a server, dnssec-lameness detection does not work - no dnssec-lameness 
+  is detected. Instead the zone that is dnssec-lame becomes bogus.
+
+o authority features.
+  This is a recursive server, and authority features are out of scope.
+  However, some authority features are expected in a recursor. Things like
+  localhost, reverse lookup for 127.0.0.1, or blocking AS112 traffic.
+  Also redirection of domain names with fixed data is needed by service
+  providers. Limited support is added specifically to address this.
+
+  Adding full authority support, requires much more code, and more complex
+  maintenance.
+
+  The limited support allows adding some static data (for localhost and so),
+  and to respond with a fixed rcode (NXDOMAIN) for domains (such as AS112).
+
+  You can put authority data on a separate server, and set the server in 
+  unbound.conf as stub for those zones, this allows clients to access data 
+  from the server without making unbound authoritative for the zones.

doc/unbound-checkconf.8

@@ -0,0 +1,45 @@
+.\"
+.\" unbound-checkconf.8 -- unbound configuration checker manual
+.\"
+.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
+.\"
+.\" See LICENSE for the license.
+.\"
+.\"
+.Dd @date@
+.Dt unbound-checkconf 8
+.Sh NAME
+unbound-checkconf
+.Nd Check unbound configuration file for errors.
+.Sh SYNOPSIS
+.Nm unbound-checkconf
+.Op Fl h
+.Op cfgfile
+.Sh DESCRIPTION
+.Ic Unbound-checkconf
+checks the configuration file for the
+.Xr unbound 8
+DNS resolver for syntax and other errors. 
+The config file syntax is
+described in 
+.Xr unbound.conf 5 .
+.Pp
+The available options are:
+.Bl -tag -width indent
+.It Fl h
+Show the version and commandline option help.
+.It cfgfile
+The config file to read with settings for unbound. It is checked.
+If omitted, the config file at the default location is checked.
+.Sh EXIT CODE
+The unbound-checkconf program exits with status code 1 on error, 
+0 for a correct config file.
+.El
+.Sh FILES
+.Bl -tag -width indent
+.It Pa /etc/unbound/unbound.conf
+unbound configuration file.
+.El
+.Sh SEE ALSO
+.Xr unbound.conf 5 , 
+.Xr unbound 8 .

doc/unbound.8

@@ -1,38 +1,11 @@
-.ig
-unbound.8 -- unbound manual
-
-Copyright (c) 2007, NLnet Labs. All rights reserved.
-
-This software is open source.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-Redistributions of source code must retain the above copyright notice,
-this list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-Neither the name of the NLNET LABS nor the names of its contributors may
-be used to endorse or promote products derived from this software without
-specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-..
+.\"
+.\" unbound.8 -- unbound manual
+.\"
+.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
+.\"
+.\" See LICENSE for the license.
+.\"
+.\"
 .Dd @date@
 .Dt unbound 8
 .Sh NAME
@@ -41,11 +14,9 @@ unbound
 .Sh SYNOPSIS
 .Nm unbound
 .Op Fl h
-.Op Fl p Ar port
-.Op Fl f Ar ip
-.Op Fl z Ar port
+.Op Fl d
 .Op Fl v
-
+.Op Fl c Ar cfgfile
 .Sh DESCRIPTION
 .Ic Unbound 
 is an implementation of a DNS resolver, that does caching and 
@@ -53,26 +24,24 @@ DNSSEC validation.
 .Pp
 The available options are:
 .Bl -tag -width indent
-
 .It Fl h
 Show the version and commandline option help.
-
-.It Fl p Ar port
-Start listening on the given port. Default is port 53(DNS).
-
-.It Fl f Ar ip
-Set forwarder address. DNS queries will be forwarded to this server.
-
-.It Fl z Ar ip
-Set forwarder port. DNS queries will be forwarded to this port.
-
+.It Fl c Ar cfgfile
+Set the config file with settings for unbound to read instead of the 
+file at default location /etc/unbound/unbound.conf. The syntax is
+described in 
+.Xr unbound.conf 5 .
+.It Fl d
+Debug flag, do not fork into the background, but stay attached to the
+console. This flag will also delay writing to the logfile until the
+thread-spawn time. So that most config and setup errors appear on stderr.
 .It Fl v
 Increase verbosity. If given multiple times, more information is logged.
-
+This is in addition to the verbosity (if any) from the config file.
 .El
 .Sh SEE ALSO
-.Xr resolv.conf 5
-
+.Xr unbound.conf 5 , 
+.Xr unbound-checkconf 8 .
 .Sh AUTHORS
 .Ic Unbound
 developers are mentioned in the CREDITS file in the distribution.

doc/unbound.conf.5

@@ -0,0 +1,393 @@
+.\"
+.\" unbound.conf.5 -- unbound.conf manual
+.\"
+.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
+.\"
+.\" See LICENSE for the license.
+.\"
+.\"
+.Dd @date@
+.Os FreeBSD
+.Dt unbound.conf 5
+.Sh NAME
+.Nm unbound.conf
+.Nd Unbound configuration file.
+.Sh SYNOPSIS
+.Nm unbound.conf
+.Sh DESCRIPTION
+.Ic unbound.conf
+is used to configure
+.Xr unbound 8 .
+The file format has attributes and values. Some attributes have attributes inside them.
+The notation is: attribute: value.
+.Pp
+Comments start with # and last to the end of line. Empty lines are
+ignored as is whitespace at the beginning of a line.
+.Pp
+The utility 
+.Xr unbound-checkconf 8
+can be used to check unbound.conf prior to usage.
+.Sh EXAMPLE
+An example config file is shown below. Copy this to /etc/unbound/unbound.conf
+and start the server with:
+.nf
+	$ unbound -c /etc/unbound/unbound.conf 
+.fi
+Most settings are the defaults. Stop the server with:
+.nf
+	$ kill `cat /etc/unbound/unbound.pid`
+.fi
+Below is a minimal config file. The source distribution contains an extensive
+example.conf file with all the options.
+.nf
+# unbound.conf(5) config file for unbound(8).
+server:
+	directory: "/etc/unbound"
+	username: unbound 	# make sure it can write to pidfile.
+	chroot: "/etc/unbound"
+	# logfile: "/etc/unbound/unbound.log"  #uncomment to use logfile.
+	pidfile: "/etc/unbound/unbound.pid"
+	# verbosity: 1		# uncomment and increase to get more logging.
+.fi
+.Sh FILE FORMAT
+There must be whitespace between keywords. Attribute keywords end with a colon ':'. An attribute
+is followed by its containing attributes, or a value.
+.Pp
+Files can be included using the
+.Ic include:
+directive. It can appear anywhere, and takes a single filename as an argument.
+Processing continues as if the text from the included file was copied into
+the config file at that point.
+.Ss Server Options
+These options are part of the
+.Ic server:
+clause.
+.Bl -tag -width indent
+.It \fBverbosity:\fR <number>
+The verbosity number, level 0 means no verbosity, only errors. Level 1 
+gives operational information. Level 2 gives query level information, 
+output per query. Level 3 gives algorithm level information.  
+Default is level 1. The verbosity can also be increased from the commandline,
+see
+.Xr unbound 8 .
+.It \fBnum-threads:\fR <number>
+The number of threads to create to serve clients. Use 1 for no threading.
+.It \fBport:\fR <port number>
+The port number, default 53, on which the server responds to queries.
+.It \fBinterface:\fR <ip address>
+Interface to use to connect to the network. This interface is listened to
+for queries from clients, and answers to clients are given from it.
+Can be given multiple times to work on several interfaces. If none are 
+given the default is to listen to localhost.
+The interfaces are not changed on a reload (kill -HUP) but only on restart.
+.It \fBoutgoing-interface:\fR <ip address>
+Interface to use to connect to the network. This interface is used to send
+queries to authoritative servers and receive their replies. Can be given 
+multiple times to work on several interfaces. If none are given the 
+default (all) is used. You can specify the same interfaces in 
+.Ic interface:
+and
+.Ic outgoing-interface:
+lines, the interfaces are then used for both purposes. Outgoing queries are 
+sent via a random outgoing interface to counter spoofing.
+.It \fBoutgoing-port:\fR <port number>
+The starting port number where the outgoing query port range is allocated.
+Default is 1053.
+.It \fBoutgoing-range:\fR <number>
+Number of ports to open. This number is opened per thread for every outgoing
+query interface. Must be at least 1. Default is 16.
+Larger numbers give more protection against spoofing attempts, but need
+extra resources from the operating system.
+.It \fBoutgoing-num-tcp:\fR <number>
+Number of outgoing TCP buffers to allocate per thread. Default is 10. If set
+to 0, or if do_tcp is "no", no TCP queries to authoritative servers are done.
+.It \fBincoming-num-tcp:\fR <number>
+Number of incoming TCP buffers to allocate per thread. Default is 10. If set
+to 0, or if do_tcp is "no", no TCP queries from clients are accepted.
+.It \fBmsg-buffer-size:\fR <number>
+Number of bytes size of the message buffers. Default is 65552 bytes, enough
+for 64 Kb packets, the maximum DNS message size. No message larger than this
+can be sent or received. Can be reduced to use less memory, but some requests
+for DNS data, such as for huge resource records, will result in a SERVFAIL 
+reply to the client.
+.It \fBmsg-cache-size:\fR <number>
+Number of bytes size of the message cache. Default is 4 megabytes.
+.It \fBmsg-cache-slabs:\fR <number>
+Number of slabs in the message cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. Setting (close) to the number of cpus is a 
+reasonable guess.
+.It \fBnum-queries-per-thread:\fR <number>
+The number of queries that every thread will service simultaneously.
+If more queries arrive that need servicing, they are dropped. This forces
+the client to resend after a timeout; allowing the server time to work on
+the existing queries. Default 1024.
+.It \fBrrset-cache-size:\fR <number>
+Number of bytes size of the RRset cache. Default is 4 megabytes.
+.It \fBrrset-cache-slabs:\fR <number>
+Number of slabs in the RRset cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. 
+.It \fBcache-max-ttl:\fR <seconds>
+Time to live maximum for RRsets and messages in the cache. Default is 
+864000 seconds (10 days). If the maximum kicks in, responses to clients 
+still get decrementing TTLs based on the original (larger) values. 
+When the internal TTL expires, the cache item has expired.
+Can be set lower to force the resolver to query for data often, and not
+trust (very large) TTL values.
+.It \fBinfra-host-ttl:\fR <seconds>
+Time to live for entries in the host cache. The host cache contains 
+roundtrip timing and EDNS support information. Default is 900.
+.It \fBinfra-lame-ttl:\fR <seconds>
+The time to live when a delegation is discovered to be lame. Default is 900.
+.It \fBinfra-cache-slabs:\fR <number>
+Number of slabs in the infrastructure cache. Slabs reduce lock contention 
+by threads. Must be set to a power of 2. 
+.It \fBinfra-cache-numhosts:\fR <number>
+Number of hosts for which information is cached. Default is 10000.
+.It \fBinfra-cache-lame-size:\fR <number>
+Number of bytes that the lameness cache per host is allowed to use. Default
+is 10 kb, which gives maximum storage for a couple score zones, depending on 
+the lame zone name lengths.
+.It \fBdo-ip4:\fR <yes or no>
+Enable or disable whether ip4 queries are answered. Default is yes.
+.It \fBdo-ip6:\fR <yes or no>
+Enable or disable whether ip6 queries are answered. Default is yes.
+.It \fBdo-udp:\fR <yes or no>
+Enable or disable whether UDP queries are answered. Default is yes.
+.It \fBdo-tcp:\fR <yes or no>
+Enable or disable whether TCP queries are answered. Default is yes.
+.It \fBchroot:\fR <directory>
+If given a chroot is done to the given directory. The default is 
+"/etc/unbound". If you give "" no chroot is performed.
+.It \fBusername:\fR <name>
+If given, after binding the port the user privileges are dropped. Default is
+"unbound". If you give username: "" no user change is performed. 
+.Pp
+If this user is not capable of binding the
+port, reloads (by signal HUP) will still retain the opened ports.
+If you change the port number in the config file, and that new port number 
+requires privileges, then a reload will fail; a restart is needed.
+.It \fBdirectory:\fR <directory>
+Sets the working directory for the program.
+.It \fBlogfile:\fR <filename>
+If "" is given, logging goes to stderr, or nowhere once daemonized.
+The logfile is appended to, in the following format: 
+[seconds since 1970] unbound[pid:tid]: type: message. 
+If this option is given, the use-syslog is option is set to "no".
+The logfile is reopened (for append) when the config file is reread, on 
+SIGHUP.
+.It \fBuse-syslog:\fR <yes or no>
+Sets unbound to send log messages to the syslogd, using 
+.Xr syslog 3 .
+The log facility LOG_DAEMON is used, with identity "unbound".
+The logfile setting is overridden when use-syslog is turned on.
+The default is to log to syslog.
+.It \fBpidfile:\fR <filename>
+The process id is written to the file. Default is "/etc/unbound/unbound.pid". 
+So,
+.nf
+kill -HUP `cat /etc/unbound/unbound.pid` 
+.fi
+triggers a reload,
+.nf
+kill -QUIT `cat /etc/unbound/unbound.pid` 
+.fi
+gracefully terminates.
+.It \fBroot-hints:\fR <filename>
+Read the root hints from this file. Default is nothing, using builtin hints
+for the IN class. The file has the format of zone files, with root 
+nameserver names and addresses only. The default may become outdated,
+when servers change, therefore it is good practice to use a root-hints file.
+.It \fBhide-identity:\fR <yes or no>
+If enabled id.server and hostname.bind queries are refused.
+.It \fBidentity:\fR <string>
+Set the identity to report. If set to "", the default, then the hostname
+of the server is returned.
+.It \fBhide-version:\fR <yes or no>
+If enabled version.server and version.bind queries are refused.
+.It \fBversion:\fR <string>
+Set the version to report. If set to "", the default, then the package
+version is returned.
+.It \fBtarget-fetch-policy:\fR <"list of numbers">
+Set the target fetch policy used by unbound to determine if it should fetch
+nameserver target addresses opportunistically. The policy is described per
+dependency depth. 
+.Pp
+The number of values determines the maximum dependency depth
+that unbound will pursue in answering a query.  
+A value of -1 means to fetch all targets opportunistically for that dependency
+depth. A value of 0 means to fetch on demand only. A positive value fetches
+that many targets opportunistically. 
+.Pp
+Enclose the list between quotes ("") and put spaces between numbers.
+The default is "3 2 1 0 0". Setting all zeroes, "0 0 0 0 0" gives behaviour
+closer to that of BIND 9, while setting "-1 -1 -1 -1 -1" gives behaviour 
+rumoured to be closer to that of BIND 8.
+.It \fBharden-short-bufsize:\fR <yes or no>
+Very small EDNS buffer sizes from queries are ignored. Default is off, since
+it is legal protocol wise to send these, and unbound tries to give very 
+small answers to these queries, where possible.
+.It \fBharden-large-queries:\fR <yes or no>
+Very large queries are ignored. Default is off, since it is legal protocol
+wise to send these, and could be necessary for operation if TSIG or EDNS
+payload is very large.
+.It \fBharden-glue:\fR <yes or no>
+Will trust glue only if it is within the servers authority. Default is on.
+.It \fBharden-dnssec-stripped:\fR <yes or no>
+Require DNSSEC data for trust-anchored zones, if such data is absent,
+the zone becomes bogus. If turned off, and no DNSSEC data is received
+(or the DNSKEY data fails to validate), then the zone is made insecure, 
+this behaves like there is no trust anchor. You could turn this off if 
+you are sometimes behind an intrusive firewall (of some sort) that 
+removes DNSSEC data from packets, or a zone changes from signed to 
+unsigned to badly signed often. If turned off you run the risk of a 
+downgrade attack that disables security for a zone. Default is on.
+.It \fBdo-not-query-address:\fR <IP address>
+Do not query the given IP address. Can be IP4 or IP6. Append /num to 
+indicate a classless delegation netblock, for example like
+10.2.3.4/24 or 2001::11/64.
+.It \fBdo-not-query-localhost:\fR <yes or no>
+If yes, localhost is added to the do-not-query-address entries, both
+IP6 ::1 and IP4 127.0.0.1/8. If no, then localhost can be used to send
+queries to. Default is yes.
+.It \fBmodule-config:\fR <"module names">
+Module configuration, a list of module names separated by spaces, surround
+the string with quotes (""). The modules can be validator, iterator.
+Setting this to "iterator" will result in a non-validating server.
+Setting this to "validator iterator" will turn on DNSSEC validation.
+You must also set trust-anchors for validation to be useful.
+.It \fBtrust-anchor-file:\fR <filename>
+File with trusted keys for validation. Both DS and DNSKEY entries can appear
+in the file. The format of the file is the standard DNS Zone file format.
+Default is "", or no trust anchor file.
+.It \fBtrust-anchor:\fR <"Resource Record">
+A DS or DNSKEY RR for a key to use for validation. Multiple entries can be
+given to specify multiple trusted keys, in addition to the trust-anchor-files.
+The resource record is entered in the same format as 'dig' or 'drill' prints
+them, the same format as in the zone file. Has to be on a single line, with
+"" around it. A TTL can be specified for ease of cut and paste, but is ignored. 
+A class can be specified, but class IN is default.
+.It \fBtrusted-keys-file:\fR <filename>
+File with trusted keys for validation. Specify more than one file
+with several entries, one file per entry. Like \fBtrust-anchor-file\fR
+but has a different file format. Format is BIND-9 style format, 
+the trusted-keys { name flag proto algo "key"; }; clauses are read.
+.It \fBval-override-date:\fR <rrsig-style date spec>
+Default is "" or "0", which disables this debugging feature. If enabled by
+giving a RRSIG style date, that date is used for verifying RRSIG inception
+and expiration dates, instead of the current date. Do not set this unless 
+you are debugging signature inception and expiration.
+.It \fBval-bogus-ttl:\fR <number>
+The time to live for bogus data. This is data that has failed validation;
+due to invalid signatures or other checks. The TTL from that data cannot be
+trusted, and this value is used instead. The value is in seconds, default 900.
+The time interval prevents repeated revalidation of bogus data.
+.It \fBval-clean-additional:\fR <yes or no>
+Instruct the validator to remove data from the additional section of secure
+messages that are not signed properly. Messages that are insecure, bogus,
+indeterminate or unchecked are not affected. Default is yes. Use this setting
+to protect the users that rely on this validator for authentication from 
+protentially bad data in the additional section.
+.It \fBval-permissive-mode:\fR <yes or no>
+Instruct the validator to mark bogus messages as indeterminate. The security
+checks are performed, but if the result is bogus (failed security), the
+reply is not withheld from the client with SERVFAIL as usual. The client 
+receives the bogus data. For messages that are found to be secure the AD bit 
+is set in replies. Also logging is performed as for full validation.
+The default value is "no". 
+.It \fBval-nsec3-keysize-iterations:\fR <"list of values">
+List of keysize and iteration count values, separated by spaces, surrounded
+by quotes. Default is "1024 150 2048 500 4096 2500". This determines the
+maximum allowed NSEC3 iteration count before a message is simply marked
+insecure instead of performing the many hashing iterations. The list must
+be in ascending order and have at least one entry. If you set it to 
+"1024 65535" there is no restriction to NSEC3 iteration values.
+This table must be kept short; a very long list could cause slower operation.
+.It \fBkey-cache-size:\fR <number>
+Number of bytes size of the key cache. Default is 4 megabytes.
+.It \fBkey-cache-slabs:\fR <number>
+Number of slabs in the key cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. Setting (close) to the number of cpus is a 
+reasonable guess.
+.El
+.Ss Stub Zone Options
+There may be multiple
+.Ic stub-zone:
+clauses. Each with a name: and zero or more hostnames or IP addresses.
+For the stub zone this list of nameservers is used. Class IN is assumed.
+.Bl -tag -width indent
+.It \fBname:\fR <domain name>
+Name of the stub zone.
+.It \fBstub-host:\fR <domain name>
+Name of stub zone nameserver. Is itself resolved before it is used.
+.It \fBstub-addr:\fR <IP address>
+IP address of stub zone nameserver. Can be IP 4 or IP 6.
+To use a nondefault port for DNS communication append '@' with the port number.
+.El
+.Ss Forward Zone Options
+There may be multiple
+.Ic forward-zone:
+clauses. Each with a name: and zero or more hostnames or IP addresses.
+For the forward zone this list of nameservers is used to forward the queries
+to. The servers have to handle further recursion for the query. Class IN is 
+assumed. A forward-zone entry with name "." and a forward-addr target will 
+forward all queries to that other server (unless it can answer from the cache).
+.Bl -tag -width indent
+.It \fBname:\fR <domain name>
+Name of the forward zone.
+.It \fBforward-host:\fR <domain name>
+Name of server to forward to. Is itself resolved before it is used.
+.It \fBforward-addr:\fR <IP address>
+IP address of server to forward to. Can be IP 4 or IP 6.
+To use a nondefault port for DNS communication append '@' with the port number.
+.El
+.Sh MEMORY CONTROL EXAMPLE
+In the example config settings below memory usage is reduced. Some service
+levels are lower, notable very large data and a high TCP load are no longer
+supported. Very large data and high TCP loads are exceptional for the DNS.
+DNSSEC validation is enabled, just add trust anchors.
+If you do not have to worry about programs using more than 1 meg of memory,
+the below example is not for you. Use the defaults to receive full service.
+.Pp
+.nf
+# example settings that reduce memory usage
+server:
+	num-threads: 1
+	outgoing-num-tcp: 1	# this limits TCP service, uses less buffers.
+	incoming-num-tcp: 1
+	outgoing-range: 1	# uses less memory, but less port randomness.
+	msg-buffer-size: 8192   # note this limits service, 'no huge stuff'.
+	msg-cache-size: 102400  # 100 Kb.
+	msg-cache-slabs: 1
+	rrset-cache-size: 102400  # 100 Kb.
+	rrset-cache-slabs: 1
+	infra-cache-numhosts: 200
+	infra-cache-numlame: 10
+	key-cache-size: 102400  # 100 Kb.
+	key-cache-slabs: 1
+	num-queries-per-thread: 30
+	target-fetch-policy: "2 1 0 0 0 0"
+	harden-large-queries: "yes"
+	harden-short-bufsize: "yes"
+	do-ip6: no	# save a bit of memory if not used.
+.fi
+.Sh FILES
+.Bl -tag -width indent
+.It Pa /etc/unbound
+default unbound working directory and default 
+.Xr chroot 2 
+location.
+.It Pa unbound.conf
+unbound configuration file.
+.It Pa unbound.pid
+default unbound pidfile with process ID of the running daemon.
+.It Pa unbound.log
+unbound log file. default is to log to 
+.Xr syslog 3 .
+.El
+.Sh SEE ALSO
+.Xr unbound 8 , 
+.Xr unbound-checkconf 8 .
+.Sh AUTHORS
+.Ic Unbound 
+was written by NLnet Labs. Please see CREDITS file
+in the distribution for further details.

doc/unbound.doxygen

@@ -1,4 +1,4 @@
-# Doxyfile 1.4.6
+# Doxyfile 1.5.2
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project
@@ -14,6 +14,14 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 
+# This tag specifies the encoding used for all characters in the config file that 
+# follow. The default is UTF-8 which is also the encoding used for all text before 
+# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into 
+# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of 
+# possible encodings.
+
+# DOXYFILE_ENCODING      = UTF-8
+
 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded 
 # by quotes) that should identify the project.
 
@@ -45,24 +53,14 @@ CREATE_SUBDIRS         = NO
 # documentation generated by doxygen is written. Doxygen will use this 
 # information to generate all constant output in the proper language. 
 # The default language is English, other supported languages are: 
-# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, 
-# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, 
-# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, 
-# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, 
-# Swedish, and Ukrainian.
+# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, 
+# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, 
+# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, 
+# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, 
+# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian.
 
 OUTPUT_LANGUAGE        = English
 
-# This tag can be used to specify the encoding used in the generated output. 
-# The encoding is not always determined by the language that is chosen, 
-# but also whether or not the output is meant for Windows or non-Windows users. 
-# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES 
-# forces the Windows encoding (this is the default for the Windows binary), 
-# whereas setting the tag to NO uses a Unix-style encoding (the default for 
-# all platforms other than Windows).
-
-USE_WINDOWS_ENCODING   = NO
-
 # If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will 
 # include brief member descriptions after the members that are listed in 
 # the file and class documentation (similar to JavaDoc). 
@@ -204,6 +202,11 @@ OPTIMIZE_OUTPUT_JAVA   = NO
 
 BUILTIN_STL_SUPPORT    = NO
 
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+
+# CPP_CLI_SUPPORT        = NO
+
 # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC 
 # tag is set to YES, then doxygen will reuse the documentation of the first 
 # member in the group (if any) for the other members of the group. By default 
@@ -459,7 +462,14 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories 
 # with spaces.
 
-INPUT                  =  .
+INPUT                  = .
+
+# This tag can be used to specify the character encoding of the source files that 
+# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default 
+# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. 
+# See http://www.gnu.org/software/libiconv for the list of possible encodings.
+
+# INPUT_ENCODING         = UTF-8
 
 # If the value of the INPUT tag contains directories, you can use the 
 # FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
@@ -468,7 +478,7 @@ INPUT                  =  .
 # *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx 
 # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py
 
-#FILE_PATTERNS          = *.h *.c *.dox
+FILE_PATTERNS          = 
 
 # The RECURSIVE tag can be used to turn specify whether or not subdirectories 
 # should be searched for input files as well. Possible values are YES and NO. 
@@ -480,7 +490,13 @@ RECURSIVE              = YES
 # excluded from the INPUT source files. This way you can easily exclude a 
 # subdirectory from a directory tree whose root is specified with the INPUT tag.
 
-EXCLUDE                =  ./build ./compat
+EXCLUDE                = ./build \
+                         ./compat \
+                         util/configparser.c \
+                         util/configparser.h \
+                         util/configlexer.c \
+                         util/locks.h \
+			 ./ldns-src
 
 # The EXCLUDE_SYMLINKS tag can be used select whether or not files or 
 # directories that are symbolic links (a Unix filesystem feature) are excluded 
@@ -496,6 +512,13 @@ EXCLUDE_SYMLINKS       = NO
 
 EXCLUDE_PATTERNS       = 
 
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names 
+# (namespaces, classes, functions, etc.) that should be excluded from the output. 
+# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, 
+# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test
+
+# EXCLUDE_SYMBOLS        = 
+
 # The EXAMPLE_PATH tag can be used to specify one or more files or 
 # directories that contain example code fragments that are included (see 
 # the \include command).
@@ -581,6 +604,13 @@ REFERENCED_BY_RELATION = YES
 
 REFERENCES_RELATION    = YES
 
+# If the REFERENCES_LINK_SOURCE tag is set to YES (the default)
+# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from
+# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will
+# link to the source code.  Otherwise they will link to the documentstion.
+
+# REFERENCES_LINK_SOURCE = YES
+
 # If the USE_HTAGS tag is set to YES then the references to source code 
 # will point to the HTML generated by the htags(1) tool instead of doxygen 
 # built-in source browser. The htags tool is part of GNU's global source 
@@ -1078,6 +1108,14 @@ PERL_PATH              = /usr/bin/perl
 
 CLASS_DIAGRAMS         = YES
 
+# You can define message sequence charts within doxygen comments using the \msc 
+# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to 
+# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to 
+# specify the directory where the mscgen tool resides. If left empty the tool is assumed to 
+# be found in the default search path.
+
+# MSCGEN_PATH            = 
+
 # If set to YES, the inheritance and collaboration graphs will hide 
 # inheritance and usage relations if the target is undocumented 
 # or is not a class.
@@ -1143,6 +1181,14 @@ INCLUDED_BY_GRAPH      = YES
 
 CALL_GRAPH             = NO
 
+# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will 
+# generate a caller dependency graph for every global function or class method. 
+# Note that enabling this option will significantly increase the time of a run. 
+# So in most cases it will be better to enable caller graphs for selected 
+# functions only using the \callergraph command.
+
+# CALLER_GRAPH           = NO
+
 # If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen 
 # will graphical hierarchy of all classes instead of a textual one.
 
@@ -1172,33 +1218,13 @@ DOT_PATH               =
 
 DOTFILE_DIRS           = 
 
-# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width 
-# (in pixels) of the graphs generated by dot. If a graph becomes larger than 
-# this value, doxygen will try to truncate the graph, so that it fits within 
-# the specified constraint. Beware that most browsers cannot cope with very 
-# large images.
+# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of 
+# nodes that will be shown in the graph. If the number of nodes in a graph 
+# becomes larger than this value, doxygen will truncate the graph, which is 
+# visualized by representing a node as a red box. Note that doxygen will always 
+# show the root nodes and its direct children regardless of this setting.
 
-MAX_DOT_GRAPH_WIDTH    = 1024
-
-# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height 
-# (in pixels) of the graphs generated by dot. If a graph becomes larger than 
-# this value, doxygen will try to truncate the graph, so that it fits within 
-# the specified constraint. Beware that most browsers cannot cope with very 
-# large images.
-
-MAX_DOT_GRAPH_HEIGHT   = 1024
-
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the 
-# graphs generated by dot. A depth value of 3 means that only nodes reachable 
-# from the root by following a path via at most 3 edges will be shown. Nodes 
-# that lay further from the root node will be omitted. Note that setting this 
-# option to 1 or 2 may greatly reduce the computation time needed for large 
-# code bases. Also note that a graph may be further truncated if the graph's 
-# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH 
-# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), 
-# the graph is not depth-constrained.
-
-MAX_DOT_GRAPH_DEPTH    = 0
+# DOT_GRAPH_MAX_NODES    = 50
 
 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent 
 # background. This is disabled by default, which results in a white background. 

iterator/iter_delegpt.c

@@ -0,0 +1,387 @@
+/*
+ * iterator/iter_delegpt.c - delegation point with NS and address information.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file implements the Delegation Point. It contains a list of name servers
+ * and their addresses if known.
+ */
+#include "config.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "util/regional.h"
+#include "util/data/dname.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/msgreply.h"
+#include "util/net_help.h"
+
+struct delegpt* 
+delegpt_create(struct regional* region)
+{
+	struct delegpt* dp=(struct delegpt*)regional_alloc(
+		region, sizeof(*dp));
+	if(!dp)
+		return NULL;
+	memset(dp, 0, sizeof(*dp));
+	return dp;
+}
+
+struct delegpt* delegpt_copy(struct delegpt* dp, struct regional* region)
+{
+	struct delegpt* copy = delegpt_create(region);
+	struct delegpt_ns* ns;
+	struct delegpt_addr* a;
+	if(!copy) 
+		return NULL;
+	if(!delegpt_set_name(copy, region, dp->name))
+		return NULL;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(!delegpt_add_ns(copy, region, ns->name))
+			return NULL;
+		copy->nslist->resolved = ns->resolved;
+	}
+	for(a = dp->target_list; a; a = a->next_target) {
+		if(!delegpt_add_addr(copy, region, &a->addr, a->addrlen))
+			return NULL;
+	}
+	return copy;
+}
+
+int 
+delegpt_set_name(struct delegpt* dp, struct regional* region, uint8_t* name)
+{
+	dp->namelabs = dname_count_size_labels(name, &dp->namelen);
+	dp->name = regional_alloc_init(region, name, dp->namelen);
+	return dp->name != 0;
+}
+
+int 
+delegpt_add_ns(struct delegpt* dp, struct regional* region, uint8_t* name)
+{
+	struct delegpt_ns* ns;
+	size_t len;
+	(void)dname_count_size_labels(name, &len);
+	/* slow check for duplicates to avoid counting failures when
+	 * adding the same server as a dependency twice */
+	if(delegpt_find_ns(dp, name, len))
+		return 1;
+	ns = (struct delegpt_ns*)regional_alloc(region,
+		sizeof(struct delegpt_ns));
+	if(!ns)
+		return 0;
+	ns->next = dp->nslist;
+	ns->namelen = len;
+	dp->nslist = ns;
+	ns->name = regional_alloc_init(region, name, ns->namelen);
+	ns->resolved = 0;
+	return 1;
+}
+
+struct delegpt_ns*
+delegpt_find_ns(struct delegpt* dp, uint8_t* name, size_t namelen)
+{
+	struct delegpt_ns* p = dp->nslist;
+	while(p) {
+		if(namelen == p->namelen && 
+			query_dname_compare(name, p->name) == 0) {
+			return p;
+		}
+		p = p->next;
+	}
+	return NULL;
+}
+
+int 
+delegpt_add_target(struct delegpt* dp, struct regional* region, 
+	uint8_t* name, size_t namelen, struct sockaddr_storage* addr, 
+	socklen_t addrlen)
+{
+	struct delegpt_ns* ns = delegpt_find_ns(dp, name, namelen);
+	if(!ns) {
+		/* ignore it */
+		return 1;
+	}
+	ns->resolved = 1;
+	return delegpt_add_addr(dp, region, addr, addrlen);
+}
+
+int 
+delegpt_add_addr(struct delegpt* dp, struct regional* region, 
+	struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	struct delegpt_addr* a = (struct delegpt_addr*)regional_alloc(region,
+		sizeof(struct delegpt_addr));
+	if(!a)
+		return 0;
+	a->next_target = dp->target_list;
+	dp->target_list = a;
+	a->next_result = 0;
+	a->next_usable = dp->usable_list;
+	dp->usable_list = a;
+	memcpy(&a->addr, addr, addrlen);
+	a->addrlen = addrlen;
+	a->attempts = 0;
+	return 1;
+}
+
+/** count NS and number missing */
+static void
+delegpt_count_ns(struct delegpt* dp, size_t* numns, size_t* missing)
+{
+	struct delegpt_ns* ns;
+	*numns = 0;
+	*missing = 0;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		(*numns)++;
+		if(!ns->resolved)
+			(*missing)++;
+	}
+}
+
+/** count addresses, and number in result and available lists */
+static void
+delegpt_count_addr(struct delegpt* dp, size_t* numaddr, size_t* numres, 
+	size_t* numavail)
+{
+	struct delegpt_addr* a;
+	*numaddr = 0;
+	*numres = 0;
+	*numavail = 0;
+	for(a = dp->target_list; a; a = a->next_target) {
+		(*numaddr)++;
+	}
+	for(a = dp->result_list; a; a = a->next_result) {
+		(*numres)++;
+	}
+	for(a = dp->usable_list; a; a = a->next_usable) {
+		(*numavail)++;
+	}
+}
+
+void delegpt_log(enum verbosity_value v, struct delegpt* dp)
+{
+	char buf[LDNS_MAX_DOMAINLEN+1];
+	struct delegpt_ns* ns;
+	struct delegpt_addr* a;
+	size_t missing=0, numns=0, numaddr=0, numres=0, numavail=0;
+	if(verbosity < v)
+		return;
+	dname_str(dp->name, buf);
+	if(dp->nslist == NULL && dp->target_list == NULL) {
+		log_info("DelegationPoint<%s>: empty", buf);
+		return;
+	}
+	delegpt_count_ns(dp, &numns, &missing);
+	delegpt_count_addr(dp, &numaddr, &numres, &numavail);
+	log_info("DelegationPoint<%s>: %u names (%u missing), "
+		"%u addrs (%u result, %u avail)", 
+		buf, (unsigned)numns, (unsigned)missing, 
+		(unsigned)numaddr, (unsigned)numres, (unsigned)numavail);
+	if(verbosity >= VERB_ALGO) {
+		for(ns = dp->nslist; ns; ns = ns->next) {
+			dname_str(ns->name, buf);
+			log_info("  %s%s", buf, (ns->resolved?"*":""));
+		}
+		for(a = dp->target_list; a; a = a->next_target) {
+			log_addr(VERB_ALGO, "  ", &a->addr, a->addrlen);
+		}
+	}
+}
+
+void 
+delegpt_add_unused_targets(struct delegpt* dp)
+{
+	struct delegpt_addr* usa = dp->usable_list;
+	dp->usable_list = NULL;
+	while(usa) {
+		usa->next_result = dp->result_list;
+		dp->result_list = usa;
+		usa = usa->next_usable;
+	}
+}
+
+size_t 
+delegpt_count_missing_targets(struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	size_t n = 0;
+	for(ns = dp->nslist; ns; ns = ns->next)
+		if(!ns->resolved)
+			n++;
+	return n;
+}
+
+/** find NS rrset in given list */
+static struct ub_packed_rrset_key*
+find_NS(struct reply_info* rep, size_t from, size_t to)
+{
+	size_t i;
+	for(i=from; i<to; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == LDNS_RR_TYPE_NS)
+			return rep->rrsets[i];
+	}
+	return NULL;
+}
+
+struct delegpt* 
+delegpt_from_message(struct dns_msg* msg, struct regional* region)
+{
+	struct ub_packed_rrset_key* ns_rrset = NULL;
+	struct delegpt* dp;
+	size_t i;
+	/* look for NS records in the authority section... */
+	ns_rrset = find_NS(msg->rep, msg->rep->an_numrrsets, 
+		msg->rep->an_numrrsets+msg->rep->ns_numrrsets);
+
+	/* In some cases (even legitimate, perfectly legal cases), the 
+	 * NS set for the "referral" might be in the answer section. */
+	if(!ns_rrset)
+		ns_rrset = find_NS(msg->rep, 0, msg->rep->an_numrrsets);
+	
+	/* If there was no NS rrset in the authority section, then this 
+	 * wasn't a referral message. (It might not actually be a 
+	 * referral message anyway) */
+	if(!ns_rrset)
+		return NULL;
+	
+	/* If we found any, then Yay! we have a delegation point. */
+	dp = delegpt_create(region);
+	if(!dp)
+		return NULL;
+	if(!delegpt_set_name(dp, region, ns_rrset->rk.dname))
+		return NULL;
+	if(!delegpt_rrset_add_ns(dp, region, ns_rrset))
+		return NULL;
+
+	/* add glue, A and AAAA in answer and additional section */
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+		/* skip auth section. FIXME really needed?*/
+		if(msg->rep->an_numrrsets <= i && 
+			i < (msg->rep->an_numrrsets+msg->rep->ns_numrrsets))
+			continue;
+
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_A) {
+			if(!delegpt_add_rrset_A(dp, region, s))
+				return NULL;
+		} else if(ntohs(s->rk.type) == LDNS_RR_TYPE_AAAA) {
+			if(!delegpt_add_rrset_AAAA(dp, region, s))
+				return NULL;
+		}
+	}
+	return dp;
+}
+
+int 
+delegpt_rrset_add_ns(struct delegpt* dp, struct regional* region,
+        struct ub_packed_rrset_key* ns_rrset)
+{
+	struct packed_rrset_data* nsdata = (struct packed_rrset_data*)
+		ns_rrset->entry.data;
+	size_t i;
+	for(i=0; i<nsdata->count; i++) {
+		if(nsdata->rr_len[i] < 2+1) continue; /* len + root label */
+		if(dname_valid(nsdata->rr_data[i]+2, nsdata->rr_len[i]-2) !=
+			(size_t)ldns_read_uint16(nsdata->rr_data[i]))
+			continue; /* bad format */
+		/* add rdata of NS (= wirefmt dname), skip rdatalen bytes */
+		if(!delegpt_add_ns(dp, region, nsdata->rr_data[i]+2))
+			return 0;
+	}
+	return 1;
+}
+
+int 
+delegpt_add_rrset_A(struct delegpt* dp, struct regional* region,
+	struct ub_packed_rrset_key* ak)
+{
+        struct packed_rrset_data* d=(struct packed_rrset_data*)ak->entry.data;
+        size_t i;
+        struct sockaddr_in sa;
+        socklen_t len = (socklen_t)sizeof(sa);
+        memset(&sa, 0, len);
+        sa.sin_family = AF_INET;
+        sa.sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+        for(i=0; i<d->count; i++) {
+                if(d->rr_len[i] != 2 + INET_SIZE)
+                        continue;
+                memmove(&sa.sin_addr, d->rr_data[i]+2, INET_SIZE);
+                if(!delegpt_add_target(dp, region, ak->rk.dname,
+                        ak->rk.dname_len, (struct sockaddr_storage*)&sa,
+                        len))
+                        return 0;
+        }
+        return 1;
+}
+
+int 
+delegpt_add_rrset_AAAA(struct delegpt* dp, struct regional* region,
+	struct ub_packed_rrset_key* ak)
+{
+        struct packed_rrset_data* d=(struct packed_rrset_data*)ak->entry.data;
+        size_t i;
+        struct sockaddr_in6 sa;
+        socklen_t len = (socklen_t)sizeof(sa);
+        memset(&sa, 0, len);
+        sa.sin6_family = AF_INET6;
+        sa.sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+        for(i=0; i<d->count; i++) {
+                if(d->rr_len[i] != 2 + INET6_SIZE) /* rdatalen + len of IP6 */
+                        continue;
+                memmove(&sa.sin6_addr, d->rr_data[i]+2, INET6_SIZE);
+                if(!delegpt_add_target(dp, region, ak->rk.dname,
+                        ak->rk.dname_len, (struct sockaddr_storage*)&sa,
+                        len))
+                        return 0;
+        }
+        return 1;
+}
+
+int 
+delegpt_add_rrset(struct delegpt* dp, struct regional* region,
+        struct ub_packed_rrset_key* rrset)
+{
+	if(!rrset)
+		return 1;
+	if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_NS)
+		return delegpt_rrset_add_ns(dp, region, rrset);
+	else if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_A)
+		return delegpt_add_rrset_A(dp, region, rrset);
+	else if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_AAAA)
+		return delegpt_add_rrset_AAAA(dp, region, rrset);
+	log_warn("Unknown rrset type added to delegpt");
+	return 1;
+}

iterator/iter_delegpt.h

@@ -0,0 +1,263 @@
+/*
+ * iterator/iter_delegpt.h - delegation point with NS and address information.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file implements the Delegation Point. It contains a list of name servers
+ * and their addresses if known.
+ */
+
+#ifndef ITERATOR_ITER_DELEGPT_H
+#define ITERATOR_ITER_DELEGPT_H
+#include "util/log.h"
+struct regional;
+struct delegpt_ns;
+struct delegpt_addr;
+struct dns_msg;
+struct ub_packed_rrset_key;
+
+/**
+ * Delegation Point.
+ * For a domain name, the NS rrset, and the A and AAAA records for those.
+ */
+struct delegpt {
+	/** the domain name of the delegation point. */
+	uint8_t* name;
+	/** length of the delegation point name */
+	size_t namelen;
+	/** number of labels in delegation point */
+	int namelabs;
+
+	/** the nameservers, names from the NS RRset rdata. */
+	struct delegpt_ns* nslist;
+	/** the target addresses for delegation */
+	struct delegpt_addr* target_list;
+	/** the list of usable targets; subset of target_list 
+	 * the items in this list are not part of the result list.  */
+	struct delegpt_addr* usable_list;
+	/** the list of returned targets; subset of target_list */
+	struct delegpt_addr* result_list;
+};
+
+/**
+ * Nameservers for a delegation point.
+ */
+struct delegpt_ns {
+	/** next in list */
+	struct delegpt_ns* next;
+	/** name of nameserver */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** 
+	 * If the name has been resolved. false if not queried for yet.
+	 * true if the address is known, or marked true if failed.
+	 */
+	int resolved;
+};
+
+/**
+ * Address of target nameserver in delegation point.
+ */
+struct delegpt_addr {
+	/** next delegation point in results */
+	struct delegpt_addr* next_result;
+	/** next delegation point in usable list */
+	struct delegpt_addr* next_usable;
+	/** next delegation point in all targets list */
+	struct delegpt_addr* next_target;
+
+	/** delegation point address */
+	struct sockaddr_storage addr;
+	/** length of addr */
+	socklen_t addrlen;
+	/** number of attempts for this addr */
+	int attempts;
+};
+
+/**
+ * Create new delegation point.
+ * @param regional: where to allocate it.
+ * @return new delegation point or NULL on error.
+ */
+struct delegpt* delegpt_create(struct regional* regional);
+
+/**
+ * Create a copy of a delegation point.
+ * @param dp: delegation point to copy.
+ * @param regional: where to allocate it.
+ * @return new delegation point or NULL on error.
+ */
+struct delegpt* delegpt_copy(struct delegpt* dp, struct regional* regional);
+
+/**
+ * Set name of delegation point.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the name copy.
+ * @param name: name to use.
+ * @return false on error.
+ */
+int delegpt_set_name(struct delegpt* dp, struct regional* regional, 
+	uint8_t* name);
+
+/**
+ * Add a name to the delegation point.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param name: domain name in wire format.
+ * @return false on error.
+ */
+int delegpt_add_ns(struct delegpt* dp, struct regional* regional, 
+	uint8_t* name);
+
+/**
+ * Add NS rrset; calls add_ns repeatedly.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param ns_rrset: NS rrset.
+ * return 0 on alloc error.
+ */
+int delegpt_rrset_add_ns(struct delegpt* dp, struct regional* regional,
+	struct ub_packed_rrset_key* ns_rrset);
+
+/**
+ * Add target address to the delegation point.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param name: name for which target was found (must be in nslist).
+ *	This name is marked resolved.
+ * @param namelen: length of name.
+ * @param addr: the address.
+ * @param addrlen: the length of addr.
+ * @return false on error.
+ */
+int delegpt_add_target(struct delegpt* dp, struct regional* regional, 
+	uint8_t* name, size_t namelen, struct sockaddr_storage* addr, 
+	socklen_t addrlen);
+
+/**
+ * Add A RRset to delegpt.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param rrset: RRset A to add.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset_A(struct delegpt* dp, struct regional* regional, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add AAAA RRset to delegpt.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param rrset: RRset AAAA to add.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset_AAAA(struct delegpt* dp, struct regional* regional, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add any RRset to delegpt.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param rrset: RRset to add, NS, A, AAAA.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset(struct delegpt* dp, struct regional* regional, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add address to the delegation point. No servername is associated or checked.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param addr: the address.
+ * @param addrlen: the length of addr.
+ * @return false on error.
+ */
+int delegpt_add_addr(struct delegpt* dp, struct regional* regional, 
+	struct sockaddr_storage* addr, socklen_t addrlen);
+
+/** 
+ * Find NS record in name list of delegation point.
+ * @param dp: delegation point.
+ * @param name: name of nameserver to look for, uncompressed wireformat.
+ * @param namelen: length of name.
+ * @return the ns structure or NULL if not found.
+ */
+struct delegpt_ns* delegpt_find_ns(struct delegpt* dp, uint8_t* name, 
+	size_t namelen);
+
+/**
+ * Print the delegation point to the log. For debugging.
+ * @param v: verbosity value that is needed to emit to log.
+ * @param dp: delegation point.
+ */
+void delegpt_log(enum verbosity_value v, struct delegpt* dp);
+
+/**
+ * Add all usable targets to the result list.
+ * @param dp: delegation point.
+ */
+void delegpt_add_unused_targets(struct delegpt* dp);
+
+/**
+ * Count number of missing targets. These are ns names with no resolved flag.
+ * @param dp: delegation point.
+ * @return number of missing targets (or 0).
+ */
+size_t delegpt_count_missing_targets(struct delegpt* dp);
+
+/**
+ * Create new delegation point from a dns message
+ *
+ * Note that this method does not actually test to see if the message is an
+ * actual referral. It really is just checking to see if it can construct a
+ * delegation point, so the message could be of some other type (some ANSWER
+ * messages, some CNAME messages, generally.) Note that the resulting
+ * DelegationPoint will contain targets for all "relevant" glue (i.e.,
+ * address records whose ownernames match the target of one of the NS
+ * records), so if policy dictates that some glue should be discarded beyond
+ * that, discard it before calling this method. Note that this method will
+ * find "glue" in either the ADDITIONAL section or the ANSWER section.
+ *
+ * @param msg: the dns message, referral.
+ * @param regional: where to allocate delegation point.
+ * @return new delegation point or NULL on alloc error, or if the
+ *         message was not appropriate.
+ */
+struct delegpt* delegpt_from_message(struct dns_msg* msg, 
+	struct regional* regional);
+
+#endif /* ITERATOR_ITER_DELEGPT_H */

iterator/iter_donotq.c

@@ -0,0 +1,309 @@
+/*
+ * iterator/iter_donotq.c - iterative resolver donotqueryaddresses storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * The donotqueryaddresses are stored and looked up. These addresses
+ * (like 127.0.0.1) must not be used to send queries to, and can be
+ * discarded immediately from the server selection.
+ */
+#include "config.h"
+#include "iterator/iter_donotq.h"
+#include "util/regional.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+
+int
+donotq_cmp(const void* k1, const void* k2)
+{
+	struct iter_donotq_addr* n1 = (struct iter_donotq_addr*)k1;
+	struct iter_donotq_addr* n2 = (struct iter_donotq_addr*)k2;
+	int r = sockaddr_cmp_addr(&n1->addr, n1->addrlen, &n2->addr, 
+		n2->addrlen);
+	if(r != 0) return r;
+	if(n1->net < n2->net)
+		return -1;
+	if(n1->net > n2->net)
+		return 1;
+	return 0;
+}
+
+struct iter_donotq* 
+donotq_create()
+{
+	struct iter_donotq* dq = (struct iter_donotq*)calloc(1,
+		sizeof(struct iter_donotq));
+	if(!dq)
+		return NULL;
+	dq->region = regional_create();
+	if(!dq->region) {
+		donotq_delete(dq);
+		return NULL;
+	}
+	return dq;
+}
+
+void 
+donotq_delete(struct iter_donotq* dq)
+{
+	if(!dq) 
+		return;
+	regional_destroy(dq->region);
+	free(dq->tree);
+	free(dq);
+}
+
+/** insert new address into donotq structure */
+static int
+donotq_insert(struct iter_donotq* dq, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int net)
+{
+	struct iter_donotq_addr* node = regional_alloc(dq->region,
+		sizeof(struct iter_donotq_addr));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	memcpy(&node->addr, addr, addrlen);
+	node->addrlen = addrlen;
+	node->net = net;
+	node->parent = NULL;
+	if(!rbtree_insert(dq->tree, &node->node)) {
+		verbose(VERB_DETAIL, "duplicate donotquery address ignored.");
+	}
+	return 1;
+}
+
+/** make sure the netblock ends in zeroes for compare in tree */
+static void
+mask_block(int ip6, struct sockaddr_storage* addr, int net)
+{
+	uint8_t mask[8] = {0x0, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	int i, max;
+	uint8_t* s;
+	if(ip6) {
+		s = (uint8_t*)&((struct sockaddr_in6*)addr)->sin6_addr;
+		max = 128;
+	} else {
+		s = (uint8_t*)&((struct sockaddr_in*)addr)->sin_addr;
+		max = 32;
+	}
+	if(net >= max)
+		return;
+	for(i=net/8+1; i<max/8; i++) {
+		s[i] = 0;
+	}
+	s[net/8] &= mask[net&0x7];
+}
+
+/** apply donotq string */
+static int
+donotq_str_cfg(struct iter_donotq* dq, const char* str)
+{
+	struct sockaddr_storage addr;
+	int net;
+	char* s = NULL;
+	socklen_t addrlen;
+	net = (str_is_ip6(str)?128:32);
+	if((s=strchr(str, '/'))) {
+		if(atoi(s+1) > net) {
+			log_err("netblock too large: %s", str);
+			return 0;
+		}
+		net = atoi(s+1);
+		if(net == 0 && strcmp(s+1, "0") != 0) {
+			log_err("cannot parse donotquery netblock:"
+				" '%s'", str);
+			return 0;
+		}
+		if(!(s = strdup(str))) {
+			log_err("out of memory");
+			return 0;
+		}
+		*strchr(s, '/') = '\0';
+	}
+	if(!ipstrtoaddr(s?s:str, UNBOUND_DNS_PORT, &addr, &addrlen)) {
+		free(s);
+		log_err("cannot parse donotquery ip address: '%s'", str);
+		return 0;
+	}
+	if(s) {
+		free(s);
+		mask_block(str_is_ip6(str), &addr, net);
+	}
+	if(!donotq_insert(dq, &addr, addrlen, net)) {
+		log_err("out of memory");
+		return 0;
+	}
+	return 1;
+}
+
+/** read donotq config */
+static int 
+read_donotq(struct iter_donotq* dq, struct config_file* cfg)
+{
+	struct config_strlist* p;
+	for(p = cfg->donotqueryaddrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!donotq_str_cfg(dq, p->str))
+			return 0;
+	}
+	return 1;
+}
+
+/** number of bits that two addrs share (are equal) */
+static int
+addr_in_common(struct sockaddr_storage* addr1, int net1, 
+	struct sockaddr_storage* addr2, int net2, socklen_t addrlen)
+{
+	int min = (net1<net2)?net1:net2;
+	int i, to;
+	int match = 0;
+	uint8_t* s1, *s2;
+	if(addr_is_ip6(addr1, addrlen)) {
+		s1 = (uint8_t*)&((struct sockaddr_in6*)addr1)->sin6_addr;
+		s2 = (uint8_t*)&((struct sockaddr_in6*)addr2)->sin6_addr;
+		to = 16;
+	} else {
+		s1 = (uint8_t*)&((struct sockaddr_in*)addr1)->sin_addr;
+		s2 = (uint8_t*)&((struct sockaddr_in*)addr2)->sin_addr;
+		to = 4;
+	}
+	/* match = bits_in_common(s1, s2, to); */
+	for(i=0; i<to; i++) {
+		if(s1[i] == s2[i]) {
+			match += 8;
+		} else {
+			uint8_t z = s1[i]^s2[i];
+			log_assert(z);
+			while(!(z&0x80)) {
+				match++;
+				z<<=1;
+			}
+			break;
+		}
+	}
+	if(match > min) match = min;
+	return match;
+}
+
+/** initialise parent pointers in the tree */
+static void
+donotq_init_parents(struct iter_donotq* donotq)
+{
+	struct iter_donotq_addr* node, *prev = NULL, *p;
+	int m;
+	RBTREE_FOR(node, struct iter_donotq_addr*, donotq->tree) {
+		node->parent = NULL;
+		if(!prev || prev->addrlen != node->addrlen) {
+			prev = node;
+			continue;
+		}
+		m = addr_in_common(&prev->addr, prev->net, &node->addr, 
+			node->net, node->addrlen);
+		/* sort order like: ::/0, 1::/2, 1::/4, ... 2::/2 */
+		/* find the previous, or parent-parent-parent */
+		for(p = prev; p; p = p->parent)
+			if(p->net <= m) {
+				/* ==: since prev matched m, this is closest*/
+				/* <: prev matches more, but is not a parent,
+				* this one is a (grand)parent */
+				node->parent = p;
+				break;
+			}
+		prev = node;
+	}
+}
+
+int 
+donotq_apply_cfg(struct iter_donotq* dq, struct config_file* cfg)
+{
+	free(dq->tree);
+	dq->tree = rbtree_create(donotq_cmp);
+	if(!dq->tree)
+		return 0;
+	if(!read_donotq(dq, cfg))
+		return 0;
+	if(cfg->donotquery_localhost) {
+		if(!donotq_str_cfg(dq, "127.0.0.0/8"))
+			return 0;
+		if(!donotq_str_cfg(dq, "::1"))
+			return 0;
+	}
+	donotq_init_parents(dq);
+	return 1;
+}
+
+int 
+donotq_lookup(struct iter_donotq* donotq, struct sockaddr_storage* addr,
+        socklen_t addrlen)
+{
+	/* lookup in the tree */
+	rbnode_t* res = NULL;
+	struct iter_donotq_addr* result;
+	struct iter_donotq_addr key;
+	key.node.key = &key;
+	memcpy(&key.addr, addr, addrlen);
+	key.addrlen = addrlen;
+	key.net = (addr_is_ip6(addr, addrlen)?128:32);
+	if(rbtree_find_less_equal(donotq->tree, &key, &res)) {
+		/* exact */
+		return 1;
+	} else {
+		/* smaller element (or no element) */
+		int m;
+		result = (struct iter_donotq_addr*)res;
+		if(!result || result->addrlen != addrlen)
+			return 0;
+		/* count number of bits matched */
+		m = addr_in_common(&result->addr, result->net, addr, 
+			key.net, addrlen);
+		while(result) { /* go up until addr is inside netblock */
+			if(result->net <= m)
+				return 1;
+			result = result->parent;
+		}
+	}
+	return 0;
+}
+
+size_t 
+donotq_get_mem(struct iter_donotq* donotq)
+{
+	if(!donotq) return 0;
+	return sizeof(*donotq) + regional_get_mem(donotq->region);
+}

iterator/iter_donotq.h

@@ -0,0 +1,120 @@
+/*
+ * iterator/iter_donotq.h - iterative resolver donotqueryaddresses storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of the donotquery addresses and lookup fast.
+ */
+
+#ifndef ITERATOR_ITER_DONOTQ_H
+#define ITERATOR_ITER_DONOTQ_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct regional;
+
+/**
+ * Iterator donotqueryaddresses structure
+ */
+struct iter_donotq {
+	/** regional for allocation */
+	struct regional* region;
+	/** 
+	 * Tree of the single addresses that are blocked.
+	 * contents of type iter_donotq_addr.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator donotquery address.
+ * A single address that must not be used to send queries to.
+ */
+struct iter_donotq_addr {
+	/** redblacktree node, key is this structure: addr and addrlen */
+	rbnode_t node;
+	/** address */
+	struct sockaddr_storage addr;
+	/** length of addr */
+	socklen_t addrlen;
+	/** netblock size */
+	int net;
+	/** parent node in donotq tree that encompasses this entry */
+	struct iter_donotq_addr* parent;
+};
+
+/**
+ * Create donotqueryaddresses structure 
+ * @return new structure or NULL on error.
+ */
+struct iter_donotq* donotq_create();
+
+/**
+ * Delete donotqueryaddresses structure.
+ * @param donotq: to delete.
+ */
+void donotq_delete(struct iter_donotq* donotq);
+
+/**
+ * Process donotqueryaddresses config.
+ * @param donotq: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int donotq_apply_cfg(struct iter_donotq* donotq, struct config_file* cfg);
+
+/**
+ * See if an address is blocked.
+ * @param donotq: structure for address storage.
+ * @param addr: address to check
+ * @param addrlen: length of addr.
+ * @return: true if the address must not be queried. false if unlisted.
+ */
+int donotq_lookup(struct iter_donotq* donotq, struct sockaddr_storage* addr,
+	socklen_t addrlen);
+
+/**
+ * Get memory used by donotqueryaddresses structure.
+ * @param donotq: structure for address storage.
+ * @return bytes in use.
+ */
+size_t donotq_get_mem(struct iter_donotq* donotq);
+
+/** compare two donotq entries */
+int donotq_cmp(const void* k1, const void* k2);
+
+#endif /* ITERATOR_ITER_DONOTQ_H */

iterator/iter_fwd.c

@@ -0,0 +1,292 @@
+/*
+ * iterator/iter_fwd.c - iterative resolver module forward zones.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of forward zones and config settings.
+ */
+#include "config.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_delegpt.h"
+#include "util/regional.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+int
+fwd_cmp(const void* k1, const void* k2)
+{
+	int m;
+	struct iter_forward_zone* n1 = (struct iter_forward_zone*)k1;
+	struct iter_forward_zone* n2 = (struct iter_forward_zone*)k2;
+	if(n1->dclass != n2->dclass) {
+		if(n1->dclass < n2->dclass)
+			return -1;
+		return 1;
+	}
+	return dname_lab_cmp(n1->name, n1->namelabs, n2->name, n2->namelabs, 
+		&m);
+}
+
+struct iter_forwards* 
+forwards_create()
+{
+	struct iter_forwards* fwd = (struct iter_forwards*)calloc(1,
+		sizeof(struct iter_forwards));
+	if(!fwd)
+		return NULL;
+	fwd->region = regional_create();
+	if(!fwd->region) {
+		forwards_delete(fwd);
+		return NULL;
+	}
+	return fwd;
+}
+
+void 
+forwards_delete(struct iter_forwards* fwd)
+{
+	if(!fwd) 
+		return;
+	regional_destroy(fwd->region);
+	free(fwd->tree);
+	free(fwd);
+}
+
+/** insert new info into forward structure */
+static int
+forwards_insert(struct iter_forwards* fwd, uint16_t c, struct delegpt* dp)
+{
+	struct iter_forward_zone* node = regional_alloc(fwd->region,
+		sizeof(struct iter_forward_zone));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	node->dclass = c;
+	node->name = regional_alloc_init(fwd->region, dp->name, dp->namelen);
+	if(!node->name)
+		return 0;
+	node->namelen = dp->namelen;
+	node->namelabs = dp->namelabs;
+	node->dp = dp;
+	if(!rbtree_insert(fwd->tree, &node->node)) {
+		log_err("duplicate forward zone ignored.");
+	}
+	return 1;
+}
+
+/** initialise parent pointers in the tree */
+static void
+fwd_init_parents(struct iter_forwards* fwd)
+{
+	struct iter_forward_zone* node, *prev = NULL, *p;
+	int m;
+	RBTREE_FOR(node, struct iter_forward_zone*, fwd->tree) {
+		node->parent = NULL;
+		if(!prev || prev->dclass != node->dclass) {
+			prev = node;
+			continue;
+		}
+		(void)dname_lab_cmp(prev->name, prev->namelabs, node->name,
+			node->namelabs, &m); /* we know prev is smaller */
+		/* sort order like: . com. bla.com. zwb.com. net. */
+		/* find the previous, or parent-parent-parent */
+		for(p = prev; p; p = p->parent)
+			/* looking for name with few labels, a parent */
+			if(p->namelabs <= m) {
+				/* ==: since prev matched m, this is closest*/
+				/* <: prev matches more, but is not a parent,
+				 * this one is a (grand)parent */
+				node->parent = p;
+				break;
+			}
+		prev = node;
+	}
+}
+
+/** set zone name */
+static int 
+read_fwds_name(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	ldns_rdf* rdf;
+	if(!s->name) {
+		log_err("forward zone without a name (use name \".\" to forward everything)");
+		return 0;
+	}
+	rdf = ldns_dname_new_frm_str(s->name);
+	if(!rdf) {
+		log_err("cannot parse forward zone name %s", s->name);
+		return 0;
+	}
+	if(!delegpt_set_name(dp, fwd->region, ldns_rdf_data(rdf))) {
+		ldns_rdf_deep_free(rdf);
+		log_err("out of memory");
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** set fwd host names */
+static int 
+read_fwds_host(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	ldns_rdf* rdf;
+	for(p = s->hosts; p; p = p->next) {
+		log_assert(p->str);
+		rdf = ldns_dname_new_frm_str(p->str);
+		if(!rdf) {
+			log_err("cannot parse forward %s server name: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_ns(dp, fwd->region, ldns_rdf_data(rdf))) {
+			ldns_rdf_deep_free(rdf);
+			log_err("out of memory");
+			return 0;
+		}
+		ldns_rdf_deep_free(rdf);
+	}
+	return 1;
+}
+
+/** set fwd server addresses */
+static int 
+read_fwds_addr(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	for(p = s->addrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!extstrtoaddr(p->str, &addr, &addrlen)) {
+			log_err("cannot parse forward %s ip address: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_addr(dp, fwd->region, &addr, addrlen)) {
+			log_err("out of memory");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/** read forwards config */
+static int 
+read_forwards(struct iter_forwards* fwd, struct config_file* cfg)
+{
+	struct config_stub* s;
+	for(s = cfg->forwards; s; s = s->next) {
+		struct delegpt* dp = delegpt_create(fwd->region);
+		if(!dp) {
+			log_err("out of memory");
+			return 0;
+		}
+		if(!read_fwds_name(fwd, s, dp) ||
+			!read_fwds_host(fwd, s, dp) ||
+			!read_fwds_addr(fwd, s, dp))
+			return 0;
+		if(!forwards_insert(fwd, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		verbose(VERB_DETAIL, "Forward zone server list:");
+		delegpt_log(VERB_DETAIL, dp);
+	}
+	return 1;
+}
+
+int 
+forwards_apply_cfg(struct iter_forwards* fwd, struct config_file* cfg)
+{
+	free(fwd->tree);
+	fwd->tree = rbtree_create(fwd_cmp);
+	if(!fwd->tree)
+		return 0;
+
+	/* read forward zones */
+	if(!read_forwards(fwd, cfg))
+		return 0;
+	fwd_init_parents(fwd);
+	return 1;
+}
+
+struct delegpt* 
+forwards_lookup(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass)
+{
+	/* lookup the forward zone in the tree */
+	rbnode_t* res = NULL;
+	struct iter_forward_zone *result;
+	struct iter_forward_zone key;
+	key.node.key = &key;
+	key.dclass = qclass;
+	key.name = qname;
+	key.namelabs = dname_count_size_labels(qname, &key.namelen);
+	if(rbtree_find_less_equal(fwd->tree, &key, &res)) {
+		/* exact */
+		result = (struct iter_forward_zone*)res;
+	} else {
+		/* smaller element (or no element) */
+		int m;
+		result = (struct iter_forward_zone*)res;
+		if(!result || result->dclass != qclass)
+			return NULL;
+		/* count number of labels matched */
+		(void)dname_lab_cmp(result->name, result->namelabs, key.name,
+			key.namelabs, &m);
+		while(result) { /* go up until qname is subdomain of stub */
+			if(result->namelabs <= m)
+				break;
+			result = result->parent;
+		}
+	}
+	if(result)
+		return result->dp;
+	return NULL;
+}
+
+size_t 
+forwards_get_mem(struct iter_forwards* fwd)
+{
+	if(!fwd)
+		return 0;
+	return sizeof(*fwd) + regional_get_mem(fwd->region);
+}

iterator/iter_fwd.h

@@ -0,0 +1,131 @@
+/*
+ * iterator/iter_fwd.h - iterative resolver module forward zones.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of forward zones, and read those from config.
+ */
+
+#ifndef ITERATOR_ITER_FWD_H
+#define ITERATOR_ITER_FWD_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct delegpt;
+struct regional;
+
+/**
+ * Iterator forward zones structure
+ */
+struct iter_forwards {
+	/** regional where forward zone server addresses are allocated */
+	struct regional* region;
+	/** 
+	 * Zones are stored in this tree. Sort order is specially chosen.
+	 * first sorted on qtype. Then on dname in nsec-like order, so that
+	 * a lookup on class, name will return an exact match or the closest
+	 * match which gives the ancestor needed.
+	 * contents of type iter_forward_zone.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator forward servers for a particular zone.
+ */
+struct iter_forward_zone {
+	/** redblacktree node, key is this structure: class and name */
+	rbnode_t node;
+	/** name */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** number of labels in name */
+	int namelabs;
+	/** delegation point with forward server information for this zone. */
+	struct delegpt* dp;
+	/** pointer to parent in tree (or NULL if none) */
+	struct iter_forward_zone* parent;
+	/** class. host order. */
+	uint16_t dclass;
+};
+
+/**
+ * Create forwards 
+ * @return new forwards or NULL on error.
+ */
+struct iter_forwards* forwards_create();
+
+/**
+ * Delete forwards.
+ * @param fwd: to delete.
+ */
+void forwards_delete(struct iter_forwards* fwd);
+
+/**
+ * Process forwards config.
+ * @param fwd: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int forwards_apply_cfg(struct iter_forwards* fwd, struct config_file* cfg);
+
+/**
+ * Find forward zone information
+ * For this qname/qclass find forward zone information, returns delegation
+ * point with server names and addresses, or NULL if no forwarding is needed.
+ *
+ * @param fwd: forward storage.
+ * @param qname: The qname of the query.
+ * @param qclass: The qclass of the query.
+ * @return: A delegation point if the query has to be forwarded to that list,
+ *         otherwise null.
+ */
+struct delegpt* forwards_lookup(struct iter_forwards* fwd, 
+	uint8_t* qname, uint16_t qclass);
+
+/**
+ * Get memory in use by forward storage
+ * @param fwd: forward storage.
+ * @return bytes in use
+ */
+size_t forwards_get_mem(struct iter_forwards* fwd);
+
+/** compare two fwd entries */
+int fwd_cmp(const void* k1, const void* k2);
+
+#endif /* ITERATOR_ITER_FWD_H */

iterator/iter_hints.c

@@ -0,0 +1,506 @@
+/*
+ * iterator/iter_hints.c - iterative resolver module stub and root hints.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of stub and root hints, and read those from config.
+ */
+#include "config.h"
+#include "iterator/iter_hints.h"
+#include "iterator/iter_delegpt.h"
+#include "util/regional.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+int
+stub_cmp(const void* k1, const void* k2)
+{
+	int m;
+	struct iter_hints_stub* n1 = (struct iter_hints_stub*)k1;
+	struct iter_hints_stub* n2 = (struct iter_hints_stub*)k2;
+	if(n1->hint_class != n2->hint_class) {
+		if(n1->hint_class < n2->hint_class)
+			return -1;
+		return 1;
+	}
+	return dname_lab_cmp(n1->name, n1->namelabs, n2->name, n2->namelabs, 
+		&m);
+}
+
+struct iter_hints* 
+hints_create()
+{
+	struct iter_hints* hints = (struct iter_hints*)calloc(1,
+		sizeof(struct iter_hints));
+	if(!hints)
+		return NULL;
+	hints->region = regional_create();
+	if(!hints->region) {
+		hints_delete(hints);
+		return NULL;
+	}
+	return hints;
+}
+
+void 
+hints_delete(struct iter_hints* hints)
+{
+	if(!hints) 
+		return;
+	regional_destroy(hints->region);
+	free(hints->tree);
+	free(hints);
+}
+
+/** add hint to delegation hints */
+static int
+ah(struct delegpt* dp, struct regional* r, const char* sv, const char* ip)
+{
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	ldns_rdf* rdf = ldns_dname_new_frm_str(sv);
+	if(!rdf) {
+		log_err("could not parse %s", sv);
+		return 0;
+	}
+	if(!delegpt_add_ns(dp, r, ldns_rdf_data(rdf)) ||
+	   !extstrtoaddr(ip, &addr, &addrlen) ||
+	   !delegpt_add_target(dp, r, ldns_rdf_data(rdf), ldns_rdf_size(rdf),
+		&addr, addrlen)) {
+		ldns_rdf_deep_free(rdf);
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** obtain compiletime provided root hints */
+static struct delegpt* 
+compile_time_root_prime(struct regional* r)
+{
+	/* from:
+	 ;       This file is made available by InterNIC
+	 ;       under anonymous FTP as
+	 ;           file                /domain/named.cache
+	 ;           on server           FTP.INTERNIC.NET
+	 ;       -OR-                    RS.INTERNIC.NET
+	 ;
+	 ;       last update:    Jan 29, 2004
+	 ;       related version of root zone:   2004012900
+	 */
+	struct delegpt* dp = delegpt_create(r);
+	if(!dp)
+		return NULL;
+	if(!delegpt_set_name(dp, r, (uint8_t*)"\000"))
+		return NULL;
+	if(!ah(dp, r, "A.ROOT-SERVERS.NET.", "198.41.0.4"))	return 0;
+	if(!ah(dp, r, "B.ROOT-SERVERS.NET.", "192.228.79.201")) return 0;
+	if(!ah(dp, r, "C.ROOT-SERVERS.NET.", "192.33.4.12"))	return 0;
+	if(!ah(dp, r, "D.ROOT-SERVERS.NET.", "128.8.10.90"))	return 0;
+	if(!ah(dp, r, "E.ROOT-SERVERS.NET.", "192.203.230.10")) return 0;
+	if(!ah(dp, r, "F.ROOT-SERVERS.NET.", "192.5.5.241"))	return 0;
+	if(!ah(dp, r, "G.ROOT-SERVERS.NET.", "192.112.36.4"))	return 0;
+	if(!ah(dp, r, "H.ROOT-SERVERS.NET.", "128.63.2.53"))	return 0;
+	if(!ah(dp, r, "I.ROOT-SERVERS.NET.", "192.36.148.17"))	return 0;
+	if(!ah(dp, r, "J.ROOT-SERVERS.NET.", "192.58.128.30"))	return 0;
+	if(!ah(dp, r, "K.ROOT-SERVERS.NET.", "193.0.14.129"))	return 0;
+	if(!ah(dp, r, "L.ROOT-SERVERS.NET.", "199.7.83.42"))	return 0;
+	if(!ah(dp, r, "M.ROOT-SERVERS.NET.", "202.12.27.33"))	return 0;
+	return dp;
+}
+
+/** insert new hint info into hint structure */
+static int
+hints_insert(struct iter_hints* hints, uint16_t c, struct delegpt* dp)
+{
+	struct iter_hints_stub* node = regional_alloc(hints->region,
+		sizeof(struct iter_hints_stub));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	node->hint_class = c;
+	node->name = regional_alloc_init(hints->region, dp->name, dp->namelen);
+	if(!node->name)
+		return 0;
+	node->namelen = dp->namelen;
+	node->namelabs = dp->namelabs;
+	node->dp = dp;
+	if(!rbtree_insert(hints->tree, &node->node)) {
+		log_err("second hints ignored.");
+	}
+	return 1;
+}
+
+/** initialise parent pointers in the tree */
+static void
+init_parents(struct iter_hints* hints)
+{
+	struct iter_hints_stub* node, *prev = NULL, *p;
+	int m;
+	RBTREE_FOR(node, struct iter_hints_stub*, hints->tree) {
+		node->parent = NULL;
+		if(!prev || prev->hint_class != node->hint_class) {
+			prev = node;
+			continue;
+		}
+		(void)dname_lab_cmp(prev->name, prev->namelabs, node->name,
+			node->namelabs, &m); /* we know prev is smaller */
+		/* sort order like: . com. bla.com. zwb.com. net. */
+		/* find the previous, or parent-parent-parent */
+		for(p = prev; p; p = p->parent)
+			/* looking for name with few labels, a parent */
+			if(p->namelabs <= m) {
+				/* ==: since prev matched m, this is closest*/
+				/* <: prev matches more, but is not a parent,
+				 * this one is a (grand)parent */
+				node->parent = p;
+				break;
+			}
+		prev = node;
+	}
+}
+
+/** set stub name */
+static int 
+read_stubs_name(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	ldns_rdf* rdf;
+	if(!s->name) {
+		log_err("stub zone without a name");
+		return 0;
+	}
+	rdf = ldns_dname_new_frm_str(s->name);
+	if(!rdf) {
+		log_err("cannot parse stub zone name %s", s->name);
+		return 0;
+	}
+	if(!delegpt_set_name(dp, hints->region, ldns_rdf_data(rdf))) {
+		ldns_rdf_deep_free(rdf);
+		log_err("out of memory");
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** set stub host names */
+static int 
+read_stubs_host(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	ldns_rdf* rdf;
+	for(p = s->hosts; p; p = p->next) {
+		log_assert(p->str);
+		rdf = ldns_dname_new_frm_str(p->str);
+		if(!rdf) {
+			log_err("cannot parse stub %s nameserver name: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_ns(dp, hints->region, ldns_rdf_data(rdf))) {
+			ldns_rdf_deep_free(rdf);
+			log_err("out of memory");
+			return 0;
+		}
+		ldns_rdf_deep_free(rdf);
+	}
+	return 1;
+}
+
+/** set stub server addresses */
+static int 
+read_stubs_addr(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	for(p = s->addrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!extstrtoaddr(p->str, &addr, &addrlen)) {
+			log_err("cannot parse stub %s ip address: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_addr(dp, hints->region, &addr, addrlen)) {
+			log_err("out of memory");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/** read stubs config */
+static int 
+read_stubs(struct iter_hints* hints, struct config_file* cfg)
+{
+	struct config_stub* s;
+	for(s = cfg->stubs; s; s = s->next) {
+		struct delegpt* dp = delegpt_create(hints->region);
+		if(!dp) {
+			log_err("out of memory");
+			return 0;
+		}
+		if(!read_stubs_name(hints, s, dp) ||
+			!read_stubs_host(hints, s, dp) ||
+			!read_stubs_addr(hints, s, dp))
+			return 0;
+		if(!hints_insert(hints, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		delegpt_log(VERB_DETAIL, dp);
+	}
+	return 1;
+}
+
+/** read root hints from file */
+static int 
+read_root_hints(struct iter_hints* hints, char* fname)
+{
+	int lineno = 0;
+	uint32_t default_ttl = 0;
+	ldns_rdf* origin = NULL;
+	ldns_rdf* prev_rr = NULL;
+	struct delegpt* dp;
+	ldns_rr* rr = NULL;
+	ldns_status status;
+	uint16_t c = LDNS_RR_CLASS_IN;
+	FILE* f = fopen(fname, "r");
+	if(!f) {
+		log_err("could not read root hints %s: %s",
+			fname, strerror(errno));
+		return 0;
+	}
+	dp = delegpt_create(hints->region);
+	if(!dp) {
+		log_err("out of memory reading root hints");
+		fclose(f);
+		return 0;
+	}
+	verbose(VERB_DETAIL, "Reading root hints from %s", fname);
+	while(!feof(f)) {
+		status = ldns_rr_new_frm_fp_l(&rr, f, 
+			&default_ttl, &origin, &prev_rr, &lineno);
+		if(status == LDNS_STATUS_SYNTAX_EMPTY ||
+			status == LDNS_STATUS_SYNTAX_TTL ||
+			status == LDNS_STATUS_SYNTAX_ORIGIN)
+			continue;
+		if(status != LDNS_STATUS_OK) {
+			log_err("reading root hints %s %d: %s", fname,
+				lineno, ldns_get_errorstr_by_id(status));
+			fclose(f);
+			return 0;
+		}
+		if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_NS) {
+			if(!delegpt_add_ns(dp, hints->region,
+				ldns_rdf_data(ldns_rr_rdf(rr, 0)))) {
+				log_err("out of memory reading root hints");
+				fclose(f);
+				return 0;
+			}
+			c = ldns_rr_get_class(rr);
+			if(!dp->name) {
+				if(!delegpt_set_name(dp, hints->region, 
+					ldns_rdf_data(ldns_rr_owner(rr)))){
+					log_err("out of memory.");
+					fclose(f);
+					return 0;
+				}
+			}
+		} else if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_A) {
+			struct sockaddr_in sa;
+			socklen_t len = (socklen_t)sizeof(sa);
+			memset(&sa, 0, len);
+			sa.sin_family = AF_INET;
+			sa.sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+			memmove(&sa.sin_addr, 
+				ldns_rdf_data(ldns_rr_rdf(rr, 0)), INET_SIZE);
+			if(!delegpt_add_target(dp, hints->region,
+					ldns_rdf_data(ldns_rr_owner(rr)),
+					ldns_rdf_size(ldns_rr_owner(rr)),
+					(struct sockaddr_storage*)&sa, len)) {
+				log_err("out of memory reading root hints");
+				fclose(f);
+				return 0;
+			}
+		} else if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_AAAA) {
+			struct sockaddr_in6 sa;
+			socklen_t len = (socklen_t)sizeof(sa);
+			memset(&sa, 0, len);
+			sa.sin6_family = AF_INET6;
+			sa.sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+			memmove(&sa.sin6_addr, 
+				ldns_rdf_data(ldns_rr_rdf(rr, 0)), INET6_SIZE);
+			if(!delegpt_add_target(dp, hints->region,
+					ldns_rdf_data(ldns_rr_owner(rr)),
+					ldns_rdf_size(ldns_rr_owner(rr)),
+					(struct sockaddr_storage*)&sa, len)) {
+				log_err("out of memory reading root hints");
+				fclose(f);
+				return 0;
+			}
+		} else {
+			log_warn("root hints %s:%d skipping type %d",
+				fname, lineno, ldns_rr_get_type(rr));
+		}
+
+		ldns_rr_free(rr);
+	}
+	fclose(f);
+	if(!dp->name) {
+		log_warn("root hints %s: no NS content", fname);
+		return 1;
+	}
+	if(!hints_insert(hints, c, dp)) {
+		return 0;
+	}
+	delegpt_log(VERB_DETAIL, dp);
+	return 1;
+}
+
+/** read root hints list */
+static int 
+read_root_hints_list(struct iter_hints* hints, struct config_file* cfg)
+{
+	struct config_strlist* p;
+	for(p = cfg->root_hints; p; p = p->next) {
+		log_assert(p->str);
+		if(p->str && p->str[0]) {
+			if(!read_root_hints(hints, p->str))
+				return 0;
+		}
+	}
+	return 1;
+}
+
+int 
+hints_apply_cfg(struct iter_hints* hints, struct config_file* cfg)
+{
+	free(hints->tree);
+	hints->tree = rbtree_create(stub_cmp);
+	if(!hints->tree)
+		return 0;
+	
+	/* read root hints */
+	if(!read_root_hints_list(hints, cfg))
+		return 0;
+
+	/* read stub hints */
+	if(!read_stubs(hints, cfg))
+		return 0;
+
+	/* use fallback compiletime root hints */
+	if(!hints_lookup_root(hints, LDNS_RR_CLASS_IN)) {
+		struct delegpt* dp = compile_time_root_prime(hints->region);
+		verbose(VERB_ALGO, "no config, using builtin root hints.");
+		if(!dp) 
+			return 0;
+		if(!hints_insert(hints, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		delegpt_log(VERB_DETAIL, dp);
+	}
+
+	init_parents(hints);
+	return 1;
+}
+
+struct delegpt* 
+hints_lookup_root(struct iter_hints* hints, uint16_t qclass)
+{
+	uint8_t rootlab = 0;
+	struct iter_hints_stub key, *stub;
+	key.node.key = &key;
+	key.hint_class = qclass;
+	key.name = &rootlab;
+	key.namelen = 1;
+	key.namelabs = 1;
+	stub = (struct iter_hints_stub*)rbtree_search(hints->tree, &key);
+	if(!stub)
+		return NULL;
+	return stub->dp;
+}
+
+struct delegpt* 
+hints_lookup_stub(struct iter_hints* hints, uint8_t* qname, 
+	uint16_t qclass, struct delegpt* cache_dp)
+{
+	/* first lookup the stub */
+	rbnode_t* res = NULL;
+	struct iter_hints_stub *result;
+	struct iter_hints_stub key;
+	key.node.key = &key;
+	key.hint_class = qclass;
+	key.name = qname;
+	key.namelabs = dname_count_size_labels(qname, &key.namelen);
+	if(rbtree_find_less_equal(hints->tree, &key, &res)) {
+		/* exact */
+		result = (struct iter_hints_stub*)res;
+	} else {
+		/* smaller element (or no element) */
+		int m;
+		result = (struct iter_hints_stub*)res;
+		if(!result || result->hint_class != qclass)
+			return NULL;
+		/* count number of labels matched */
+		(void)dname_lab_cmp(result->name, result->namelabs, key.name,
+			key.namelabs, &m);
+		while(result) { /* go up until qname is subdomain of stub */
+			if(result->namelabs <= m)
+				break;
+			result = result->parent;
+		}
+		if(!result)
+			return NULL;
+	}
+	/* 
+	 * If our cached delegation point is above the hint, we need to prime.
+	 */
+	if(dname_strict_subdomain(result->dp->name, result->dp->namelabs,
+		cache_dp->name, cache_dp->namelabs))
+		return result->dp; /* need to prime this stub */
+	return NULL;
+}
+
+size_t 
+hints_get_mem(struct iter_hints* hints)
+{
+	if(!hints) return 0;
+	return sizeof(*hints) + regional_get_mem(hints->region);
+}

iterator/iter_hints.h

@@ -0,0 +1,140 @@
+/*
+ * iterator/iter_hints.h - iterative resolver module stub and root hints.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of stub and root hints, and read those from config.
+ */
+
+#ifndef ITERATOR_ITER_HINTS_H
+#define ITERATOR_ITER_HINTS_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct delegpt;
+struct regional;
+
+/**
+ * Iterator hints structure
+ */
+struct iter_hints {
+	/** regional where hints are allocated */
+	struct regional* region;
+	/** 
+	 * Hints are stored in this tree. Sort order is specially chosen.
+	 * first sorted on qtype. Then on dname in nsec-like order, so that
+	 * a lookup on class, name will return an exact match or the closest
+	 * match which gives the ancestor needed.
+	 * contents of type iter_hints_stub. The class IN root is in here.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator hints for a particular stub.
+ */
+struct iter_hints_stub {
+	/** redblacktree node, key is this structure: class and name */
+	rbnode_t node;
+	/** name */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** number of labels in name */
+	int namelabs;
+	/** delegation point with hint information for this stub. */
+	struct delegpt* dp;
+	/** pointer to parent in stub hint tree (or NULL if none) */
+	struct iter_hints_stub* parent;
+	/** class of hints. host order. */
+	uint16_t hint_class;
+};
+
+/**
+ * Create hints 
+ * @return new hints or NULL on error.
+ */
+struct iter_hints* hints_create();
+
+/**
+ * Delete hints.
+ * @param hints: to delete.
+ */
+void hints_delete(struct iter_hints* hints);
+
+/**
+ * Process hints config. Sets default values for root hints if no config.
+ * @param hints: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int hints_apply_cfg(struct iter_hints* hints, struct config_file* cfg);
+
+/**
+ * Find root hints for the given class.
+ * @param hints: hint storage.
+ * @param qclass: class for which root hints are requested. host order.
+ * @return: NULL if no hints, or a ptr to stored hints.
+ */
+struct delegpt* hints_lookup_root(struct iter_hints* hints, uint16_t qclass);
+
+/**
+ * Given a qname/qclass combination, and the delegation point from the cache
+ * for this qname/qclass, determine if this combination indicates that a
+ * stub hint exists and must be primed.
+ *
+ * @param hints: hint storage.
+ * @param qname: The qname that generated the delegation point.
+ * @param qclass: The qclass that generated the delegation point.
+ * @param dp: The cache generated delegation point.
+ * @return: A priming delegation point if there is a stub hint that must
+ *         be primed, otherwise null.
+ */
+struct delegpt* hints_lookup_stub(struct iter_hints* hints, 
+	uint8_t* qname, uint16_t qclass, struct delegpt* dp);
+
+/**
+ * Get memory in use by hints
+ * @param hints: hint storage.
+ * @return bytes in use
+ */
+size_t hints_get_mem(struct iter_hints* hints);
+
+/** compare two hint entries */
+int stub_cmp(const void* k1, const void* k2);
+
+#endif /* ITERATOR_ITER_HINTS_H */

iterator/iter_resptype.c

@@ -0,0 +1,252 @@
+/*
+ * iterator/iter_resptype.c - response type information and classification.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file defines the response type. DNS Responses can be classified as
+ * one of the response types.
+ */
+#include "config.h"
+#include "iterator/iter_resptype.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+enum response_type 
+response_type_from_cache(struct dns_msg* msg, 
+	struct query_info* request)
+{
+	/* If the message is NXDOMAIN, then it is an ANSWER. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_NXDOMAIN)
+		return RESPONSE_TYPE_ANSWER;
+	
+	/* First we look at the answer section. This can tell us if this is
+	 * CNAME or positive ANSWER. */
+	if(msg->rep->an_numrrsets > 0) {
+		/* Now look at the answer section first. 3 states: 
+		 *	o our answer is there directly,
+		 *	o our answer is there after a cname,
+		 *	o or there is just a cname. */
+		uint8_t* mname = request->qname;
+		size_t mname_len = request->qname_len;
+		size_t i;
+		for(i=0; i<msg->rep->an_numrrsets; i++) {
+			struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+
+			/* If we have encountered an answer (before or 
+			 * after a CNAME), then we are done! Note that 
+			 * if qtype == CNAME then this will be noted as 
+			 * an ANSWER before it gets treated as a CNAME, 
+			 * as it should */
+			if(ntohs(s->rk.type) == request->qtype &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				return RESPONSE_TYPE_ANSWER;
+			}
+
+			/* If we have encountered a CNAME, make sure that 
+			 * it is relevant. */
+			if(ntohs(s->rk.type) == LDNS_RR_TYPE_CNAME &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				get_cname_target(s, &mname, &mname_len);
+			}
+		}
+
+		/* if we encountered a CNAME (or a bunch of CNAMEs), and 
+		 * still got to here, then it is a CNAME response. (i.e., 
+		 * the CNAME chain didn't terminate in an answer rrset.) */
+		if(mname != request->qname) {
+			return RESPONSE_TYPE_CNAME;
+		}
+	}
+
+	/* At this point, since we don't need to detect REFERRAL or LAME 
+	 * messages, it can only be an ANSWER. */
+	return RESPONSE_TYPE_ANSWER;
+}
+
+enum response_type 
+response_type_from_server(int rdset,
+	struct dns_msg* msg, struct query_info* request, struct delegpt* dp)
+{
+	uint8_t* origzone = (uint8_t*)"\000"; /* the default */
+	size_t origzonelen = 1;
+	size_t i;
+
+	if(!msg || !request)
+		return RESPONSE_TYPE_THROWAWAY;
+	
+	/* If the message is NXDOMAIN, then it answers the question. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_NXDOMAIN) {
+		/* make sure its not recursive when we don't want it to */
+		if( (msg->rep->flags&BIT_RA) &&
+			!(msg->rep->flags&BIT_AA) && !rdset)
+				return RESPONSE_TYPE_LAME;
+		return RESPONSE_TYPE_ANSWER;
+	}
+	
+	/* Other response codes mean (so far) to throw the response away as
+	 * meaningless and move on to the next nameserver. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) != LDNS_RCODE_NOERROR)
+		return RESPONSE_TYPE_THROWAWAY;
+
+	/* Note: TC bit has already been handled */
+
+	if(dp) {
+		origzone = dp->name;
+		origzonelen = dp->namelen;
+	}
+
+	/* First we look at the answer section. This can tell us if this is a
+	 * CNAME or ANSWER or (provisional) ANSWER. */
+	if(msg->rep->an_numrrsets > 0) {
+		uint8_t* mname = request->qname;
+		size_t mname_len = request->qname_len;
+
+		/* Now look at the answer section first. 3 states: our 
+		 * answer is there directly, our answer is there after 
+		 * a cname, or there is just a cname. */
+		for(i=0; i<msg->rep->an_numrrsets; i++) {
+			struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+			
+			/* if the answer section has NS rrset, and qtype ANY 
+		 	 * and the delegation is lower, and no CNAMEs followed,
+		 	 * this is a referral where the NS went to AN section */
+			if((request->qtype == LDNS_RR_TYPE_ANY ||
+				request->qtype == LDNS_RR_TYPE_NS) &&
+				ntohs(s->rk.type) == LDNS_RR_TYPE_NS &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				dname_strict_subdomain_c(s->rk.dname, 
+				origzone)) {
+				return RESPONSE_TYPE_REFERRAL;
+			}
+
+			/* If we have encountered an answer (before or 
+			 * after a CNAME), then we are done! Note that 
+			 * if qtype == CNAME then this will be noted as an
+			 * ANSWER before it gets treated as a CNAME, as 
+			 * it should. */
+			if(ntohs(s->rk.type) == request->qtype &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				if((msg->rep->flags&BIT_AA))
+					return RESPONSE_TYPE_ANSWER;
+				/* If the AA bit isn't on, and we've seen 
+				 * the answer, we only provisionally say 
+				 * 'ANSWER' -- it very well could be a 
+				 * REFERRAL. */
+				break;
+			}
+
+			/* If we have encountered a CNAME, make sure that 
+			 * it is relevant. */
+			if(ntohs(s->rk.type) == LDNS_RR_TYPE_CNAME &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				get_cname_target(s, &mname, &mname_len);
+			}
+		}
+		/* if we encountered a CNAME (or a bunch of CNAMEs), and 
+		 * still got to here, then it is a CNAME response. 
+		 * (This is regardless of the AA bit at this point) */
+		if(mname != request->qname) {
+			return RESPONSE_TYPE_CNAME;
+		}
+	}
+
+	/* Looking at the authority section, we just look and see if 
+	 * there is a delegation NS set, turning it into a delegation. 
+	 * Otherwise, we will have to conclude ANSWER (either it is 
+	 * NOERROR/NODATA, or an non-authoritative answer). */
+	for(i = msg->rep->an_numrrsets; i < (msg->rep->an_numrrsets +
+		msg->rep->ns_numrrsets); i++) {
+		struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+
+		/* The normal way of detecting NOERROR/NODATA. */
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_SOA &&
+			dname_subdomain_c(request->qname, s->rk.dname)) {
+			/* we do our own recursion, thank you */
+			if( (msg->rep->flags&BIT_RA) &&
+				!(msg->rep->flags&BIT_AA) && !rdset)
+				return RESPONSE_TYPE_LAME;
+			return RESPONSE_TYPE_ANSWER;
+		}
+
+		/* Detect REFERRAL/LAME/ANSWER based on the relationship 
+		 * of the NS set to the originating zone name. */
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_NS) {
+			/* If we are getting an NS set for the zone we 
+			 * thought we were contacting, then it is an answer.*/
+			/* FIXME: is this correct? */
+			if(query_dname_compare(s->rk.dname, origzone) == 0) {
+				/* see if mistakenly a recursive server was
+				 * deployed and is responding nonAA */
+				if( (msg->rep->flags&BIT_RA) &&
+					!(msg->rep->flags&BIT_AA) && !rdset)
+					return RESPONSE_TYPE_LAME;
+				return RESPONSE_TYPE_ANSWER;
+			}
+			/* If we are getting a referral upwards (or to 
+			 * the same zone), then the server is 'lame'. */
+			if(dname_subdomain_c(origzone, s->rk.dname)) {
+				return RESPONSE_TYPE_LAME;
+			}
+			/* If the NS set is below the delegation point we 
+			 * are on, and it is non-authoritative, then it is 
+			 * a referral, otherwise it is an answer. */
+			if(dname_subdomain_c(s->rk.dname, origzone)) {
+				/* NOTE: I no longer remember in what case 
+				 * we would like this to be an answer. 
+				 * NODATA should have a SOA or nothing, 
+				 * not an NS rrset. 
+				 * True, referrals should not have the AA 
+				 * bit set, but... */
+				 
+				/* if((msg->rep->flags&BIT_AA))
+					return RESPONSE_TYPE_ANSWER; */
+				return RESPONSE_TYPE_REFERRAL;
+			}
+			/* Otherwise, the NS set is irrelevant. */
+		}
+	}
+
+	/* If we've gotten this far, this is NOERROR/NODATA (which could 
+	 * be an entirely empty message) */
+	/* check if recursive answer; saying it has empty cache */
+	if( (msg->rep->flags&BIT_RA) && !(msg->rep->flags&BIT_AA) && !rdset)
+		return RESPONSE_TYPE_LAME;
+	return RESPONSE_TYPE_ANSWER;
+}

iterator/iter_resptype.h

@@ -0,0 +1,120 @@
+/*
+ * iterator/iter_resptype.h - response type information and classification.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file defines the response type. DNS Responses can be classified as
+ * one of the response types.
+ */
+
+#ifndef ITERATOR_ITER_RESPTYPE_H
+#define ITERATOR_ITER_RESPTYPE_H
+struct dns_msg;
+struct query_info;
+struct delegpt;
+
+/**
+ * The response type is used to interpret the response.
+ */
+enum response_type {
+	/** 
+	 * 'untyped' means that the type of this response hasn't been 
+	 * assigned. 
+	 */
+	RESPONSE_TYPE_UNTYPED   = 0,
+
+	/** 
+	 * 'answer' means that the response terminates the resolution 
+	 * process. 
+	 */
+	RESPONSE_TYPE_ANSWER,
+
+	/** 'delegation' means that the response is a delegation. */
+	RESPONSE_TYPE_REFERRAL,
+
+	/**
+	 * 'cname' means that the response is a cname without the final 
+	 * answer, and thus must be restarted.
+	 */
+	RESPONSE_TYPE_CNAME,
+
+	/**
+	 * 'throwaway' means that this particular response should be 
+	 * discarded and the next nameserver should be contacted
+	 */
+	RESPONSE_TYPE_THROWAWAY,
+
+	/**
+	 * 'lame' means that this particular response indicates that 
+	 * the nameserver knew nothing about the question.
+	 */
+	RESPONSE_TYPE_LAME
+};
+
+/**
+ * Classifies a response message from cache based on the current request.
+ * Note that this routine assumes that THROWAWAY or LAME responses will not
+ * occur. Also, it will not detect REFERRAL type messages, since those are
+ * (currently) automatically classified based on how they came from the
+ * cache (findDelegation() instead of lookup()).
+ *
+ * @param msg: the message from the cache.
+ * @param request: the request that generated the response.
+ * @return the response type (CNAME or ANSWER).
+ */
+enum response_type response_type_from_cache(struct dns_msg* msg, 
+	struct query_info* request);
+
+/**
+ * Classifies a response message (from the wire) based on the current
+ * request.
+ *
+ * NOTE: currently this routine uses the AA bit in the response to help
+ * distinguish between some non-standard referrals and answers. It also
+ * relies somewhat on the originating zone to be accurate (for lameness
+ * detection, mostly).
+ *
+ * @param rdset: if RD bit was sent in query sent by unbound.
+ * @param msg: the message from the cache.
+ * @param request: the request that generated the response.
+ * @param dp: The delegation point that was being queried
+ *          when the response was returned.
+ * @return the response type (CNAME or ANSWER).
+ */
+enum response_type response_type_from_server(int rdset, 
+	struct dns_msg* msg, struct query_info* request, struct delegpt* dp);
+
+#endif /* ITERATOR_ITER_RESPTYPE_H */

iterator/iter_scrub.c

@@ -0,0 +1,575 @@
+/*
+ * iterator/iter_scrub.c - scrubbing, normalization, sanitization of DNS msgs.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file has routine(s) for cleaning up incoming DNS messages from 
+ * possible useless or malicious junk in it.
+ */
+#include "config.h"
+#include "iterator/iter_scrub.h"
+#include "services/cache/rrset.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/regional.h"
+#include "util/config_file.h"
+#include "util/module.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/data/msgreply.h"
+#include "util/alloc.h"
+
+/** RRset flag used during scrubbing. The RRset is OK. */
+#define RRSET_SCRUB_OK	0x80
+
+/** remove rrset, update loop variables */
+static void
+remove_rrset(const char* str, ldns_buffer* pkt, struct msg_parse* msg, 
+	struct rrset_parse* prev, struct rrset_parse** rrset)
+{
+	if(verbosity >= VERB_DETAIL 
+		&& (*rrset)->dname_len <= LDNS_MAX_DOMAINLEN) {
+		uint8_t buf[LDNS_MAX_DOMAINLEN+1];
+		dname_pkt_copy(pkt, buf, (*rrset)->dname);
+		log_nametypeclass(VERB_DETAIL, str, buf, 
+			(*rrset)->type, ntohs((*rrset)->rrset_class));
+	}
+	if(prev)
+		prev->rrset_all_next = (*rrset)->rrset_all_next;
+	else	msg->rrset_first = (*rrset)->rrset_all_next;
+	if(msg->rrset_last == *rrset)
+		msg->rrset_last = prev;
+	msg->rrset_count --;
+	switch((*rrset)->section) {
+		case LDNS_SECTION_ANSWER: msg->an_rrsets--; break;
+		case LDNS_SECTION_AUTHORITY: msg->ns_rrsets--; break;
+		case LDNS_SECTION_ADDITIONAL: msg->ar_rrsets--; break;
+		default: log_assert(0);
+	}
+	msgparse_bucket_remove(msg, *rrset);
+	*rrset = (*rrset)->rrset_all_next;
+}
+
+/** return true if rr type has additional names in it */
+static int
+has_additional(uint16_t t)
+{
+	switch(t) {
+		case LDNS_RR_TYPE_MB:
+		case LDNS_RR_TYPE_MD:
+		case LDNS_RR_TYPE_MF:
+		case LDNS_RR_TYPE_NS:
+		case LDNS_RR_TYPE_MX:
+		case LDNS_RR_TYPE_KX:
+		case LDNS_RR_TYPE_SRV:
+			return 1;
+		case LDNS_RR_TYPE_NAPTR:
+			/* TODO: NAPTR not supported, glue stripped off */
+			return 0;
+		default:
+			return 0;
+	}
+	return 0;
+}
+
+/** get additional name from rrset RR, return false if no name present */
+static int
+get_additional_name(struct rrset_parse* rrset, struct rr_parse* rr, 
+	uint8_t** nm, size_t* nmlen, ldns_buffer* pkt) 
+{
+	size_t offset = 0;
+	size_t len, oldpos;
+	switch(rrset->type) {
+		case LDNS_RR_TYPE_MB:
+		case LDNS_RR_TYPE_MD:
+		case LDNS_RR_TYPE_MF:
+		case LDNS_RR_TYPE_NS:
+			offset = 0;
+			break;
+		case LDNS_RR_TYPE_MX:
+		case LDNS_RR_TYPE_KX:
+			offset = 2;
+			break;
+		case LDNS_RR_TYPE_SRV:
+			offset = 6;
+			break;
+		case LDNS_RR_TYPE_NAPTR:
+			/* TODO: NAPTR not supported, glue stripped off */
+			return 0;
+		default:
+			return 0;
+	}
+	len = ldns_read_uint16(rr->ttl_data+sizeof(uint32_t));
+	if(len < offset+1)
+		return 0; /* rdata field too small */
+	*nm = rr->ttl_data+sizeof(uint32_t)+sizeof(uint16_t)+offset;
+	oldpos = ldns_buffer_position(pkt);
+	ldns_buffer_set_position(pkt, (size_t)(*nm - ldns_buffer_begin(pkt)));
+	*nmlen = pkt_dname_len(pkt);
+	ldns_buffer_set_position(pkt, oldpos);
+	if(*nmlen == 0)
+		return 0;
+	return 1;
+}
+
+/** Place mark on rrsets in additional section they are OK */
+static void
+mark_additional_rrset(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct rrset_parse* rrset)
+{
+	/* Mark A and AAAA for NS as appropriate additional section info. */
+	uint8_t* nm = NULL;
+	size_t nmlen = 0;
+	struct rr_parse* rr;
+
+	if(!has_additional(rrset->type))
+		return;
+	for(rr = rrset->rr_first; rr; rr = rr->next) {
+		if(get_additional_name(rrset, rr, &nm, &nmlen, pkt)) {
+			/* mark A */
+			hashvalue_t h = pkt_hash_rrset(pkt, nm, LDNS_RR_TYPE_A, 
+				rrset->rrset_class, 0);
+			struct rrset_parse* r = msgparse_hashtable_lookup(
+				msg, pkt, h, 0, nm, nmlen, 
+				LDNS_RR_TYPE_A, rrset->rrset_class);
+			if(r && r->section == LDNS_SECTION_ADDITIONAL) {
+				r->flags |= RRSET_SCRUB_OK;
+			}
+			
+			/* mark AAAA */
+			h = pkt_hash_rrset(pkt, nm, LDNS_RR_TYPE_AAAA, 
+				rrset->rrset_class, 0);
+			r = msgparse_hashtable_lookup(msg, pkt, h, 0, nm, 
+				nmlen, LDNS_RR_TYPE_AAAA, rrset->rrset_class);
+			if(r && r->section == LDNS_SECTION_ADDITIONAL) {
+				r->flags |= RRSET_SCRUB_OK;
+			}
+		}
+	}
+}
+
+/** Get target name of a CNAME */
+static int
+parse_get_cname_target(struct rrset_parse* rrset, uint8_t** sname, 
+	size_t* snamelen)
+{
+	if(rrset->rr_count != 1) {
+		verbose(VERB_ALGO, "Found CNAME rrset with "
+			"size > 1: %u", (unsigned)rrset->rr_count);
+		return 0;
+	}
+	if(rrset->rr_first->size < sizeof(uint16_t)+1)
+		return 0; /* CNAME rdata too small */
+	*sname = rrset->rr_first->ttl_data + sizeof(uint32_t)
+		+ sizeof(uint16_t); /* skip ttl, rdatalen */
+	*snamelen = rrset->rr_first->size - sizeof(uint16_t);
+	return 1;
+}
+
+/** Synthesize CNAME from DNAME, false if too long */
+static int 
+synth_cname(uint8_t* qname, size_t qnamelen, struct rrset_parse* dname_rrset, 
+	uint8_t* alias, size_t* aliaslen, ldns_buffer* pkt)
+{
+	/* we already know that sname is a strict subdomain of DNAME owner */
+	uint8_t* dtarg = NULL;
+	size_t dtarglen;
+	if(!parse_get_cname_target(dname_rrset, &dtarg, &dtarglen))
+		return 0; 
+	log_assert(qnamelen > dname_rrset->dname_len);
+	/* DNAME from com. to net. with qname example.com. -> example.net. */
+	/* so: \3com\0 to \3net\0 and qname \7example\3com\0 */
+	*aliaslen = qnamelen + dtarglen - dname_rrset->dname_len;
+	if(*aliaslen > LDNS_MAX_DOMAINLEN)
+		return 0; /* should have been RCODE YXDOMAIN */
+	/* decompress dnames into buffer, we know it fits */
+	dname_pkt_copy(pkt, alias, qname);
+	dname_pkt_copy(pkt, alias+(qnamelen-dname_rrset->dname_len), dtarg);
+	return 1;
+}
+
+/** synthesize a CNAME rrset */
+static struct rrset_parse*
+synth_cname_rrset(uint8_t** sname, size_t* snamelen, uint8_t* alias, 
+	size_t aliaslen, struct regional* region, struct msg_parse* msg, 
+	struct rrset_parse* rrset, struct rrset_parse* prev,
+	struct rrset_parse* nx, ldns_buffer* pkt)
+{
+	struct rrset_parse* cn = (struct rrset_parse*)regional_alloc(region,
+		sizeof(struct rrset_parse));
+	if(!cn)
+		return NULL;
+	memset(cn, 0, sizeof(*cn));
+	cn->rr_first = (struct rr_parse*)regional_alloc(region, 
+		sizeof(struct rr_parse));
+	if(!cn->rr_first)
+		return NULL;
+	cn->rr_last = cn->rr_first;
+	/* CNAME from sname to alias */
+	cn->dname = (uint8_t*)regional_alloc(region, *snamelen);
+	if(!cn->dname)
+		return NULL;
+	dname_pkt_copy(pkt, cn->dname, *sname);
+	cn->dname_len = *snamelen;
+	cn->type = LDNS_RR_TYPE_CNAME;
+	cn->section = rrset->section;
+	cn->rrset_class = rrset->rrset_class;
+	cn->rr_count = 1;
+	cn->size = sizeof(uint16_t) + aliaslen;
+	cn->hash=pkt_hash_rrset(pkt, cn->dname, cn->type, cn->rrset_class, 0);
+	/* allocate TTL + rdatalen + uncompressed dname */
+	memset(cn->rr_first, 0, sizeof(struct rr_parse));
+	cn->rr_first->outside_packet = 1;
+	cn->rr_first->ttl_data = (uint8_t*)regional_alloc(region, 
+		sizeof(uint32_t)+sizeof(uint16_t)+aliaslen);
+	if(!cn->rr_first->ttl_data)
+		return NULL;
+	ldns_write_uint32(cn->rr_first->ttl_data, 0); /* TTL = 0 */
+	ldns_write_uint16(cn->rr_first->ttl_data+4, aliaslen);
+	memmove(cn->rr_first->ttl_data+6, alias, aliaslen);
+	cn->rr_first->size = sizeof(uint16_t)+aliaslen;
+
+	/* link it in */
+	cn->rrset_all_next = nx;
+	if(prev)
+		prev->rrset_all_next = cn;
+	else	msg->rrset_first = cn;
+	if(nx == NULL)
+		msg->rrset_last = cn;
+	msg->rrset_count ++;
+	msg->an_rrsets++;
+	/* it is not inserted in the msg hashtable. */
+
+	*sname = cn->rr_first->ttl_data + sizeof(uint32_t)+sizeof(uint16_t);
+	*snamelen = aliaslen;
+	return cn;
+}
+
+/** check if DNAME applies to a name */
+static int
+pkt_strict_sub(ldns_buffer* pkt, uint8_t* sname, uint8_t* dr)
+{
+	uint8_t buf1[LDNS_MAX_DOMAINLEN+1];
+	uint8_t buf2[LDNS_MAX_DOMAINLEN+1];
+	/* decompress names */
+	dname_pkt_copy(pkt, buf1, sname);
+	dname_pkt_copy(pkt, buf2, dr);
+	return dname_strict_subdomain_c(buf1, buf2);
+}
+
+/** check subdomain with decompression */
+static int
+pkt_sub(ldns_buffer* pkt, uint8_t* comprname, uint8_t* zone)
+{
+	uint8_t buf[LDNS_MAX_DOMAINLEN+1];
+	dname_pkt_copy(pkt, buf, comprname);
+	return dname_subdomain_c(buf, zone);
+}
+
+/**
+ * This routine normalizes a response. This includes removing "irrelevant"
+ * records from the answer and additional sections and (re)synthesizing
+ * CNAMEs from DNAMEs, if present.
+ *
+ * @param pkt: packet.
+ * @param msg: msg to normalize.
+ * @param qinfo: original query.
+ * @param region: where to allocate synthesized CNAMEs.
+ * @return 0 on error.
+ */
+static int
+scrub_normalize(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, struct regional* region)
+{
+	uint8_t* sname = qinfo->qname;
+	size_t snamelen = qinfo->qname_len;
+	struct rrset_parse* rrset, *prev;
+
+	if(FLAGS_GET_RCODE(msg->flags) != LDNS_RCODE_NOERROR &&
+		FLAGS_GET_RCODE(msg->flags) != LDNS_RCODE_NXDOMAIN)
+		return 1;
+
+	/* For the ANSWER section, remove all "irrelevant" records and add
+	 * synthesized CNAMEs from DNAMEs
+	 * This will strip out-of-order CNAMEs as well. */
+
+	/* walk through the parse packet rrset list, keep track of previous
+	 * for insert and delete ease, and examine every RRset */
+	prev = NULL;
+	rrset = msg->rrset_first;
+	while(rrset && rrset->section == LDNS_SECTION_ANSWER) {
+		if(rrset->type == LDNS_RR_TYPE_DNAME && 
+			pkt_strict_sub(pkt, sname, rrset->dname)) {
+			/* check if next rrset is correct CNAME. else,
+			 * synthesize a CNAME */
+			struct rrset_parse* nx = rrset->rrset_all_next;
+			uint8_t alias[LDNS_MAX_DOMAINLEN+1];
+			size_t aliaslen = 0;
+			if(rrset->rr_count != 1) {
+				verbose(VERB_ALGO, "Found DNAME rrset with "
+					"size > 1: %u", 
+					(unsigned)rrset->rr_count);
+				return 0;
+			}
+			if(!synth_cname(sname, snamelen, rrset, alias, 
+				&aliaslen, pkt)) {
+				verbose(VERB_ALGO, "synthesized CNAME "
+					"too long");
+				return 0;
+			}
+			if(nx && nx->type == LDNS_RR_TYPE_CNAME && 
+			   dname_pkt_compare(pkt, sname, nx->dname) == 0) {
+				/* check next cname */
+				uint8_t* t = NULL;
+				size_t tlen = 0;
+				if(!parse_get_cname_target(rrset, &t, &tlen))
+					return 0;
+				if(dname_pkt_compare(pkt, alias, t) == 0) {
+					/* it's OK and better capitalized */
+					prev = rrset;
+					rrset = nx;
+					continue;
+				}
+				/* synth ourselves */
+			}
+			/* synth a CNAME rrset */
+			prev = synth_cname_rrset(&sname, &snamelen, alias, 
+				aliaslen, region, msg, rrset, rrset, nx, pkt);
+			if(!prev) {
+				log_err("out of memory synthesizing CNAME");
+				return 0;
+			}
+			/* FIXME: resolve the conflict between synthesized 
+			 * CNAME ttls and the cache. */
+			rrset = nx;
+			continue;
+
+		}
+
+		/* The only records in the ANSWER section not allowed to */
+		if(dname_pkt_compare(pkt, sname, rrset->dname) != 0) {
+			remove_rrset("normalize: removing irrelevant RRset:", 
+				pkt, msg, prev, &rrset);
+			continue;
+		}
+
+		/* Follow the CNAME chain. */
+		if(rrset->type == LDNS_RR_TYPE_CNAME) {
+			if(!parse_get_cname_target(rrset, &sname, &snamelen))
+				return 0;
+			prev = rrset;
+			rrset = rrset->rrset_all_next;
+			continue;
+		}
+
+		/* Otherwise, make sure that the RRset matches the qtype. */
+		if(qinfo->qtype != LDNS_RR_TYPE_ANY && 
+			qinfo->qtype != rrset->type) {
+			remove_rrset("normalize: removing irrelevant RRset:", 
+				pkt, msg, prev, &rrset);
+			continue;
+		}
+
+		/* Mark the additional names from relevant rrset as OK. */
+		mark_additional_rrset(pkt, msg, rrset);
+		
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+
+	/* Mark additional names from AUTHORITY */
+	while(rrset && rrset->section == LDNS_SECTION_AUTHORITY) {
+		mark_additional_rrset(pkt, msg, rrset);
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+
+	/* For each record in the additional section, remove it if it is an
+	 * address record and not in the collection of additional names 
+	 * found in ANSWER and AUTHORITY. */
+	/* These records have not been marked OK previously */
+	while(rrset && rrset->section == LDNS_SECTION_ADDITIONAL) {
+		/* FIXME: what about other types? */
+		if(rrset->type==LDNS_RR_TYPE_A || 
+			rrset->type==LDNS_RR_TYPE_AAAA) 
+		{
+			if((rrset->flags & RRSET_SCRUB_OK)) {
+				/* remove flag to clean up flags variable */
+				rrset->flags &= ~RRSET_SCRUB_OK;
+			} else {
+				remove_rrset("normalize: removing irrelevant "
+					"RRset:", pkt, msg, prev, &rrset);
+				continue;
+			}
+		}
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+	
+	return 1;
+}
+
+/**
+ * Store potential poison in the cache (only if hardening disabled).
+ * The rrset is stored in the cache but removed from the message.
+ * So that it will be used for infrastructure purposes, but not be 
+ * returned to the client.
+ * @param pkt: packet
+ * @param msg: message parsed
+ * @param env: environment with cache
+ * @param rrset: to store.
+ */
+static void
+store_rrset(ldns_buffer* pkt, struct msg_parse* msg, struct module_env* env,
+	struct rrset_parse* rrset)
+{
+	struct ub_packed_rrset_key* k;
+	struct packed_rrset_data* d;
+	struct rrset_ref ref;
+	uint32_t now = time(NULL);
+
+	k = alloc_special_obtain(env->alloc);
+	if(!k)
+		return;
+	k->entry.data = NULL;
+	if(!parse_copy_decompress_rrset(pkt, msg, rrset, NULL, k)) {
+		alloc_special_release(env->alloc, k);
+		return;
+	}
+	d = (struct packed_rrset_data*)k->entry.data;
+	packed_rrset_ttl_add(d, now);
+	ref.key = k;
+	ref.id = k->id;
+	/*ignore ret: it was in the cache, ref updated */
+	(void)rrset_cache_update(env->rrset_cache, &ref,
+		env->alloc, now);
+}
+
+/**
+ * Given a response event, remove suspect RRsets from the response.
+ * "Suspect" rrsets are potentially poison. Note that this routine expects
+ * the response to be in a "normalized" state -- that is, all "irrelevant"
+ * RRsets have already been removed, CNAMEs are in order, etc.
+ *
+ * @param pkt: packet.
+ * @param msg: msg to normalize.
+ * @param zonename: name of server zone.
+ * @param env: module environment with config and cache.
+ * @return 0 on error.
+ */
+static int
+scrub_sanitize(ldns_buffer* pkt, struct msg_parse* msg, uint8_t* zonename,
+	struct module_env* env)
+{
+	struct rrset_parse* rrset, *prev;
+	prev = NULL;
+	rrset = msg->rrset_first;
+
+	/* At this point, we brutally remove ALL rrsets that aren't 
+	 * children of the originating zone. The idea here is that, 
+	 * as far as we know, the server that we contacted is ONLY 
+	 * authoritative for the originating zone. It, of course, MAY 
+	 * be authoriative for any other zones, and of course, MAY 
+	 * NOT be authoritative for some subdomains of the originating 
+	 * zone. */
+	while(rrset) {
+		
+		/* skip DNAME records -- they will always be followed by a 
+		 * synthesized CNAME, which will be relevant.
+		 * FIXME: should this do something differently with DNAME 
+		 * rrsets NOT in Section.ANSWER? */
+		/* But since DNAME records are also subdomains of the zone,
+		 * same check can be used */
+
+		if(!pkt_sub(pkt, rrset->dname, zonename)) {
+			if(msg->an_rrsets == 0 && 
+				rrset->type == LDNS_RR_TYPE_NS && 
+				rrset->section == LDNS_SECTION_AUTHORITY &&
+				FLAGS_GET_RCODE(msg->flags) == 
+				LDNS_RCODE_NOERROR) {
+				/* noerror, nodata and this NS rrset is above
+				 * the zone. This is LAME! 
+				 * Leave in the NS for lame classification. */
+			} else if(!env->cfg->harden_glue) {
+				/* store in cache! Since it is relevant
+				 * (from normalize) it will be picked up 
+				 * from the cache to be used later */
+				store_rrset(pkt, msg, env, rrset);
+				remove_rrset("sanitize: storing potential "
+				"poison RRset:", pkt, msg, prev, &rrset);
+				continue;
+			} else {
+				remove_rrset("sanitize: removing potential "
+				"poison RRset:", pkt, msg, prev, &rrset);
+				continue;
+			}
+		}
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+	return 1;
+}
+
+int 
+scrub_message(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, uint8_t* zonename, struct regional* region,
+	struct module_env* env)
+{
+	/* basic sanity checks */
+	log_nametypeclass(VERB_ALGO, "scrub for", zonename, LDNS_RR_TYPE_NS, 
+		qinfo->qclass);
+	if(msg->qdcount > 1)
+		return 0;
+	if( !(msg->flags&BIT_QR) )
+		return 0;
+	
+	/* if a query is echoed back, make sure it is correct. Otherwise,
+	 * this may be not a reply to our query. */
+	if(msg->qdcount == 1) {
+		if(dname_pkt_compare(pkt, msg->qname, qinfo->qname) != 0)
+			return 0;
+		if(msg->qtype != qinfo->qtype || msg->qclass != qinfo->qclass)
+			return 0;
+	}
+
+	/* normalize the response, this cleans up the additional.  */
+	if(!scrub_normalize(pkt, msg, qinfo, region))
+		return 0;
+	/* delete all out-of-zone information */
+	if(!scrub_sanitize(pkt, msg, zonename, env))
+		return 0;
+	return 1;
+}

iterator/iter_scrub.h

@@ -0,0 +1,66 @@
+/*
+ * iterator/iter_scrub.h - scrubbing, normalization, sanitization of DNS msgs.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file has routine(s) for cleaning up incoming DNS messages from 
+ * possible useless or malicious junk in it.
+ */
+
+#ifndef ITERATOR_ITER_SCRUB_H
+#define ITERATOR_ITER_SCRUB_H
+struct msg_parse;
+struct query_info;
+struct regional;
+struct module_env;
+
+/**
+ * Cleanup the passed dns message.
+ * @param pkt: the packet itself, for resolving name compression pointers.
+ *	the packet buffer is unaltered.
+ * @param msg: the parsed packet, this structure is cleaned up.
+ * @param qinfo: the query info that was sent to the server. Checked.
+ * @param zonename: the name of the last delegation point.
+ *	Used to determine out of bailiwick information.
+ * @param regional: where to allocate (new) parts of the message.
+ * @param env: module environment with config settings and cache. 
+ * @return: false if the message is total waste. true if scrubbed with success.
+ */
+int scrub_message(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, uint8_t* zonename, struct regional* regional,
+	struct module_env* env);
+
+#endif /* ITERATOR_ITER_SCRUB_H */

iterator/iter_utils.c

@@ -0,0 +1,455 @@
+/*
+ * iterator/iter_utils.c - iterative resolver module utility functions.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Configuration options. Forward zones. 
+ */
+#include "config.h"
+#include "iterator/iter_utils.h"
+#include "iterator/iterator.h"
+#include "iterator/iter_hints.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_donotq.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/infra.h"
+#include "services/cache/dns.h"
+#include "services/cache/rrset.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/regional.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/random.h"
+#include "util/fptr_wlist.h"
+#include "validator/val_anchor.h"
+
+/** fillup fetch policy array */
+static void
+fetch_fill(struct iter_env* ie, const char* str)
+{
+	char* s = (char*)str, *e;
+	int i;
+	for(i=0; i<ie->max_dependency_depth+1; i++) {
+		ie->target_fetch_policy[i] = strtol(s, &e, 10);
+		if(s == e)
+			fatal_exit("cannot parse fetch policy number %s", s);
+		s = e;
+	}
+}
+
+/** Read config string that represents the target fetch policy */
+static int
+read_fetch_policy(struct iter_env* ie, const char* str)
+{
+	int count = cfg_count_numbers(str);
+	if(count < 1) {
+		log_err("Cannot parse target fetch policy: \"%s\"", str);
+		return 0;
+	}
+	ie->max_dependency_depth = count - 1;
+	ie->target_fetch_policy = (int*)calloc(
+		(size_t)ie->max_dependency_depth+1, sizeof(int));
+	if(!ie->target_fetch_policy) {
+		log_err("alloc fetch policy: out of memory");
+		return 0;
+	}
+	fetch_fill(ie, str);
+	return 1;
+}
+
+int 
+iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
+{
+	int i;
+	/* target fetch policy */
+	if(!read_fetch_policy(iter_env, cfg->target_fetch_policy))
+		return 0;
+	for(i=0; i<iter_env->max_dependency_depth+1; i++)
+		verbose(VERB_DETAIL, "target fetch policy for level %d is %d",
+			i, iter_env->target_fetch_policy[i]);
+	
+	if(!iter_env->hints)
+		iter_env->hints = hints_create();
+	if(!iter_env->hints || !hints_apply_cfg(iter_env->hints, cfg)) {
+		log_err("Could not set root or stub hints");
+		return 0;
+	}
+	if(!iter_env->fwds)
+		iter_env->fwds = forwards_create();
+	if(!iter_env->fwds || !forwards_apply_cfg(iter_env->fwds, cfg)) {
+		log_err("Could not set forward zones");
+		return 0;
+	}
+	if(!iter_env->donotq)
+		iter_env->donotq = donotq_create();
+	if(!iter_env->donotq || !donotq_apply_cfg(iter_env->donotq, cfg)) {
+		log_err("Could not set donotqueryaddresses");
+		return 0;
+	}
+	iter_env->supports_ipv6 = cfg->do_ip6;
+	return 1;
+}
+
+/** filter out unsuitable targets, return rtt or -1 */
+static int
+iter_filter_unsuitable(struct iter_env* iter_env, struct module_env* env,
+	uint8_t* name, size_t namelen, time_t now, struct delegpt_addr* a)
+{
+	int rtt;
+	int lame;
+	int dnsseclame;
+	if(donotq_lookup(iter_env->donotq, &a->addr, a->addrlen)) {
+		return -1; /* server is on the donotquery list */
+	}
+	if(!iter_env->supports_ipv6 && addr_is_ip6(&a->addr, a->addrlen)) {
+		return -1; /* there is no ip6 available */
+	}
+	/* check lameness - need zone , class info */
+	if(infra_get_lame_rtt(env->infra_cache, &a->addr, a->addrlen, 
+		name, namelen, &lame, &dnsseclame, &rtt, now)) {
+		if(lame)
+			return -1; /* server is lame */
+		else if(rtt >= USEFUL_SERVER_TOP_TIMEOUT)
+			return -1; /* server is unresponsive */
+		else if(dnsseclame)
+			return rtt+USEFUL_SERVER_TOP_TIMEOUT; /* nonpref */
+		else	return rtt;
+	}
+	/* no server information present */
+	return UNKNOWN_SERVER_NICENESS;
+}
+
+/** filter the addres list, putting best targets at front,
+ * returns number of best targets (or 0, no suitable targets) */
+static int
+iter_filter_order(struct iter_env* iter_env, struct module_env* env,
+	uint8_t* name, size_t namelen, time_t now, struct delegpt* dp,
+	int* best_rtt)
+{
+	int got_num = 0, got_rtt = 0, thisrtt, swap_to_front;
+	struct delegpt_addr* a, *n, *prev=NULL;
+
+	a = dp->result_list;
+	while(a) {
+		/* filter out unsuitable targets */
+		thisrtt = iter_filter_unsuitable(iter_env, env, name, namelen, 
+			now, a);
+		if(thisrtt == -1) {
+			prev = a;
+			a = a->next_result;
+			continue;
+		}
+		/* classify the server address and determine what to do */
+		swap_to_front = 0;
+		if(got_num == 0) {
+			got_rtt = thisrtt;
+			got_num = 1;
+			swap_to_front = 1;
+		} else if(thisrtt == got_rtt) {
+			got_num++;
+			swap_to_front = 1;
+		} else if(thisrtt < got_rtt) {
+			got_rtt = thisrtt;
+			got_num = 1; /* start back at count of 1 */
+			swap_to_front = 1;
+		}
+		/* swap to front if necessary, or move to next result */
+		if(swap_to_front && prev) {
+			n = a->next_result;
+			prev->next_result = n;
+			a->next_result = dp->result_list;
+			dp->result_list = a;
+			a = n;
+		} else {
+			prev = a;
+			a = a->next_result;
+		}
+	}
+	*best_rtt = got_rtt;
+	return got_num;
+}
+
+struct delegpt_addr* 
+iter_server_selection(struct iter_env* iter_env, 
+	struct module_env* env, struct delegpt* dp, 
+	uint8_t* name, size_t namelen, int* dnssec_expected)
+{
+	time_t now = time(NULL);
+	int sel;
+	int selrtt;
+	struct delegpt_addr* a, *prev;
+	int num = iter_filter_order(iter_env, env, name, namelen, now, dp,
+		&selrtt);
+
+	if(num == 0)
+		return NULL;
+	if(selrtt >= USEFUL_SERVER_TOP_TIMEOUT)
+		*dnssec_expected = 0;
+	if(num == 1) {
+		a = dp->result_list;
+		if(++a->attempts < OUTBOUND_MSG_RETRY)
+			return a;
+		dp->result_list = a->next_result;
+		return a;
+	}
+	/* randomly select a target from the list */
+	log_assert(num > 1);
+	/* we do not need secure random numbers here, but
+	 * we do need it to be threadsafe, so we use this */
+	sel = ub_random(env->rnd) % num; 
+	a = dp->result_list;
+	prev = NULL;
+	while(sel > 0 && a) {
+		prev = a;
+		a = a->next_result;
+		sel--;
+	}
+	if(!a)  /* robustness */
+		return NULL;
+	if(++a->attempts < OUTBOUND_MSG_RETRY)
+		return a;
+	/* remove it from the delegation point result list */
+	if(prev)
+		prev->next_result = a->next_result;
+	else	dp->result_list = a->next_result;
+	return a;
+}
+
+struct dns_msg* 
+dns_alloc_msg(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct regional* region)
+{
+	struct dns_msg* m = (struct dns_msg*)regional_alloc(region,
+		sizeof(struct dns_msg));
+	if(!m)
+		return NULL;
+	memset(m, 0, sizeof(*m));
+	if(!parse_create_msg(pkt, msg, NULL, &m->qinfo, &m->rep, region)) {
+		log_err("malloc failure: allocating incoming dns_msg");
+		return NULL;
+	}
+	return m;
+}
+
+struct dns_msg* 
+dns_copy_msg(struct dns_msg* from, struct regional* region)
+{
+	struct dns_msg* m = (struct dns_msg*)regional_alloc(region,
+		sizeof(struct dns_msg));
+	if(!m)
+		return NULL;
+	m->qinfo = from->qinfo;
+	if(!(m->qinfo.qname = regional_alloc_init(region, from->qinfo.qname,
+		from->qinfo.qname_len)))
+		return NULL;
+	if(!(m->rep = reply_info_copy(from->rep, NULL, region)))
+		return NULL;
+	return m;
+}
+
+int 
+iter_dns_store(struct module_env* env, struct query_info* msgqinf,
+	struct reply_info* msgrep, int is_referral)
+{
+	return dns_cache_store(env, msgqinf, msgrep, is_referral);
+}
+
+int 
+iter_ns_probability(struct ub_randstate* rnd, int n, int m)
+{
+	int sel;
+	if(n == m) /* 100% chance */
+		return 1;
+	/* we do not need secure random numbers here, but
+	 * we do need it to be threadsafe, so we use this */
+	sel = ub_random(rnd) % m; 
+	return (sel < n);
+}
+
+/** detect dependency cycle for query and target */
+static int
+causes_cycle(struct module_qstate* qstate, uint8_t* name, size_t namelen,
+	uint16_t t, uint16_t c)
+{
+	struct query_info qinf;
+	qinf.qname = name;
+	qinf.qname_len = namelen;
+	qinf.qtype = t;
+	qinf.qclass = c;
+	log_assert(fptr_whitelist_modenv_detect_cycle(
+		qstate->env->detect_cycle));
+	return (*qstate->env->detect_cycle)(qstate, &qinf, 
+		(uint16_t)(BIT_RD|BIT_CD), qstate->is_priming);
+}
+
+void 
+iter_mark_cycle_targets(struct module_qstate* qstate, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved)
+			continue;
+		/* see if this ns as target causes dependency cycle */
+		if(causes_cycle(qstate, ns->name, ns->namelen, 
+			LDNS_RR_TYPE_AAAA, qstate->qinfo.qclass) ||
+		   causes_cycle(qstate, ns->name, ns->namelen, 
+			LDNS_RR_TYPE_A, qstate->qinfo.qclass)) {
+			log_nametypeclass(VERB_DETAIL, "skipping target due "
+			 	"to dependency cycle (harden-glue: no may "
+				"fix some of the cycles)", 
+				ns->name, LDNS_RR_TYPE_A, 
+				qstate->qinfo.qclass);
+			ns->resolved = 1;
+		}
+	}
+}
+
+int 
+iter_dp_is_useless(struct module_qstate* qstate, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	/* check:
+	 *      o all NS items are required glue.
+	 *      o no addresses are provided.
+	 *      o RD qflag is on.
+	 * OR
+	 *      o no addresses are provided.
+	 *      o RD qflag is on.
+	 *      o the query is for one of the nameservers in dp,
+	 *        and that nameserver is a glue-name for this dp.
+	 */
+	if(!(qstate->query_flags&BIT_RD))
+		return 0;
+	/* either available or unused targets */
+	if(dp->usable_list || dp->result_list) 
+		return 0;
+	
+	/* see if query is for one of the nameservers, which is glue */
+	if( (qstate->qinfo.qtype == LDNS_RR_TYPE_A ||
+		qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA) &&
+		dname_subdomain_c(qstate->qinfo.qname, dp->name) &&
+		delegpt_find_ns(dp, qstate->qinfo.qname, 
+			qstate->qinfo.qname_len))
+		return 1;
+	
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved) /* skip failed targets */
+			continue;
+		if(!dname_subdomain_c(ns->name, dp->name))
+			return 0; /* one address is not required glue */
+	}
+	return 1;
+}
+
+int 
+iter_indicates_dnssec(struct module_env* env, struct delegpt* dp,
+        struct dns_msg* msg, uint16_t dclass)
+{
+	/* information not available, !env->anchors can be common */
+	if(!env || !env->anchors || !dp || !dp->name || !msg || !msg->rep)
+		return 0;
+	/* a trust anchor exists with this name, RRSIGs expected */
+	if(anchor_find(env->anchors, dp->name, dp->namelabs, dp->namelen,
+		dclass))
+		return 1;
+	/* see if DS rrset was given, in AUTH section */
+	if(reply_find_rrset_section_ns(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_DS, dclass))
+		return 1;
+	return 0;
+}
+
+int 
+iter_msg_has_dnssec(struct dns_msg* msg)
+{
+	size_t i;
+	if(!msg || !msg->rep)
+		return 0;
+	for(i=0; i<msg->rep->an_numrrsets + msg->rep->ns_numrrsets; i++) {
+		if(((struct packed_rrset_data*)msg->rep->rrsets[i]->
+			entry.data)->rrsig_count > 0)
+			return 1;
+	}
+	/* empty message has no DNSSEC info, with DNSSEC the reply is
+	 * not empty (NSEC) */
+	return 0;
+}
+
+int iter_msg_from_zone(struct dns_msg* msg, struct delegpt* dp,
+        enum response_type type, uint16_t dclass)
+{
+	if(!msg || !dp || !msg->rep || !dp->name)
+		return 0;
+	/* SOA RRset - always from reply zone */
+	if(reply_find_rrset_section_an(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_SOA, dclass) ||
+	   reply_find_rrset_section_ns(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_SOA, dclass))
+		return 1;
+	if(type == RESPONSE_TYPE_REFERRAL) {
+		size_t i;
+		/* if it adds a single label, i.e. we expect .com,
+		 * and referral to example.com. NS ... , then origin zone
+		 * is .com. For a referral to sub.example.com. NS ... then
+		 * we do not know, since example.com. may be in between. */
+		for(i=0; i<msg->rep->an_numrrsets+msg->rep->ns_numrrsets; 
+			i++) {
+			struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+			if(ntohs(s->rk.type) == LDNS_RR_TYPE_NS &&
+				ntohs(s->rk.rrset_class) == dclass) {
+				int l = dname_count_labels(s->rk.dname);
+				if(l == dp->namelabs + 1 &&
+					dname_strict_subdomain(s->rk.dname,
+					l, dp->name, dp->namelabs))
+					return 1;
+			}
+		}
+		return 0;
+	}
+	log_assert(type==RESPONSE_TYPE_ANSWER || type==RESPONSE_TYPE_CNAME);
+	/* not a referral, and not lame delegation (upwards), so, 
+	 * any NS rrset must be from the zone itself */
+	if(reply_find_rrset_section_an(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_NS, dclass) ||
+	   reply_find_rrset_section_ns(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_NS, dclass))
+		return 1;
+	return 0;
+}

iterator/iter_utils.h

@@ -0,0 +1,180 @@
+/*
+ * iterator/iter_utils.h - iterative resolver module utility functions.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Configuration options. Forward zones. 
+ */
+
+#ifndef ITERATOR_ITER_UTILS_H
+#define ITERATOR_ITER_UTILS_H
+#include "iterator/iter_resptype.h"
+struct iter_env;
+struct config_file;
+struct module_env;
+struct delegpt_addr;
+struct delegpt;
+struct regional;
+struct msg_parse;
+struct ub_randstate;
+struct query_info;
+struct reply_info;
+struct module_qstate;
+
+/**
+ * Process config options and set iterator module state.
+ * Sets default values if no config is found.
+ * @param iter_env: iterator module state.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg);
+
+/**
+ * Select a valid, nice target to send query to.
+ * Sorting and removing unsuitable targets is combined.
+ *
+ * @param iter_env: iterator module global state, with ip6 enabled and 
+ *	do-not-query-addresses.
+ * @param env: environment with infra cache (lameness, rtt info).
+ * @param dp: delegation point with result list.
+ * @param name: zone name (for lameness check).
+ * @param namelen: length of name.
+ * @param dnssec_expected: set to 0, if a known dnssec-lame server is selected
+ *	these are not preferred, but are used as a last resort.
+ * @return best target or NULL if no target.
+ *	if not null, that target is removed from the result list in the dp.
+ */
+struct delegpt_addr* iter_server_selection(struct iter_env* iter_env, 
+	struct module_env* env, struct delegpt* dp, uint8_t* name, 
+	size_t namelen, int* dnssec_expected);
+
+/**
+ * Allocate dns_msg from parsed msg, in regional.
+ * @param pkt: packet.
+ * @param msg: parsed message (cleaned and ready for regional allocation).
+ * @param regional: regional to use for allocation.
+ * @return newly allocated dns_msg, or NULL on memory error.
+ */
+struct dns_msg* dns_alloc_msg(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct regional* regional);
+
+/**
+ * Copy a dns_msg to this regional.
+ * @param from: dns message, also in regional.
+ * @param regional: regional to use for allocation.
+ * @return newly allocated dns_msg, or NULL on memory error.
+ */
+struct dns_msg* dns_copy_msg(struct dns_msg* from, struct regional* regional);
+
+/**
+ * Allocate a dns_msg with malloc/alloc structure and store in dns cache.
+ * @param env: environment, with alloc structure and dns cache.
+ * @param qinf: query info, the query for which answer is stored.
+ * @param rep: reply in dns_msg from dns_alloc_msg for example.
+ * @param is_referral: If true, then the given message to be stored is a
+ *	referral. The cache implementation may use this as a hint.
+ * @return 0 on alloc error (out of memory).
+ */
+int iter_dns_store(struct module_env* env, struct query_info* qinf,
+	struct reply_info* rep, int is_referral);
+
+/**
+ * Select randomly with n/m probability.
+ * For shuffle NS records for address fetching.
+ * @param rnd: random table
+ * @param n: probability.
+ * @param m: divisor for probability.
+ * @return true with n/m probability.
+ */
+int iter_ns_probability(struct ub_randstate* rnd, int n, int m);
+
+/**
+ * Mark targets that result in a dependency cycle as done, so they
+ * will not get selected as targets.
+ * @param qstate: query state.
+ * @param dp: delegpt to mark ns in.
+ */
+void iter_mark_cycle_targets(struct module_qstate* qstate, struct delegpt* dp);
+
+/**
+ * See if delegation is useful or offers immediately no targets for 
+ * further recursion.
+ * @param qstate: query state with RD flag and query name.
+ * @param dp: delegpt to check.
+ */
+int iter_dp_is_useless(struct module_qstate* qstate, struct delegpt* dp);
+
+/**
+ * See if delegation is expected to have DNSSEC information (RRSIGs) in 
+ * its answers, or not. Inspects delegation point (name), trust anchors,
+ * and delegation message (DS RRset) to determine this.
+ * @param env: module env with trust anchors.
+ * @param dp: delegation point.
+ * @param msg: delegation message, with DS if a secure referral.
+ * @param dclass: class of query.
+ * @return 1 if dnssec is expected, 0 if not.
+ */
+int iter_indicates_dnssec(struct module_env* env, struct delegpt* dp,
+	struct dns_msg* msg, uint16_t dclass);
+
+/**
+ * See if a message contains DNSSEC.
+ * This is examined by looking for RRSIGs. With DNSSEC a valid answer, 
+ * nxdomain, nodata, referral or cname reply has RRSIGs in answer or auth 
+ * sections, sigs on answer data, SOA, DS, or NSEC/NSEC3 records.
+ * @param msg: message to examine.
+ * @return true if DNSSEC information was found.
+ */
+int iter_msg_has_dnssec(struct dns_msg* msg);
+
+/**
+ * See if a message is known to be from a certain zone.
+ * This looks for SOA or NS rrsets, for answers.
+ * For referrals, when one label is delegated, the zone is detected.
+ * Does not look at signatures.
+ * @param msg: the message to inspect.
+ * @param dp: delegation point with zone name to look for.
+ * @param type: type of message.
+ * @param dclass: class of query.
+ * @return true if message is certain to be from zone in dp->name.
+ *	false if not sure (empty msg), or not from the zone.
+ */
+int iter_msg_from_zone(struct dns_msg* msg, struct delegpt* dp, 
+	enum response_type type, uint16_t dclass);
+
+#endif /* ITERATOR_ITER_UTILS_H */

iterator/iterator.h

@@ -0,0 +1,314 @@
+/*
+ * iterator/iterator.h - iterative resolver DNS query response module
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains a module that performs recusive iterative DNS query
+ * processing.
+ */
+
+#ifndef ITERATOR_ITERATOR_H
+#define ITERATOR_ITERATOR_H
+#include "services/outbound_list.h"
+#include "util/data/msgreply.h"
+#include "util/module.h"
+struct delegpt;
+struct iter_hints;
+struct iter_forwards;
+struct iter_donotq;
+struct iter_prep_list;
+
+/** max number of query restarts. Determines max number of CNAME chain. */
+#define MAX_RESTART_COUNT       8
+/** max number of referrals. Makes sure resolver does not run away */
+#define MAX_REFERRAL_COUNT	30
+/** how nice is a server without further information, in msec 
+ * Equals rtt initial timeout value.
+ */
+#define UNKNOWN_SERVER_NICENESS 376
+/** maximum timeout before a host is deemed unsuitable, in msec. 
+ * After host_ttl this will be timed out and the host will be tried again. 
+ * Equals RTT_MAX_TIMEOUT
+ */
+#define USEFUL_SERVER_TOP_TIMEOUT	120000
+/** number of retries on outgoing queries */
+#define OUTBOUND_MSG_RETRY 4
+
+/**
+ * Global state for the iterator. 
+ */
+struct iter_env {
+	/** 
+	 * The hints -- these aren't stored in the cache because they don't 
+	 * expire. The hints are always used to "prime" the cache. Note 
+	 * that both root hints and stub zone "hints" are stored in this 
+	 * data structure.
+	 */
+	struct iter_hints* hints;
+
+	/** A flag to indicate whether or not we have an IPv6 route */
+	int supports_ipv6;
+
+	/** Mapping of forwarding zones to targets. */
+	struct iter_forwards* fwds;
+
+	/** A set of inetaddrs that should never be queried. */
+	struct iter_donotq* donotq;
+
+	/** The maximum dependency depth that this resolver will pursue. */
+	int max_dependency_depth;
+
+	/**
+	 * The target fetch policy for each dependency level. This is 
+	 * described as a simple number (per dependency level): 
+	 *	negative numbers (usually just -1) mean fetch-all, 
+	 *	0 means only fetch on demand, and 
+	 *	positive numbers mean to fetch at most that many targets.
+	 * array of max_dependency_depth+1 size.
+	 */
+	int* target_fetch_policy;
+};
+
+/**
+ * State of the iterator for a query.
+ */
+enum iter_state {
+	/**
+	 * Externally generated queries start at this state. Query restarts are
+	 * reset to this state.
+	 */
+	INIT_REQUEST_STATE = 0,
+
+	/**
+	 * Root priming events reactivate here, most other events pass 
+	 * through this naturally as the 2nd part of the INIT_REQUEST_STATE.
+	 */
+	INIT_REQUEST_2_STATE,
+
+	/**
+	 * Stub priming events reactivate here, most other events pass 
+	 * through this naturally as the 3rd part of the INIT_REQUEST_STATE.
+	 */
+	INIT_REQUEST_3_STATE,
+
+	/**
+	 * Each time a delegation point changes for a given query or a 
+	 * query times out and/or wakes up, this state is (re)visited. 
+	 * This state is reponsible for iterating through a list of 
+	 * nameserver targets.
+	 */
+	QUERYTARGETS_STATE,
+
+	/**
+	 * Responses to queries start at this state. This state handles 
+	 * the decision tree associated with handling responses.
+	 */
+	QUERY_RESP_STATE,
+
+	/** Responses to priming queries finish at this state. */
+	PRIME_RESP_STATE,
+
+	/** Responses that are to be returned upstream end at this state. 
+	 * As well as responses to target queries. */
+	FINISHED_STATE
+};
+
+/**
+ * Per query state for the iterator module.
+ */
+struct iter_qstate {
+	/** 
+	 * State of the iterator module.
+	 * This is the state that event is in or should sent to -- all 
+	 * requests should start with the INIT_REQUEST_STATE. All 
+	 * responses should start with QUERY_RESP_STATE. Subsequent 
+	 * processing of the event will change this state.
+	 */
+	enum iter_state state;
+
+	/** 
+	 * Final state for the iterator module.
+	 * This is the state that responses should be routed to once the 
+	 * response is final. For externally initiated queries, this 
+	 * will be FINISHED_STATE, locally initiated queries will have 
+	 * different final states.
+	 */
+	enum iter_state final_state;
+
+	/** 
+	 * The depth of this query, this means the depth of recursion.
+	 * This address is needed for another query, which is an address
+	 * needed for another query, etc. Original client query has depth 0.
+	 */
+	int depth;
+
+	/**
+	 * The response
+	 */
+	struct dns_msg* response;
+
+	/** 
+	 * This is a list of RRsets that must be prepended to the 
+	 * ANSWER section of a response before being sent upstream.
+	 */
+	struct iter_prep_list* an_prepend_list;
+	/** Last element of the prepend list */
+	struct iter_prep_list* an_prepend_last;
+
+	/**
+	 * This is the list of RRsets that must be prepended to the
+	 * AUTHORITY section of the response before being sent upstream.
+	 */
+	struct iter_prep_list* ns_prepend_list;
+	/** Last element of the authority prepend list */
+	struct iter_prep_list* ns_prepend_last;
+
+	/** query name used for chasing the results. Initially the same as
+	 * the state qinfo, but after CNAMEs this will be different. 
+	 * The query info used to elicit the results needed. */
+	struct query_info qchase;
+	/** query flags to use when chasing the answer (i.e. RD flag) */
+	uint16_t chase_flags;
+
+	/** 
+	 * This is the current delegation point for an in-progress query. This
+	 * object retains state as to which delegation targets need to be
+	 * (sub)queried for vs which ones have already been visited.
+	 */
+	struct delegpt* dp;
+
+	/** Current delegation message - returned for non-RD queries */
+	struct dns_msg* deleg_msg;
+
+	/** number of outstanding target sub queries */
+	int num_target_queries;
+
+	/** outstanding direct queries */
+	int num_current_queries;
+
+	/** the number of times this query has been restarted. */
+	int query_restart_count;
+
+	/** the number of times this query as followed a referral. */
+	int referral_count;
+
+	/** 
+	 * expected dnssec information for this iteration step. 
+	 * If dnssec rrsigs are expected and not given, the server is marked
+	 * lame (dnssec-lame).
+	 */
+	int dnssec_expected;
+
+	/**
+	 * This is flag that, if true, means that this event is 
+	 * waiting for a stub priming query. 
+	 */
+	int wait_priming_stub;
+
+	/**
+	 * This is a flag that, if true, means that this query is
+	 * for (re)fetching glue from a zone. Since the address should
+	 * have been glue, query again to the servers that should have
+	 * been returning it as glue.
+	 * The delegation point must be set to the one that should *not*
+	 * be used when creating the state. A higher one will be attempted.
+	 */
+	int refetch_glue;
+
+	/** list of pending queries to authoritative servers. */
+	struct outbound_list outlist;
+};
+
+/**
+ * List of prepend items
+ */
+struct iter_prep_list {
+	/** next in list */
+	struct iter_prep_list* next;
+	/** rrset */
+	struct ub_packed_rrset_key* rrset;
+};
+
+/**
+ * Get the iterator function block.
+ * @return: function block with function pointers to iterator methods.
+ */
+struct module_func_block* iter_get_funcblock();
+
+/**
+ * Get iterator state as a string
+ * @param state: to convert
+ * @return constant string that is printable.
+ */
+const char* iter_state_to_string(enum iter_state state);
+
+/**
+ * See if iterator state is a response state
+ * @param s: to inspect
+ * @return true if response state.
+ */
+int iter_state_is_responsestate(enum iter_state s);
+
+/** iterator init */
+int iter_init(struct module_env* env, int id);
+
+/** iterator deinit */
+void iter_deinit(struct module_env* env, int id);
+
+/** iterator operate on a query */
+void iter_operate(struct module_qstate* qstate, enum module_ev event, int id,
+	struct outbound_entry* outbound);
+
+/**
+ * Return priming query results to interestes super querystates.
+ * 
+ * Sets the delegation point and delegation message (not nonRD queries).
+ * This is a callback from walk_supers.
+ *
+ * @param qstate: query state that finished.
+ * @param id: module id.
+ * @param super: the qstate to inform.
+ */
+void iter_inform_super(struct module_qstate* qstate, int id, 
+	struct module_qstate* super);
+
+/** iterator cleanup query state */
+void iter_clear(struct module_qstate* qstate, int id);
+
+/** iterator alloc size routine */
+size_t iter_get_mem(struct module_env* env, int id);
+
+#endif /* ITERATOR_ITERATOR_H */

makedist.sh

@@ -134,7 +134,7 @@ done
 # Check if SVNROOT is specified.
 if [ -z "$SVNROOT" ]; then
     if test -f .svn/entries; then
-	  eval `grep 'url=' .svn/entries`
+	  eval `svn info | grep 'URL:' | sed -e 's/URL: /url=/' | head -1`
 	  SVNROOT="$url"
     fi
     if test -z "$SVNROOT"; then
@@ -148,15 +148,11 @@ if test -z "$LDNSDIR"; then
 	  eval `grep 'ldnsdir=' Makefile`
 	  LDNSDIR="$ldnsdir"
     fi
-    if test -z "$LDNSDIR"; then
-	error "LDNSDIR not detected in Makefile, specify manually (using -l)"
-    fi
 fi
 
 # Start the packaging process.
 info "SVNROOT  is $SVNROOT"
 info "SNAPSHOT is $SNAPSHOT"
-info "LDNSDIR  is $LDNSDIR"
 
 #question "Do you wish to continue with these settings?" || error "User abort."
 
@@ -175,19 +171,25 @@ cd unbound || error_cleanup "Unbound not exported correctly from SVN"
 info "Adding libtool utils (libtoolize)."
 libtoolize -c || error_cleanup "libtoolize failed"
 
-info "Building configure script (autoconf)."
-autoconf || error_cleanup "Autoconf failed."
-
-info "Building config.h.in (autoheader)."
-autoheader || error_cleanup "Autoheader failed."
+info "Building configure script (autoreconf)."
+autoreconf || error_cleanup "Autoconf failed."
 
 rm -r autom4te* || error_cleanup "Failed to remove autoconf cache directory."
 
-# info "Building lexer and parser."
+info "Building lexer and parser."
+echo "#include \"util/configyyrename.h\"" > util/configlexer.c || error_cleanup "Failed to create configlexer"
+flex -i -t util/configlexer.lex >> util/configlexer.c  || error_cleanup "Failed to create configlexer"
+bison -y -d -o util/configparser.c util/configparser.y || error_cleanup "Failed to create configparser"
 
-# copy ldns-testpkts from ldns examples
-#cp $LDNSDIR/examples/ldns-testpkts.c testcode/ldns-testpkts.c || error_cleanup "copy ldns/examples/.. failed"
-#cp $LDNSDIR/examples/ldns-testpkts.h testcode/ldns-testpkts.h || error_cleanup "copy ldns/examples/.. failed"
+# check shared code, ldns-testpkts from ldns examples, if possible.
+if test ! -z "$LDNSDIR"; then
+	if diff -q $LDNSDIR/examples/ldns-testpkts.c testcode/ldns-testpkts.c &&
+	   diff -q $LDNSDIR/examples/ldns-testpkts.h testcode/ldns-testpkts.h; then
+	   	info "ldns-testpkts.c and ldns-testpkts.h are OK"
+	else
+		error_cleanup "ldns-testpkts is different in ldns and unbound"
+	fi
+fi
 
 find . -name .c-mode-rc.el -exec rm {} \;
 find . -name .cvsignore -exec rm {} \;

services/cache/dns.c

@@ -0,0 +1,669 @@
+/*
+ * services/cache/dns.c - Cache services for DNS using msg and rrset caches.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the DNS cache.
+ */
+#include "config.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "services/cache/rrset.h"
+#include "util/data/msgreply.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/dname.h"
+#include "util/module.h"
+#include "util/net_help.h"
+#include "util/regional.h"
+
+/** store rrsets in the rrset cache. 
+ * @param env: module environment with caches.
+ * @param rep: contains list of rrsets to store.
+ * @param now: current time(NULL).
+ */
+static void
+store_rrsets(struct module_env* env, struct reply_info* rep, uint32_t now)
+{
+        size_t i;
+        /* see if rrset already exists in cache, if not insert it. */
+        for(i=0; i<rep->rrset_count; i++) {
+                rep->ref[i].key = rep->rrsets[i];
+                rep->ref[i].id = rep->rrsets[i]->id;
+                if(rrset_cache_update(env->rrset_cache, &rep->ref[i],
+                        env->alloc, now)) /* it was in the cache */
+                        rep->rrsets[i] = rep->ref[i].key;
+        }
+}
+
+void 
+dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
+	hashvalue_t hash, struct reply_info* rep)
+{
+	struct msgreply_entry* e;
+	uint32_t now = time(NULL), ttl = rep->ttl;
+	size_t i;
+
+	/* store RRsets */
+        for(i=0; i<rep->rrset_count; i++) {
+		rep->ref[i].key = rep->rrsets[i];
+		rep->ref[i].id = rep->rrsets[i]->id;
+	}
+	reply_info_sortref(rep);
+	reply_info_set_ttls(rep, now);
+	store_rrsets(env, rep, now);
+	if(ttl == 0) {
+		/* we do not store the message, but we did store the RRs,
+		 * which could be useful for delegation information */
+		verbose(VERB_ALGO, "TTL 0: dropped msg from cache");
+		free(rep);
+		return;
+	}
+
+	/* store msg in the cache */
+	reply_info_sortref(rep);
+	if(!(e = query_info_entrysetup(qinfo, rep, hash))) {
+		log_err("store_msg: malloc failed");
+		return;
+	}
+	slabhash_insert(env->msg_cache, hash, &e->entry, rep, env->alloc);
+}
+
+/** allocate rrset in region - no more locks needed */
+static struct ub_packed_rrset_key*
+copy_rrset(struct ub_packed_rrset_key* key, struct regional* region, 
+	uint32_t now)
+{
+	struct ub_packed_rrset_key* ck = regional_alloc(region, 
+		sizeof(struct ub_packed_rrset_key));
+	struct packed_rrset_data* d;
+	struct packed_rrset_data* data = (struct packed_rrset_data*)
+		key->entry.data;
+	size_t dsize, i;
+	if(!ck)
+		return NULL;
+	ck->id = key->id;
+	memset(&ck->entry, 0, sizeof(ck->entry));
+	ck->entry.hash = key->entry.hash;
+	ck->entry.key = ck;
+	ck->rk = key->rk;
+	ck->rk.dname = regional_alloc_init(region, key->rk.dname, 
+		key->rk.dname_len);
+	if(!ck->rk.dname)
+		return NULL;
+	dsize = packed_rrset_sizeof(data);
+	d = (struct packed_rrset_data*)regional_alloc_init(region, data, dsize);
+	if(!d)
+		return NULL;
+	ck->entry.data = d;
+	packed_rrset_ptr_fixup(d);
+	/* make TTLs relative - once per rrset */
+	for(i=0; i<d->count + d->rrsig_count; i++)
+		d->rr_ttl[i] -= now;
+	d->ttl -= now;
+	return ck;
+}
+
+/** find closest NS or DNAME and returns the rrset (locked) */
+static struct ub_packed_rrset_key*
+find_closest_of_type(struct module_env* env, uint8_t* qname, size_t qnamelen, 
+	uint16_t qclass, uint32_t now, uint16_t searchtype, int stripfront)
+{
+	struct ub_packed_rrset_key *rrset;
+	uint8_t lablen;
+
+	if(stripfront) {
+		/* strip off so that DNAMEs have strict subdomain match */
+		lablen = *qname;
+		qname += lablen + 1;
+		qnamelen -= lablen + 1;
+	}
+
+	/* snip off front part of qname until the type is found */
+	while(qnamelen > 0) {
+		if((rrset = rrset_cache_lookup(env->rrset_cache, qname, 
+			qnamelen, searchtype, qclass, 0, now, 0)))
+			return rrset;
+
+		/* snip off front label */
+		lablen = *qname;
+		qname += lablen + 1;
+		qnamelen -= lablen + 1;
+	}
+	return NULL;
+}
+
+/** add addr to additional section */
+static void
+addr_to_additional(struct ub_packed_rrset_key* rrset, struct regional* region,
+	struct dns_msg* msg, uint32_t now)
+{
+	if((msg->rep->rrsets[msg->rep->rrset_count] = 
+		copy_rrset(rrset, region, now))) {
+		msg->rep->ar_numrrsets++;
+		msg->rep->rrset_count++;
+	}
+}
+
+/** find and add A and AAAA records for nameservers in delegpt */
+static int
+find_add_addrs(struct module_env* env, uint16_t qclass, 
+	struct regional* region, struct delegpt* dp, uint32_t now, 
+	struct dns_msg** msg)
+{
+	struct delegpt_ns* ns;
+	struct ub_packed_rrset_key* akey;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_A, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_A(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			if(msg)
+				addr_to_additional(akey, region, *msg, now);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_AAAA(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			if(msg)
+				addr_to_additional(akey, region, *msg, now);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+	}
+	return 1;
+}
+
+/** find and add A and AAAA records for missing nameservers in delegpt */
+int
+cache_fill_missing(struct module_env* env, uint16_t qclass, 
+	struct regional* region, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	struct ub_packed_rrset_key* akey;
+	uint32_t now = time(NULL);
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved)
+			continue;
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_A, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_A(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			log_nametypeclass(VERB_ALGO, "found in cache",
+				ns->name, LDNS_RR_TYPE_A, qclass);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_AAAA(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			log_nametypeclass(VERB_ALGO, "found in cache",
+				ns->name, LDNS_RR_TYPE_AAAA, qclass);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+	}
+	return 1;
+}
+
+/** find and add DS or NSEC to delegation msg */
+static void
+find_add_ds(struct module_env* env, struct regional* region, 
+	struct dns_msg* msg, struct delegpt* dp, uint32_t now)
+{
+	/* Lookup the DS or NSEC at the delegation point. */
+	struct ub_packed_rrset_key* rrset = rrset_cache_lookup(
+		env->rrset_cache, dp->name, dp->namelen, LDNS_RR_TYPE_DS, 
+		msg->qinfo.qclass, 0, now, 0);
+	if(!rrset) {
+		/* NOTE: this won't work for alternate NSEC schemes 
+		 *	(opt-in, NSEC3) */
+		rrset = rrset_cache_lookup(env->rrset_cache, dp->name, 
+			dp->namelen, LDNS_RR_TYPE_NSEC, msg->qinfo.qclass, 
+			0, now, 0);
+		/* Note: the PACKED_RRSET_NSEC_AT_APEX flag is not used.
+		 * since this is a referral, we need the NSEC at the parent
+		 * side of the zone cut, not the NSEC at apex side. */
+	}
+	if(rrset) {
+		/* add it to auth section. This is the second rrset. */
+		if((msg->rep->rrsets[msg->rep->rrset_count] = 
+			copy_rrset(rrset, region, now))) {
+			msg->rep->ns_numrrsets++;
+			msg->rep->rrset_count++;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+}
+
+/** create referral message with NS and query */
+static struct dns_msg*
+create_msg(uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct regional* region, struct ub_packed_rrset_key* nskey, 
+	struct packed_rrset_data* nsdata, uint32_t now)
+{
+	struct dns_msg* msg = (struct dns_msg*)regional_alloc(region,
+		sizeof(struct dns_msg));
+	if(!msg)
+		return NULL;
+	msg->qinfo.qname = regional_alloc_init(region, qname, qnamelen);
+	if(!msg->qinfo.qname)
+		return NULL;
+	msg->qinfo.qname_len = qnamelen;
+	msg->qinfo.qtype = qtype;
+	msg->qinfo.qclass = qclass;
+	/* non-packed reply_info, because it needs to grow the array */
+	msg->rep = (struct reply_info*)regional_alloc(region, 
+		sizeof(struct reply_info)-sizeof(struct rrset_ref));
+	if(!msg->rep)
+		return NULL;
+	memset(msg->rep, 0, 
+		sizeof(struct reply_info)-sizeof(struct rrset_ref));
+	msg->rep->flags = BIT_QR; /* with QR, no AA */
+	msg->rep->qdcount = 1;
+	/* allocate the array to as much as we could need:
+	 *	NS rrset + DS/NSEC rrset +
+	 *	A rrset for every NS RR
+	 *	AAAA rrset for every NS RR
+	 */
+	msg->rep->rrsets = (struct ub_packed_rrset_key**)
+		regional_alloc(region, 
+		(2 + nsdata->count*2)*sizeof(struct ub_packed_rrset_key*));
+	if(!msg->rep->rrsets)
+		return NULL;
+	msg->rep->rrsets[0] = copy_rrset(nskey, region, now);
+	if(!msg->rep->rrsets[0])
+		return NULL;
+	msg->rep->ns_numrrsets++;
+	msg->rep->rrset_count++;
+	return msg;
+}
+
+struct delegpt* 
+dns_cache_find_delegation(struct module_env* env, uint8_t* qname, 
+	size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct regional* region, struct dns_msg** msg, uint32_t now)
+{
+	/* try to find closest NS rrset */
+	struct ub_packed_rrset_key* nskey;
+	struct packed_rrset_data* nsdata;
+	struct delegpt* dp;
+
+	nskey = find_closest_of_type(env, qname, qnamelen, qclass, now,
+		LDNS_RR_TYPE_NS, 0);
+	if(!nskey) /* hope the caller has hints to prime or something */
+		return NULL;
+	nsdata = (struct packed_rrset_data*)nskey->entry.data;
+	/* got the NS key, create delegation point */
+	dp = delegpt_create(region);
+	if(!dp || !delegpt_set_name(dp, region, nskey->rk.dname)) {
+		lock_rw_unlock(&nskey->entry.lock);
+		log_err("find_delegation: out of memory");
+		return NULL;
+	}
+	/* create referral message */
+	if(msg) {
+		*msg = create_msg(qname, qnamelen, qtype, qclass, region, 
+			nskey, nsdata, now);
+		if(!*msg) {
+			lock_rw_unlock(&nskey->entry.lock);
+			log_err("find_delegation: out of memory");
+			return NULL;
+		}
+	}
+	if(!delegpt_rrset_add_ns(dp, region, nskey))
+		log_err("find_delegation: addns out of memory");
+	lock_rw_unlock(&nskey->entry.lock); /* first unlock before next lookup*/
+	/* find and add DS/NSEC (if any) */
+	if(msg)
+		find_add_ds(env, region, *msg, dp, now);
+	/* find and add A entries */
+	if(!find_add_addrs(env, qclass, region, dp, now, msg))
+		log_err("find_delegation: addrs out of memory");
+	return dp;
+}
+
+/** allocate dns_msg from query_info and reply_info */
+static struct dns_msg*
+gen_dns_msg(struct regional* region, struct query_info* q, size_t num)
+{
+	struct dns_msg* msg = (struct dns_msg*)regional_alloc(region, 
+		sizeof(struct dns_msg));
+	if(!msg)
+		return NULL;
+	memcpy(&msg->qinfo, q, sizeof(struct query_info));
+	msg->qinfo.qname = regional_alloc_init(region, q->qname, q->qname_len);
+	if(!msg->qinfo.qname)
+		return NULL;
+	/* allocate replyinfo struct and rrset key array separately */
+	msg->rep = (struct reply_info*)regional_alloc(region,
+		sizeof(struct reply_info) - sizeof(struct rrset_ref));
+	if(!msg->rep)
+		return NULL;
+	msg->rep->rrsets = (struct ub_packed_rrset_key**)
+		regional_alloc(region,
+		num * sizeof(struct ub_packed_rrset_key*));
+	if(!msg->rep->rrsets)
+		return NULL;
+	return msg;
+}
+
+/** generate dns_msg from cached message */
+static struct dns_msg*
+tomsg(struct module_env* env, struct msgreply_entry* e, struct reply_info* r, 
+	struct regional* region, uint32_t now, struct regional* scratch)
+{
+	struct dns_msg* msg;
+	size_t i;
+	if(now > r->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, &e->key, r->rrset_count);
+	if(!msg)
+		return NULL;
+	msg->rep->flags = r->flags;
+	msg->rep->qdcount = r->qdcount;
+	msg->rep->ttl = r->ttl;
+	msg->rep->security = r->security;
+	msg->rep->an_numrrsets = r->an_numrrsets;
+	msg->rep->ns_numrrsets = r->ns_numrrsets;
+	msg->rep->ar_numrrsets = r->ar_numrrsets;
+	msg->rep->rrset_count = r->rrset_count;
+	if(!rrset_array_lock(r->ref, r->rrset_count, now))
+		return NULL;
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		msg->rep->rrsets[i] = copy_rrset(r->rrsets[i], region, now);
+		if(!msg->rep->rrsets[i]) {
+			rrset_array_unlock(r->ref, r->rrset_count);
+			return NULL;
+		}
+	}
+	rrset_array_unlock_touch(env->rrset_cache, scratch, r->ref, 
+		r->rrset_count);
+	return msg;
+}
+
+/** synthesize RRset-only response from cached RRset item */
+static struct dns_msg*
+rrset_msg(struct ub_packed_rrset_key* rrset, struct regional* region, 
+	uint32_t now, struct query_info* q)
+{
+	struct dns_msg* msg;
+	struct packed_rrset_data* d = (struct packed_rrset_data*)
+		rrset->entry.data;
+	if(now > d->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, q, 1); /* only the CNAME (or other) RRset */
+	if(!msg)
+		return NULL;
+	msg->rep->flags = BIT_QR; /* reply, no AA, no error */
+	msg->rep->qdcount = 1;
+	msg->rep->ttl = d->ttl - now;
+	msg->rep->security = sec_status_unchecked;
+	msg->rep->an_numrrsets = 1;
+	msg->rep->ns_numrrsets = 0;
+	msg->rep->ar_numrrsets = 0;
+	msg->rep->rrset_count = 1;
+	msg->rep->rrsets[0] = copy_rrset(rrset, region, now);
+	if(!msg->rep->rrsets[0]) /* copy CNAME */
+		return NULL;
+	return msg;
+}
+
+/** synthesize DNAME+CNAME response from cached DNAME item */
+static struct dns_msg*
+synth_dname_msg(struct ub_packed_rrset_key* rrset, struct regional* region, 
+	uint32_t now, struct query_info* q)
+{
+	struct dns_msg* msg;
+	struct ub_packed_rrset_key* ck;
+	struct packed_rrset_data* newd, *d = (struct packed_rrset_data*)
+		rrset->entry.data;
+	uint8_t* newname, *dtarg = NULL;
+	size_t newlen, dtarglen;
+	if(now > d->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, q, 2); /* DNAME + CNAME RRset */
+	if(!msg)
+		return NULL;
+	msg->rep->flags = BIT_QR; /* reply, no AA, no error */
+	msg->rep->qdcount = 1;
+	msg->rep->ttl = d->ttl - now;
+	msg->rep->security = sec_status_unchecked;
+	msg->rep->an_numrrsets = 1;
+	msg->rep->ns_numrrsets = 0;
+	msg->rep->ar_numrrsets = 0;
+	msg->rep->rrset_count = 1;
+	msg->rep->rrsets[0] = copy_rrset(rrset, region, now);
+	if(!msg->rep->rrsets[0]) /* copy DNAME */
+		return NULL;
+	/* synth CNAME rrset */
+	get_cname_target(rrset, &dtarg, &dtarglen);
+	if(!dtarg)
+		return NULL;
+	newlen = q->qname_len + dtarglen - rrset->rk.dname_len;
+	if(newlen > LDNS_MAX_DOMAINLEN) {
+		msg->rep->flags |= LDNS_RCODE_YXDOMAIN;
+		return msg;
+	}
+	newname = (uint8_t*)regional_alloc(region, newlen);
+	if(!newname)
+		return NULL;
+	/* new name is concatenation of qname front (without DNAME owner)
+	 * and DNAME target name */
+	memcpy(newname, q->qname, q->qname_len-rrset->rk.dname_len);
+	memmove(newname+(q->qname_len-rrset->rk.dname_len), dtarg, dtarglen);
+	/* create rest of CNAME rrset */
+	ck = (struct ub_packed_rrset_key*)regional_alloc(region, 
+		sizeof(struct ub_packed_rrset_key));
+	if(!ck)
+		return NULL;
+	memset(&ck->entry, 0, sizeof(ck->entry));
+	msg->rep->rrsets[1] = ck;
+	ck->entry.key = ck;
+	ck->rk.type = htons(LDNS_RR_TYPE_CNAME);
+	ck->rk.rrset_class = rrset->rk.rrset_class;
+	ck->rk.flags = 0;
+	ck->rk.dname = regional_alloc_init(region, q->qname, q->qname_len);
+	if(!ck->rk.dname)
+		return NULL;
+	ck->rk.dname_len = q->qname_len;
+	ck->entry.hash = rrset_key_hash(&ck->rk);
+	newd = (struct packed_rrset_data*)regional_alloc(region,
+		sizeof(struct packed_rrset_data) + sizeof(size_t) + 
+		sizeof(uint8_t*) + sizeof(uint32_t) + sizeof(uint16_t) 
+		+ newlen);
+	if(!newd)
+		return NULL;
+	ck->entry.data = newd;
+	newd->ttl = 0; /* 0 for synthesized CNAME TTL */
+	newd->count = 1;
+	newd->rrsig_count = 0;
+	newd->trust = rrset_trust_ans_noAA;
+	newd->rr_len = (size_t*)((uint8_t*)newd + 
+		sizeof(struct packed_rrset_data));
+	newd->rr_len[0] = newlen + sizeof(uint16_t);
+	packed_rrset_ptr_fixup(newd);
+	newd->rr_ttl[0] = newd->ttl;
+	msg->rep->ttl = newd->ttl;
+	ldns_write_uint16(newd->rr_data[0], newlen);
+	memmove(newd->rr_data[0] + sizeof(uint16_t), newname, newlen);
+	msg->rep->an_numrrsets ++;
+	msg->rep->rrset_count ++;
+	return msg;
+}
+
+struct dns_msg* 
+dns_cache_lookup(struct module_env* env,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	struct regional* region, struct regional* scratch)
+{
+	struct lruhash_entry* e;
+	struct query_info k;
+	hashvalue_t h;
+	uint32_t now = (uint32_t)time(NULL);
+	struct ub_packed_rrset_key* rrset;
+
+	/* lookup first, this has both NXdomains and ANSWER responses */
+	k.qname = qname;
+	k.qname_len = qnamelen;
+	k.qtype = qtype;
+	k.qclass = qclass;
+	h = query_info_hash(&k);
+	e = slabhash_lookup(env->msg_cache, h, &k, 0);
+	if(e) {
+		struct msgreply_entry* key = (struct msgreply_entry*)e->key;
+		struct reply_info* data = (struct reply_info*)e->data;
+		struct dns_msg* msg = tomsg(env, key, data, region, now, 
+			scratch);
+		if(msg) {
+			lock_rw_unlock(&e->lock);
+			return msg;
+		}
+		/* could be msg==NULL; due to TTL or not all rrsets available */
+		lock_rw_unlock(&e->lock);
+	}
+
+	/* see if a DNAME exists. Checked for first, to enforce that DNAMEs
+	 * are more important, the CNAME is resynthesized and thus 
+	 * consistent with the DNAME */
+	if( (rrset=find_closest_of_type(env, qname, qnamelen, qclass, now,
+		LDNS_RR_TYPE_DNAME, 1))) {
+		/* synthesize a DNAME+CNAME message based on this */
+		struct dns_msg* msg = synth_dname_msg(rrset, region, now, &k);
+		if(msg) {
+			lock_rw_unlock(&rrset->entry.lock);
+			return msg;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+
+	/* see if we have CNAME for this domain */
+	if( (rrset=rrset_cache_lookup(env->rrset_cache, qname, qnamelen, 
+		LDNS_RR_TYPE_CNAME, qclass, 0, now, 0))) {
+		struct dns_msg* msg = rrset_msg(rrset, region, now, &k);
+		if(msg) {
+			lock_rw_unlock(&rrset->entry.lock);
+			return msg;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+
+	/* construct DS, DNSKEY messages from rrset cache. */
+	if((qtype == LDNS_RR_TYPE_DS || qtype == LDNS_RR_TYPE_DNSKEY) &&
+		(rrset=rrset_cache_lookup(env->rrset_cache, qname, qnamelen, 
+		qtype, qclass, 0, now, 0))) {
+		/* if the rrset is from the additional section, and the
+		 * signatures have fallen off, then do not synthesize a msg
+		 * instead, allow a full query for signed results to happen.
+		 * Forego all rrset data from additional section, because
+		 * some signatures may not be present and cause validation
+		 * failure.
+		 */
+		struct packed_rrset_data *d = (struct packed_rrset_data*)
+			rrset->entry.data;
+		if(d->trust != rrset_trust_add_noAA && 
+			d->trust != rrset_trust_add_AA) {
+			struct dns_msg* msg = rrset_msg(rrset, region, now, &k);
+			if(msg) {
+				lock_rw_unlock(&rrset->entry.lock);
+				return msg;
+			}
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+	return NULL;
+}
+
+int 
+dns_cache_store(struct module_env* env, struct query_info* msgqinf,
+        struct reply_info* msgrep, int is_referral)
+{
+	struct reply_info* rep = NULL;
+	/* alloc, malloc properly (not in region, like msg is) */
+	rep = reply_info_copy(msgrep, env->alloc, NULL);
+	if(!rep)
+		return 0;
+
+	if(is_referral) {
+		/* store rrsets */
+		struct rrset_ref ref;
+		uint32_t now = time(NULL);
+		size_t i;
+		for(i=0; i<rep->rrset_count; i++) {
+			packed_rrset_ttl_add((struct packed_rrset_data*)
+				rep->rrsets[i]->entry.data, now);
+			ref.key = rep->rrsets[i];
+			ref.id = rep->rrsets[i]->id;
+			/*ignore ret: it was in the cache, ref updated */
+			(void)rrset_cache_update(env->rrset_cache, &ref, 
+				env->alloc, now);
+		}
+		free(rep);
+		return 1;
+	} else {
+		/* store msg, and rrsets */
+		struct query_info qinf;
+		hashvalue_t h;
+
+		qinf = *msgqinf;
+		qinf.qname = memdup(msgqinf->qname, msgqinf->qname_len);
+		if(!qinf.qname) {
+			reply_info_parsedelete(rep, env->alloc);
+			return 0;
+		}
+		/* fixup flags to be sensible for a reply based on the cache */
+		/* this module means that RA is available. It is an answer QR. 
+		 * Not AA from cache. Not CD in cache (depends on client bit). */
+		rep->flags |= (BIT_RA | BIT_QR);
+		rep->flags &= ~(BIT_AA | BIT_CD);
+		h = query_info_hash(&qinf);
+		dns_cache_store_msg(env, &qinf, h, rep);
+		/* qname is used inside query_info_entrysetup, and set to 
+		 * NULL. If it has not been used, free it. free(0) is safe. */
+		free(qinf.qname);
+	}
+	return 1;
+}

services/cache/dns.h

@@ -0,0 +1,142 @@
+/*
+ * services/cache/dns.h - Cache services for DNS using msg and rrset caches.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the DNS cache.
+ */
+
+#ifndef SERVICES_CACHE_DNS_H
+#define SERVICES_CACHE_DNS_H
+#include "util/storage/lruhash.h"
+#include "util/data/msgreply.h"
+struct module_env;
+struct query_info;
+struct reply_info;
+struct regional;
+struct delegpt;
+
+/**
+ * Region allocated message reply
+ */
+struct dns_msg {
+	/** query info */
+	struct query_info qinfo;
+	/** reply info - ptr to packed repinfo structure */
+	struct reply_info *rep;
+};
+
+/**
+ * Allocate a dns_msg with malloc/alloc structure and store in dns cache.
+ *
+ * @param env: environment, with alloc structure and dns cache.
+ * @param qinf: query info, the query for which answer is stored.
+ * 	this is allocated in a region, and will be copied to malloc area
+ * 	before insertion.
+ * @param rep: reply in dns_msg from dns_alloc_msg for example.
+ * 	this is allocated in a region, and will be copied to malloc area
+ * 	before insertion.
+ * @param is_referral: If true, then the given message to be stored is a
+ *      referral. The cache implementation may use this as a hint.
+ *      It will store only the RRsets, not the message.
+ * @return 0 on alloc error (out of memory).
+ */
+int dns_cache_store(struct module_env* env, struct query_info* qinf,
+        struct reply_info* rep, int is_referral); 
+
+/**
+ * Store message in the cache. Stores in message cache and rrset cache.
+ * Both qinfo and rep should be malloced and are put in the cache.
+ * They should not be used after this call, as they are then in shared cache.
+ * Does not return errors, they are logged and only lead to less cache.
+ *
+ * @param env: module environment with the DNS cache.
+ * @param qinfo: query info
+ * @param hash: hash over qinfo.
+ * @param rep: reply info, together with qinfo makes up the message.
+ *	Adjusts the reply info TTLs to absolute time.
+ */
+void dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
+	hashvalue_t hash, struct reply_info* rep);
+
+/**
+ * Find a delegation from the cache.
+ * @param env: module environment with the DNS cache.
+ * @param qname: query name.
+ * @param qnamelen: length of qname.
+ * @param qtype: query type.
+ * @param qclass: query class.
+ * @param region: where to allocate result delegation.
+ * @param msg: if not NULL, delegation message is returned here, synthesized
+ *	from the cache.
+ * @param timenow: the time now, for checking if TTL on cache entries is OK.
+ * @return new delegation or NULL on error or if not found in cache.
+ */
+struct delegpt* dns_cache_find_delegation(struct module_env* env, 
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct regional* region, struct dns_msg** msg, uint32_t timenow);
+
+/** 
+ * Find cached message 
+ * @param env: module environment with the DNS cache.
+ * @param qname: query name.
+ * @param qnamelen: length of qname.
+ * @param qtype: query type.
+ * @param qclass: query class.
+ * @param region: where to allocate result.
+ * @param scratch: where to allocate temporary data.
+ * @return new response message (alloced in region, rrsets do not have IDs).
+ * 	or NULL on error or if not found in cache.
+ *	TTLs are made relative to the current time.
+ */
+struct dns_msg* dns_cache_lookup(struct module_env* env,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	struct regional* region, struct regional* scratch);
+
+/** 
+ * find and add A and AAAA records for missing nameservers in delegpt 
+ * @param env: module environment with rrset cache
+ * @param qclass: which class to look in.
+ * @param region: where to store new dp info.
+ * @param dp: delegation point to fill missing entries.
+ * @return false on alloc failure.
+ */
+int cache_fill_missing(struct module_env* env, uint16_t qclass, 
+	struct regional* region, struct delegpt* dp);
+
+/** Find covering DNAME */
+
+#endif /* SERVICES_CACHE_DNS_H */

services/cache/infra.c

@@ -0,0 +1,550 @@
+/*
+ * services/cache/infra.c - infrastructure cache, server rtt and capabilities
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the infrastructure cache.
+ */
+#include "config.h"
+#include "services/cache/infra.h"
+#include "util/storage/slabhash.h"
+#include "util/storage/lookup3.h"
+#include "util/data/dname.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/config_file.h"
+
+size_t 
+infra_host_sizefunc(void* k, void* ATTR_UNUSED(d))
+{
+	struct infra_host_key* key = (struct infra_host_key*)k;
+	return sizeof(*key) + sizeof(struct infra_host_data) 
+		+ lock_get_mem(&key->entry.lock);
+}
+
+int 
+infra_host_compfunc(void* key1, void* key2)
+{
+	struct infra_host_key* k1 = (struct infra_host_key*)key1;
+	struct infra_host_key* k2 = (struct infra_host_key*)key2;
+	return sockaddr_cmp(&k1->addr, k1->addrlen, &k2->addr, k2->addrlen);
+}
+
+void 
+infra_host_delkeyfunc(void* k, void* ATTR_UNUSED(arg), int il)
+{
+	struct infra_host_key* key = (struct infra_host_key*)k;
+	if(!key)
+		return;
+	if(il) { lock_rw_unlock(&key->entry.lock); }
+	lock_rw_destroy(&key->entry.lock);
+	free(key);
+}
+
+void 
+infra_host_deldatafunc(void* d, void* ATTR_UNUSED(arg))
+{
+	struct infra_host_data* data = (struct infra_host_data*)d;
+	lruhash_delete(data->lameness);
+	free(data);
+}
+
+struct infra_cache* 
+infra_create(struct config_file* cfg)
+{
+	struct infra_cache* infra = (struct infra_cache*)calloc(1, 
+		sizeof(struct infra_cache));
+	/* the size of the lameness tables are not counted */
+	size_t maxmem = cfg->infra_cache_numhosts * 
+		(sizeof(struct infra_host_key)+sizeof(struct infra_host_data));
+	infra->hosts = slabhash_create(cfg->infra_cache_slabs,
+		INFRA_HOST_STARTSIZE, maxmem, &infra_host_sizefunc,
+		&infra_host_compfunc, &infra_host_delkeyfunc,
+		&infra_host_deldatafunc, NULL);
+	if(!infra->hosts) {
+		free(infra);
+		return NULL;
+	}
+	infra->host_ttl = cfg->host_ttl;
+	infra->lame_ttl = cfg->lame_ttl;
+	infra->max_lame_size = cfg->infra_cache_lame_size;
+	return infra;
+}
+
+void 
+infra_delete(struct infra_cache* infra)
+{
+	if(!infra)
+		return;
+	slabhash_delete(infra->hosts);
+	free(infra);
+}
+
+struct infra_cache* 
+infra_adjust(struct infra_cache* infra, struct config_file* cfg)
+{
+	size_t maxmem;
+	if(!infra)
+		return infra_create(cfg);
+	infra->host_ttl = cfg->host_ttl;
+	infra->lame_ttl = cfg->lame_ttl;
+	infra->max_lame_size = cfg->infra_cache_lame_size;
+	maxmem = cfg->infra_cache_numhosts * 
+		(sizeof(struct infra_host_key)+sizeof(struct infra_host_data));
+	if(maxmem != slabhash_get_size(infra->hosts) ||
+		cfg->infra_cache_slabs != infra->hosts->size) {
+		infra_delete(infra);
+		infra = infra_create(cfg);
+	}
+	return infra;
+}
+
+/** calculate the hash value for a host key */
+static hashvalue_t
+hash_addr(struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	hashvalue_t h = 0xab;
+	h = hashlittle(&addrlen, sizeof(addrlen), h);
+	h = hashlittle(addr, addrlen, h);
+	return h;
+}
+
+/** lookup version that does not check host ttl (you check it) */
+static struct lruhash_entry* 
+infra_lookup_host_nottl(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, int wr)
+{
+	struct infra_host_key k;
+	k.addrlen = addrlen;
+	memcpy(&k.addr, addr, addrlen);
+	k.entry.hash = hash_addr(addr, addrlen);
+	k.entry.key = (void*)&k;
+	k.entry.data = NULL;
+	return slabhash_lookup(infra->hosts, k.entry.hash, &k, wr);
+}
+
+struct infra_host_data* 
+infra_lookup_host(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, int wr,
+        time_t timenow, struct infra_host_key** key)
+{
+	struct infra_host_data* data;
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, wr);
+	*key = NULL;
+	if(!e)
+		return NULL;
+	/* check TTL */
+	data = (struct infra_host_data*)e->data;
+	if(data->ttl < timenow) {
+		lock_rw_unlock(&e->lock);
+		return NULL;
+	}
+	*key = (struct infra_host_key*)e->key;
+	return data;
+}
+
+/** 
+ * Create and init a new entry for a host 
+ * @param infra: infra structure with config parameters.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param tm: time now.
+ * @return: the new entry or NULL on malloc failure.
+ */
+static struct lruhash_entry*
+new_host_entry(struct infra_cache* infra, struct sockaddr_storage* addr, 
+	socklen_t addrlen, time_t tm)
+{
+	struct infra_host_data* data;
+	struct infra_host_key* key = (struct infra_host_key*)malloc(
+		sizeof(struct infra_host_key));
+	if(!key)
+		return NULL;
+	data = (struct infra_host_data*)malloc(
+		sizeof(struct infra_host_data));
+	if(!data) {
+		free(key);
+		return NULL;
+	}
+	lock_rw_init(&key->entry.lock);
+	key->entry.hash = hash_addr(addr, addrlen);
+	key->entry.key = (void*)key;
+	key->entry.data = (void*)data;
+	key->addrlen = addrlen;
+	memcpy(&key->addr, addr, addrlen);
+	data->ttl = tm + infra->host_ttl;
+	data->lameness = NULL;
+	data->edns_version = 0;
+	rtt_init(&data->rtt);
+	return &key->entry;
+}
+
+int 
+infra_host(struct infra_cache* infra, struct sockaddr_storage* addr,
+        socklen_t addrlen, time_t timenow, int* edns_vs, int* to)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 0);
+	struct infra_host_data* data;
+	if(e && ((struct infra_host_data*)e->data)->ttl < timenow) {
+		/* it expired, try to reuse existing entry */
+		lock_rw_unlock(&e->lock);
+		e = infra_lookup_host_nottl(infra, addr, addrlen, 1);
+		if(e) {
+			/* if its still there we have a writelock, init */
+			/* re-initialise */
+			data = (struct infra_host_data*)e->data;
+			data->ttl = timenow + infra->host_ttl;
+			rtt_init(&data->rtt);
+			/* do not touch lameness, it may be valid still */
+			data->edns_version = 0;
+		}
+	}
+	if(!e) {
+		/* insert new entry */
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		data = (struct infra_host_data*)e->data;
+		*to = rtt_timeout(&data->rtt);
+		*edns_vs = data->edns_version;
+		slabhash_insert(infra->hosts, e->hash, e, data, NULL);
+		return 1;
+	}
+	/* use existing entry */
+	data = (struct infra_host_data*)e->data;
+	*to = rtt_timeout(&data->rtt);
+	*edns_vs = data->edns_version;
+	lock_rw_unlock(&e->lock);
+	return 1;
+}
+
+/** hash lameness key */
+static hashvalue_t
+hash_lameness(uint8_t* name, size_t namelen)
+{
+	return hashlittle(name, namelen, 0xab);
+}
+
+int 
+infra_lookup_lame(struct infra_host_data* host,
+        uint8_t* name, size_t namelen, time_t timenow)
+{
+	struct lruhash_entry* e;
+	struct infra_lame_key k;
+	struct infra_lame_data *d;
+	int dl;
+	if(!host->lameness)
+		return 0;
+	k.entry.hash = hash_lameness(name, namelen);
+	k.zonename = name;
+	k.namelen = namelen;
+	k.entry.key = (void*)&k;
+	k.entry.data = NULL;
+	e = lruhash_lookup(host->lameness, k.entry.hash, &k, 0);
+	if(!e)
+		return 0;
+	d = (struct infra_lame_data*)e->data;
+	if(d->ttl < timenow) {
+		lock_rw_unlock(&e->lock);
+		return 0;
+	}
+	dl = d->isdnsseclame;
+	lock_rw_unlock(&e->lock);
+	return dl?2:1;
+}
+
+size_t 
+infra_lame_sizefunc(void* k, void* ATTR_UNUSED(d))
+{
+	struct infra_lame_key* key = (struct infra_lame_key*)k;
+	return sizeof(*key) + sizeof(struct infra_lame_data)
+		+ key->namelen + lock_get_mem(&key->entry.lock);
+}
+
+int 
+infra_lame_compfunc(void* key1, void* key2) 
+{
+	struct infra_lame_key* k1 = (struct infra_lame_key*)key1;
+	struct infra_lame_key* k2 = (struct infra_lame_key*)key2;
+	if(k1->namelen != k2->namelen) {
+		if(k1->namelen < k2->namelen)
+			return -1;
+		return 1;
+	}
+	return query_dname_compare(k1->zonename, k2->zonename);
+}
+
+void 
+infra_lame_delkeyfunc(void* k, void* ATTR_UNUSED(arg), int il)
+{
+	struct infra_lame_key* key = (struct infra_lame_key*)k;
+	if(!key) 
+		return;
+	if(il) { lock_rw_unlock(&key->entry.lock); }
+	lock_rw_destroy(&key->entry.lock);
+	free(key->zonename);
+	free(key);
+}
+
+void 
+infra_lame_deldatafunc(void* d, void* ATTR_UNUSED(arg))
+{
+	if(!d) 
+		return;
+	free(d);
+}
+
+int 
+infra_set_lame(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        uint8_t* name, size_t namelen, time_t timenow, int dnsseclame)
+{
+	struct infra_host_data* data;
+	struct lruhash_entry* e;
+	int needtoinsert = 0;
+	struct infra_lame_key* k;
+	struct infra_lame_data* d;
+	/* allocate at start, easier cleanup (no locks held) */
+	k = (struct infra_lame_key*)malloc(sizeof(*k));
+	if(!k) {
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	d = (struct infra_lame_data*)malloc(sizeof(*d));
+	if(!d) {
+		free(k);
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	k->zonename = memdup(name, namelen);
+	if(!k->zonename) {
+		free(d);
+		free(k);
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	lock_rw_init(&k->entry.lock);
+	k->entry.hash = hash_lameness(name, namelen);
+	k->entry.key = (void*)k;
+	k->entry.data = (void*)d;
+	d->ttl = timenow + infra->lame_ttl;
+	d->isdnsseclame = dnsseclame;
+	k->namelen = namelen;
+	e = infra_lookup_host_nottl(infra, addr, addrlen, 1);
+	if(!e) {
+		/* insert it */
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow))) {
+			free(k->zonename);
+			free(k);
+			free(d);
+			log_err("set_lame: malloc failure");
+			return 0;
+		}
+		needtoinsert = 1;
+	}
+	/* got an entry, now set the zone lame */
+	data = (struct infra_host_data*)e->data;
+	if(!data->lameness) {
+		/* create hash table if not there already */
+		data->lameness = lruhash_create(INFRA_LAME_STARTSIZE,
+			infra->max_lame_size, infra_lame_sizefunc, 
+			infra_lame_compfunc, infra_lame_delkeyfunc,
+			infra_lame_deldatafunc, NULL);
+		if(!data->lameness) {
+			log_err("set_lame: malloc failure");
+			if(needtoinsert) slabhash_insert(infra->hosts, 
+				e->hash, e, e->data, NULL);
+			else 	{ lock_rw_unlock(&e->lock); }
+			free(k->zonename);
+			free(k);
+			free(d);
+			return 0;
+		}
+	}
+	/* inserts new entry, or updates TTL of older entry */
+	lruhash_insert(data->lameness, k->entry.hash, &k->entry, d, NULL);
+	
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return 1;
+}
+
+void 
+infra_update_tcp_works(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 1);
+	struct infra_host_data* data;
+	if(!e)
+		return; /* doesn't exist */
+	data = (struct infra_host_data*)e->data;
+	if(data->rtt.rto >= RTT_MAX_TIMEOUT)
+		/* do not disqualify this server altogether, it is better
+		 * than nothing */
+		data->rtt.rto = RTT_MAX_TIMEOUT-1;
+	lock_rw_unlock(&e->lock);
+}
+
+int 
+infra_rtt_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        int roundtrip, time_t timenow)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 1);
+	struct infra_host_data* data;
+	int needtoinsert = 0;
+	int rto = 1;
+	if(!e) {
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		needtoinsert = 1;
+	}
+	/* have an entry, update the rtt, and the ttl */
+	data = (struct infra_host_data*)e->data;
+	data->ttl = timenow + infra->host_ttl;
+	if(roundtrip == -1)
+		rtt_lost(&data->rtt);
+	else	rtt_update(&data->rtt, roundtrip);
+	if(data->rtt.rto > 0)
+		rto = data->rtt.rto;
+
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return rto;
+}
+
+int 
+infra_edns_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        int edns_version, time_t timenow)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 1);
+	struct infra_host_data* data;
+	int needtoinsert = 0;
+	if(!e) {
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		needtoinsert = 1;
+	}
+	/* have an entry, update the rtt, and the ttl */
+	data = (struct infra_host_data*)e->data;
+	data->ttl = timenow + infra->host_ttl;
+	data->edns_version = edns_version;
+
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return 1;
+}
+
+int 
+infra_get_lame_rtt(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        uint8_t* name, size_t namelen, int* lame, int* dnsseclame,
+	int* rtt, time_t timenow)
+{
+	struct infra_host_data* host;
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 0);
+	int lm;
+	if(!e) 
+		return 0;
+	host = (struct infra_host_data*)e->data;
+	*rtt = rtt_unclamped(&host->rtt);
+	/* check lameness first, if so, ttl on host does not matter anymore */
+	if((lm=infra_lookup_lame(host, name, namelen, timenow))) {
+		lock_rw_unlock(&e->lock);
+		if(lm == 1) {
+			*lame = 1;
+			*dnsseclame = 0;
+		} else {
+			*lame = 0;
+			*dnsseclame = 1;
+		}
+		return 1;
+	}
+	*lame = 0;
+	*dnsseclame = 0;
+	if(timenow > host->ttl) {
+		lock_rw_unlock(&e->lock);
+		return 0;
+	}
+	lock_rw_unlock(&e->lock);
+	return 1;
+}
+
+/** helper memory count for a host lame cache */
+static size_t
+count_host_lame(struct lruhash_entry* e)
+{
+	struct infra_host_data* host_data = (struct infra_host_data*)e->data;
+	if(!host_data->lameness)
+		return 0;
+	return lruhash_get_mem(host_data->lameness);
+}
+
+size_t 
+infra_get_mem(struct infra_cache* infra)
+{
+	size_t i, bin;
+	size_t s = sizeof(*infra) +
+		slabhash_get_mem(infra->hosts);
+	struct lruhash_entry* e;
+	for(i=0; i<infra->hosts->size; i++) {
+		lock_quick_lock(&infra->hosts->array[i]->lock);
+		for(bin=0; bin<infra->hosts->array[i]->size; bin++) {
+			lock_quick_lock(&infra->hosts->array[i]->
+				array[bin].lock);
+			/* count data size in bin items. */
+			for(e = infra->hosts->array[i]->array[bin].
+				overflow_list; e; e = e->overflow_next) {
+				lock_rw_rdlock(&e->lock);
+				s += count_host_lame(e);
+				lock_rw_unlock(&e->lock);
+			}
+			lock_quick_unlock(&infra->hosts->array[i]->
+				array[bin].lock);
+		}
+		lock_quick_unlock(&infra->hosts->array[i]->lock);
+	}
+	return s;
+}

services/cache/infra.h

@@ -0,0 +1,288 @@
+/*
+ * services/cache/infra.h - infrastructure cache, server rtt and capabilities
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the infrastructure cache.
+ */
+
+#ifndef SERVICES_CACHE_INFRA_H
+#define SERVICES_CACHE_INFRA_H
+#include "util/storage/lruhash.h"
+#include "util/rtt.h"
+struct slabhash;
+struct config_file;
+
+/**
+ * Host information kept for every server.
+ */
+struct infra_host_key {
+	/** the host address. */
+	struct sockaddr_storage addr;
+	/** length of addr. */
+	socklen_t addrlen;
+	/** hash table entry, data of type infra_host_data. */
+	struct lruhash_entry entry;
+};
+
+/**
+ * Host information encompasses host capabilities and retransmission timeouts.
+ */
+struct infra_host_data {
+	/** TTL value for this entry. absolute time. */
+	time_t ttl;
+	/** round trip times for timeout calculation */
+	struct rtt_info rtt;
+	/** Names of the zones that are lame. NULL=no lame zones. */
+	struct lruhash* lameness;
+	/** edns version that the host supports, -1 means no EDNS */
+	int edns_version;
+};
+
+/**
+ * Lameness information, per host, per zone. 
+ */
+struct infra_lame_key {
+	/** key is zone name in wireformat */
+	uint8_t* zonename;
+	/** length of zonename */
+	size_t namelen;
+	/** lruhash entry */
+	struct lruhash_entry entry;
+};
+
+/**
+ * Lameness information. Expires.
+ * This host is lame because it is in the cache.
+ */
+struct infra_lame_data {
+	/** TTL of this entry. absolute time. */
+	time_t ttl;
+	/** is the host lame (does not serve the zone authoritatively),
+	 * or is the host dnssec lame (does not serve DNSSEC data) */
+	int isdnsseclame;
+};
+
+/**
+ * Infra cache 
+ */
+struct infra_cache {
+	/** The hash table with hosts */
+	struct slabhash* hosts;
+	/** TTL value for host information, in seconds */
+	int host_ttl;
+	/** TTL for Lameness information, in seconds */
+	int lame_ttl;
+	/** infra lame cache max memory per host, in bytes */
+	size_t max_lame_size;
+};
+
+/** infra host cache default hash lookup size */
+#define INFRA_HOST_STARTSIZE 32
+/** infra lame cache default hash lookup size */
+#define INFRA_LAME_STARTSIZE 2
+
+/**
+ * Create infra cache.
+ * @param cfg: config parameters or NULL for defaults.
+ * @return: new infra cache, or NULL.
+ */
+struct infra_cache* infra_create(struct config_file* cfg);
+
+/**
+ * Delete infra cache.
+ * @param infra: infrastructure cache to delete.
+ */
+void infra_delete(struct infra_cache* infra);
+
+/**
+ * Adjust infra cache to use updated configuration settings.
+ * This may clean the cache. Operates a bit like realloc.
+ * There may be no threading or use by other threads.
+ * @param infra: existing cache. If NULL a new infra cache is returned.
+ * @param cfg: config options.
+ * @return the new infra cache pointer or NULL on error.
+ */
+struct infra_cache* infra_adjust(struct infra_cache* infra, 
+	struct config_file* cfg);
+
+/**
+ * Lookup host data
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param wr: set to true to get a writelock on the entry.
+ * @param timenow: what time it is now.
+ * @param key: the key for the host, returned so caller can unlock when done.
+ * @return: host data or NULL if not found or expired.
+ */
+struct infra_host_data* infra_lookup_host(struct infra_cache* infra, 
+	struct sockaddr_storage* addr, socklen_t addrlen, int wr, 
+	time_t timenow, struct infra_host_key** key);
+
+/**
+ * Find host information to send a packet. Creates new entry if not found.
+ * Lameness is empty. EDNS is 0 (try with first), and rtt is returned for 
+ * the first message to it.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param timenow: what time it is now.
+ * @param edns_vs: edns version it supports, is returned.
+ * @param to: timeout to use, is returned.
+ * @return: 0 on error.
+ */
+int infra_host(struct infra_cache* infra, struct sockaddr_storage* addr, 
+	socklen_t addrlen, time_t timenow, int* edns_vs, int* to);
+
+/**
+ * Check for lameness of this server for a particular zone.
+ * You must have a lock on the host structure.
+ * @param host: infrastructure cache data for the host. Caller holds lock.
+ * @param name: domain name of zone apex.
+ * @param namelen: length of domain name.
+ * @param timenow: what time it is now.
+ * @return: 0 if not lame or unknown or timed out, 1 if lame, 2 if dnsseclame.
+ */
+int infra_lookup_lame(struct infra_host_data* host,
+	uint8_t* name, size_t namelen, time_t timenow);
+
+/**
+ * Set a host to be lame for the given zone.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param name: domain name of zone apex.
+ * @param namelen: length of domain name.
+ * @param timenow: what time it is now.
+ * @param dnsseclame: if true the host is set dnssec lame.
+ *	if false, the host is marked lame (not serving the zone).
+ * @return: 0 on error.
+ */
+int infra_set_lame(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	uint8_t* name, size_t namelen, time_t timenow, int dnsseclame);
+
+/**
+ * Update rtt information for the host.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param roundtrip: estimate of roundtrip time in milliseconds or -1 for 
+ * 	timeout.
+ * @param timenow: what time it is now.
+ * @return: 0 on error. new rto otherwise.
+ */
+int infra_rtt_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	int roundtrip, time_t timenow);
+
+/**
+ * Update information for the host, store that a TCP transaction works.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ */
+void infra_update_tcp_works(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen);
+
+/**
+ * Update edns information for the host.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param edns_version: the version that it publishes.
+ * @param timenow: what time it is now.
+ * @return: 0 on error.
+ */
+int infra_edns_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	int edns_version, time_t timenow);
+
+/**
+ * Get Lameness information and average RTT if host is in the cache.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param name: zone name.
+ * @param namelen: zone name length.
+ * @param lame: if function returns true, this returns lameness of the zone.
+ * @param dnsseclame: if function returns true, this returns if the zone
+ *	is dnssec-lame.
+ * @param rtt: if function returns true, this returns avg rtt of the server.
+ * 	The rtt value is unclamped and reflects recent timeouts.
+ * @param timenow: what time it is now.
+ * @return if found in cache, or false if not (or TTL bad).
+ */
+int infra_get_lame_rtt(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, 
+	uint8_t* name, size_t namelen, int* lame, int* dnsseclame,
+	int* rtt, time_t timenow);
+
+/**
+ * Get memory used by the infra cache.
+ * @param infra: infrastructure cache.
+ * @return memory in use in bytes.
+ */
+size_t infra_get_mem(struct infra_cache* infra);
+
+/** calculate size for the hashtable, does not count size of lameness,
+ * so the hashtable is a fixed number of items */
+size_t infra_host_sizefunc(void* k, void* d);
+
+/** compare two addresses, returns -1, 0, or +1 */
+int infra_host_compfunc(void* key1, void* key2);
+
+/** delete key, and destroy the lock */
+void infra_host_delkeyfunc(void* k, void* arg, int il);
+
+/** delete data and destroy the lameness hashtable */
+void infra_host_deldatafunc(void* d, void* arg);
+
+/** calculate size, which is fixed, zonename does not count so that
+ * a fixed number of items is stored */
+size_t infra_lame_sizefunc(void* k, void* d);
+
+/** compare zone names, returns -1, 0, +1 */
+int infra_lame_compfunc(void* key1, void* key2);
+
+/** free key, lock and zonename */
+void infra_lame_delkeyfunc(void* k, void* arg, int il);
+
+/** free the lameness data */
+void infra_lame_deldatafunc(void* d, void* arg);
+
+#endif /* SERVICES_CACHE_INFRA_H */

services/cache/rrset.c

@@ -0,0 +1,359 @@
+/*
+ * services/cache/rrset.c - Resource record set cache.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the rrset cache.
+ */
+#include "config.h"
+#include "services/cache/rrset.h"
+#include "util/storage/slabhash.h"
+#include "util/config_file.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/msgreply.h"
+#include "util/regional.h"
+#include "util/alloc.h"
+
+struct rrset_cache* rrset_cache_create(struct config_file* cfg, 
+	struct alloc_cache* alloc)
+{
+	size_t slabs = (cfg?cfg->rrset_cache_slabs:HASH_DEFAULT_SLABS);
+	size_t startarray = HASH_DEFAULT_STARTARRAY;
+	size_t maxmem = (cfg?cfg->rrset_cache_size:HASH_DEFAULT_MAXMEM);
+
+	struct rrset_cache *r = (struct rrset_cache*)slabhash_create(slabs,
+		startarray, maxmem, ub_rrset_sizefunc, ub_rrset_compare,
+		ub_rrset_key_delete, rrset_data_delete, alloc);
+	return r;
+}
+
+void rrset_cache_delete(struct rrset_cache* r)
+{
+	if(!r) 
+		return;
+	slabhash_delete(&r->table);
+}
+
+struct rrset_cache* rrset_cache_adjust(struct rrset_cache *r, 
+	struct config_file* cfg, struct alloc_cache* alloc)
+{
+	if(!r || !cfg || cfg->rrset_cache_slabs != r->table.size ||
+		cfg->rrset_cache_size != slabhash_get_size(&r->table))
+	{
+		rrset_cache_delete(r);
+		r = rrset_cache_create(cfg, alloc);
+	}
+	return r;
+}
+
+void 
+rrset_cache_touch(struct rrset_cache* r, struct ub_packed_rrset_key* key,
+        hashvalue_t hash, rrset_id_t id)
+{
+	struct lruhash* table = slabhash_gettable(&r->table, hash);
+	/* 
+	 * This leads to locking problems, deadlocks, if the caller is 
+	 * holding any other rrset lock.
+	 * Because a lookup through the hashtable does:
+	 *	tablelock -> entrylock  (for that entry caller holds)
+	 * And this would do
+	 *	entrylock(already held) -> tablelock
+	 * And if two threads do this, it results in deadlock.
+	 * So, the caller must not hold entrylock.
+	 */
+	lock_quick_lock(&table->lock);
+	/* we have locked the hash table, the item can still be deleted.
+	 * because it could already have been reclaimed, but not yet set id=0.
+	 * This is because some lruhash routines have lazy deletion.
+	 * so, we must acquire a lock on the item to verify the id != 0.
+	 * also, with hash not changed, we are using the right slab.
+	 */
+	lock_rw_rdlock(&key->entry.lock);
+	if(key->id == id && key->entry.hash == hash) {
+		lru_touch(table, &key->entry);
+	}
+	lock_rw_unlock(&key->entry.lock);
+	lock_quick_unlock(&table->lock);
+}
+
+/** see if rrset needs to be updated in the cache */
+static int
+need_to_update_rrset(void* nd, void* cd, uint32_t timenow, int equal)
+{
+	struct packed_rrset_data* newd = (struct packed_rrset_data*)nd;
+	struct packed_rrset_data* cached = (struct packed_rrset_data*)cd;
+	/* 	o store if rrset has been validated */
+	if( newd->security > cached->security) {
+		return 1;
+	}
+        /*      o if current RRset is more trustworthy - insert it */
+        if( newd->trust > cached->trust ) {
+		/* if the cached rrset is bogus, and this one equal,
+		 * do not update the TTL - let it expire. */
+		if(equal && cached->ttl >= timenow && 
+			cached->security == sec_status_bogus)
+			return 0;
+                return 1;
+	}
+	/*	o item in cache has expired */
+	if( cached->ttl < timenow )
+		return 1;
+        /*      o same trust, but different in data - insert it */
+        if( newd->trust == cached->trust && !equal )
+                return 1;
+        return 0;
+}
+
+/** Update RRSet special key ID */
+static void
+rrset_update_id(struct rrset_ref* ref, struct alloc_cache* alloc)
+{
+	/* this may clear the cache and invalidate lock below */
+	uint64_t newid = alloc_get_id(alloc);
+	/* obtain writelock */
+	lock_rw_wrlock(&ref->key->entry.lock);
+	/* check if it was deleted in the meantime, if so, skip update */
+	if(ref->key->id == ref->id) {
+		ref->key->id = newid;
+		ref->id = newid;
+	}
+	lock_rw_unlock(&ref->key->entry.lock);
+}
+
+int 
+rrset_cache_update(struct rrset_cache* r, struct rrset_ref* ref,
+	struct alloc_cache* alloc, uint32_t timenow)
+{
+	struct lruhash_entry* e;
+	struct ub_packed_rrset_key* k = ref->key;
+	hashvalue_t h = k->entry.hash;
+	uint16_t rrset_type = ntohs(k->rk.type);
+	int equal = 0;
+	log_assert(ref->id != 0 && k->id != 0);
+	/* looks up item with a readlock - no editing! */
+	if((e=slabhash_lookup(&r->table, h, k, 0)) != 0) {
+		/* return id and key as they will be used in the cache
+		 * since the lruhash_insert, if item already exists, deallocs
+		 * the passed key in favor of the already stored key.
+		 * because of the small gap (see below) this key ptr and id
+		 * may prove later to be already deleted, which is no problem
+		 * as it only makes a cache miss. 
+		 */
+		ref->key = (struct ub_packed_rrset_key*)e->key;
+		ref->id = ref->key->id;
+		equal = rrsetdata_equal((struct packed_rrset_data*)k->entry.
+			data, (struct packed_rrset_data*)e->data);
+		if(!need_to_update_rrset(k->entry.data, e->data, timenow,
+			equal)) {
+			/* cache is superior, return that value */
+			lock_rw_unlock(&e->lock);
+			ub_packed_rrset_parsedelete(k, alloc);
+			return 1;
+		}
+		lock_rw_unlock(&e->lock);
+		/* Go on and insert the passed item.
+		 * small gap here, where entry is not locked.
+		 * possibly entry is updated with something else.
+		 * we then overwrite that with our data.
+		 * this is just too bad, its cache anyway. */
+		/* use insert to update entry to manage lruhash
+		 * cache size values nicely. */
+	}
+	log_assert(ref->key->id != 0);
+	slabhash_insert(&r->table, h, &k->entry, k->entry.data, alloc);
+	if(e) {
+		/* For NSEC, NSEC3, DNAME, when rdata is updated, update 
+		 * the ID number so that proofs in message cache are 
+		 * invalidated */
+		if((rrset_type == LDNS_RR_TYPE_NSEC 
+			|| rrset_type == LDNS_RR_TYPE_NSEC3
+			|| rrset_type == LDNS_RR_TYPE_DNAME) && !equal) {
+			rrset_update_id(ref, alloc);
+		}
+		return 1;
+	}
+	return 0;
+}
+
+struct ub_packed_rrset_key* 
+rrset_cache_lookup(struct rrset_cache* r, uint8_t* qname, size_t qnamelen, 
+	uint16_t qtype, uint16_t qclass, uint32_t flags, uint32_t timenow,
+	int wr)
+{
+	struct lruhash_entry* e;
+	struct ub_packed_rrset_key key;
+	
+	key.entry.key = &key;
+	key.entry.data = NULL;
+	key.rk.dname = qname;
+	key.rk.dname_len = qnamelen;
+	key.rk.type = htons(qtype);
+	key.rk.rrset_class = htons(qclass);
+	key.rk.flags = flags;
+
+	key.entry.hash = rrset_key_hash(&key.rk);
+
+	if((e = slabhash_lookup(&r->table, key.entry.hash, &key, wr))) {
+		/* check TTL */
+		struct packed_rrset_data* data = 
+			(struct packed_rrset_data*)e->data;
+		if(timenow > data->ttl) {
+			lock_rw_unlock(&e->lock);
+			return NULL;
+		}
+		/* we're done */
+		return (struct ub_packed_rrset_key*)e->key;
+	}
+	return NULL;
+}
+
+int 
+rrset_array_lock(struct rrset_ref* ref, size_t count, uint32_t timenow)
+{
+	size_t i;
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only lock items once */
+		lock_rw_rdlock(&ref[i].key->entry.lock);
+		if(ref[i].id != ref[i].key->id || timenow >
+			((struct packed_rrset_data*)(ref[i].key->entry.data))
+			->ttl) {
+			/* failure! rollback our readlocks */
+			rrset_array_unlock(ref, i+1);
+			return 0;
+		}
+	}
+	return 1;
+}
+
+void 
+rrset_array_unlock(struct rrset_ref* ref, size_t count)
+{
+	size_t i;
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only unlock items once */
+		lock_rw_unlock(&ref[i].key->entry.lock);
+	}
+}
+
+void 
+rrset_array_unlock_touch(struct rrset_cache* r, struct regional* scratch,
+	struct rrset_ref* ref, size_t count)
+{
+	hashvalue_t* h;
+	size_t i;
+	if(!(h = (hashvalue_t*)regional_alloc(scratch, 
+		sizeof(hashvalue_t)*count)))
+		log_warn("rrset LRU: memory allocation failed");
+	else 	/* store hash values */
+		for(i=0; i<count; i++)
+			h[i] = ref[i].key->entry.hash;
+	/* unlock */
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only unlock items once */
+		lock_rw_unlock(&ref[i].key->entry.lock);
+	}
+	if(h) {
+		/* LRU touch, with no rrset locks held */
+		for(i=0; i<count; i++) {
+			if(i>0 && ref[i].key == ref[i-1].key)
+				continue; /* only touch items once */
+			rrset_cache_touch(r, ref[i].key, h[i], ref[i].id);
+		}
+	}
+}
+
+void 
+rrset_update_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset)
+{
+	uint32_t now = (uint32_t)time(0);
+	struct packed_rrset_data* updata = 
+		(struct packed_rrset_data*)rrset->entry.data;
+	struct lruhash_entry* e;
+	struct packed_rrset_data* cachedata;
+
+	/* hash it again to make sure it has a hash */
+	rrset->entry.hash = rrset_key_hash(&rrset->rk);
+
+	e = slabhash_lookup(&r->table, rrset->entry.hash, rrset, 1);
+	if(!e)
+		return; /* not in the cache anymore */
+	cachedata = (struct packed_rrset_data*)e->data;
+	if(!rrsetdata_equal(updata, cachedata)) {
+		lock_rw_unlock(&e->lock);
+		return; /* rrset has changed in the meantime */
+	}
+	/* update the cached rrset */
+	if(updata->security > cachedata->security) {
+		cachedata->trust = updata->trust;
+		cachedata->security = updata->security;
+		cachedata->ttl = updata->ttl + now;
+	}
+	lock_rw_unlock(&e->lock);
+}
+
+void 
+rrset_check_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset)
+{
+	uint32_t now = (uint32_t)time(0);
+	struct packed_rrset_data* updata = 
+		(struct packed_rrset_data*)rrset->entry.data;
+	struct lruhash_entry* e;
+	struct packed_rrset_data* cachedata;
+
+	/* hash it again to make sure it has a hash */
+	rrset->entry.hash = rrset_key_hash(&rrset->rk);
+
+	e = slabhash_lookup(&r->table, rrset->entry.hash, rrset, 0);
+	if(!e)
+		return; /* not in the cache anymore */
+	cachedata = (struct packed_rrset_data*)e->data;
+	if(now > cachedata->ttl || !rrsetdata_equal(updata, cachedata)) {
+		lock_rw_unlock(&e->lock);
+		return; /* expired, or rrset has changed in the meantime */
+	}
+	if(cachedata->security > updata->security) {
+		updata->security = cachedata->security;
+		if(cachedata->security == sec_status_bogus)
+			updata->ttl = cachedata->ttl - now;
+		updata->trust = cachedata->trust;
+	}
+	lock_rw_unlock(&e->lock);
+}

services/cache/rrset.h

@@ -0,0 +1,210 @@
+/*
+ * services/cache/rrset.h - Resource record set cache.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the rrset cache.
+ */
+
+#ifndef SERVICES_CACHE_RRSET_H
+#define SERVICES_CACHE_RRSET_H
+#include "util/storage/lruhash.h"
+#include "util/storage/slabhash.h"
+#include "util/data/packed_rrset.h"
+struct config_file;
+struct alloc_cache;
+struct rrset_ref;
+struct regional;
+
+/**
+ * The rrset cache
+ * Thin wrapper around hashtable, like a typedef.
+ */
+struct rrset_cache {
+	/** uses partitioned hash table */
+	struct slabhash table;
+};
+
+/**
+ * Create rrset cache
+ * @param cfg: config settings or NULL for defaults.
+ * @param alloc: initial default rrset key allocation.
+ * @return: NULL on error.
+ */
+struct rrset_cache* rrset_cache_create(struct config_file* cfg, 
+	struct alloc_cache* alloc);
+
+/**
+ * Delete rrset cache
+ * @param r: rrset cache to delete.
+ */
+void rrset_cache_delete(struct rrset_cache* r);
+
+/**
+ * Adjust settings of the cache to settings from the config file.
+ * May purge the cache. May recreate the cache.
+ * There may be no threading or use by other threads.
+ * @param r: rrset cache to adjust (like realloc).
+ * @param cfg: config settings or NULL for defaults.
+ * @param alloc: initial default rrset key allocation.
+ * @return 0 on error, or new rrset cache pointer on success.
+ */
+struct rrset_cache* rrset_cache_adjust(struct rrset_cache* r, 
+	struct config_file* cfg, struct alloc_cache* alloc);
+
+/**
+ * Touch rrset, with given pointer and id.
+ * Caller may not hold a lock on ANY rrset, this could give deadlock.
+ *
+ * This routine is faster than a hashtable lookup:
+ *	o no bin_lock is acquired.
+ *	o no walk through the bin-overflow-list. 
+ *	o no comparison of the entry key to find it.
+ *
+ * @param r: rrset cache.
+ * @param key: rrset key. Marked recently used (if it was not deleted
+ *	before the lock is acquired, in that case nothing happens).
+ * @param hash: hash value of the item. Please read it from the key when
+ *	you have it locked. Used to find slab from slabhash.
+ * @param id: used to check that the item is unchanged and not deleted.
+ */
+void rrset_cache_touch(struct rrset_cache* r, struct ub_packed_rrset_key* key,
+	hashvalue_t hash, rrset_id_t id);
+
+/**
+ * Update an rrset in the rrset cache. Stores the information for later use.
+ * Will lookup if the rrset is in the cache and perform an update if necessary.
+ * If the item was present, and superior, references are returned to that.
+ * The passed item is then deallocated with rrset_parsedelete.
+ *
+ * A superior rrset is:
+ *	o rrset with better trust value.
+ *	o same trust value, different rdata, newly passed rrset is inserted.
+ * If rdata is the same, TTL in the cache is updated.
+ *
+ * @param r: the rrset cache.
+ * @param ref: reference (ptr and id) to the rrset. Pass reference setup for
+ *	the new rrset. The reference may be changed if the cached rrset is
+ *	superior.
+ *	Before calling the rrset is presumed newly allocated and changeable.
+ *	Afer calling you do not hold a lock, and the rrset is inserted in
+ *	the hashtable so you need a lock to change it.
+ * @param alloc: how to allocate (and deallocate) the special rrset key.
+ * @param timenow: current time (to see if ttl in cache is expired).
+ * @return: true if the passed reference is updated, false if it is unchanged.
+ */
+int rrset_cache_update(struct rrset_cache* r, struct rrset_ref* ref, 
+	struct alloc_cache* alloc, uint32_t timenow);
+
+/**
+ * Lookup rrset. You obtain read/write lock. You must unlock before lookup
+ * anything of else.
+ * @param r: the rrset cache.
+ * @param qname: name of rrset to lookup.
+ * @param qnamelen: length of name of rrset to lookup.
+ * @param qtype: type of rrset to lookup (host order).
+ * @param qclass: class of rrset to lookup (host order).
+ * @param flags: rrset flags, or 0.
+ * @param timenow: used to compare with TTL.
+ * @param wr: set true to get writelock.
+ * @return packed rrset key pointer. Remember to unlock the key.entry.lock.
+ * 	or NULL if could not be found or it was timed out.
+ */
+struct ub_packed_rrset_key* rrset_cache_lookup(struct rrset_cache* r,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint32_t flags, uint32_t timenow, int wr);
+
+/**
+ * Obtain readlock on a (sorted) list of rrset references.
+ * Checks TTLs and IDs of the rrsets and rollbacks locking if not Ok.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ * @param timenow: used to compare with TTL.
+ * @return true on success, false on a failure, which can be that some
+ * 	RRsets have timed out, or that they do not exist any more, the
+ *	RRsets have been purged from the cache.
+ *	If true, you hold readlocks on all the ref items. 
+ */
+int rrset_array_lock(struct rrset_ref* ref, size_t count, uint32_t timenow);
+
+/**
+ * Unlock array (sorted) of rrset references.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ */
+void rrset_array_unlock(struct rrset_ref* ref, size_t count);
+
+/**
+ * Unlock array (sorted) of rrset references and at the same time
+ * touch LRU on the rrsets. It needs the scratch region for temporary
+ * storage as it uses the initial locks to obtain hash values.
+ * @param r: the rrset cache. In this cache LRU is updated.
+ * @param scratch: region for temporary storage of hash values.
+ *	if memory allocation fails, the lru touch fails silently,
+ *	but locks are released. memory errors are logged.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ */
+void rrset_array_unlock_touch(struct rrset_cache* r, struct regional* scratch,
+	struct rrset_ref* ref, size_t count);
+
+/**
+ * Update security status of an rrset. Looks up the rrset.
+ * If found, checks if rdata is equal.
+ * If so, it will update the security, trust and rrset-ttl values.
+ * The values are only updated if security is increased (towards secure).
+ * @param r: the rrset cache. 
+ * @param rrset: which rrset to attempt to update. This rrset is left 
+ * 	untouched. The rrset in the cache is updated in-place.
+ */
+void rrset_update_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Looks up security status of an rrset. Looks up the rrset.
+ * If found, checks if rdata is equal, and entry did not expire.
+ * If so, it will update the security, trust and rrset-ttl values.
+ * @param r: the rrset cache. 
+ * @param rrset: This rrset may change security status due to the cache.
+ * 	But its status will only improve, towards secure.
+ */
+void rrset_check_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset);
+
+#endif /* SERVICES_CACHE_RRSET_H */

services/listen_dnsport.c

@@ -43,19 +43,17 @@
 #include "services/outside_network.h"
 #include "util/netevent.h"
 #include "util/log.h"
+#include "util/config_file.h"
 #include "util/net_help.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #  include <sys/types.h>
 #endif
 #include <netdb.h>
-#include <errno.h>
 #include <fcntl.h>
 
 /** number of queued TCP connections for listen() */
 #define TCP_BACKLOG 5 
-/** number of simultaneous open TCP connections for queries */
-#define TCP_COUNT 10 
 
 /**
  * Debug print of the getaddrinfo returned address.
@@ -76,6 +74,7 @@ verbose_print_addr(struct addrinfo *addr)
 			(socklen_t)sizeof(buf)) == 0) {
 			strncpy(buf, "(null)", sizeof(buf));
 		}
+		buf[sizeof(buf)-1] = 0;
 		verbose(VERB_ALGO, "creating %s%s socket %s %d", 
 			addr->ai_socktype==SOCK_DGRAM?"udp":
 			addr->ai_socktype==SOCK_STREAM?"tcp":"otherproto",
@@ -89,7 +88,7 @@ verbose_print_addr(struct addrinfo *addr)
 int
 create_udp_sock(struct addrinfo *addr)
 {
-	int s, flag;
+	int s;
 # if defined(IPV6_V6ONLY)
 	int on=1;
 # endif
@@ -128,15 +127,8 @@ create_udp_sock(struct addrinfo *addr)
 		log_err("can't bind socket: %s", strerror(errno));
 		return -1;
 	}
-	if((flag = fcntl(s, F_GETFL)) == -1) {
-		log_err("can't fcntl F_GETFL: %s", strerror(errno));
-		flag = 0;
-	}
-	flag |= O_NONBLOCK;
-	if(fcntl(s, F_SETFL, flag) == -1) {
-		log_err("can't fcntl F_SETFL: %s", strerror(errno));
+	if(!fd_set_nonblock(s))
 		return -1;
-	}
 	return s;
 }
 
@@ -219,87 +211,87 @@ make_sock(int stype, const char* ifname, const char* port,
 }
 
 /**
- * Helper for listen_create. Creates one interface (or NULL for default).
+ * Add port to open ports list.
+ * @param list: list head. changed.
+ * @param s: fd.
+ * @param is_udp: if fd is UDP.
+ * @return false on failure. list in unchanged then.
+ */
+static int
+port_insert(struct listen_port** list, int s, int is_udp)
+{
+	struct listen_port* item = (struct listen_port*)malloc(
+		sizeof(struct listen_port));
+	if(!item)
+		return 0;
+	item->next = *list;
+	item->fd = s;
+	item->is_udp = is_udp;
+	*list = item;
+	return 1;
+}
+
+/**
+ * Helper for ports_open. Creates one interface (or NULL for default).
  * @param ifname: The interface ip address.
- * @param front: The the listening info.
- * @param base: Event base.
- * @param port: Port number to use (as string).
  * @param do_udp: if udp should be used.
  * @param do_tcp: if udp should be used.
  * @param hints: for getaddrinfo. family and flags have to be set by caller.
- * @param bufsize: TCP buffer size.
- * @param cb: callback function
- * @param cb_arg: user parameter for callback function.
+ * @param port: Port number to use (as string).
+ * @param list: list of open ports, appended to, changed to point to list head.
  * @return: returns false on error.
  */
 static int
-listen_create_if(const char* ifname, struct listen_dnsport* front, 
-	struct comm_base* base, const char* port, int do_udp, int do_tcp, 
-	struct addrinfo *hints, size_t bufsize, comm_point_callback_t* cb, 
-	void *cb_arg)
+ports_create_if(const char* ifname, int do_udp, int do_tcp, 
+	struct addrinfo *hints, const char* port, struct listen_port** list)
 {
-	struct comm_point *cp_udp = NULL, *cp_tcp = NULL;
-	struct listen_list *el_udp, *el_tcp;
 	int s;
 	if(!do_udp && !do_tcp)
 		return 0;
 	if(do_udp) {
 		if((s = make_sock(SOCK_DGRAM, ifname, port, hints)) == -1)
 			return 0;
-		cp_udp = comm_point_create_udp(base, s, front->udp_buff, 
-			cb, cb_arg);
-		if(!cp_udp) {
-			log_err("can't create commpoint");	
+		if(!port_insert(list, s, 1)) {
 			close(s);
 			return 0;
 		}
 	}
 	if(do_tcp) {
 		if((s = make_sock(SOCK_STREAM, ifname, port, hints)) == -1) {
-			comm_point_delete(cp_udp);
 			return 0;
 		}
-		cp_tcp = comm_point_create_tcp(base, s, TCP_COUNT, bufsize, 
-			cb, cb_arg);
-		if(!cp_tcp) {
-			log_err("can't create commpoint");	
-			comm_point_delete(cp_udp);
+		if(!port_insert(list, s, 0)) {
 			close(s);
 			return 0;
 		}
 	}
-	/* add commpoints to the listen structure */
-	el_udp = (struct listen_list*)malloc(sizeof(struct listen_list));
-	if(!el_udp) {
-		log_err("out of memory");
-		comm_point_delete(cp_udp);
-		comm_point_delete(cp_tcp);
+	return 1;
+}
+
+/** 
+ * Add items to commpoint list in front.
+ * @param c: commpoint to add.
+ * @param front: listen struct.
+ * @return: false on failure.
+ */
+static int
+listen_cp_insert(struct comm_point* c, struct listen_dnsport* front)
+{
+	struct listen_list* item = (struct listen_list*)malloc(
+		sizeof(struct listen_list));
+	if(!item)
 		return 0;
-	}
-	el_tcp = (struct listen_list*)malloc(sizeof(struct listen_list));
-	if(!el_tcp) {
-		log_err("out of memory");
-		free(el_udp);
-		comm_point_delete(cp_udp);
-		comm_point_delete(cp_tcp);
-		return 0;
-	}
-	el_udp->com = cp_udp;
-	el_udp->next = front->cps;
-	front->cps = el_udp;
-	el_tcp->com = cp_tcp;
-	el_tcp->next = front->cps;
-	front->cps = el_tcp;
+	item->com = c;
+	item->next = front->cps;
+	front->cps = item;
 	return 1;
 }
 
 struct listen_dnsport* 
-listen_create(struct comm_base* base, int num_ifs, const char* ifs[], 
-	const char* port, int do_ip4, int do_ip6, int do_udp, int do_tcp,
-	size_t bufsize, comm_point_callback_t* cb, void *cb_arg)
+listen_create(struct comm_base* base, struct listen_port* ports,
+	size_t bufsize, int tcp_accept_count,
+	comm_point_callback_t* cb, void *cb_arg)
 {
-	struct addrinfo hints;
-	int i;
 	struct listen_dnsport* front = (struct listen_dnsport*)
 		malloc(sizeof(struct listen_dnsport));
 	if(!front)
@@ -311,59 +303,27 @@ listen_create(struct comm_base* base, int num_ifs, const char* ifs[],
 		return NULL;
 	}
 	
-	/* getaddrinfo */
-	memset(&hints, 0, sizeof(hints));
-	hints.ai_flags = AI_PASSIVE;
-	/* no name lookups on our listening ports */
-	if(num_ifs > 0)
-		hints.ai_flags |= AI_NUMERICHOST;
-	hints.ai_family = AF_UNSPEC;
-#ifndef INET6
-	do_ip6 = 0;
-#endif
-	if(!do_ip4 && !do_ip6) {
-		listen_delete(front);
-		return NULL;
-	}
-
-	/* create ip4 and ip6 ports so that return addresses are nice. */
-	if(num_ifs == 0) {
-		if(do_ip6) {
-			hints.ai_family = AF_INET6;
-			if(!listen_create_if(NULL, front, base, port, 
-				do_udp, do_tcp, &hints, bufsize, cb, cb_arg)) {
-				listen_delete(front);
-				return NULL;
-			}
+	/* create comm points as needed */
+	while(ports) {
+		struct comm_point* cp = NULL;
+		if(ports->is_udp) 
+			cp = comm_point_create_udp(base, ports->fd, 
+				front->udp_buff, cb, cb_arg);
+		else 	cp = comm_point_create_tcp(base, ports->fd, 
+				tcp_accept_count, bufsize, cb, cb_arg);
+		if(!cp) {
+			log_err("can't create commpoint");	
+			listen_delete(front);
+			return NULL;
 		}
-		if(do_ip4) {
-			hints.ai_family = AF_INET;
-			if(!listen_create_if(NULL, front, base, port, 
-				do_udp, do_tcp, &hints, bufsize, cb, cb_arg)) {
-				listen_delete(front);
-				return NULL;
-			}
-		}
-	} else for(i = 0; i<num_ifs; i++) {
-		if(str_is_ip6(ifs[i])) {
-			if(!do_ip6)
-				continue;
-			hints.ai_family = AF_INET6;
-			if(!listen_create_if(ifs[i], front, base, port, 
-				do_udp, do_tcp, &hints, bufsize, cb, cb_arg)) {
-				listen_delete(front);
-				return NULL;
-			}
-		} else {
-			if(!do_ip4)
-				continue;
-			hints.ai_family = AF_INET;
-			if(!listen_create_if(ifs[i], front, base, port, 
-				do_udp, do_tcp, &hints, bufsize, cb, cb_arg)) {
-				listen_delete(front);
-				return NULL;
-			}
+		cp->do_not_close = 1;
+		if(!listen_cp_insert(cp, front)) {
+			log_err("malloc failed");
+			comm_point_delete(cp);
+			listen_delete(front);
+			return NULL;
 		}
+		ports = ports->next;
 	}
 	if(!front->cps) {
 		log_err("Could not open sockets to accept queries.");
@@ -390,3 +350,124 @@ listen_delete(struct listen_dnsport* front)
 	ldns_buffer_free(front->udp_buff);
 	free(front);
 }
+
+void listen_pushback(struct listen_dnsport* listen)
+{
+	struct listen_list *p;
+	log_assert(listen);
+	for(p = listen->cps; p; p = p->next)
+	{
+		if(p->com->type != comm_udp &&
+			p->com->type != comm_tcp_accept)
+			continue;
+		comm_point_stop_listening(p->com);
+	}
+}
+
+void listen_resume(struct listen_dnsport* listen)
+{
+	struct listen_list *p;
+	log_assert(listen);
+	for(p = listen->cps; p; p = p->next)
+	{
+		if(p->com->type != comm_udp &&
+			p->com->type != comm_tcp_accept)
+			continue;
+		comm_point_start_listening(p->com, -1, -1);
+	}
+}
+
+struct listen_port* 
+listening_ports_open(struct config_file* cfg)
+{
+	struct listen_port* list = NULL;
+	struct addrinfo hints;
+	int i, do_ip4, do_ip6;
+	int do_tcp;
+	char portbuf[32];
+	snprintf(portbuf, sizeof(portbuf), "%d", cfg->port);
+	do_ip4 = cfg->do_ip4;
+	do_ip6 = cfg->do_ip6;
+	do_tcp = cfg->do_tcp;
+	if(cfg->incoming_num_tcp == 0)
+		do_tcp = 0;
+
+	/* getaddrinfo */
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_flags = AI_PASSIVE;
+	/* no name lookups on our listening ports */
+	if(cfg->num_ifs > 0)
+		hints.ai_flags |= AI_NUMERICHOST;
+	hints.ai_family = AF_UNSPEC;
+#ifndef INET6
+	do_ip6 = 0;
+#endif
+	if(!do_ip4 && !do_ip6) {
+		return NULL;
+	}
+	/* create ip4 and ip6 ports so that return addresses are nice. */
+	if(cfg->num_ifs == 0) {
+		if(do_ip6) {
+			hints.ai_family = AF_INET6;
+			if(!ports_create_if("::1", cfg->do_udp, do_tcp, 
+				&hints, portbuf, &list)) {
+				listening_ports_free(list);
+				return NULL;
+			}
+		}
+		if(do_ip4) {
+			hints.ai_family = AF_INET;
+			if(!ports_create_if("127.0.0.1", cfg->do_udp, do_tcp, 
+				&hints, portbuf, &list)) {
+				listening_ports_free(list);
+				return NULL;
+			}
+		}
+	} else for(i = 0; i<cfg->num_ifs; i++) {
+		if(str_is_ip6(cfg->ifs[i])) {
+			if(!do_ip6)
+				continue;
+			hints.ai_family = AF_INET6;
+			if(!ports_create_if(cfg->ifs[i], cfg->do_udp, 
+				do_tcp, &hints, portbuf, &list)) {
+				listening_ports_free(list);
+				return NULL;
+			}
+		} else {
+			if(!do_ip4)
+				continue;
+			hints.ai_family = AF_INET;
+			if(!ports_create_if(cfg->ifs[i], cfg->do_udp, 
+				do_tcp, &hints, portbuf, &list)) {
+				listening_ports_free(list);
+				return NULL;
+			}
+		}
+	}
+	return list;
+}
+
+void listening_ports_free(struct listen_port* list)
+{
+	struct listen_port* nx;
+	while(list) {
+		nx = list->next;
+		if(list->fd != -1)
+			close(list->fd);
+		free(list);
+		list = nx;
+	}
+}
+
+size_t listen_get_mem(struct listen_dnsport* listen)
+{
+	size_t s = sizeof(*listen) + sizeof(*listen->base) + 
+		sizeof(*listen->udp_buff) + 
+		ldns_buffer_capacity(listen->udp_buff);
+	struct listen_list* p;
+	for(p = listen->cps; p; p = p->next) {
+		s += sizeof(*p);
+		s += comm_point_get_mem(p->com);
+	}
+	return s;
+}

services/listen_dnsport.h

@@ -46,6 +46,7 @@
 #include "util/netevent.h"
 struct listen_list;
 struct addrinfo;
+struct config_file;
 
 /**
  * Listening for queries structure.
@@ -74,34 +75,82 @@ struct listen_list {
 };
 
 /**
+ * Single linked list to store shared ports that have been 
+ * opened for use by all threads.
+ */
+struct listen_port {
+	/** next in list */
+	struct listen_port* next;
+	/** file descriptor, open and ready for use */
+	int fd;
+	/** type of file descriptor, udp or tcp */
+	int is_udp;
+};
+
+/**
+ * Create shared listening ports
  * Getaddrinfo, create socket, bind and listen to zero or more 
  * interfaces for IP4 and/or IP6, for UDP and/or TCP.
- * On the given port number. It creates the listening sockets.
+ * On the given port number. It creates the sockets.
+ * @param cfg: settings on what ports to open.
+ * @return: linked list of ports or NULL on error.
+ */
+struct listen_port* listening_ports_open(struct config_file* cfg);
+
+/**
+ * Close and delete the (list of) listening ports.
+ */
+void listening_ports_free(struct listen_port* list);
+
+/**
+ * Create commpoints with for this thread for the shared ports.
  * @param base: the comm_base that provides event functionality.
- * @param num_ifs: number of interfaces to listen on. Can be 0,
  *	for default all ifs.
- * @param ifs: array of strings with interface specs, IP addresses.
- * @param port: the port number to bind to.
- * @param do_ip4: listen to ip4 queries.
- * @param do_ip6: listen to ip6 queries.
- * @param do_udp: listen to udp queries.
- * @param do_tcp: listen to tcp queries.
+ * @param ports: the list of shared ports.
  * @param bufsize: size of datagram buffer.
+ * @param tcp_accept_count: max number of simultaneous TCP connections 
+ * 	from clients.
  * @param cb: callback function when a request arrives. It is passed
  *	  the packet and user argument. Return true to send a reply.
  * @param cb_arg: user data argument for callback function.
  * @return: the malloced listening structure, ready for use. NULL on error.
  */
 struct listen_dnsport* listen_create(struct comm_base* base,
-	int num_ifs, const char* ifs[], const char* port,
-	int do_ip4, int do_ip6, int do_udp, int do_tcp,
-	size_t bufsize, comm_point_callback_t* cb, void* cb_arg);
+	struct listen_port* ports, size_t bufsize, int tcp_accept_count,
+	comm_point_callback_t* cb, void* cb_arg);
+
+/**
+ * Stop listening to the dnsports. Ports are still open but not checked
+ * for readability - performs pushback of the load.
+ * @param listen: the listening structs to stop listening on. Note that
+ *    udp and tcp-accept handlers stop, but ongoing tcp-handlers are kept
+ *    going, since its rude to 'reset connection by peer' them, instead,
+ *    we keep them and the callback will be called when its ready. It can
+ *    be dropped at that time. New tcp and udp queries can be served by 
+ *    other threads.
+ */
+void listen_pushback(struct listen_dnsport* listen);
+
+/**
+ * Start listening again to the dnsports.
+ * Call after the listen_pushback has been called.
+ * @param listen: the listening structs to stop listening on.
+ */
+void listen_resume(struct listen_dnsport* listen);
 
 /**
  * delete the listening structure
+ * @param listen: listening structure.
  */
 void listen_delete(struct listen_dnsport* listen);
 
+/**
+ * get memory size used by the listening structs
+ * @param listen: listening structure.
+ * @return: size in bytes.
+ */
+size_t listen_get_mem(struct listen_dnsport* listen);
+
 /**
  * Create and bind nonblocking UDP socket
  * @param addr: address info ready to make socket.

services/mesh.c

@@ -0,0 +1,708 @@
+/*
+ * services/mesh.c - deal with mesh of query states and handle events for that.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist in dealing with a mesh of
+ * query states. This mesh is supposed to be thread-specific.
+ * It consists of query states (per qname, qtype, qclass) and connections
+ * between query states and the super and subquery states, and replies to
+ * send back to clients.
+ */
+#include "config.h"
+#include "services/mesh.h"
+#include "services/outbound_list.h"
+#include "services/cache/dns.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/regional.h"
+#include "util/data/msgencode.h"
+#include "util/timehist.h"
+#include "util/fptr_wlist.h"
+#include "util/alloc.h"
+
+int
+mesh_state_compare(const void* ap, const void* bp)
+{
+	struct mesh_state* a = (struct mesh_state*)ap;
+	struct mesh_state* b = (struct mesh_state*)bp;
+
+	if(a->s.is_priming && !b->s.is_priming)
+		return -1;
+	if(!a->s.is_priming && b->s.is_priming)
+		return 1;
+
+	if((a->s.query_flags&BIT_RD) && !(b->s.query_flags&BIT_RD))
+		return -1;
+	if(!(a->s.query_flags&BIT_RD) && (b->s.query_flags&BIT_RD))
+		return 1;
+
+	if((a->s.query_flags&BIT_CD) && !(b->s.query_flags&BIT_CD))
+		return -1;
+	if(!(a->s.query_flags&BIT_CD) && (b->s.query_flags&BIT_CD))
+		return 1;
+
+	return query_info_compare(&a->s.qinfo, &b->s.qinfo);
+}
+
+int
+mesh_state_ref_compare(const void* ap, const void* bp)
+{
+	struct mesh_state_ref* a = (struct mesh_state_ref*)ap;
+	struct mesh_state_ref* b = (struct mesh_state_ref*)bp;
+	return mesh_state_compare(a->s, b->s);
+}
+
+struct mesh_area* 
+mesh_create(int num_modules, struct module_func_block** modfunc,
+	struct module_env* env)
+{
+	struct mesh_area* mesh = calloc(1, sizeof(struct mesh_area));
+	if(!mesh) {
+		log_err("mesh area alloc: out of memory");
+		return NULL;
+	}
+	mesh->histogram = timehist_setup();
+	if(!mesh->histogram) {
+		free(mesh);
+		log_err("mesh area alloc: out of memory");
+		return NULL;
+	}
+	mesh->num_modules = num_modules;
+	mesh->modfunc = modfunc;
+	mesh->env = env;
+	rbtree_init(&mesh->run, &mesh_state_compare);
+	rbtree_init(&mesh->all, &mesh_state_compare);
+	mesh->num_reply_addrs = 0;
+	mesh->num_reply_states = 0;
+	mesh->num_detached_states = 0;
+	return mesh;
+}
+
+/** help mesh delete delete mesh states */
+static void
+mesh_delete_helper(rbnode_t* n, void* ATTR_UNUSED(arg))
+{
+	struct mesh_state* mstate = (struct mesh_state*)n->key;
+	mesh_state_cleanup(mstate);
+}
+
+void 
+mesh_delete(struct mesh_area* mesh)
+{
+	if(!mesh)
+		return;
+	/* free all query states */
+	traverse_postorder(&mesh->all, &mesh_delete_helper, NULL);
+	timehist_delete(mesh->histogram);
+	free(mesh);
+}
+
+void mesh_new_client(struct mesh_area* mesh, struct query_info* qinfo,
+        uint16_t qflags, struct edns_data* edns, struct comm_reply* rep,
+        uint16_t qid)
+{
+	struct mesh_state* s = mesh_area_find(mesh, qinfo, qflags, 0);
+	int was_detached = 0;
+	int was_noreply = 0;
+	int added = 0;
+	/* see if it already exists, if not, create one */
+	if(!s) {
+		struct rbnode_t* n;
+		s = mesh_state_create(mesh->env,qinfo, qflags, 0);
+		if(!s) {
+			log_err("mesh_state_create: out of memory; SERVFAIL");
+			error_encode(rep->c->buffer, LDNS_RCODE_SERVFAIL,
+				qinfo, qid, qflags, edns);
+			comm_point_send_reply(rep);
+			return;
+		}
+		n = rbtree_insert(&mesh->all, &s->node);
+		log_assert(n != NULL);
+		/* set detached (it is now) */
+		mesh->num_detached_states++;
+		added = 1;
+	}
+	if(!s->reply_list && s->super_set.count == 0)
+		was_detached = 1;
+	if(!s->reply_list)
+		was_noreply = 1;
+	/* add reply to s */
+	if(!mesh_state_add_reply(s, edns, rep, qid, qflags)) {
+			log_err("mesh_new_client: out of memory; SERVFAIL");
+			error_encode(rep->c->buffer, LDNS_RCODE_SERVFAIL,
+				qinfo, qid, qflags, edns);
+			comm_point_send_reply(rep);
+			if(added)
+				mesh_state_delete(&s->s);
+			return;
+	}
+	/* update statistics */
+	if(was_detached) {
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	if(was_noreply) {
+		mesh->num_reply_states ++;
+	}
+	mesh->num_reply_addrs++;
+	if(added)
+		mesh_run(mesh, s, module_event_new, NULL);
+}
+
+void mesh_report_reply(struct mesh_area* mesh, struct outbound_entry* e,
+        int is_ok, struct comm_reply* reply)
+{
+	e->qstate->reply = reply;
+	mesh_run(mesh, e->qstate->mesh_info,
+		is_ok?module_event_reply:module_event_noreply, e);
+}
+
+struct mesh_state* 
+mesh_state_create(struct module_env* env, struct query_info* qinfo, 
+	uint16_t qflags, int prime)
+{
+	struct regional* region = alloc_reg_obtain(env->alloc);
+	struct mesh_state* mstate;
+	int i;
+	if(!region)
+		return NULL;
+	mstate = (struct mesh_state*)regional_alloc(region, 
+		sizeof(struct mesh_state));
+	if(!mstate) {
+		alloc_reg_release(env->alloc, region);
+		return NULL;
+	}
+	memset(mstate, 0, sizeof(*mstate));
+	mstate->node = *RBTREE_NULL;
+	mstate->run_node = *RBTREE_NULL;
+	mstate->node.key = mstate;
+	mstate->run_node.key = mstate;
+	mstate->reply_list = NULL;
+	rbtree_init(&mstate->super_set, &mesh_state_ref_compare);
+	rbtree_init(&mstate->sub_set, &mesh_state_ref_compare);
+	mstate->num_activated = 0;
+	/* init module qstate */
+	mstate->s.qinfo.qtype = qinfo->qtype;
+	mstate->s.qinfo.qclass = qinfo->qclass;
+	mstate->s.qinfo.qname_len = qinfo->qname_len;
+	mstate->s.qinfo.qname = regional_alloc_init(region, qinfo->qname,
+		qinfo->qname_len);
+	if(!mstate->s.qinfo.qname) {
+		alloc_reg_release(env->alloc, region);
+		return NULL;
+	}
+	/* remove all weird bits from qflags */
+	mstate->s.query_flags = (qflags & (BIT_RD|BIT_CD));
+	mstate->s.is_priming = prime;
+	mstate->s.reply = NULL;
+	mstate->s.region = region;
+	mstate->s.curmod = 0;
+	mstate->s.return_msg = 0;
+	mstate->s.return_rcode = LDNS_RCODE_NOERROR;
+	mstate->s.env = env;
+	mstate->s.mesh_info = mstate;
+	/* init modules */
+	for(i=0; i<env->mesh->num_modules; i++) {
+		mstate->s.minfo[i] = NULL;
+		mstate->s.ext_state[i] = module_state_initial;
+	}
+	return mstate;
+}
+
+void 
+mesh_state_cleanup(struct mesh_state* mstate)
+{
+	struct mesh_area* mesh;
+	int i;
+	if(!mstate)
+		return;
+	/* de-init modules */
+	mesh = mstate->s.env->mesh;
+	for(i=0; i<mesh->num_modules; i++) {
+		log_assert(fptr_whitelist_mod_clear(mesh->modfunc[i]->clear));
+		(*mesh->modfunc[i]->clear)(&mstate->s, i);
+		mstate->s.minfo[i] = NULL;
+		mstate->s.ext_state[i] = module_finished;
+	}
+	alloc_reg_release(mstate->s.env->alloc, mstate->s.region);
+}
+
+void 
+mesh_state_delete(struct module_qstate* qstate)
+{
+	struct mesh_area* mesh;
+	struct mesh_state_ref* super, ref;
+	struct mesh_state* mstate;
+	if(!qstate)
+		return;
+	mstate = qstate->mesh_info;
+	mesh = mstate->s.env->mesh;
+	mesh_detach_subs(&mstate->s);
+	if(!mstate->reply_list && mstate->super_set.count == 0) {
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	if(mstate->reply_list) {
+		log_assert(mesh->num_reply_states > 0);
+		mesh->num_reply_states--;
+	}
+	ref.node.key = &ref;
+	ref.s = mstate;
+	RBTREE_FOR(super, struct mesh_state_ref*, &mstate->super_set) {
+		(void)rbtree_delete(&super->s->sub_set, &ref);
+	}
+	(void)rbtree_delete(&mesh->run, mstate);
+	(void)rbtree_delete(&mesh->all, mstate);
+	mesh_state_cleanup(mstate);
+}
+
+void mesh_detach_subs(struct module_qstate* qstate)
+{
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state_ref* ref, lookup;
+	struct rbnode_t* n;
+	lookup.node.key = &lookup;
+	lookup.s = qstate->mesh_info;
+	RBTREE_FOR(ref, struct mesh_state_ref*, &qstate->mesh_info->sub_set) {
+		n = rbtree_delete(&ref->s->super_set, &lookup);
+		log_assert(n != NULL); /* must have been present */
+		if(!ref->s->reply_list && ref->s->super_set.count == 0) {
+			mesh->num_detached_states++;
+			log_assert(mesh->num_detached_states + 
+				mesh->num_reply_states <= mesh->all.count);
+		}
+	}
+	rbtree_init(&qstate->mesh_info->sub_set, &mesh_state_ref_compare);
+}
+
+int mesh_attach_sub(struct module_qstate* qstate, struct query_info* qinfo,
+        uint16_t qflags, int prime, struct module_qstate** newq)
+{
+	/* find it, if not, create it */
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state* sub = mesh_area_find(mesh, qinfo, qflags, prime);
+	if(!sub) {
+		struct rbnode_t* n;
+		/* create a new one */
+		sub = mesh_state_create(qstate->env, qinfo, qflags, prime);
+		if(!sub) {
+			log_err("mesh_attach_sub: out of memory");
+			return 0;
+		}
+		n = rbtree_insert(&mesh->all, &sub->node);
+		log_assert(n != NULL);
+		/* set detached (it is now) */
+		mesh->num_detached_states++;
+		/* set new query state to run */
+		n = rbtree_insert(&mesh->run, &sub->run_node);
+		log_assert(n != NULL);
+		*newq = &sub->s;
+	} else
+		*newq = NULL;
+	if(!mesh_state_attachment(qstate->mesh_info, sub))
+		return 0;
+	if(!sub->reply_list && sub->super_set.count == 1) {
+		/* it used to be detached, before this one got added */
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	/* *newq will be run when inited after the current module stops */
+	return 1;
+}
+
+int mesh_state_attachment(struct mesh_state* super, struct mesh_state* sub)
+{
+	struct rbnode_t* n;
+	struct mesh_state_ref* subref; /* points to sub, inserted in super */
+	struct mesh_state_ref* superref; /* points to super, inserted in sub */
+	if( !(subref = regional_alloc(super->s.region,
+		sizeof(struct mesh_state_ref))) ||
+		!(superref = regional_alloc(sub->s.region,
+		sizeof(struct mesh_state_ref))) ) {
+		log_err("mesh_state_attachment: out of memory");
+		return 0;
+	}
+	superref->node.key = superref;
+	superref->s = super;
+	subref->node.key = subref;
+	subref->s = sub;
+	n = rbtree_insert(&sub->super_set, &superref->node);
+	log_assert(n != NULL);
+	n = rbtree_insert(&super->sub_set, &subref->node);
+	log_assert(n != NULL);
+	return 1;
+}
+
+/** subtract timers and the values do not overflow or become negative */
+static void
+timeval_subtract(struct timeval* d, struct timeval* end, struct timeval* start)
+{
+#ifndef S_SPLINT_S
+	d->tv_sec = end->tv_sec - start->tv_sec;
+	while(end->tv_usec < start->tv_usec) {
+		end->tv_usec += 1000000;
+		d->tv_sec--;
+	}
+	d->tv_usec = end->tv_usec - start->tv_usec;
+#endif
+}
+
+/** add timers and the values do not overflow or become negative */
+static void
+timeval_add(struct timeval* d, struct timeval* add)
+{
+#ifndef S_SPLINT_S
+	d->tv_sec += add->tv_sec;
+	d->tv_usec += add->tv_usec;
+	while(d->tv_usec > 1000000 ) {
+		d->tv_usec -= 1000000;
+		d->tv_sec++;
+	}
+#endif
+}
+
+/** divide sum of timers to get average */
+static void
+timeval_divide(struct timeval* avg, struct timeval* sum, size_t d)
+{
+#ifndef S_SPLINT_S
+	size_t leftover;
+	if(d == 0) {
+		avg->tv_sec = 0;
+		avg->tv_usec = 0;
+		return;
+	}
+	avg->tv_sec = sum->tv_sec / d;
+	avg->tv_usec = sum->tv_usec / d;
+	/* handle fraction from seconds divide */
+	leftover = sum->tv_sec - avg->tv_sec*d;
+	avg->tv_usec += (leftover*1000000)/d;
+#endif
+}
+
+/**
+ * Send reply to mesh reply entry
+ * @param m: mesh state to send it for.
+ * @param rcode: if not 0, error code.
+ * @param rep: reply to send (or NULL if rcode is set).
+ * @param r: reply entry
+ */
+static void
+mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
+	struct mesh_reply* r)
+{
+	struct timeval end_time;
+	int secure;
+	/* examine security status */
+	if(m->s.env->need_to_validate && !(r->qflags&BIT_CD) && rep && 
+		rep->security <= sec_status_bogus) {
+		rcode = LDNS_RCODE_SERVFAIL;
+	}
+	if(rep && rep->security == sec_status_secure)
+		secure = 1;
+	else	secure = 0;
+	/* send the reply */
+	if(rcode) {
+		error_encode(r->query_reply.c->buffer, rcode, &m->s.qinfo,
+			r->qid, r->qflags, &r->edns);
+		comm_point_send_reply(&r->query_reply);
+	} else {
+		size_t udp_size = r->edns.udp_size;
+		r->edns.edns_version = EDNS_ADVERTISED_VERSION;
+		r->edns.udp_size = EDNS_ADVERTISED_SIZE;
+		r->edns.ext_rcode = 0;
+		r->edns.bits &= EDNS_DO;
+		if(!reply_info_answer_encode(&m->s.qinfo, rep, r->qid, 
+			r->qflags, r->query_reply.c->buffer, 0, 1, 
+			m->s.env->scratch, udp_size, &r->edns, 
+			(int)(r->edns.bits & EDNS_DO), secure)) 
+		{
+			error_encode(r->query_reply.c->buffer, 
+				LDNS_RCODE_SERVFAIL, &m->s.qinfo, r->qid, 
+				r->qflags, &r->edns);
+		}
+		comm_point_send_reply(&r->query_reply);
+	}
+	/* account */
+	m->s.env->mesh->num_reply_addrs--;
+	if(gettimeofday(&end_time, NULL) < 0) {
+		log_err("gettimeofday: %s", strerror(errno));
+		return;
+	} else {
+		struct timeval duration;
+		timeval_subtract(&duration, &end_time, &r->start_time);
+		verbose(VERB_ALGO, "query took %d.%6.6d sec",
+			(int)duration.tv_sec, (int)duration.tv_usec);
+		m->s.env->mesh->replies_sent++;
+		timeval_add(&m->s.env->mesh->replies_sum_wait, &duration);
+		timehist_insert(m->s.env->mesh->histogram, &duration);
+	}
+}
+
+void mesh_query_done(struct mesh_state* mstate)
+{
+	struct mesh_reply* r;
+	struct reply_info* rep = (mstate->s.return_msg?
+		mstate->s.return_msg->rep:NULL);
+	for(r = mstate->reply_list; r; r = r->next) {
+		mesh_send_reply(mstate, mstate->s.return_rcode, rep, r);
+	}
+}
+
+void mesh_walk_supers(struct mesh_area* mesh, struct mesh_state* mstate)
+{
+	struct mesh_state_ref* ref;
+	RBTREE_FOR(ref, struct mesh_state_ref*, &mstate->super_set)
+	{
+		/* make super runnable */
+		(void)rbtree_insert(&mesh->run, &ref->s->run_node);
+		/* callback the function to inform super of result */
+		log_assert(fptr_whitelist_mod_inform_super(
+			mesh->modfunc[ref->s->s.curmod]->inform_super));
+		(*mesh->modfunc[ref->s->s.curmod]->inform_super)(&mstate->s, 
+			ref->s->s.curmod, &ref->s->s);
+	}
+}
+
+struct mesh_state* mesh_area_find(struct mesh_area* mesh,
+	struct query_info* qinfo, uint16_t qflags, int prime)
+{
+	struct mesh_state key;
+	struct mesh_state* result;
+
+	key.node.key = &key;
+	key.s.is_priming = prime;
+	key.s.qinfo = *qinfo;
+	key.s.query_flags = qflags;
+	
+	result = (struct mesh_state*)rbtree_search(&mesh->all, &key);
+	return result;
+}
+
+int mesh_state_add_reply(struct mesh_state* s, struct edns_data* edns,
+        struct comm_reply* rep, uint16_t qid, uint16_t qflags)
+{
+	struct mesh_reply* r = regional_alloc(s->s.region, 
+		sizeof(struct mesh_reply));
+	if(!r)
+		return 0;
+	r->query_reply = *rep;
+	r->edns = *edns;
+	r->qid = qid;
+	r->qflags = qflags;
+	if(gettimeofday(&r->start_time, NULL) < 0) {
+		log_err("addrep: gettimeofday: %s", strerror(errno));
+		memset(&r->start_time, 0, sizeof(r->start_time));
+	}
+	r->next = s->reply_list;
+	s->reply_list = r;
+	return 1;
+
+}
+
+/**
+ * Continue processing the mesh state at another module.
+ * Handles module to modules tranfer of control.
+ * Handles module finished.
+ * @param mesh: the mesh area.
+ * @param mstate: currently active mesh state.
+ * 	Deleted if finished, calls _done and _supers to 
+ * 	send replies to clients and inform other mesh states.
+ * 	This in turn may create additional runnable mesh states.
+ * @param s: state at which the current module exited.
+ * @param ev: the event sent to the module.
+ * 	returned is the event to send to the next module.
+ * @return true if continue processing at the new module.
+ * 	false if not continued processing is needed.
+ */
+static int
+mesh_continue(struct mesh_area* mesh, struct mesh_state* mstate,
+	enum module_ext_state s, enum module_ev* ev)
+{
+	mstate->num_activated++;
+	if(mstate->num_activated > MESH_MAX_ACTIVATION) {
+		/* module is looping. Stop it. */
+		log_err("internal error: looping module stopped");
+		log_query_info(VERB_DETAIL, "pass error for qstate",
+			&mstate->s.qinfo);
+		s = module_error;
+	}
+	if(s == module_wait_module) {
+		/* start next module */
+		mstate->s.curmod++;
+		if(mesh->num_modules == mstate->s.curmod) {
+			log_err("Cannot pass to next module; at last module");
+			log_query_info(VERB_DETAIL, "pass error for qstate",
+				&mstate->s.qinfo);
+			mstate->s.curmod--;
+			return mesh_continue(mesh, mstate, module_error, ev);
+		}
+		*ev = module_event_pass;
+		return 1;
+	}
+	if(s == module_error && mstate->s.return_rcode == LDNS_RCODE_NOERROR) {
+		/* error is bad, handle pass back up below */
+		mstate->s.return_rcode = LDNS_RCODE_SERVFAIL;
+	}
+	if(s == module_error || s == module_finished) {
+		if(mstate->s.curmod == 0) {
+			mesh_query_done(mstate);
+			mesh_walk_supers(mesh, mstate);
+			mesh_state_delete(&mstate->s);
+			return 0;
+		}
+		/* pass along the locus of control */
+		mstate->s.curmod --;
+		*ev = module_event_moddone;
+		return 1;
+	}
+	return 0;
+}
+
+void mesh_run(struct mesh_area* mesh, struct mesh_state* mstate,
+	enum module_ev ev, struct outbound_entry* e)
+{
+	enum module_ext_state s;
+	verbose(VERB_ALGO, "mesh_run: start");
+	while(mstate) {
+		/* run the module */
+		log_assert(fptr_whitelist_mod_operate(
+			mesh->modfunc[mstate->s.curmod]->operate));
+		(*mesh->modfunc[mstate->s.curmod]->operate)
+			(&mstate->s, ev, mstate->s.curmod, e);
+
+		/* examine results */
+		mstate->s.reply = NULL;
+		regional_free_all(mstate->s.env->scratch);
+		s = mstate->s.ext_state[mstate->s.curmod];
+		verbose(VERB_ALGO, "mesh_run: %s module exit state is %s", 
+			mesh->modfunc[mstate->s.curmod]->name, strextstate(s));
+		e = NULL;
+		if(mesh_continue(mesh, mstate, s, &ev))
+			continue;
+
+		/* run more modules */
+		ev = module_event_pass;
+		if(mesh->run.count > 0) {
+			/* pop random element off the runnable tree */
+			mstate = (struct mesh_state*)mesh->run.root->key;
+			(void)rbtree_delete(&mesh->run, mstate);
+		} else mstate = NULL;
+	}
+	if(verbosity >= VERB_ALGO) {
+		mesh_stats(mesh, "mesh_run: end");
+		mesh_log_list(mesh);
+	}
+}
+
+void 
+mesh_log_list(struct mesh_area* mesh)
+{
+	char buf[30];
+	struct mesh_state* m;
+	int num = 0;
+	RBTREE_FOR(m, struct mesh_state*, &mesh->all) {
+		snprintf(buf, sizeof(buf), "%d%s%s%s%s%s mod%d %s", 
+			num++, (m->s.is_priming)?"p":"",  /* prime */
+			(m->s.query_flags&BIT_RD)?"RD":"",
+			(m->s.query_flags&BIT_CD)?"CD":"",
+			(m->super_set.count==0)?"d":"", /* detached */
+			(m->sub_set.count!=0)?"c":"",  /* children */
+			m->s.curmod, (m->reply_list)?"hr":"nr"); /*hasreply*/
+		log_query_info(VERB_ALGO, buf, &m->s.qinfo);
+	}
+}
+
+void 
+mesh_stats(struct mesh_area* mesh, const char* str)
+{
+	log_info("%s %u states (%u with reply, %u detached), "
+		"%u waiting replies", str, (unsigned)mesh->all.count, 
+		(unsigned)mesh->num_reply_states,
+		(unsigned)mesh->num_detached_states,
+		(unsigned)mesh->num_reply_addrs);
+	if(mesh->replies_sent > 0) {
+		struct timeval avg;
+		timeval_divide(&avg, &mesh->replies_sum_wait, 
+			mesh->replies_sent);
+		log_info("sent %u replies, with average wait "
+			"of %d.%6.6d sec", (unsigned)mesh->replies_sent,
+			(int)avg.tv_sec, (int)avg.tv_usec);
+		log_info("histogram of reply wait times");
+		timehist_log(mesh->histogram);
+	}
+}
+
+size_t 
+mesh_get_mem(struct mesh_area* mesh)
+{
+	struct mesh_state* m;
+	size_t s = sizeof(*mesh) + sizeof(struct timehist) +
+		sizeof(struct th_buck)*mesh->histogram->num;
+	RBTREE_FOR(m, struct mesh_state*, &mesh->all) {
+		/* all, including m itself allocated in qstate region */
+		s += regional_get_mem(m->s.region);
+	}
+	return s;
+}
+
+/** helper recursive rbtree find routine */
+static int
+find_in_subsub(struct mesh_state* m, struct mesh_state* tofind)
+{
+	struct mesh_state_ref* r;
+	RBTREE_FOR(r, struct mesh_state_ref*, &m->sub_set) {
+		if(r->s == tofind || find_in_subsub(r->s, tofind))
+			return 1;
+	}
+	return 0;
+}
+
+int 
+mesh_detect_cycle(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t flags, int prime)
+{
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state* cyc_m = qstate->mesh_info;
+	struct mesh_state* dep_m = mesh_area_find(mesh, qinfo, flags, prime);
+	if(!dep_m)
+		return 0;
+	if(dep_m == cyc_m || find_in_subsub(dep_m, cyc_m))
+		return 1;
+	return 0;
+}

services/mesh.h

@@ -0,0 +1,387 @@
+/*
+ * services/mesh.h - deal with mesh of query states and handle events for that.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist in dealing with a mesh of
+ * query states. This mesh is supposed to be thread-specific.
+ * It consists of query states (per qname, qtype, qclass) and connections
+ * between query states and the super and subquery states, and replies to
+ * send back to clients.
+ */
+
+#ifndef SERVICES_MESH_H
+#define SERVICES_MESH_H
+
+#include "util/rbtree.h"
+#include "util/netevent.h"
+#include "util/data/msgparse.h"
+#include "util/module.h"
+struct mesh_state;
+struct mesh_reply;
+struct query_info;
+struct reply_info;
+struct outbound_entry;
+struct timehist;
+
+/**
+ * Maximum number of mesh state activations. Any more is likely an
+ * infinite loop in the module. It is then terminated.
+ */
+#define MESH_MAX_ACTIVATION 1000
+
+/** 
+ * Mesh of query states
+ */
+struct mesh_area {
+	/** the number of modules */
+	int num_modules;
+	/** the module callbacks, array of num_modules length (ref only) */
+	struct module_func_block** modfunc;
+	/** environment for new states */
+	struct module_env* env;
+
+	/** set of runnable queries (mesh_state.run_node) */
+	rbtree_t run;
+	/** rbtree of all current queries (mesh_state.node)*/
+	rbtree_t all;
+
+	/** count of the total number of mesh_reply entries */
+	size_t num_reply_addrs;
+	/** count of the number of mesh_states that have mesh_replies 
+	 * Because a state can send results to multiple reply addresses,
+	 * this number must be equal or lower than num_reply_addrs. */
+	size_t num_reply_states;
+	/** number of mesh_states that have no mesh_replies, and also
+	 * an empty set of super-states, thus are 'toplevel' or detached
+	 * internal opportunistic queries */
+	size_t num_detached_states;
+
+	/** number of replies sent */
+	size_t replies_sent;
+	/** sum of waiting times for the replies */
+	struct timeval replies_sum_wait;
+	/** histogram of time values */
+	struct timehist* histogram;
+};
+
+/**
+ * A mesh query state
+ * Unique per qname, qtype, qclass (from the qstate).
+ * And RD / CD flag; in case a client turns it off.
+ * And priming queries are different from ordinary queries (because of hints).
+ *
+ * The entire structure is allocated in a region, this region is the qstate
+ * region. All parts (rbtree nodes etc) are also allocated in the region.
+ */
+struct mesh_state {
+	/** node in mesh_area all tree, key is this struct. Must be first. */
+	rbnode_t node;
+	/** node in mesh_area runnable tree, key is this struct */
+	rbnode_t run_node;
+	/** the query state. Note that the qinfo and query_flags 
+	 * may not change. */
+	struct module_qstate s;
+	/** the list of replies to clients for the results */
+	struct mesh_reply* reply_list;
+	/** set of superstates (that want this state's result) 
+	 * contains struct mesh_state_ref* */
+	rbtree_t super_set;
+	/** set of substates (that this state needs to continue)
+	 * contains struct mesh_state_ref* */
+	rbtree_t sub_set;
+	/** number of activations for the mesh state */
+	size_t num_activated;
+};
+
+/**
+ * Rbtree reference to a mesh_state.
+ * Used in super_set and sub_set. 
+ */
+struct mesh_state_ref {
+	/** node in rbtree for set, key is this structure */
+	rbnode_t node;
+	/** the mesh state */
+	struct mesh_state* s;
+};
+
+/**
+ * Reply to a client
+ */
+struct mesh_reply {
+	/** next in reply list */
+	struct mesh_reply* next;
+	/** the query reply destination, packet buffer and where to send. */
+	struct comm_reply query_reply;
+	/** edns data from query */
+	struct edns_data edns;
+	/** the time when request was entered */
+	struct timeval start_time;
+	/** id of query, in network byteorder. */
+	uint16_t qid;
+	/** flags of query, for reply flags */
+	uint16_t qflags;
+};
+
+/* ------------------- Functions for worker -------------------- */
+
+/**
+ * Allocate mesh, to empty.
+ * @param num_modules: number of modules that are present.
+ * @param modfunc: array passed (alloced and deleted by caller), that has
+ * 	num_modules function callbacks for the modules.
+ * @param env: environment for new queries.
+ * @return mesh: the new mesh or NULL on error.
+ */
+struct mesh_area* mesh_create(int num_modules, 
+	struct module_func_block** modfunc, struct module_env* env);
+
+/**
+ * Delete mesh, and all query states and replies in it.
+ * @param mesh: the mesh to delete.
+ */
+void mesh_delete(struct mesh_area* mesh);
+
+/**
+ * New query incoming from clients. Create new query state if needed, and
+ * add mesh_reply to it. Returns error to client on malloc failures.
+ * Will run the mesh area queries to process if a new query state is created.
+ *
+ * @param mesh: the mesh.
+ * @param qinfo: query from client.
+ * @param qflags: flags from client query.
+ * @param edns: edns data from client query.
+ * @param rep: where to reply to.
+ * @param qid: query id to reply with.
+ */
+void mesh_new_client(struct mesh_area* mesh, struct query_info* qinfo,
+	uint16_t qflags, struct edns_data* edns, struct comm_reply* rep, 
+	uint16_t qid);
+
+/**
+ * Handle new event from the wire. A serviced query has returned.
+ * The query state will be made runnable, and the mesh_area will process
+ * query states until processing is complete.
+ *
+ * @param mesh: the query mesh.
+ * @param e: outbound entry, with query state to run and reply pointer.
+ * @param is_ok: if true, reply is OK, otherwise a timeout happened.
+ * @param reply: the comm point reply info.
+ */
+void mesh_report_reply(struct mesh_area* mesh, struct outbound_entry* e,
+	int is_ok, struct comm_reply* reply);
+
+/* ------------------- Functions for module environment --------------- */
+
+/**
+ * Detach-subqueries.
+ * Remove all sub-query references from this query state.
+ * Keeps super-references of those sub-queries correct.
+ * Updates stat items in mesh_area structure.
+ * @param qstate: used to find mesh state.
+ */
+void mesh_detach_subs(struct module_qstate* qstate);
+
+/**
+ * Attach subquery.
+ * Creates it if it does not exist already.
+ * Keeps sub and super references correct.
+ * Updates stat items in mesh_area structure.
+ * Pass if it is priming query or not.
+ * return:
+ * 	o if error (malloc) happened.
+ * 	o need to initialise the new state (module init; it is a new state).
+ * 	  so that the next run of the query with this module is successful.
+ * 	o no init needed, attachment successful.
+ *
+ * @param qstate: the state to find mesh state, and that wants to receive
+ * 	the results from the new subquery.
+ * @param qinfo: what to query for (copied).
+ * @param qflags: what flags to use (RD / CD flag or not).
+ * @param prime: if it is a (stub) priming query.
+ * @param newq: If the new subquery needs initialisation, it is returned,
+ * 	otherwise NULL is returned.
+ * @return: false on error, true if success (and init may be needed).
+ */
+int mesh_attach_sub(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t qflags, int prime, struct module_qstate** newq);
+
+/**
+ * Query state is done, send messages to reply entries.
+ * Encode messages using reply entry values and the querystate (with original
+ * qinfo), using given reply_info.
+ * Pass errcode != 0 if an error reply is needed.
+ * If no reply entries, nothing is done.
+ * Must be called before a module can module_finished or return module_error.
+ * The module must handle the super query states itself as well.
+ *
+ * @param mstate: mesh state that is done. return_rcode and return_msg
+ * 	are used for replies.
+ * 	return_rcode: if not 0 (NOERROR) an error is sent back (and 
+ * 		return_msg is ignored).
+ * 	return_msg: reply to encode and send back to clients.
+ */
+void mesh_query_done(struct mesh_state* mstate);
+
+/**
+ * Call inform_super for the super query states that are interested in the 
+ * results from this query state. These can then be changed for error 
+ * or results.
+ * Called when a module is module_finished or returns module_error.
+ * The super query states become runnable with event module_event_pass.
+ *
+ * @param mesh: mesh area to add newly runnable modules to.
+ * @param mstate: the state that has results, used to find mesh state.
+ */
+void mesh_walk_supers(struct mesh_area* mesh, struct mesh_state* mstate);
+
+/**
+ * Delete mesh state, cleanup and also rbtrees and so on.
+ * Will detach from all super/subnodes.
+ * @param qstate: to remove.
+ */
+void mesh_state_delete(struct module_qstate* qstate);
+
+/* ------------------- Functions for mesh -------------------- */
+
+/**
+ * Create and initialize a new mesh state and its query state
+ * Does not put the mesh state into rbtrees and so on.
+ * @param env: module environment to set.
+ * @param qinfo: query info that the mesh is for.
+ * @param qflags: flags for query (RD / CD flag).
+ * @param prime: if true, it is a priming query, set is_priming on mesh state.
+ * @return: new mesh state or NULL on allocation error.
+ */
+struct mesh_state* mesh_state_create(struct module_env* env, 
+	struct query_info* qinfo, uint16_t qflags, int prime);
+
+/**
+ * Cleanup a mesh state and its query state. Does not do rbtree or 
+ * reference cleanup.
+ * @param mstate: mesh state to cleanup. Its pointer may no longer be used
+ * 	afterwards. Cleanup rbtrees before calling this function.
+ */
+void mesh_state_cleanup(struct mesh_state* mstate);
+
+/**
+ * Find a mesh state in the mesh area. Pass relevant flags.
+ *
+ * @param mesh: the mesh area to look in.
+ * @param qinfo: what query
+ * @param qflags: if RD / CD bit is set or not.
+ * @param prime: if it is a priming query.
+ * @return: mesh state or NULL if not found.
+ */
+struct mesh_state* mesh_area_find(struct mesh_area* mesh, 
+	struct query_info* qinfo, uint16_t qflags, int prime);
+
+/**
+ * Setup attachment super/sub relation between super and sub mesh state.
+ * The relation must not be present when calling the function.
+ * Does not update stat items in mesh_area.
+ * @param super: super state.
+ * @param sub: sub state.
+ * @return: 0 on alloc error.
+ */
+int mesh_state_attachment(struct mesh_state* super, struct mesh_state* sub);
+
+/**
+ * Create new reply structure and attach it to a mesh state.
+ * Does not update stat items in mesh area.
+ * @param s: the mesh state.
+ * @param edns: edns data for reply (bufsize).
+ * @param rep: comm point reply info.
+ * @param qid: ID of reply.
+ * @param qflags: original query flags.
+ * @return: 0 on alloc error.
+ */
+int mesh_state_add_reply(struct mesh_state* s, struct edns_data* edns, 
+	struct comm_reply* rep, uint16_t qid, uint16_t qflags);
+
+/**
+ * Run the mesh. Run all runnable mesh states. Which can create new
+ * runnable mesh states. Until completion. Automatically called by
+ * mesh_report_reply and mesh_new_client as needed.
+ * @param mesh: mesh area.
+ * @param mstate: first mesh state to run.
+ * @param ev: event the mstate. Others get event_pass.
+ * @param e: if a reply, its outbound entry.
+ */
+void mesh_run(struct mesh_area* mesh, struct mesh_state* mstate, 
+	enum module_ev ev, struct outbound_entry* e);
+
+/**
+ * Print some stats about the mesh to the log.
+ * @param mesh: the mesh to print it for.
+ * @param str: descriptive string to go with it.
+ */
+void mesh_stats(struct mesh_area* mesh, const char* str);
+
+/**
+ * Print all the states in the mesh to the log.
+ * @param mesh: the mesh to print all states of.
+ */
+void mesh_log_list(struct mesh_area* mesh);
+
+/**
+ * Calculate memory size in use by mesh and all queries inside it.
+ * @param mesh: the mesh to examine.
+ * @return size in bytes.
+ */
+size_t mesh_get_mem(struct mesh_area* mesh);
+
+/**
+ * Find cycle; see if the given mesh is in the targets sub, or sub-sub, ...
+ * trees.
+ * @param qstate: given mesh querystate.
+ * @param qinfo: query info for dependency.
+ * @param flags: query flags of dependency.
+ * @param prime: if dependency is a priming query or not.
+ * @return true if the name,type,class exists and the given qstate mesh exists
+ * 	as a dependency of that name. Thus if qstate becomes dependent on
+ * 	name,type,class then a cycle is created.
+ */
+int mesh_detect_cycle(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t flags, int prime);
+
+/** compare two mesh_states */
+int mesh_state_compare(const void* ap, const void* bp);
+
+/** compare two mesh references */
+int mesh_state_ref_compare(const void* ap, const void* bp);
+
+#endif /* SERVICES_MESH_H */

services/outbound_list.c

@@ -0,0 +1,88 @@
+/*
+ * services/outbound_list.c - keep list of outbound serviced queries.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to help a module keep track of the
+ * queries it has outstanding to authoritative servers.
+ */
+#include "config.h"
+#include "services/outbound_list.h"
+#include "services/outside_network.h"
+
+void 
+outbound_list_init(struct outbound_list* list)
+{
+	list->first = NULL;
+}
+
+void 
+outbound_list_clear(struct outbound_list* list)
+{
+	struct outbound_entry *p, *np;
+	p = list->first;
+	while(p) {
+		np = p->next;
+		outnet_serviced_query_stop(p->qsent, p);
+		/* in region, no free needed */
+		p = np;
+	}
+	outbound_list_init(list);
+}
+
+void 
+outbound_list_insert(struct outbound_list* list, struct outbound_entry* e)
+{
+	if(list->first)
+		list->first->prev = e;
+	e->next = list->first;
+	e->prev = NULL;
+	list->first = e;
+}
+
+void 
+outbound_list_remove(struct outbound_list* list, struct outbound_entry* e)
+{
+	if(!e)
+		return;
+	outnet_serviced_query_stop(e->qsent, e);
+	if(e->next)
+		e->next->prev = e->prev;
+	if(e->prev)
+		e->prev->next = e->next;
+	else	list->first = e->next;
+	/* in region, no free needed */
+}

services/outbound_list.h

@@ -0,0 +1,105 @@
+/*
+ * services/outbound_list.h - keep list of outbound serviced queries.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to help a module keep track of the
+ * queries it has outstanding to authoritative servers.
+ */
+#ifndef SERVICES_OUTBOUND_LIST_H
+#define SERVICES_OUTBOUND_LIST_H
+struct outbound_entry;
+struct serviced_query;
+struct module_qstate;
+
+/**
+ * The outbound list. This structure is part of the module specific query
+ * state.
+ */
+struct outbound_list {
+	/** The linked list of outbound query entries. */
+	struct outbound_entry* first;
+};
+
+/**
+ * Outbound list entry. A serviced query sent by a module processing the
+ * query from the qstate. Double linked list to aid removal.
+ */
+struct outbound_entry {
+	/** next in list */
+	struct outbound_entry* next;
+	/** prev in list */
+	struct outbound_entry* prev;
+	/** The query that was sent out */
+	struct serviced_query* qsent;
+	/** the module query state that sent it */
+	struct module_qstate* qstate;
+};
+
+/**
+ * Init the user allocated outbound list structure
+ * @param list: the list structure.
+ */
+void outbound_list_init(struct outbound_list* list);
+
+/**
+ * Clear the user owner outbound list structure.
+ * Deletes serviced queries.
+ * @param list: the list structure. It is cleared, but the list struct itself
+ * 	is callers responsability to delete.
+ */
+void outbound_list_clear(struct outbound_list* list);
+
+/**
+ * Insert new entry into the list. Caller must allocate the entry with malloc.
+ * qstate and qsent are set by caller.
+ * @param list: the list to add to.
+ * @param e: entry to add, it is only half initialised at call start, fully
+ *	initialised at call end.
+ */
+void outbound_list_insert(struct outbound_list* list, 
+	struct outbound_entry* e);
+
+/**
+ * Remove an entry from the list, and deletes it. 
+ * Deletes serviced query in the entry.
+ * @param list: the list to remove from.
+ * @param e: the entry to remove.
+ */
+void outbound_list_remove(struct outbound_list* list, 
+	struct outbound_entry* e);
+
+#endif /* SERVICES_OUTBOUND_LIST_H */

services/outside_network.h

@@ -48,6 +48,10 @@
 #include "util/netevent.h"
 struct pending;
 struct pending_timeout;
+struct ub_randstate;
+struct pending_tcp;
+struct waiting_tcp;
+struct infra_cache;
 
 /**
  * Send queries to outside servers and wait for answers from servers.
@@ -60,24 +64,49 @@ struct outside_network {
 	/** buffer shared by UDP connections, since there is only one
 	    datagram at any time. */
 	ldns_buffer* udp_buff;
+	/** serviced_callbacks malloc overhead when processing multiple
+	 * identical serviced queries to the same server. */
+	size_t svcd_overhead;
 
 	/** 
 	 * Array of udp comm point* that are used to listen to pending events.
 	 * Each is on a different port. This is for ip4 ports.
 	 */
-	struct comm_point **udp4_ports;
+	struct comm_point** udp4_ports;
 	/** number of udp4 ports */
 	size_t num_udp4;
 
 	/**
 	 * The opened ip6 ports.
 	 */
-	struct comm_point **udp6_ports;
+	struct comm_point** udp6_ports;
 	/** number of udp6 ports */
 	size_t num_udp6;
 
-	/** pending answers. sorted by id, addr */
-	rbtree_t *pending;
+	/** pending udp answers. sorted by id, addr */
+	rbtree_t* pending;
+	/** serviced queries, sorted by qbuf, addr, dnssec */
+	rbtree_t* serviced;
+	/** host cache, pointer but not owned by outnet. */
+	struct infra_cache* infra;
+	/** where to get random numbers */
+	struct ub_randstate* rnd;
+
+	/**
+	 * Array of tcp pending used for outgoing TCP connections.
+	 * Each can be used to establish a TCP connection with a server.
+	 * The file descriptors are -1 if they are free, and need to be 
+	 * opened for the tcp connection. Can be used for ip4 and ip6.
+	 */
+	struct pending_tcp **tcp_conns;
+	/** number of tcp communication points. */
+	size_t num_tcp;
+	/** list of tcp comm points that are free for use */
+	struct pending_tcp* tcp_free;
+	/** list of tcp queries waiting for a buffer */
+	struct waiting_tcp* tcp_wait_first;
+	/** last of waiting query list */
+	struct waiting_tcp* tcp_wait_last;
 };
 
 /**
@@ -104,6 +133,113 @@ struct pending {
 	struct outside_network* outnet;
 };
 
+/**
+ * Pending TCP query to server.
+ */
+struct pending_tcp {
+	/** next in list of free tcp comm points, or NULL. */
+	struct pending_tcp* next_free;
+	/** the ID for the query; checked in reply */
+	uint16_t id;
+	/** tcp comm point it was sent on (and reply must come back on). */
+	struct comm_point* c;
+	/** the query being serviced, NULL if the pending_tcp is unused. */
+	struct waiting_tcp* query;
+};
+
+/**
+ * Query waiting for TCP buffer.
+ */
+struct waiting_tcp {
+	/** 
+	 * next in waiting list.
+	 * if pkt==0, this points to the pending_tcp structure.
+	 */
+	struct waiting_tcp* next_waiting;
+	/** timeout event; timer keeps running whether the query is
+	 * waiting for a buffer or the tcp reply is pending */
+	struct comm_timer* timer;
+	/** the outside network it is part of */
+	struct outside_network* outnet;
+	/** remote address. */
+	struct sockaddr_storage addr;
+	/** length of addr field in use. */
+	socklen_t addrlen;
+	/** 
+	 * The query itself, the query packet to send.
+	 * allocated after the waiting_tcp structure.
+	 * set to NULL when the query is serviced and it part of pending_tcp.
+	 * if this is NULL, the next_waiting points to the pending_tcp.
+	 */
+	uint8_t* pkt;
+	/** length of query packet. */
+	size_t pkt_len;
+	/** callback for the timeout, error or reply to the message */
+	comm_point_callback_t* cb;
+	/** callback user argument */
+	void* cb_arg;
+};
+
+/**
+ * Callback to party interested in serviced query results.
+ */
+struct service_callback {
+	/** next in callback list */
+	struct service_callback* next;
+	/** callback function */
+	comm_point_callback_t* cb;
+	/** user argument for callback function */
+	void* cb_arg;
+};
+
+/**
+ * Query service record.
+ * Contains query and destination. UDP, TCP, EDNS are all tried.
+ * complete with retries and timeouts. A number of interested parties can
+ * receive a callback.
+ */
+struct serviced_query {
+	/** The rbtree node, key is this record */
+	rbnode_t node;
+	/** The query that needs to be answered. Starts with flags u16,
+	 * then qdcount, ..., including qname, qtype, qclass. Does not include
+	 * EDNS record. */
+	uint8_t* qbuf;
+	/** length of qbuf. */
+	size_t qbuflen;
+	/** If an EDNS section is included, the DO/CD bit will be turned on. */
+	int dnssec;
+	/** where to send it */
+	struct sockaddr_storage addr;
+	/** length of addr field in use. */
+	socklen_t addrlen;
+	/** current status */
+	enum serviced_query_status {
+		/** initial status */
+		serviced_initial,
+		/** UDP with EDNS sent */
+		serviced_query_UDP_EDNS,
+		/** UDP without EDNS sent */
+		serviced_query_UDP,
+		/** TCP with EDNS sent */
+		serviced_query_TCP_EDNS,
+		/** TCP without EDNS sent */
+		serviced_query_TCP
+	} status;
+	/** true if serviced_query is scheduled for deletion already */
+	int to_be_deleted;
+	/** number of UDP retries */
+	int retry;
+	/** time last UDP was sent */
+	struct timeval last_sent_time;
+	/** outside network this is part of */
+	struct outside_network* outnet;
+	/** list of interested parties that need callback on results. */
+	struct service_callback* cblist;
+	/** the UDP or TCP query that is pending, see status which */
+	void* pending;
+};
+
 /**
  * Create outside_network structure with N udp ports.
  * @param base: the communication base to use for event handling.
@@ -116,11 +252,15 @@ struct pending {
  * @param do_ip6: service IP6.
  * @param port_base: if -1 system assigns ports, otherwise try to get
  *    the ports numbered from this starting number.
+ * @param num_tcp: number of outgoing tcp buffers to preallocate.
+ * @param infra: pointer to infra cached used for serviced queries.
+ * @param rnd: stored to create random numbers for serviced queries.
  * @return: the new structure (with no pending answers) or NULL on error.
  */
 struct outside_network* outside_network_create(struct comm_base* base,
-	size_t bufsize, size_t num_ports, const char** ifs, int num_ifs,
-	int do_ip4, int do_ip6, int port_base);
+	size_t bufsize, size_t num_ports, char** ifs, int num_ifs,
+	int do_ip4, int do_ip6, int port_base, size_t num_tcp, 
+	struct infra_cache* infra, struct ub_randstate* rnd);
 
 /**
  * Delete outside_network structure.
@@ -135,15 +275,36 @@ void outside_network_delete(struct outside_network* outnet);
  * @param packet: wireformat query to send to destination.
  * @param addr: address to send to.
  * @param addrlen: length of addr.
- * @param timeout: in seconds from now.
+ * @param timeout: in milliseconds from now.
  * @param callback: function to call on error, timeout or reply.
- *    The routine does not return an error, instead it calls the callback,
- *    with an error code if an error happens.
  * @param callback_arg: user argument for callback function.
+ * @param rnd: random state for generating ID and port.
+ * @return: NULL on error for malloc or socket. Else the pending query object.
  */
-void pending_udp_query(struct outside_network* outnet, ldns_buffer* packet, 
-	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* callback, void* callback_arg);
+struct pending* pending_udp_query(struct outside_network* outnet, 
+	ldns_buffer* packet, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+	void* callback_arg, struct ub_randstate* rnd);
+
+/**
+ * Send TCP query. May wait for TCP buffer. Selects ID to be random, and 
+ * checks id.
+ * @param outnet: provides the event handling.
+ * @param packet: wireformat query to send to destination. copied from.
+ * @param addr: address to send to.
+ * @param addrlen: length of addr.
+ * @param timeout: in seconds from now.
+ *    Timer starts running now. Timer may expire if all buffers are used,
+ *    without any query been sent to the server yet.
+ * @param callback: function to call on error, timeout or reply.
+ * @param callback_arg: user argument for callback function.
+ * @param rnd: random state for generating ID.
+ * @return: false on error for malloc or socket. Else the pending TCP object.
+ */
+struct waiting_tcp* pending_tcp_query(struct outside_network* outnet, 
+	ldns_buffer* packet, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+	void* callback_arg, struct ub_randstate* rnd);
 
 /**
  * Delete pending answer.
@@ -153,4 +314,87 @@ void pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
  */
 void pending_delete(struct outside_network* outnet, struct pending* p);
 
+/**
+ * Perform a serviced query to the authoritative servers.
+ * Duplicate efforts are detected, and EDNS, TCP and UDP retry is performed.
+ * @param outnet: outside network, with rbtree of serviced queries.
+ * @param qname: what qname to query.
+ * @param qnamelen: length of qname in octets including 0 root label.
+ * @param qtype: rrset type to query (host format)
+ * @param qclass: query class. (host format)
+ * @param flags: flags u16 (host format), includes opcode, CD bit.
+ * @param dnssec: if set, DO bit is set in EDNS queries.
+ *	If the value includes BIT_CD, CD bit is set when in EDNS queries.
+ *	If the value includes BIT_DO, DO bit is set when in EDNS queries.
+ * @param callback: callback function.
+ * @param callback_arg: user argument to callback function.
+ * @param addr: to which server to send the query.
+ * @param addrlen: length of addr.
+ * @param buff: scratch buffer to create query contents in. Empty on exit.
+ * @param arg_compare: function to compare callback args, return true if 
+ * 	identical. It is given the callback_arg and args that are listed.
+ * @return 0 on error, or pointer to serviced query that is used to answer
+ *	this serviced query may be shared with other callbacks as well.
+ */
+struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint16_t flags, int dnssec, struct sockaddr_storage* addr, 
+	socklen_t addrlen, comm_point_callback_t* callback, 
+	void* callback_arg, ldns_buffer* buff, 
+	int (*arg_compare)(void*,void*));
+
+/**
+ * Remove service query callback.
+ * If that leads to zero callbacks, the query is completely cancelled.
+ * @param sq: serviced query to adjust.
+ * @param cb_arg: callback argument of callback that needs removal.
+ *	same as the callback_arg to outnet_serviced_query().
+ */
+void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg);
+
+/**
+ * Get memory size in use by outside network.
+ * Counts buffers and outstanding query (serviced queries) malloced data.
+ * @param outnet: outside network structure.
+ * @return size in bytes.
+ */
+size_t outnet_get_mem(struct outside_network* outnet);
+
+/**
+ * Get memory size in use by serviced query while it is servicing callbacks.
+ * This takes into account the pre-deleted status of it; it will be deleted
+ * when the callbacks are done.
+ * @param sq: serviced query. 
+ * @return size in bytes.
+ */
+size_t serviced_get_mem(struct serviced_query* sq);
+
+/** callback for incoming udp answers from the network */
+int outnet_udp_cb(struct comm_point* c, void* arg, int error,
+	struct comm_reply *reply_info);
+
+/** callback for pending tcp connections */
+int outnet_tcp_cb(struct comm_point* c, void* arg, int error,
+	struct comm_reply *reply_info);
+
+/** callback for udp timeout */
+void pending_udp_timer_cb(void *arg);
+
+/** callback for outgoing TCP timer event */
+void outnet_tcptimer(void* arg);
+
+/** callback for serviced query UDP answers */
+int serviced_udp_callback(struct comm_point* c, void* arg, int error,
+        struct comm_reply* rep);
+
+/** TCP reply or error callback for serviced queries */
+int serviced_tcp_callback(struct comm_point* c, void* arg, int error,
+        struct comm_reply* rep);
+
+/** compare function of pending rbtree */
+int pending_cmp(const void* key1, const void* key2);
+
+/** compare function of serviced query rbtree */
+int serviced_cmp(const void* key1, const void* key2);
+
 #endif /* OUTSIDE_NETWORK_H */

testcode/checklocks.c

@@ -0,0 +1,849 @@
+/**
+ * testcode/checklocks.c - wrapper on locks that checks access.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ * 
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include <signal.h>
+#include "util/locks.h"   /* include before checklocks.h */
+#include "testcode/checklocks.h"
+
+/**
+ * \file
+ * Locks that are checked.
+ *
+ * Ugly hack: uses the fact that workers start with an int thread_num, and
+ * are passed to thread_create to make the thread numbers here the same as 
+ * those used for logging which is nice.
+ *
+ * Todo: 
+ *	 - debug status print, of thread lock stacks, and current waiting.
+ */
+#ifdef USE_THREAD_DEBUG
+
+/** How long to wait before lock attempt is a failure. */
+#define CHECK_LOCK_TIMEOUT 30 /* seconds */
+/** How long to wait before join attempt is a failure. */
+#define CHECK_JOIN_TIMEOUT 120 /* seconds */
+
+/** if key has been created */
+static int key_created = 0;
+/** we hide the thread debug info with this key. */
+static ub_thread_key_t thr_debug_key;
+/** the list of threads, so all threads can be examined. NULL if unused. */
+static struct thr_check* thread_infos[THRDEBUG_MAX_THREADS];
+/** do we check locking order */
+int check_locking_order = 1;
+/** the pid of this runset, reasonably unique. */
+static pid_t check_lock_pid;
+
+/** print all possible debug info on the state of the system */
+static void total_debug_info();
+
+/** print pretty lock error and exit */
+static void lock_error(struct checked_lock* lock, 
+	const char* func, const char* file, int line, const char* err)
+{
+	log_err("lock error (description follows)");
+	log_err("Created at %s %s:%d", lock->create_func, 
+		lock->create_file, lock->create_line);
+	if(lock->holder_func && lock->holder_file)
+		log_err("Previously %s %s:%d", lock->holder_func, 
+			lock->holder_file, lock->holder_line);
+	log_err("At %s %s:%d", func, file, line);
+	log_err("Error for %s lock: %s",
+		(lock->type==check_lock_mutex)?"mutex": (
+		(lock->type==check_lock_spinlock)?"spinlock": (
+		(lock->type==check_lock_rwlock)?"rwlock": "badtype")), err);
+	log_err("complete status display:");
+	total_debug_info();
+	fatal_exit("bailing out");
+}
+
+/** 
+ * Obtain lock on debug lock structure. This could be a deadlock by the caller.
+ * The debug code itself does not deadlock. Anyway, check with timeouts. 
+ * @param lock: on what to acquire lock.
+ * @param func: user level caller identification.
+ * @param file: user level caller identification.
+ * @param line: user level caller identification.
+ */
+static void
+acquire_locklock(struct checked_lock* lock, 
+	const char* func, const char* file, int line)
+{
+	struct timespec to;
+	int err;
+	int contend = 0;
+	/* first try; inc contention counter if not immediately */
+	if((err = pthread_mutex_trylock(&lock->lock))) {
+		if(err==EBUSY)
+			contend++;
+		else fatal_exit("error in mutex_trylock: %s", strerror(err));
+	}
+	if(!err)
+		return; /* immediate success */
+	to.tv_sec = time(NULL) + CHECK_LOCK_TIMEOUT;
+	to.tv_nsec = 0;
+	err = pthread_mutex_timedlock(&lock->lock, &to);
+	if(err) {
+		log_err("in acquiring locklock: %s", strerror(err));
+		lock_error(lock, func, file, line, "acquire locklock");
+	}
+	/* since we hold the lock, we can edit the contention_count */
+	lock->contention_count += contend;
+}
+
+/** add protected region */
+void 
+lock_protect(void *p, void* area, size_t size)
+{
+	struct checked_lock* lock = *(struct checked_lock**)p;
+	struct protected_area* e = (struct protected_area*)malloc(
+		sizeof(struct protected_area));
+	if(!e)
+		fatal_exit("lock_protect: out of memory");
+	e->region = area;
+	e->size = size;
+	e->hold = malloc(size);
+	if(!e->hold)
+		fatal_exit("lock_protect: out of memory");
+	memcpy(e->hold, e->region, e->size);
+
+	acquire_locklock(lock, __func__, __FILE__, __LINE__);
+	e->next = lock->prot;
+	lock->prot = e;
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+}
+
+/** remove protected region */
+void
+lock_unprotect(void* mangled, void* area)
+{
+	struct checked_lock* lock = *(struct checked_lock**)mangled;
+	struct protected_area* p, **prevp;
+	if(!lock) 
+		return;
+	acquire_locklock(lock, __func__, __FILE__, __LINE__);
+	p = lock->prot;
+	prevp = &lock->prot;
+	while(p) {
+		if(p->region == area) {
+			*prevp = p->next;
+			free(p->hold);
+			free(p);
+			LOCKRET(pthread_mutex_unlock(&lock->lock));
+			return;
+		}
+		prevp = &p->next;
+		p = p->next;
+	}
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+}
+
+/** 
+ * Check protected memory region. Memory compare. Exit on error. 
+ * @param lock: which lock to check.
+ * @param func: location we are now (when failure is detected).
+ * @param file: location we are now (when failure is detected).
+ * @param line: location we are now (when failure is detected).
+ */
+static void 
+prot_check(struct checked_lock* lock,
+	const char* func, const char* file, int line)
+{
+	struct protected_area* p = lock->prot;
+	while(p) {
+		if(memcmp(p->hold, p->region, p->size) != 0) {
+			log_hex("memory prev", p->hold, p->size);
+			log_hex("memory here", p->region, p->size);
+			lock_error(lock, func, file, line, 
+				"protected area modified");
+		}
+		p = p->next;
+	}
+}
+
+/** Copy protected memory region */
+static void 
+prot_store(struct checked_lock* lock)
+{
+	struct protected_area* p = lock->prot;
+	while(p) {
+		memcpy(p->hold, p->region, p->size);
+		p = p->next;
+	}
+}
+
+/** get memory held by lock */
+size_t 
+lock_get_mem(void* pp)
+{
+	size_t s;
+	struct checked_lock* lock = *(struct checked_lock**)pp;
+	struct protected_area* p;
+	s = sizeof(struct checked_lock);
+	acquire_locklock(lock, __func__, __FILE__, __LINE__);
+	for(p = lock->prot; p; p = p->next) {
+		s += sizeof(struct protected_area);
+		s += p->size;
+	}
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+	return s;
+}
+
+/** write lock trace info to file, while you hold those locks */
+static void
+ordercheck_locklock(struct thr_check* thr, struct checked_lock* lock)
+{
+	int info[4];
+	if(!check_locking_order) return;
+	if(!thr->holding_first) return; /* no older lock, no info */
+	/* write: <lock id held> <lock id new> <file> <line> */
+	info[0] = thr->holding_first->create_thread;
+	info[1] = thr->holding_first->create_instance;
+	info[2] = lock->create_thread;
+	info[3] = lock->create_instance;
+	if(fwrite(info, 4*sizeof(int), 1, thr->order_info) != 1 ||
+		fwrite(lock->holder_file, strlen(lock->holder_file)+1, 1, 
+		thr->order_info) != 1 ||
+		fwrite(&lock->holder_line, sizeof(int), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
+
+/** write ordercheck lock creation details to file */
+static void 
+ordercheck_lockcreate(struct thr_check* thr, struct checked_lock* lock)
+{
+	/* write: <ffff = create> <lock id> <file> <line> */
+	int cmd = -1;
+	if(!check_locking_order) return;
+
+	if( fwrite(&cmd, sizeof(int), 1, thr->order_info) != 1 ||
+		fwrite(&lock->create_thread, sizeof(int), 1, 
+			thr->order_info) != 1 ||
+		fwrite(&lock->create_instance, sizeof(int), 1, 
+			thr->order_info) != 1 ||
+		fwrite(lock->create_file, strlen(lock->create_file)+1, 1, 
+			thr->order_info) != 1 ||
+		fwrite(&lock->create_line, sizeof(int), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
+
+/** alloc struct, init lock empty */
+void 
+checklock_init(enum check_lock_type type, struct checked_lock** lock,
+        const char* func, const char* file, int line)
+{
+	struct checked_lock* e = (struct checked_lock*)calloc(1, 
+		sizeof(struct checked_lock));
+	struct thr_check *thr = (struct thr_check*)pthread_getspecific(
+		thr_debug_key);
+	if(!e)
+		fatal_exit("%s %s %d: out of memory", func, file, line);
+	if(!thr)
+		fatal_exit("%s %s %d: lock_init no thread info", func, file,
+			line);
+	*lock = e;
+	e->type = type;
+	e->create_func = func;
+	e->create_file = file;
+	e->create_line = line;
+	e->create_thread = thr->num;
+	e->create_instance = thr->locks_created++;
+	ordercheck_lockcreate(thr, e);
+	LOCKRET(pthread_mutex_init(&e->lock, NULL));
+	switch(e->type) {
+		case check_lock_mutex:
+			LOCKRET(pthread_mutex_init(&e->u.mutex, NULL));
+			break;
+		case check_lock_spinlock:
+			LOCKRET(pthread_spin_init(&e->u.spinlock, PTHREAD_PROCESS_PRIVATE));
+			break;
+		case check_lock_rwlock:
+			LOCKRET(pthread_rwlock_init(&e->u.rwlock, NULL));
+			break;
+		default:
+			log_assert(0);
+	}
+}
+
+/** delete prot items */
+static void 
+prot_clear(struct checked_lock* lock)
+{
+	struct protected_area* p=lock->prot, *np;
+	while(p) {
+		np = p->next;
+		free(p->hold);
+		free(p);
+		p = np;
+	}
+}
+
+/** check if type is OK for the lock given */
+static void 
+checktype(enum check_lock_type type, struct checked_lock* lock,
+        const char* func, const char* file, int line)
+{
+	if(!lock) 
+		fatal_exit("use of null/deleted lock at %s %s:%d", 
+			func, file, line);
+	if(type != lock->type) {
+		lock_error(lock, func, file, line, "wrong lock type");
+	}
+}
+
+/** check if OK, free struct */
+void 
+checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
+        const char* func, const char* file, int line)
+{
+	const size_t contention_interest = 1; /* promille contented locks */
+	struct checked_lock* e;
+	if(!lock) 
+		return;
+	e = *lock;
+	if(!e)
+		return;
+	checktype(type, e, func, file, line);
+
+	/* check if delete is OK */
+	acquire_locklock(e, func, file, line);
+	if(e->hold_count != 0)
+		lock_error(e, func, file, line, "delete while locked.");
+	if(e->wait_count != 0)
+		lock_error(e, func, file, line, "delete while waited on.");
+	prot_check(e, func, file, line);
+	*lock = NULL; /* use after free will fail */
+	LOCKRET(pthread_mutex_unlock(&e->lock));
+
+	/* contention, look at fraction in trouble. */
+	if(e->history_count > 1 &&
+	   1000*e->contention_count/e->history_count > contention_interest) {
+		log_info("lock created %s %s %d has contention %u of %u (%d%%)",
+			e->create_func, e->create_file, e->create_line,
+			(unsigned int)e->contention_count, 
+			(unsigned int)e->history_count,
+			(int)(100*e->contention_count/e->history_count));
+	}
+
+	/* delete it */
+	LOCKRET(pthread_mutex_destroy(&e->lock));
+	prot_clear(e);
+	/* since nobody holds the lock - see check above, no need to unlink 
+	 * from the thread-held locks list. */
+	switch(e->type) {
+		case check_lock_mutex:
+			LOCKRET(pthread_mutex_destroy(&e->u.mutex));
+			break;
+		case check_lock_spinlock:
+			LOCKRET(pthread_spin_destroy(&e->u.spinlock));
+			break;
+		case check_lock_rwlock:
+			LOCKRET(pthread_rwlock_destroy(&e->u.rwlock));
+			break;
+		default:
+			log_assert(0);
+	}
+	memset(e, 0, sizeof(struct checked_lock));
+	free(e);
+}
+
+/** finish acquiring lock, shared between _(rd|wr||)lock() routines */
+static void 
+finish_acquire_lock(struct thr_check* thr, struct checked_lock* lock,
+        const char* func, const char* file, int line)
+{
+	thr->waiting = NULL;
+	lock->wait_count --;
+	lock->holder = thr;
+	lock->hold_count ++;
+	lock->holder_func = func;
+	lock->holder_file = file;
+	lock->holder_line = line;
+	ordercheck_locklock(thr, lock);
+	
+	/* insert in thread lock list, as first */
+	lock->prev_held_lock[thr->num] = NULL;
+	lock->next_held_lock[thr->num] = thr->holding_first;
+	if(thr->holding_first)
+		/* no need to lock it, since this thread already holds the
+		 * lock (since it is on this list) and we only edit thr->num
+		 * member in array. So it is safe.  */
+		thr->holding_first->prev_held_lock[thr->num] = lock;
+	else	thr->holding_last = lock;
+	thr->holding_first = lock;
+}
+
+/**
+ * Locking routine.
+ * @param type: as passed by user.
+ * @param lock: as passed by user.
+ * @param func: caller location.
+ * @param file: caller location.
+ * @param line: caller location.
+ * @param tryfunc: the pthread_mutex_trylock or similar function.
+ * @param timedfunc: the pthread_mutex_timedlock or similar function.
+ *	Uses absolute timeout value.
+ * @param arg: what to pass to tryfunc and timedlock.
+ * @param exclusive: if lock must be exlusive (only one allowed).
+ * @param getwr: if attempts to get writelock (or readlock) for rwlocks.
+ */
+static void 
+checklock_lockit(enum check_lock_type type, struct checked_lock* lock,
+        const char* func, const char* file, int line,
+	int (*tryfunc)(void*), int (*timedfunc)(void*, struct timespec*),
+	void* arg, int exclusive, int getwr)
+{
+	int err;
+	int contend = 0;
+	struct thr_check *thr = (struct thr_check*)pthread_getspecific(
+		thr_debug_key);
+	checktype(type, lock, func, file, line);
+	if(!thr) lock_error(lock, func, file, line, "no thread info");
+	
+	acquire_locklock(lock, func, file, line);
+	lock->wait_count ++;
+	thr->waiting = lock;
+	if(exclusive && lock->hold_count > 0 && lock->holder == thr) 
+		lock_error(lock, func, file, line, "thread already owns lock");
+	if(type==check_lock_rwlock && getwr && lock->writeholder == thr)
+		lock_error(lock, func, file, line, "thread already has wrlock");
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+
+	/* first try; if busy increase contention counter */
+	if((err=tryfunc(arg))) {
+		struct timespec to;
+		if(err != EBUSY) log_err("trylock: %s", strerror(err));
+		to.tv_sec = time(NULL) + CHECK_LOCK_TIMEOUT;
+		to.tv_nsec = 0;
+		if((err=timedfunc(arg, &to))) {
+			if(err == ETIMEDOUT)
+				lock_error(lock, func, file, line, 
+					"timeout possible deadlock");
+			log_err("timedlock: %s", strerror(err));
+		}
+		contend ++;
+	}
+	/* got the lock */
+
+	acquire_locklock(lock, func, file, line);
+	lock->contention_count += contend;
+	lock->history_count++;
+	if(exclusive && lock->hold_count > 0)
+		lock_error(lock, func, file, line, "got nonexclusive lock");
+	if(type==check_lock_rwlock && getwr && lock->writeholder)
+		lock_error(lock, func, file, line, "got nonexclusive wrlock");
+	if(type==check_lock_rwlock && getwr)
+		lock->writeholder = thr;
+	/* check the memory areas for unauthorized changes,
+	 * between last unlock time and current lock time.
+	 * we check while holding the lock (threadsafe).
+	 */
+	if(getwr || exclusive)
+		prot_check(lock, func, file, line);
+	finish_acquire_lock(thr, lock, func, file, line);
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+}
+
+/** helper for rdlock: try */
+static int try_rd(void* arg)
+{ return pthread_rwlock_tryrdlock((pthread_rwlock_t*)arg); }
+/** helper for rdlock: timed */
+static int timed_rd(void* arg, struct timespec* to)
+{ return pthread_rwlock_timedrdlock((pthread_rwlock_t*)arg, to); }
+
+/** check if OK, lock */
+void 
+checklock_rdlock(enum check_lock_type type, struct checked_lock* lock,
+        const char* func, const char* file, int line)
+{
+
+	log_assert(type == check_lock_rwlock);
+	checklock_lockit(type, lock, func, file, line,
+		try_rd, timed_rd, &lock->u.rwlock, 0, 0);
+}
+
+/** helper for wrlock: try */
+static int try_wr(void* arg)
+{ return pthread_rwlock_trywrlock((pthread_rwlock_t*)arg); }
+/** helper for wrlock: timed */
+static int timed_wr(void* arg, struct timespec* to)
+{ return pthread_rwlock_timedwrlock((pthread_rwlock_t*)arg, to); }
+
+/** check if OK, lock */
+void 
+checklock_wrlock(enum check_lock_type type, struct checked_lock* lock,
+        const char* func, const char* file, int line)
+{
+	log_assert(type == check_lock_rwlock);
+	checklock_lockit(type, lock, func, file, line,
+		try_wr, timed_wr, &lock->u.rwlock, 0, 1);
+}
+
+/** helper for lock mutex: try */
+static int try_mutex(void* arg)
+{ return pthread_mutex_trylock((pthread_mutex_t*)arg); }
+/** helper for lock mutex: timed */
+static int timed_mutex(void* arg, struct timespec* to)
+{ return pthread_mutex_timedlock((pthread_mutex_t*)arg, to); }
+
+/** helper for lock spinlock: try */
+static int try_spinlock(void* arg)
+{ return pthread_spin_trylock((pthread_spinlock_t*)arg); }
+/** helper for lock spinlock: timed */
+static int timed_spinlock(void* arg, struct timespec* to)
+{
+	int err;
+	/* spin for 5 seconds. (ouch for the CPU, but it beats forever) */
+	while( (err=try_spinlock(arg)) == EBUSY) {
+#ifndef S_SPLINT_S
+		if(time(NULL) >= to->tv_sec)
+			return ETIMEDOUT;
+		usleep(1000); /* in 1/1000000s of a second */
+#endif
+	}
+	return err;
+}
+
+/** check if OK, lock */
+void 
+checklock_lock(enum check_lock_type type, struct checked_lock* lock,
+        const char* func, const char* file, int line)
+{
+	log_assert(type != check_lock_rwlock);
+	switch(type) {
+		case check_lock_mutex:
+			checklock_lockit(type, lock, func, file, line,
+				try_mutex, timed_mutex, &lock->u.mutex, 1, 0);
+			break;
+		case check_lock_spinlock:
+			/* void* cast needed because 'volatile' on some OS */
+			checklock_lockit(type, lock, func, file, line,
+				try_spinlock, timed_spinlock, 
+				(void*)&lock->u.spinlock, 1, 0);
+			break;
+		default:
+			log_assert(0);
+	}
+}
+
+/** check if OK, unlock */
+void 
+checklock_unlock(enum check_lock_type type, struct checked_lock* lock,
+        const char* func, const char* file, int line)
+{
+	struct thr_check *thr = (struct thr_check*)pthread_getspecific(
+		thr_debug_key);
+	checktype(type, lock, func, file, line);
+	if(!thr) lock_error(lock, func, file, line, "no thread info");
+
+	acquire_locklock(lock, func, file, line);
+	/* was this thread even holding this lock? */
+	if(thr->holding_first != lock &&
+		lock->prev_held_lock[thr->num] == NULL) {
+		lock_error(lock, func, file, line, "unlock nonlocked lock");
+	}
+	if(lock->hold_count <= 0)
+		lock_error(lock, func, file, line, "too many unlocks");
+
+	/* store this point as last touched by */
+	lock->holder = thr;
+	lock->hold_count --;
+	lock->holder_func = func;
+	lock->holder_file = file;
+	lock->holder_line = line;
+
+	/* delete from thread holder list */
+	/* no need to lock other lockstructs, because they are all on the
+	 * held-locks list, and this thread holds their locks.
+	 * we only touch the thr->num members, so it is safe.  */
+	if(thr->holding_first == lock)
+		thr->holding_first = lock->next_held_lock[thr->num];
+	if(thr->holding_last == lock)
+		thr->holding_last = lock->prev_held_lock[thr->num];
+	if(lock->next_held_lock[thr->num])
+		lock->next_held_lock[thr->num]->prev_held_lock[thr->num] =
+			lock->prev_held_lock[thr->num];
+	if(lock->prev_held_lock[thr->num])
+		lock->prev_held_lock[thr->num]->next_held_lock[thr->num] =
+			lock->next_held_lock[thr->num];
+	lock->next_held_lock[thr->num] = NULL;
+	lock->prev_held_lock[thr->num] = NULL;
+
+	if(type==check_lock_rwlock && lock->writeholder == thr) {
+		lock->writeholder = NULL;
+		prot_store(lock);
+	} else if(type != check_lock_rwlock) {
+		/* store memory areas that are protected, for later checks */
+		prot_store(lock);
+	}
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+
+	/* unlock it */
+	switch(type) {
+		case check_lock_mutex:
+			LOCKRET(pthread_mutex_unlock(&lock->u.mutex));
+			break;
+		case check_lock_spinlock:
+			LOCKRET(pthread_spin_unlock(&lock->u.spinlock));
+			break;
+		case check_lock_rwlock:
+			LOCKRET(pthread_rwlock_unlock(&lock->u.rwlock));
+			break;
+		default:
+			log_assert(0);
+	}
+}
+
+/** open order info debug file, thr->num must be valid */
+static void 
+open_lockorder(struct thr_check* thr)
+{
+	char buf[24];
+	time_t t;
+	snprintf(buf, sizeof(buf), "ublocktrace.%d", thr->num);
+	thr->order_info = fopen(buf, "w");
+	if(!thr->order_info)
+		fatal_exit("could not open %s: %s", buf, strerror(errno));
+	thr->locks_created = 0;
+	t = time(NULL);
+	/* write: <time_stamp> <runpid> <thread_num> */
+	if(fwrite(&t, sizeof(t), 1, thr->order_info) != 1 ||
+		fwrite(&thr->num, sizeof(thr->num), 1, thr->order_info) != 1 || 
+		fwrite(&check_lock_pid, sizeof(check_lock_pid), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
+
+/** checklock thread main, Inits thread structure */
+static void* checklock_main(void* arg)
+{
+	struct thr_check* thr = (struct thr_check*)arg; 
+	void* ret;
+	thr->id = pthread_self();
+	/* Hack to get same numbers as in log file */
+	thr->num = *(int*)(thr->arg);
+	log_assert(thr->num < THRDEBUG_MAX_THREADS);
+	log_assert(thread_infos[thr->num] == NULL);
+	thread_infos[thr->num] = thr;
+	LOCKRET(pthread_setspecific(thr_debug_key, thr));
+	if(check_locking_order)
+		open_lockorder(thr);
+	ret = thr->func(thr->arg);
+	thread_infos[thr->num] = NULL;
+	if(check_locking_order)
+		fclose(thr->order_info);
+	free(thr);
+	return ret;
+}
+
+/** init the main thread */
+void checklock_start()
+{
+	if(!key_created) {
+		struct thr_check* thisthr = (struct thr_check*)calloc(1, 
+			sizeof(struct thr_check));
+		if(!thisthr)
+			fatal_exit("thrcreate: out of memory");
+		key_created = 1;
+		check_lock_pid = getpid();
+		LOCKRET(pthread_key_create(&thr_debug_key, NULL));
+		LOCKRET(pthread_setspecific(thr_debug_key, thisthr));
+		thread_infos[0] = thisthr;
+		if(check_locking_order)
+			open_lockorder(thisthr);
+	}
+}
+
+/** stop checklocks */
+void checklock_stop()
+{
+	if(key_created) {
+		int i;
+		if(check_locking_order)
+			fclose(thread_infos[0]->order_info);
+		free(thread_infos[0]);
+		thread_infos[0] = NULL;
+		for(i = 0; i < THRDEBUG_MAX_THREADS; i++)
+			log_assert(thread_infos[i] == NULL);
+			/* should have been cleaned up. */
+		LOCKRET(pthread_key_delete(thr_debug_key));
+		key_created = 0;
+	}
+}
+
+/** allocate debug info and create thread */
+void 
+checklock_thrcreate(pthread_t* id, void* (*func)(void*), void* arg)
+{
+	struct thr_check* thr = (struct thr_check*)calloc(1, 
+		sizeof(struct thr_check));
+	if(!thr)
+		fatal_exit("thrcreate: out of memory");
+	if(!key_created) {
+		checklock_start();
+	}
+	thr->func = func;
+	thr->arg = arg;
+	LOCKRET(pthread_create(id, NULL, checklock_main, thr));
+}
+
+/** count number of thread infos */
+static int
+count_thread_infos()
+{
+	int cnt = 0;
+	int i;
+	for(i=0; i<THRDEBUG_MAX_THREADS; i++)
+		if(thread_infos[i])
+			cnt++;
+	return cnt;
+}
+
+/** print lots of info on a lock */
+static void
+lock_debug_info(struct checked_lock* lock)
+{
+	if(!lock) return;
+	log_info("+++ Lock %x, %d %d create %s %s %d", (int)lock, 
+		lock->create_thread, lock->create_instance, 
+		lock->create_func, lock->create_file, lock->create_line);
+	log_info("lock type: %s",
+		(lock->type==check_lock_mutex)?"mutex": (
+		(lock->type==check_lock_spinlock)?"spinlock": (
+		(lock->type==check_lock_rwlock)?"rwlock": "badtype")));
+	log_info("lock contention %u, history:%u, hold:%d, wait:%d", 
+		(unsigned)lock->contention_count, (unsigned)lock->history_count,
+		lock->hold_count, lock->wait_count);
+	log_info("last touch %s %s %d", lock->holder_func, lock->holder_file,
+		lock->holder_line);
+	log_info("holder thread %d, writeholder thread %d",
+		lock->holder?lock->holder->num:-1,
+		lock->writeholder?lock->writeholder->num:-1);
+}
+
+/** print debug locks held by a thread */
+static void
+held_debug_info(struct thr_check* thr, struct checked_lock* lock)
+{
+	if(!lock) return;
+	lock_debug_info(lock);
+	held_debug_info(thr, lock->next_held_lock[thr->num]);
+}
+
+/** print debug info for a thread */
+static void
+thread_debug_info(struct thr_check* thr)
+{
+	struct checked_lock* w = NULL;
+	struct checked_lock* f = NULL;
+	struct checked_lock* l = NULL;
+	if(!thr) return;
+	log_info("pthread id is %x", (int)thr->id);
+	log_info("thread func is %x", (int)thr->func);
+	log_info("thread arg is %x (%d)", (int)thr->arg, 
+		(thr->arg?*(int*)thr->arg:0));
+	log_info("thread num is %d", thr->num);
+	log_info("locks created %d", thr->locks_created);
+	log_info("open file for lockinfo: %s", 
+		thr->order_info?"yes, flushing":"no");
+	fflush(thr->order_info);
+	w = thr->waiting;
+	f = thr->holding_first;
+	l = thr->holding_last;
+	log_info("thread waiting for a lock: %s %x", w?"yes":"no", (int)w);
+	lock_debug_info(w);
+	log_info("thread holding first: %s, last: %s", f?"yes":"no", 
+		l?"yes":"no");
+	held_debug_info(thr, f);
+}
+
+static void
+total_debug_info()
+{
+	int i;
+	log_info("checklocks: supervising %d threads.",
+		count_thread_infos());
+	if(!key_created) {
+		log_info("No thread debug key created yet");
+	}
+	for(i=0; i<THRDEBUG_MAX_THREADS; i++) {
+		if(thread_infos[i]) {
+			log_info("*** Thread %d information: ***", i);
+			thread_debug_info(thread_infos[i]);
+		}
+	}
+}
+
+/** signal handler for join timeout, Exits */
+static RETSIGTYPE joinalarm(int ATTR_UNUSED(sig))
+{
+	log_err("join thread timeout. hangup or deadlock. Info follows.");
+	total_debug_info();
+	fatal_exit("join thread timeout. hangup or deadlock.");
+}
+
+/** wait for thread with a timeout */
+void 
+checklock_thrjoin(pthread_t thread)
+{
+	/* wait with a timeout */
+	if(signal(SIGALRM, joinalarm) == SIG_ERR)
+		fatal_exit("signal(): %s", strerror(errno));
+	(void)alarm(CHECK_JOIN_TIMEOUT);
+	LOCKRET(pthread_join(thread, NULL));
+	(void)alarm(0);
+}
+
+#endif /* USE_THREAD_DEBUG */
+
+int order_lock_cmp(const void* e1, const void* e2)
+{
+	struct order_id* o1 = (struct order_id*)e1;
+	struct order_id* o2 = (struct order_id*)e2;
+	if(o1->thr < o2->thr) return -1;
+	if(o1->thr > o2->thr) return 1;
+	if(o1->instance < o2->instance) return -1;
+	if(o1->instance > o2->instance) return 1;
+	return 0;
+}
+
+int
+codeline_cmp(const void* a, const void* b)
+{
+	return strcmp((const char*)a, (const char*)b);
+}

testcode/checklocks.h

@@ -0,0 +1,356 @@
+/**
+ * testcode/checklocks.h - wrapper on locks that checks access.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ * 
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef TESTCODE_CHECK_LOCKS_H
+#define TESTCODE_CHECK_LOCKS_H
+
+/**
+ * \file
+ * Locks that are checked.
+ *
+ * Holds information per lock and per thread.
+ * That information is protected by a mutex (unchecked).
+ *
+ * Checks:
+ *      o which func, file, line created the lock.
+ *      o contention count, measures amount of contention on the lock.
+ *      o the memory region(s) that the lock protects are
+ *        memcmp'ed to ascertain no race conditions.
+ *      o checks that locks are unlocked properly (before deletion).
+ *        keeps which func, file, line that locked it.
+ *	o checks deadlocks with timeout so it can print errors for them.
+ *
+ * Limitations:
+ *	o Detects unprotected memory access when the lock is locked or freed,
+ *	  which detects races only if they happen, and only if in protected
+ *	  memory areas.
+ *	o Detects deadlocks by timeout, so approximately, as they happen.
+ *	o Does not check order of locking.
+ *	o Uses a lot of memory.
+ *	o The checks use locks themselves, changing scheduling,
+ *	  thus changing what races you see.
+ */
+
+#ifdef USE_THREAD_DEBUG
+#ifndef HAVE_PTHREAD
+/* we need the *timed*lock() routines to use for deadlock detection. */
+#error "Need pthreads for checked locks"
+#endif
+/******************* THREAD DEBUG ************************/
+#include <pthread.h>
+
+/** How many threads to allocate for */
+#define THRDEBUG_MAX_THREADS 32 /* threads */
+/** do we check locking order */
+extern int check_locking_order;
+
+/**
+ * Protection memory area.
+ * It is copied to a holding buffer to compare against later.
+ * Note that it may encompass the lock structure.
+ */
+struct protected_area {
+	/** where the memory region starts */
+	void* region;
+	/** size of the region */
+	size_t size;
+	/** backbuffer that holds a copy, of same size. */
+	void* hold;
+	/** next protected area in list */
+	struct protected_area* next;
+};
+
+/**
+ * Per thread information for locking debug wrappers. 
+ */
+struct thr_check {
+	/** thread id */
+	pthread_t id;
+	/** real thread func */
+	void* (*func)(void*);
+	/** func user arg */
+	void* arg;
+	/** number of thread in list structure */
+	int num;
+	/** instance number - how many locks have been created by thread */
+	int locks_created;
+	/** file to write locking order information to */
+	FILE* order_info;
+	/** 
+	 * List of locks that this thread is holding, double
+	 * linked list. The first element is the most recent lock acquired.
+	 * So it represents the stack of locks acquired. (of all types).
+	 */
+	struct checked_lock *holding_first, *holding_last;
+	/** if the thread is currently waiting for a lock, which one */
+	struct checked_lock* waiting;
+};
+
+/**
+ * One structure for all types of locks.
+ */
+struct checked_lock {
+	/** mutex for exclusive access to this structure */
+	pthread_mutex_t lock;
+	/** list of memory regions protected by this checked lock */
+	struct protected_area* prot;
+	/** where was this lock created */
+	const char* create_func, *create_file;
+	/** where was this lock created */
+	int create_line;
+	/** unique instance identifier */
+	int create_thread, create_instance;
+	/** contention count */
+	size_t contention_count;
+	/** number of times locked, ever */
+	size_t history_count;
+	/** hold count (how many threads are holding this lock) */
+	int hold_count;
+	/** how many threads are waiting for this lock */
+	int wait_count;
+	/** who touched it last */
+	const char* holder_func, *holder_file;
+	/** who touched it last */
+	int holder_line;
+	/** who owns the lock now */
+	struct thr_check* holder;
+	/** for rwlocks, the writelock holder */
+	struct thr_check* writeholder;
+
+	/** next lock a thread is holding (less recent) */
+	struct checked_lock* next_held_lock[THRDEBUG_MAX_THREADS];
+	/** prev lock a thread is holding (more recent) */
+	struct checked_lock* prev_held_lock[THRDEBUG_MAX_THREADS];
+
+	/** type of lock */
+	enum check_lock_type {
+		/** basic mutex */
+		check_lock_mutex,
+		/** fast spinlock */
+		check_lock_spinlock,
+		/** rwlock */
+		check_lock_rwlock
+	} type;
+	/** the lock itself, see type to disambiguate the union */
+	union {
+		/** mutex */
+		pthread_mutex_t mutex;
+		/** spinlock */
+		pthread_spinlock_t spinlock;
+		/** rwlock */
+		pthread_rwlock_t rwlock;
+	} u;
+};
+
+/**
+ * Additional call for the user to specify what areas are protected
+ * @param lock: the lock that protects the area. It can be inside the area.
+ *	The lock must be inited. Call with user lock. (any type).
+ *	It demangles the lock itself (struct checked_lock**).
+ * @param area: ptr to mem.
+ * @param size: length of area.
+ * You can call it multiple times with the same lock to give several areas.
+ * Call it when you are done initialising the area, since it will be copied
+ * at this time and protected right away against unauthorised changes until 
+ * the next lock() call is done.
+ */
+void lock_protect(void* lock, void* area, size_t size);
+
+/**
+ * Remove protected area from lock.
+ * No need to call this when deleting the lock.
+ * @param lock: the lock, any type, (struct checked_lock**).
+ * @param area: pointer to memory.
+ */
+void lock_unprotect(void* lock, void* area);
+
+/**
+ * Get memory associated with a checked lock
+ * @param lock: the checked lock, any type. (struct checked_lock**).
+ * @return: in bytes, including protected areas.
+ */
+size_t lock_get_mem(void* lock);
+
+/**
+ * Initialise checklock. Sets up internal debug structures.
+ */
+void checklock_start();
+
+/**
+ * Cleanup internal debug state.
+ */
+void checklock_stop();
+
+/**
+ * Init locks.
+ * @param type: what type of lock this is.
+ * @param lock: ptr to user alloced ptr structure. This is inited.
+ *     So an alloc is done and the ptr is stored as result.
+ * @param func: caller function name.
+ * @param file: caller file name.
+ * @param line: caller line number.
+ */
+void checklock_init(enum check_lock_type type, struct checked_lock** lock,
+	const char* func, const char* file, int line);
+
+/**
+ * Destroy locks. Free the structure.
+ * @param type: what type of lock this is.
+ * @param lock: ptr to user alloced structure. This is destroyed.
+ * @param func: caller function name.
+ * @param file: caller file name.
+ * @param line: caller line number.
+ */
+void checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
+	const char* func, const char* file, int line);
+
+/**
+ * Acquire readlock.
+ * @param type: what type of lock this is. Had better be a rwlock.
+ * @param lock: ptr to lock.
+ * @param func: caller function name.
+ * @param file: caller file name.
+ * @param line: caller line number.
+ */
+void checklock_rdlock(enum check_lock_type type, struct checked_lock* lock,
+	const char* func, const char* file, int line);
+
+/**
+ * Acquire writelock.
+ * @param type: what type of lock this is. Had better be a rwlock.
+ * @param lock: ptr to lock.
+ * @param func: caller function name.
+ * @param file: caller file name.
+ * @param line: caller line number.
+ */
+void checklock_wrlock(enum check_lock_type type, struct checked_lock* lock,
+	const char* func, const char* file, int line);
+
+/**
+ * Locks.
+ * @param type: what type of lock this is. Had better be mutex or spinlock.
+ * @param lock: the lock.
+ * @param func: caller function name.
+ * @param file: caller file name.
+ * @param line: caller line number.
+ */
+void checklock_lock(enum check_lock_type type, struct checked_lock* lock,
+	const char* func, const char* file, int line);
+
+/**
+ * Unlocks.
+ * @param type: what type of lock this is.
+ * @param lock: the lock.
+ * @param func: caller function name.
+ * @param file: caller file name.
+ * @param line: caller line number.
+ */
+void checklock_unlock(enum check_lock_type type, struct checked_lock* lock,
+	const char* func, const char* file, int line);
+
+/**
+ * Create thread.
+ * @param thr: Thread id, where to store result.
+ * @param func: thread start function.
+ * @param arg: user argument.
+ */
+void checklock_thrcreate(pthread_t* thr, void* (*func)(void*), void* arg);
+
+/**
+ * Wait for thread to exit. Returns thread return value.
+ * @param thread: thread to wait for.
+ */
+void checklock_thrjoin(pthread_t thread);
+
+/** structures to enable compiler type checking on the locks. 
+ * Also the pointer makes it so that the lock can be part of the protected
+ * region without any possible problem (since the ptr will stay the same.)
+ * i.e. there can be contention and readlocks stored in checked_lock, while
+ * the protected area stays the same, even though it contains (ptr to) lock.
+ */
+struct checked_lock_rw { struct checked_lock* c_rw; };
+/** structures to enable compiler type checking on the locks. */
+struct checked_lock_mutex { struct checked_lock* c_m; };
+/** structures to enable compiler type checking on the locks. */
+struct checked_lock_spl { struct checked_lock* c_spl; };
+
+/** debugging rwlock */
+typedef struct checked_lock_rw lock_rw_t;
+#define lock_rw_init(lock) checklock_init(check_lock_rwlock, &((lock)->c_rw), __func__, __FILE__, __LINE__)
+#define lock_rw_destroy(lock) checklock_destroy(check_lock_rwlock, &((lock)->c_rw), __func__, __FILE__, __LINE__)
+#define lock_rw_rdlock(lock) checklock_rdlock(check_lock_rwlock, (lock)->c_rw, __func__, __FILE__, __LINE__)
+#define lock_rw_wrlock(lock) checklock_wrlock(check_lock_rwlock, (lock)->c_rw, __func__, __FILE__, __LINE__)
+#define lock_rw_unlock(lock) checklock_unlock(check_lock_rwlock, (lock)->c_rw, __func__, __FILE__, __LINE__)
+
+/** debugging mutex */
+typedef struct checked_lock_mutex lock_basic_t;
+#define lock_basic_init(lock) checklock_init(check_lock_mutex, &((lock)->c_m), __func__, __FILE__, __LINE__)
+#define lock_basic_destroy(lock) checklock_destroy(check_lock_mutex, &((lock)->c_m), __func__, __FILE__, __LINE__)
+#define lock_basic_lock(lock) checklock_lock(check_lock_mutex, (lock)->c_m, __func__, __FILE__, __LINE__)
+#define lock_basic_unlock(lock) checklock_unlock(check_lock_mutex, (lock)->c_m, __func__, __FILE__, __LINE__)
+
+/** debugging spinlock */
+typedef struct checked_lock_spl lock_quick_t;
+#define lock_quick_init(lock) checklock_init(check_lock_spinlock, &((lock)->c_spl), __func__, __FILE__, __LINE__)
+#define lock_quick_destroy(lock) checklock_destroy(check_lock_spinlock, &((lock)->c_spl), __func__, __FILE__, __LINE__)
+#define lock_quick_lock(lock) checklock_lock(check_lock_spinlock, (lock)->c_spl, __func__, __FILE__, __LINE__)
+#define lock_quick_unlock(lock) checklock_unlock(check_lock_spinlock, (lock)->c_spl, __func__, __FILE__, __LINE__)
+
+/** we use the pthread id, our thr_check structure is kept behind the scenes */
+typedef pthread_t ub_thread_t;
+#define ub_thread_create(thr, func, arg) checklock_thrcreate(thr, func, arg)
+#define ub_thread_self() pthread_self()
+#define ub_thread_join(thread) checklock_thrjoin(thread)
+
+typedef pthread_key_t ub_thread_key_t;
+#define ub_thread_key_create(key, f) LOCKRET(pthread_key_create(key, f))
+#define ub_thread_key_set(key, v) LOCKRET(pthread_setspecific(key, v))
+#define ub_thread_key_get(key) pthread_getspecific(key)
+
+#endif /* USE_THREAD_DEBUG */
+
+/** keep track of lock id in lock-verify application */
+struct order_id {
+	/** the thread id that created it */
+	int thr;
+	/** the instance number of creation */
+	int instance;
+};
+/** compare two order_ids */
+int order_lock_cmp(const void* e1, const void* e2);
+/** compare two codeline structs for rbtree */
+int codeline_cmp(const void* a, const void* b);
+
+#endif /* TESTCODE_CHECK_LOCKS_H */

testcode/do-tests.sh

@@ -2,8 +2,12 @@
 
 NEED_SPLINT='00-lint.tpkg'
 NEED_DOXYGEN='01-doc.tpkg'
+NEED_LDNS_TESTNS='fwd_no_edns.tpkg fwd_tcp_tc.tpkg fwd_tcp.tpkg fwd_three_service.tpkg fwd_three.tpkg fwd_ttlexpire.tpkg fwd_udp.tpkg fwd_tcp_tc6.tpkg fwd_compress_c00c.tpkg'
+NEED_XXD='fwd_compress_c00c.tpkg'
+NEED_NC='fwd_compress_c00c.tpkg'
 
 cd testdata;
+sh ../testcode/mini_tpkg.sh clean
 for test in `ls *.tpkg`; do
 	SKIP=0
 	if echo $NEED_SPLINT | grep $test >/dev/null; then
@@ -16,10 +20,26 @@ for test in `ls *.tpkg`; do
 			SKIP=1;
 		fi
 	fi
+	if echo $NEED_LDNS_TESTNS | grep $test >/dev/null; then
+		if test ! -x "`which ldns-testns`"; then
+			SKIP=1;
+		fi
+	fi
+	if echo $NEED_XXD | grep $test >/dev/null; then
+		if test ! -x "`which xxd`"; then
+			SKIP=1;
+		fi
+	fi
+	if echo $NEED_NC | grep $test >/dev/null; then
+		if test ! -x "`which nc`"; then
+			SKIP=1;
+		fi
+	fi
 	if test $SKIP -eq 0; then
 		echo $test
-		tpkg -a ../.. exe $test
+		sh ../testcode/mini_tpkg.sh -a ../.. exe $test
 	else
 		echo "skip $test"
 	fi
 done
+sh ../testcode/mini_tpkg.sh report

testcode/fake_event.c

@@ -47,11 +47,16 @@
 #include "config.h"
 #include "testcode/fake_event.h"
 #include "util/netevent.h"
+#include "util/net_help.h"
+#include "util/data/msgparse.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgencode.h"
 #include "services/listen_dnsport.h"
 #include "services/outside_network.h"
 #include "testcode/replay.h"
 #include "testcode/ldns-testpkts.h"
 #include "util/log.h"
+#include <signal.h>
 
 /** Global variable: the scenario. Saved here for when event_init is done. */
 static struct replay_scenario* saved_scenario = NULL;
@@ -69,7 +74,7 @@ fake_event_cleanup()
 	saved_scenario = NULL;
 }
 
-/** helper function that logs a ldns_pkt packet to logfile. */
+/** helper function that logs a ldns_pkt packet to logfile */
 static void
 log_pkt(const char* desc, ldns_pkt* pkt)
 {
@@ -139,6 +144,10 @@ pending_matches_current(struct replay_runtime* runtime,
 		return 0;
 	/* see if any of the pending queries matches */
 	for(p = runtime->pending_list; p; p = p->next) {
+		if(runtime->now->addrlen != 0 &&
+			sockaddr_cmp(&p->addr, p->addrlen, &runtime->now->addr,
+			runtime->now->addrlen) != 0)
+			continue;
 		if((e=find_match(runtime->now->match, p->pkt, p->transport))) {
 			*entry = e;
 			*pend = p;
@@ -163,10 +172,15 @@ pending_find_match(struct replay_runtime* runtime, struct entry** entry,
 	struct replay_range* p = runtime->scenario->range_list;
 	while(p) {
 		if(p->start_step <= timenow && timenow <= p->end_step &&
+		  (p->addrlen == 0 || sockaddr_cmp(&p->addr, p->addrlen,
+		  	&pend->addr, pend->addrlen) == 0) &&
 		  (*entry = find_match(p->match, pend->pkt, pend->transport))) {
 			log_info("matched query time %d in range [%d, %d] "
 				"with entry line %d", timenow, 
 				p->start_step, p->end_step, (*entry)->lineno);
+			if(p->addrlen != 0)
+				log_addr(0, "matched ip", &p->addr, p->addrlen);
+			log_pkt("matched pkt: ", (*entry)->reply_list->reply);
 			return 1;
 		}
 		p = p->next_range;
@@ -188,6 +202,7 @@ pending_matches_range(struct replay_runtime* runtime,
 	struct fake_pending* p = runtime->pending_list;
 	/* slow, O(N*N), but it works as advertised with weird matching */
 	while(p) {
+		log_info("check of pending");
 		if(pending_find_match(runtime, entry, p)) {
 			*pend = p;
 			return 1;
@@ -261,6 +276,10 @@ answer_callback_from_entry(struct replay_runtime* runtime,
         struct entry* entry, struct fake_pending* pend)
 {
 	struct comm_point c;
+	struct comm_reply repinfo;
+	void* cb_arg = pend->cb_arg;
+	comm_point_callback_t* cb = pend->callback;
+
 	memset(&c, 0, sizeof(c));
 	c.fd = -1;
 	c.buffer = ldns_buffer_new(runtime->bufsize);
@@ -268,11 +287,15 @@ answer_callback_from_entry(struct replay_runtime* runtime,
 	if(pend->transport == transport_tcp)
 		c.type = comm_tcp;
 	fill_buffer_with_reply(c.buffer, entry, pend->pkt);
-	if((*pend->callback)(&c, pend->cb_arg, NETEVENT_NOERROR, NULL)) {
+	repinfo.c = &c;
+	repinfo.addrlen = pend->addrlen;
+	memcpy(&repinfo.addr, &pend->addr, pend->addrlen);
+	if(!pend->serviced)
+		pending_list_delete(runtime, pend);
+	if((*cb)(&c, cb_arg, NETEVENT_NOERROR, &repinfo)) {
 		fatal_exit("testbound: unexpected: callback returned 1");
 	}
 	ldns_buffer_free(c.buffer);
-	pending_list_delete(runtime, pend);
 }
 
 /** Check the now moment answer check event */
@@ -280,26 +303,33 @@ static void
 answer_check_it(struct replay_runtime* runtime)
 {
 	struct replay_answer* ans = runtime->answer_list, 
-		**prev = &runtime->answer_list;
+		*prev = NULL;
 	log_assert(runtime && runtime->now && 
 		runtime->now->evt_type == repevt_front_reply);
 	while(ans) {
 		enum transport_type tr = transport_tcp;
 		if(ans->repinfo.c->type == comm_udp)
 			tr = transport_udp;
-		if(find_match(runtime->now->match, ans->pkt, tr)) {
+		if((runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&ans->repinfo.addr, ans->repinfo.addrlen) == 0) &&
+			find_match(runtime->now->match, ans->pkt, tr)) {
 			struct replay_answer *n = ans->next;
 			log_info("testbound matched event entry from line %d",
 				runtime->now->match->lineno);
 			log_info("testbound: do STEP %d %s", 
 				runtime->now->time_step,
 				repevt_string(runtime->now->evt_type));
-			*prev = ans->next;
+			if(prev)
+				prev->next = ans->next;
+			else 	runtime->answer_list = ans->next;
+			if(!ans->next)
+				runtime->answer_last = prev;
 			delete_replay_answer(ans);
 			ans = n;
 			return;
 		} else {
-			prev = &ans->next;
+			prev = ans;
 			ans = ans->next;
 		}
 	}
@@ -318,12 +348,19 @@ fake_front_query(struct replay_runtime* runtime, struct replay_moment *todo)
 	memset(&repinfo, 0, sizeof(repinfo));
 	repinfo.c = (struct comm_point*)calloc(1, sizeof(struct comm_point));
 	repinfo.addrlen = (socklen_t)sizeof(struct sockaddr_in);
+	if(todo->addrlen != 0) {
+		repinfo.addrlen = todo->addrlen;
+		memcpy(&repinfo.addr, &todo->addr, todo->addrlen);
+	}
 	repinfo.c->fd = -1;
 	repinfo.c->ev = (struct internal_event*)runtime;
 	repinfo.c->buffer = ldns_buffer_new(runtime->bufsize);
-	repinfo.c->type = comm_udp;
+	if(todo->match->match_transport == transport_tcp)
+		repinfo.c->type = comm_tcp;
+	else	repinfo.c->type = comm_udp;
 	fill_buffer_with_reply(repinfo.c->buffer, todo->match, NULL);
 	log_info("testbound: incoming QUERY");
+	log_pkt("query pkt", todo->match->reply_list->reply);
 	/* call the callback for incoming queries */
 	if((*runtime->callback_query)(repinfo.c, runtime->cb_arg, 
 		NETEVENT_NOERROR, &repinfo)) {
@@ -342,7 +379,11 @@ fake_pending_callback(struct replay_runtime* runtime,
 	struct replay_moment* todo, int error)
 {
 	struct fake_pending* p = runtime->pending_list;
+	struct comm_reply repinfo;
 	struct comm_point c;
+	void* cb_arg = p->cb_arg;
+	comm_point_callback_t* cb = p->callback;
+
 	memset(&c, 0, sizeof(c));
 	if(!p) fatal_exit("No pending queries.");
 	log_assert(todo->qname == NULL); /* or find that one */
@@ -352,13 +393,17 @@ fake_pending_callback(struct replay_runtime* runtime,
 		c.type = comm_tcp;
 	if(todo->evt_type == repevt_back_reply && todo->match) {
 		fill_buffer_with_reply(c.buffer, todo->match, p->pkt);
-	}	
-	if((*p->callback)(&c, p->cb_arg, error, NULL)) {
+	}
+	repinfo.c = &c;
+	repinfo.addrlen = p->addrlen;
+	memcpy(&repinfo.addr, &p->addr, p->addrlen);
+	if(!p->serviced)
+		pending_list_delete(runtime, p);
+	if((*cb)(&c, cb_arg, error, &repinfo)) {
 		fatal_exit("unexpected: pending callback returned 1");
 	}
 	/* delete the pending item. */
 	ldns_buffer_free(c.buffer);
-	pending_list_delete(runtime, p);
 }
 
 /**
@@ -468,22 +513,25 @@ run_scenario(struct replay_runtime* runtime)
 	} while(runtime->now);
 
 	if(runtime->pending_list) {
+		struct fake_pending* p;
+		log_err("testbound: there are still messages pending.");
+		for(p = runtime->pending_list; p; p=p->next)
+			log_pkt("pending msg", p->pkt);
 		fatal_exit("testbound: there are still messages pending.");
 	}
 	if(runtime->answer_list) {
 		fatal_exit("testbound: there are unmatched answers.");
 	}
 	log_info("testbound: exiting fake runloop.");
+	runtime->exit_cleanly = 1;
 }
 
 /*********** Dummy routines ***********/
 
 struct listen_dnsport* 
-listen_create(struct comm_base* base, int ATTR_UNUSED(num_ifs), 
-	const char* ATTR_UNUSED(ifs[]), const char* ATTR_UNUSED(port),
-	int ATTR_UNUSED(do_ip4), int ATTR_UNUSED(do_ip6), 
-	int ATTR_UNUSED(do_udp), int ATTR_UNUSED(do_tcp),
-	size_t bufsize, comm_point_callback_t* cb, void* cb_arg)
+listen_create(struct comm_base* base, struct listen_port* ATTR_UNUSED(ports),
+	size_t bufsize, int ATTR_UNUSED(tcp_accept_count),
+	comm_point_callback_t* cb, void* cb_arg)
 {
 	struct replay_runtime* runtime = (struct replay_runtime*)base;
 	struct listen_dnsport* l= calloc(1, sizeof(struct listen_dnsport));
@@ -549,13 +597,19 @@ comm_base_dispatch(struct comm_base* b)
 {
 	struct replay_runtime* runtime = (struct replay_runtime*)b;
 	run_scenario(runtime);
+	if(runtime->sig_cb)
+		(*runtime->sig_cb)(SIGTERM, runtime->sig_cb_arg);
+	else	exit(0); /* OK exit when LIBEVENT_SIGNAL_PROBLEM exists */
 }
 
 void 
-comm_base_exit(struct comm_base* ATTR_UNUSED(b))
+comm_base_exit(struct comm_base* b)
 {
-	/* some sort of failure */
-	fatal_exit("testbound: comm_base_exit was called.");
+	struct replay_runtime* runtime = (struct replay_runtime*)b;
+	if(!runtime->exit_cleanly) {
+		/* some sort of failure */
+		fatal_exit("testbound: comm_base_exit was called.");
+	}
 }
 
 struct comm_signal* 
@@ -608,6 +662,15 @@ comm_point_send_reply(struct comm_reply* repinfo)
 	log_pkt("reply pkt: ", ans->pkt);
 }
 
+void 
+comm_point_send_reply_iov(struct comm_reply* repinfo, struct iovec* iov,
+        size_t iovlen)
+{
+	/* skip tcp len at [0]. */
+	write_iov_buffer(repinfo->c->buffer, iov+1, iovlen-1);
+	comm_point_send_reply(repinfo);
+}
+
 void 
 comm_point_drop_reply(struct comm_reply* repinfo)
 {
@@ -620,9 +683,11 @@ comm_point_drop_reply(struct comm_reply* repinfo)
 
 struct outside_network* 
 outside_network_create(struct comm_base* base, size_t bufsize, 
-	size_t ATTR_UNUSED(num_ports), const char** ATTR_UNUSED(ifs), 
+	size_t ATTR_UNUSED(num_ports), char** ATTR_UNUSED(ifs), 
 	int ATTR_UNUSED(num_ifs), int ATTR_UNUSED(do_ip4), 
-	int ATTR_UNUSED(do_ip6), int ATTR_UNUSED(port_base))
+	int ATTR_UNUSED(do_ip6), int ATTR_UNUSED(port_base),
+	size_t ATTR_UNUSED(num_tcp), struct infra_cache* ATTR_UNUSED(infra),
+	struct ub_randstate* ATTR_UNUSED(rnd))
 {
 	struct outside_network* outnet =  calloc(1, 
 		sizeof(struct outside_network));
@@ -644,10 +709,64 @@ outside_network_delete(struct outside_network* outnet)
 	free(outnet);
 }
 
-void 
+struct pending* 
 pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* callback, void* callback_arg)
+	comm_point_callback_t* callback, void* callback_arg,
+	struct ub_randstate* ATTR_UNUSED(rnd))
+{
+	struct replay_runtime* runtime = (struct replay_runtime*)outnet->base;
+	struct fake_pending* pend = (struct fake_pending*)calloc(1,
+		sizeof(struct fake_pending));
+	ldns_status status;
+	log_assert(pend);
+	pend->buffer = ldns_buffer_new(ldns_buffer_capacity(packet));
+	log_assert(pend->buffer);
+	ldns_buffer_write(pend->buffer, ldns_buffer_begin(packet),
+		ldns_buffer_limit(packet));
+	ldns_buffer_flip(pend->buffer);
+	memcpy(&pend->addr, addr, addrlen);
+	pend->addrlen = addrlen;
+	pend->callback = callback;
+	pend->cb_arg = callback_arg;
+	pend->timeout = timeout/1000;
+	pend->transport = transport_udp;
+	pend->pkt = NULL;
+	pend->serviced = 0;
+	pend->runtime = runtime;
+	status = ldns_buffer2pkt_wire(&pend->pkt, packet);
+	if(status != LDNS_STATUS_OK) {
+		log_err("ldns error parsing udp output packet: %s",
+			ldns_get_errorstr_by_id(status));
+		fatal_exit("Sending unparseable DNS packets to servers!");
+	}
+	log_pkt("pending udp pkt: ", pend->pkt);
+
+	/* see if it matches the current moment */
+	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
+		find_match(runtime->now->match, pend->pkt, pend->transport)) {
+		log_info("testbound: matched pending to event. "
+			"advance time between events.");
+		log_info("testbound: do STEP %d %s", runtime->now->time_step,
+			repevt_string(runtime->now->evt_type));
+		advance_moment(runtime);
+		/* still create the pending, because we need it to callback */
+	} 
+	log_info("testbound: created fake pending");
+	/* add to list */
+	pend->next = runtime->pending_list;
+	runtime->pending_list = pend;
+	return (struct pending*)pend;
+}
+
+struct waiting_tcp* 
+pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
+	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
+	comm_point_callback_t* callback, void* callback_arg,
+	struct ub_randstate* ATTR_UNUSED(rnd))
 {
 	struct replay_runtime* runtime = (struct replay_runtime*)outnet->base;
 	struct fake_pending* pend = (struct fake_pending*)calloc(1,
@@ -664,18 +783,23 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	pend->callback = callback;
 	pend->cb_arg = callback_arg;
 	pend->timeout = timeout;
-	pend->transport = transport_udp;
+	pend->transport = transport_tcp;
 	pend->pkt = NULL;
+	pend->runtime = runtime;
+	pend->serviced = 0;
 	status = ldns_buffer2pkt_wire(&pend->pkt, packet);
 	if(status != LDNS_STATUS_OK) {
-		log_err("ldns error parsing udp output packet: %s",
+		log_err("ldns error parsing tcp output packet: %s",
 			ldns_get_errorstr_by_id(status));
 		fatal_exit("Sending unparseable DNS packets to servers!");
 	}
-	log_pkt("pending udp pkt: ", pend->pkt);
+	log_pkt("pending tcp pkt: ", pend->pkt);
 
 	/* see if it matches the current moment */
 	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
 		find_match(runtime->now->match, pend->pkt, pend->transport)) {
 		log_info("testbound: matched pending to event. "
 			"advance time between events.");
@@ -688,6 +812,255 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	/* add to list */
 	pend->next = runtime->pending_list;
 	runtime->pending_list = pend;
+	return (struct waiting_tcp*)pend;
+}
+
+struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
+        uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint16_t flags, int dnssec, struct sockaddr_storage* addr,
+	socklen_t addrlen, comm_point_callback_t* callback,
+	void* callback_arg, ldns_buffer* ATTR_UNUSED(buff),
+	int (*arg_compare)(void*,void*))
+{
+	struct replay_runtime* runtime = (struct replay_runtime*)outnet->base;
+	struct fake_pending* pend = (struct fake_pending*)calloc(1,
+		sizeof(struct fake_pending));
+	ldns_status status;
+	(void)arg_compare;
+	log_assert(pend);
+	log_nametypeclass(VERB_OPS, "pending serviced query", 
+		qname, qtype, qclass);
+	verbose(VERB_OPS, "pending serviced query flags%s%s%s%s", 
+		(flags&BIT_RD)?" RD":"", (flags&BIT_CD)?" CD":"",
+		(flags&~(BIT_RD|BIT_CD))?" MORE":"", (dnssec)?" DO":"");
+
+	/* create packet with EDNS */
+	pend->buffer = ldns_buffer_new(512);
+	log_assert(pend->buffer);
+	ldns_buffer_write_u16(pend->buffer, 0); /* id */
+	ldns_buffer_write_u16(pend->buffer, flags);
+	ldns_buffer_write_u16(pend->buffer, 1); /* qdcount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* ancount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* nscount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* arcount */
+	ldns_buffer_write(pend->buffer, qname, qnamelen);
+	ldns_buffer_write_u16(pend->buffer, qtype);
+	ldns_buffer_write_u16(pend->buffer, qclass);
+	ldns_buffer_flip(pend->buffer);
+	if(1) {
+		/* add edns */
+		struct edns_data edns;
+		edns.edns_present = 1;
+		edns.ext_rcode = 0;
+		edns.edns_version = EDNS_ADVERTISED_VERSION;
+		edns.udp_size = EDNS_ADVERTISED_SIZE;
+		edns.bits = 0;
+		if(dnssec)
+			edns.bits = EDNS_DO;
+		attach_edns_record(pend->buffer, &edns);
+	}
+	memcpy(&pend->addr, addr, addrlen);
+	pend->addrlen = addrlen;
+	pend->callback = callback;
+	pend->cb_arg = callback_arg;
+	pend->timeout = UDP_QUERY_TIMEOUT;
+	pend->transport = transport_udp; /* pretend UDP */
+	pend->pkt = NULL;
+	pend->runtime = runtime;
+	pend->serviced = 1;
+	status = ldns_buffer2pkt_wire(&pend->pkt, pend->buffer);
+	if(status != LDNS_STATUS_OK) {
+		log_err("ldns error parsing serviced output packet: %s",
+			ldns_get_errorstr_by_id(status));
+		fatal_exit("internal error");
+	}
+	/*log_pkt("pending serviced query: ", pend->pkt);*/
+
+	/* see if it matches the current moment */
+	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
+		find_match(runtime->now->match, pend->pkt, pend->transport)) {
+		log_info("testbound: matched pending to event. "
+			"advance time between events.");
+		log_info("testbound: do STEP %d %s", runtime->now->time_step,
+			repevt_string(runtime->now->evt_type));
+		advance_moment(runtime);
+		/* still create the pending, because we need it to callback */
+	} 
+	log_info("testbound: created fake pending");
+	/* add to list */
+	pend->next = runtime->pending_list;
+	runtime->pending_list = pend;
+	return (struct serviced_query*)pend;
+}
+
+void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
+{
+	struct fake_pending* pend = (struct fake_pending*)sq;
+	struct replay_runtime* runtime = pend->runtime;
+	/* delete from the list */
+	struct fake_pending* p = runtime->pending_list, *prev=NULL;
+	while(p) {
+		if(p == pend) {
+			log_assert(p->cb_arg == cb_arg);
+			log_info("serviced pending delete");
+			if(prev)
+				prev->next = p->next;
+			else 	runtime->pending_list = p->next;
+			ldns_buffer_free(p->buffer);
+			ldns_pkt_free(p->pkt);
+			free(p);
+			return;
+		}
+		prev = p;
+		p = p->next;
+	}
+	log_info("double delete of pending serviced query");
+}
+
+struct listen_port* listening_ports_open(struct config_file* ATTR_UNUSED(cfg))
+{
+	return calloc(1, 1);
+}
+
+void listening_ports_free(struct listen_port* list)
+{
+	free(list);
+}
+
+void listen_pushback(struct listen_dnsport* ATTR_UNUSED(listen))
+{
+}
+
+void listen_resume(struct listen_dnsport* ATTR_UNUSED(listen))
+{
+}
+
+struct comm_point* comm_point_create_local(struct comm_base* ATTR_UNUSED(base),
+        int ATTR_UNUSED(fd), size_t ATTR_UNUSED(bufsize),
+        comm_point_callback_t* ATTR_UNUSED(callback), 
+	void* ATTR_UNUSED(callback_arg))
+{
+	return calloc(1, 1);
+}
+
+/* only cmd com _local gets deleted */
+void comm_point_delete(struct comm_point* c)
+{
+	free(c);
+}
+
+size_t listen_get_mem(struct listen_dnsport* ATTR_UNUSED(listen))
+{
+	return 0;
+}
+
+size_t outnet_get_mem(struct outside_network* ATTR_UNUSED(outnet))
+{
+	return 0;
+}
+
+size_t comm_point_get_mem(struct comm_point* ATTR_UNUSED(c))
+{
+	return 0;
+}
+
+size_t serviced_get_mem(struct serviced_query* ATTR_UNUSED(c))
+{
+	return 0;
+}
+
+/* fake for fptr wlist */
+int outnet_udp_cb(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply *ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int outnet_tcp_cb(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply *ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+void pending_udp_timer_cb(void *ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void outnet_tcptimer(void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_point_udp_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(event), 
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_point_tcp_accept_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_point_tcp_handle_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_timer_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_signal_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_point_local_handle_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+int serviced_udp_callback(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int serviced_tcp_callback(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int pending_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
+{
+	log_assert(0);
+	return 0;
+}
+
+int serviced_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
+{
+	log_assert(0);
+	return 0;
 }
 
 /*********** End of Dummy routines ***********/

testcode/ldns-testpkts.c

@@ -31,7 +31,7 @@ struct sockaddr_storage;
 /** string to show in warnings and errors */
 static const char* prog_name = "ldns-testpkts";
 
-/** logging routine, provided by caller. */
+/** logging routine, provided by caller */
 void verbose(int lvl, const char* msg, ...);
 
 /** print error and exit */
@@ -68,7 +68,7 @@ static bool str_keyword(const char** str, const char* keyword)
 	if(strncmp(*str, keyword, len) != 0)
 		return false;
 	*str += len;
-	while(isspace(**str))
+	while(isspace((int)**str))
 		(*str)++;
 	return true;
 }
@@ -106,6 +106,10 @@ static void matchline(const char* line, struct entry* e)
 			e->match_qname = true;
 		} else if(str_keyword(&parse, "all")) {
 			e->match_all = true;
+		} else if(str_keyword(&parse, "ttl")) {
+			e->match_ttl = true;
+		} else if(str_keyword(&parse, "DO")) {
+			e->match_do = true;
 		} else if(str_keyword(&parse, "UDP")) {
 			e->match_transport = transport_udp;
 		} else if(str_keyword(&parse, "TCP")) {
@@ -116,7 +120,7 @@ static void matchline(const char* line, struct entry* e)
 				error("expected = or : in MATCH: %s", line);
 			parse++;
 			e->ixfr_soa_serial = (uint32_t)strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else {
 			error("could not parse MATCH: '%s'", parse);
@@ -153,6 +157,8 @@ static void replyline(const char* line, ldns_pkt *reply)
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_NXDOMAIN);
 		} else if(str_keyword(&parse, "NOTIMPL")) {
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_NOTIMPL);
+		} else if(str_keyword(&parse, "REFUSED")) {
+			ldns_pkt_set_rcode(reply, LDNS_RCODE_REFUSED);
 		} else if(str_keyword(&parse, "YXDOMAIN")) {
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_YXDOMAIN);
 		} else if(str_keyword(&parse, "YXRRSET")) {
@@ -178,6 +184,9 @@ static void replyline(const char* line, ldns_pkt *reply)
 			ldns_pkt_set_ra(reply, true);
 		} else if(str_keyword(&parse, "AD")) {
 			ldns_pkt_set_ad(reply, true);
+		} else if(str_keyword(&parse, "DO")) {
+			ldns_pkt_set_edns_udp_size(reply, 4096);
+			ldns_pkt_set_edns_do(reply, true);
 		} else {
 			error("could not parse REPLY: '%s'", parse);
 		}
@@ -194,13 +203,15 @@ static void adjustline(const char* line, struct entry* e,
 			return;
 		if(str_keyword(&parse, "copy_id")) {
 			e->copy_id = true;
+		} else if(str_keyword(&parse, "copy_query")) {
+			e->copy_query = true;
 		} else if(str_keyword(&parse, "sleep=")) {
 			e->sleeptime = (unsigned int) strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else if(str_keyword(&parse, "packet_sleep=")) {
 			pkt->packet_sleep = (unsigned int) strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else {
 			error("could not parse ADJUST: '%s'", parse);
@@ -217,11 +228,14 @@ static struct entry* new_entry()
 	e->match_qtype = false;
 	e->match_qname = false;
 	e->match_all = false;
+	e->match_ttl = false;
+	e->match_do = false;
 	e->match_serial = false;
 	e->ixfr_soa_serial = 0;
 	e->match_transport = transport_any;
 	e->reply_list = NULL;
 	e->copy_id = false;
+	e->copy_query = false;
 	e->sleeptime = 0;
 	e->next = NULL;
 	return e;
@@ -377,7 +391,7 @@ get_origin(const char* name, int lineno, ldns_rdf** origin, char* parse)
 	*origin = NULL;
 
 	end=parse;
-	while(!isspace(*end) && !isendline(*end))
+	while(!isspace((int)*end) && !isendline(*end))
 		end++;
 	store = *end;
 	*end = 0;
@@ -391,7 +405,7 @@ get_origin(const char* name, int lineno, ldns_rdf** origin, char* parse)
 
 /* Reads one entry from file. Returns entry or NULL on error. */
 struct entry*
-read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl, 
+read_entry(FILE* in, const char* name, int *lineno, uint32_t* default_ttl, 
 	ldns_rdf** origin, ldns_rdf** prev_rr)
 {
 	struct entry* current = NULL;
@@ -407,7 +421,7 @@ read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl,
 		parse = line;
 		(*lineno) ++;
 		
-		while(isspace(*parse))
+		while(isspace((int)*parse))
 			parse++;
 		/* test for keywords */
 		if(isendline(*parse))
@@ -425,7 +439,7 @@ read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl,
 			get_origin(name, *lineno, origin, (char*)parse);
 			continue;
 		} else if(str_keyword(&parse, "$TTL")) {
-			*default_ttl = (uint16_t)atoi(parse);
+			*default_ttl = (uint32_t)atoi(parse);
 			continue;
 		}
 
@@ -499,7 +513,7 @@ read_datafile(const char* name)
 	struct entry* current = NULL;
 	FILE *in;
 	int lineno = 0;
-	uint16_t default_ttl = 0;
+	uint32_t default_ttl = 0;
 	ldns_rdf* origin = NULL;
 	ldns_rdf* prev_rr = NULL;
 	int entry_num = 0;
@@ -520,6 +534,8 @@ read_datafile(const char* name)
 	verbose(1, "%s: Read %d entries\n", prog_name, entry_num);
 
 	fclose(in);
+	ldns_rdf_deep_free(origin);
+	ldns_rdf_deep_free(prev_rr);
 	return list;
 }
 
@@ -555,7 +571,7 @@ static uint32_t get_serial(ldns_pkt* p)
 
 /** match two rr lists */
 static int
-match_list(ldns_rr_list* q, ldns_rr_list *p)
+match_list(ldns_rr_list* q, ldns_rr_list *p, bool mttl)
 {
 	size_t i;
 	if(ldns_rr_list_rr_count(q) != ldns_rr_list_rr_count(p))
@@ -567,6 +583,11 @@ match_list(ldns_rr_list* q, ldns_rr_list *p)
 			verbose(3, "rr %d different", i);
 			return 0;
 		}
+		if(mttl && ldns_rr_ttl(ldns_rr_list_rr(q, i)) !=
+			ldns_rr_ttl(ldns_rr_list_rr(p, i))) {
+			verbose(3, "rr %d ttl different", i);
+			return 0;
+		}
 	}
 	return 1;
 }
@@ -583,7 +604,7 @@ cmp_bool(int x, int y)
 
 /** match all of the packet */
 static int
-match_all(ldns_pkt* q, ldns_pkt* p)
+match_all(ldns_pkt* q, ldns_pkt* p, bool mttl)
 {
 	if(ldns_pkt_get_opcode(q) != ldns_pkt_get_opcode(p)) 
 	{ verbose(3, "allmatch: opcode different"); return 0;}
@@ -613,13 +634,13 @@ match_all(ldns_pkt* q, ldns_pkt* p)
 	{ verbose(3, "allmatch: nscount different"); return 0;}
 	if(ldns_pkt_arcount(q) != ldns_pkt_arcount(p))
 	{ verbose(3, "allmatch: arcount different"); return 0;}
-	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p)))
+	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p), mttl))
 	{ verbose(3, "allmatch: qd section different"); return 0;}
-	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p)))
+	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p), mttl))
 	{ verbose(3, "allmatch: an section different"); return 0;}
-	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p)))
+	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p), mttl))
 	{ verbose(3, "allmatch: ar section different"); return 0;}
-	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p))) 
+	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p), mttl)) 
 	{ verbose(3, "allmatch: ns section different"); return 0;}
 	return 1;
 }
@@ -655,11 +676,15 @@ find_match(struct entry* entries, ldns_pkt* query_pkt,
 				verbose(3, "bad serial\n");
 				continue;
 		}
+		if(p->match_do && !ldns_pkt_edns_do(query_pkt)) {
+			verbose(3, "no DO bit set\n");
+			continue;
+		}
 		if(p->match_transport != transport_any && p->match_transport != transport) {
 			verbose(3, "bad transport\n");
 			continue;
 		}
-		if(p->match_all && !match_all(query_pkt, reply)) {
+		if(p->match_all && !match_all(query_pkt, reply, p->match_ttl)) {
 			verbose(3, "bad allmatch\n");
 			continue;
 		}
@@ -675,6 +700,12 @@ adjust_packet(struct entry* match, ldns_pkt* answer_pkt, ldns_pkt* query_pkt)
 	/* copy & adjust packet */
 	if(match->copy_id)
 		ldns_pkt_set_id(answer_pkt, ldns_pkt_id(query_pkt));
+	if(match->copy_query) {
+		ldns_rr_list* list = ldns_pkt_get_section_clone(query_pkt,
+			LDNS_SECTION_QUESTION);
+		ldns_rr_list_deep_free(ldns_pkt_question(answer_pkt));
+		ldns_pkt_set_question(answer_pkt, list);
+	}
 	if(match->sleeptime > 0) {
 		verbose(3, "sleeping for %d seconds\n", match->sleeptime);
 		sleep(match->sleeptime);

testcode/ldns-testpkts.h

@@ -44,19 +44,23 @@
 	; 'qname' makes the query match the qname from the reply
 	; 'serial=1023' makes the query match if ixfr serial is 1023. 
 	; 'all' has to match header byte for byte and all rrs in packet.
-	MATCH [opcode] [qtype] [qname] [serial=<value>] [all]
-	MATCH [UDP|TCP]
+	; 'ttl' used with all, rrs in packet must also have matching TTLs.
+	; 'DO' will match only queries with DO bit set.
+	MATCH [opcode] [qtype] [qname] [serial=<value>] [all] [ttl]
+	MATCH [UDP|TCP] DO
 	MATCH ...
 	; Then the REPLY header is specified.
 	REPLY opcode, rcode or flags.
 		(opcode)  QUERY IQUERY STATUS NOTIFY UPDATE
 		(rcode)   NOERROR FORMERR SERVFAIL NXDOMAIN NOTIMPL YXDOMAIN
 		 		YXRRSET NXRRSET NOTAUTH NOTZONE
-		(flags)   QR AA TC RD CD RA AD
+		(flags)   QR AA TC RD CD RA AD DO
 	REPLY ...
 	; any additional actions to do.
 	; 'copy_id' copies the ID from the query to the answer.
 	ADJUST copy_id
+	; 'copy_query' copies the query name, type and class to the answer.
+	ADJUST copy_query
 	; 'sleep=10' sleeps for 10 seconds before giving the answer (TCP is open)
 	ADJUST [sleep=<num>]    ; sleep before giving any reply
 	ADJUST [packet_sleep=<num>]  ; sleep before this packet in sequence
@@ -157,6 +161,10 @@ struct entry {
 	bool match_serial; 
 	/** match all of the packet */
 	bool match_all;
+	/** match ttls in the packet */
+	bool match_ttl;
+	/** match DO bit */
+	bool match_do;
 	/** match query serial with this value. */
 	uint32_t ixfr_soa_serial; 
 	/** match on UDP/TCP */
@@ -168,6 +176,8 @@ struct entry {
 	/** how to adjust the reply packet */
 	/** copy over the ID from the query into the answer */
 	bool copy_id; 
+	/** copy the query nametypeclass from query into the answer */
+	bool copy_query;
 	/** in seconds */
 	unsigned int sleeptime; 
 
@@ -204,7 +214,7 @@ void delete_entry(struct entry* list);
  * @return: The entry read (malloced) or NULL if no entry could be read.
  */
 struct entry* read_entry(FILE* in, const char* name, int *lineno, 
-	uint16_t* default_ttl, ldns_rdf** origin, ldns_rdf** prev_rr);
+	uint32_t* default_ttl, ldns_rdf** origin, ldns_rdf** prev_rr);
 
 /**
  * finds entry in list, or returns NULL.

testcode/lock_verify.c

@@ -0,0 +1,408 @@
+/*
+ * testcode/lock_verify.c - verifier program for lock traces, checks order.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file checks the lock traces generated by checklock.c.
+ * Checks if locks are consistently locked in the same order.
+ * If not, this can lead to deadlock if threads execute the different
+ * ordering at the same time.
+ * 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/rbtree.h"
+#include "util/locks.h"
+#include "testcode/checklocks.h"
+
+/* --- data structures --- */
+struct lock_ref;
+
+/** a lock */
+struct order_lock {
+	/** rbnode in all tree */
+	rbnode_t node;
+	/** lock id */
+	struct order_id id;
+	/** the creation file */
+	char* create_file;
+	/** creation line */
+	int create_line;
+	/** set of all locks that are smaller than this one (locked earlier) */
+	rbtree_t* smaller;
+	/** during depthfirstsearch, this is a linked list of the stack 
+	 * of locks. points to the next lock bigger than this one. */
+	struct lock_ref* dfs_next;
+	/** if lock has been visited (all smaller locks have been compared to
+	 * this lock), only need to compare this with all unvisited(bigger) 
+	 * locks */
+	int visited;
+};
+
+/** reference to a lock in a rbtree set */
+struct lock_ref {
+	/** rbnode, key is an order_id ptr */
+	rbnode_t node;
+	/** the lock referenced */
+	struct order_lock* lock;
+	/** why is this ref */
+	char* file;
+	/** line number */
+	int line;
+};
+
+/** count of errors detected */
+static int errors_detected = 0;
+/** verbose? */
+static int verb = 0;
+
+/** print program usage help */
+static void
+usage()
+{
+	printf("lock_verify <trace files>\n");
+}
+
+/** read header entry. 
+ * @param in: file to read header of.
+ * @return: False if it does not belong to the rest. */
+static int 
+read_header(FILE* in)
+{
+	time_t t;
+	pid_t p;
+	int thrno;
+	static int have_values = 0;
+	static time_t the_time;
+	static pid_t the_pid;
+	static int threads[256];
+
+	if(fread(&t, sizeof(t), 1, in) != 1 ||	
+		fread(&thrno, sizeof(thrno), 1, in) != 1 ||
+		fread(&p, sizeof(p), 1, in) != 1) {
+		fatal_exit("fread failed");
+	}
+	/* check these values are sorta OK */
+	if(!have_values) {
+		the_time = t;
+		the_pid = p;
+		memset(threads, 0, 256*sizeof(int));
+		if(thrno >= 256) {
+			fatal_exit("Thread number too big. %d", thrno);
+		}
+		threads[thrno] = 1;
+		have_values = 1;
+		printf(" trace %d from pid %u on %s", thrno, 
+			(unsigned)p, ctime(&t));
+	} else {
+		if(the_pid != p) {
+			printf(" has pid %u, not %u. Skipped.\n",
+				(unsigned)p, (unsigned)the_pid);
+			return 0;
+		}
+		if(threads[thrno])
+			fatal_exit("same threadno in two files");
+		threads[thrno] = 1;
+		if( abs((int)(the_time - t)) > 3600)
+			fatal_exit("input files from different times: %u %u",
+				(unsigned)the_time, (unsigned)t);
+		printf(" trace of thread %u:%d\n", (unsigned)p, thrno);
+	}
+	return 1;
+}
+
+/** max length of strings: filenames and function names. */
+#define STRMAX 1024
+/** read a string from file, false on error */
+static int readup_str(char** str, FILE* in)
+{
+	char buf[STRMAX];
+	int len = 0;
+	int c;
+	/* ends in zero */
+	while( (c = fgetc(in)) != 0) {
+		if(c == EOF)
+			fatal_exit("eof in readstr, file too short");
+		buf[len++] = c;
+		if(len == STRMAX) {
+			fatal_exit("string too long, bad file format");
+		}
+	}
+	buf[len] = 0;
+	*str = strdup(buf);
+	return 1;
+}
+
+/** read creation entry */
+static void read_create(rbtree_t* all, FILE* in)
+{
+	struct order_lock* o = calloc(1, sizeof(struct order_lock));
+	if(!o) fatal_exit("malloc failure");
+	if(fread(&o->id.thr, sizeof(int), 1, in) != 1 ||	
+	   fread(&o->id.instance, sizeof(int), 1, in) != 1 ||	
+	   !readup_str(&o->create_file, in) ||
+	   fread(&o->create_line, sizeof(int), 1, in) != 1)
+		fatal_exit("fread failed");
+	o->smaller = rbtree_create(order_lock_cmp);
+	o->node.key = &o->id;
+	if(!rbtree_insert(all, &o->node)) {
+		/* already inserted */
+		struct order_lock* a = (struct order_lock*)rbtree_search(all, 
+			&o->id);
+		log_assert(a);
+		a->create_file = o->create_file;
+		a->create_line = o->create_line;
+		free(o->smaller);
+		free(o);
+		o = a;
+	}
+	if(verb) printf("read create %u %u %s %d\n", 
+		(unsigned)o->id.thr, (unsigned)o->id.instance,
+		o->create_file, o->create_line);
+}
+
+/** insert lock entry (empty) into list */
+static struct order_lock* 
+insert_lock(rbtree_t* all, struct order_id* id)
+{
+	struct order_lock* o = calloc(1, sizeof(struct order_lock));
+	if(!o) fatal_exit("malloc failure");
+	o->smaller = rbtree_create(order_lock_cmp);
+	o->id = *id;
+	o->node.key = &o->id;
+	if(!rbtree_insert(all, &o->node))
+		fatal_exit("insert fail should not happen");
+	return o;
+}
+
+/** read lock entry */
+static void read_lock(rbtree_t* all, FILE* in, int val)
+{
+	struct order_id prev_id, now_id;
+	struct lock_ref* ref;
+	struct order_lock* prev, *now;
+	ref = (struct lock_ref*)calloc(1, sizeof(struct lock_ref));
+	if(!ref) fatal_exit("malloc failure");
+	prev_id.thr = val;
+	if(fread(&prev_id.instance, sizeof(int), 1, in) != 1 ||	
+	   fread(&now_id.thr, sizeof(int), 1, in) != 1 ||	
+	   fread(&now_id.instance, sizeof(int), 1, in) != 1 ||	
+	   !readup_str(&ref->file, in) ||
+	   fread(&ref->line, sizeof(int), 1, in) != 1)
+		fatal_exit("fread failed");
+	if(verb) printf("read lock %u %u %u %u %s %d\n", 
+		(unsigned)prev_id.thr, (unsigned)prev_id.instance,
+		(unsigned)now_id.thr, (unsigned)now_id.instance,
+		ref->file, ref->line);
+	/* find the two locks involved */
+	prev = (struct order_lock*)rbtree_search(all, &prev_id);
+	now = (struct order_lock*)rbtree_search(all, &now_id);
+	/* if not there - insert 'em */
+	if(!prev) prev = insert_lock(all, &prev_id);
+	if(!now) now = insert_lock(all, &now_id);
+	ref->lock = prev;
+	ref->node.key = &prev->id;
+	if(!rbtree_insert(now->smaller, &ref->node)) {
+		free(ref->file);
+		free(ref);
+	}
+}
+
+/** read input file */
+static void readinput(rbtree_t* all, char* file)
+{
+	FILE *in = fopen(file, "r");
+	int fst;
+	if(!in) {
+		perror(file);
+		exit(1);
+	}
+	printf("file %s", file);
+	if(!read_header(in)) {
+		fclose(in);
+		return;
+	}
+	while(fread(&fst, sizeof(fst), 1, in) == 1) {
+		if(fst == -1)
+			read_create(all, in);
+		else	read_lock(all, in, fst);
+	}
+	fclose(in);
+}
+
+/** print cycle message */
+static void found_cycle(struct lock_ref* visit, int level)
+{
+	struct lock_ref* p;
+	int i = 0;
+	errors_detected++;
+	printf("Found inconsistent locking order of length %d\n", level);
+	printf("for lock %d %d created %s %d\n", 
+		visit->lock->id.thr, visit->lock->id.instance,
+		visit->lock->create_file, visit->lock->create_line);
+	printf("sequence is:\n");
+	p = visit;
+	while(p) {
+		struct order_lock* next = 
+			p->lock->dfs_next?p->lock->dfs_next->lock:visit->lock;
+		printf("[%d] is locked at line %s %d before lock %d %d\n",
+			i, p->file, p->line, next->id.thr, next->id.instance);
+		printf("[%d] lock %d %d is created at %s %d\n",
+			i, next->id.thr, next->id.instance,
+			next->create_file, next->create_line); 
+		i++;
+		p = p->lock->dfs_next;
+		if(p && p->lock == visit->lock)
+			break;
+	}
+}
+
+/** Detect cycle by comparing visited now with all (unvisited) bigger nodes */
+static int detect_cycle(struct lock_ref* visit, struct lock_ref* from)
+{
+	struct lock_ref* p = from;
+	while(p) {
+		if(p->lock == visit->lock)
+			return 1;
+		p = p->lock->dfs_next;
+	}
+	return 0;
+}
+
+/** recursive function to depth first search for cycles.
+ * @param visit: the lock visited at this step.
+ *	its dfs_next pointer gives the visited lock up in recursion.
+ * 	same as lookfor at level 0.
+ * @param level: depth of recursion. 0 is start.
+ * @param from: search for matches from unvisited node upwards.
+ */
+static void search_cycle(struct lock_ref* visit, int level, 
+	struct lock_ref* from)
+{
+	struct lock_ref* ref;
+	/* check for cycle */
+	if(detect_cycle(visit, from) && level != 0) {
+		found_cycle(visit, level);
+		fatal_exit("found lock order cycle");
+	}
+	/* recurse */
+	if(!visit->lock->visited)
+		from = visit;
+	if(verb > 1) fprintf(stderr, "[%d] visit lock %u %u %s %d\n", level,
+			(unsigned)visit->lock->id.thr, 
+			(unsigned)visit->lock->id.instance,
+			visit->lock->create_file, visit->lock->create_line);
+	RBTREE_FOR(ref, struct lock_ref*, visit->lock->smaller) {
+		ref->lock->dfs_next = visit;
+		search_cycle(ref, level+1, from);
+	}
+	visit->lock->visited = 1;
+}
+
+/** Check ordering of one lock */
+static void check_order_lock(struct order_lock* lock)
+{
+	struct lock_ref start;
+	if(lock->visited) return;
+
+	start.node.key = &lock->id;
+	start.lock = lock;
+	start.file = lock->create_file;
+	start.line = lock->create_line;
+
+	if(!lock->create_file)
+		log_err("lock %u %u does not have create info",
+			(unsigned)lock->id.thr, (unsigned)lock->id.instance);
+
+	/* depth first search to find cycle with this lock at head */
+	lock->dfs_next = NULL;
+	search_cycle(&start, 0, &start);
+}
+
+/** Check ordering of locks */
+static void check_order(rbtree_t* all_locks)
+{
+	/* check each lock */
+	struct order_lock* lock;
+	int i=0;
+	RBTREE_FOR(lock, struct order_lock*, all_locks) {
+		if(verb)
+		    printf("[%d/%d] Checking lock %d %d %s %d\n",
+			i, (int)all_locks->count,
+			lock->id.thr, lock->id.instance, 
+			lock->create_file, lock->create_line);
+		else if (i % ((all_locks->count/75)<1?1:all_locks->count/75) 
+			== 0) 
+		    fprintf(stderr, ".");
+		i++;
+		check_order_lock(lock);
+	}
+	fprintf(stderr, "\n");
+}
+
+/** main program to verify all traces passed */
+int
+main(int argc, char* argv[])
+{
+	rbtree_t* all_locks;
+	int i;
+	time_t starttime = time(NULL);
+	if(argc <= 1) {
+		usage();
+		return 1;
+	}
+	log_init(NULL, 0, NULL);
+	log_ident_set("lock-verify");
+	/* init */
+	all_locks = rbtree_create(order_lock_cmp);
+	errors_detected = 0;
+
+	/* read the input files */
+	for(i=1; i<argc; i++) {
+		readinput(all_locks, argv[i]);
+	}
+
+	/* check ordering */
+	check_order(all_locks);
+
+	/* do not free a thing, OS will do it */
+	printf("checked %d locks in %d seconds with %d errors.\n", 
+		(int)all_locks->count, (int)(time(NULL)-starttime),
+		errors_detected);
+	if(errors_detected) return 1;
+	return 0;
+}

testcode/memstats.c

@@ -0,0 +1,265 @@
+/*
+ * testcode/memstats.c - debug tool to show memory allocation statistics.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program reads a log file and prints the memory allocation summed
+ * up.
+ */
+#include "config.h"
+#include "util/log.h"
+#include "util/rbtree.h"
+#include "util/locks.h"
+#include "testcode/checklocks.h"
+#include <sys/stat.h>
+
+/**
+ * The allocation statistics block
+ */
+struct codeline {
+	/** rbtree node */
+	rbnode_t node;
+	/** the name of the file:linenumber */
+	char* codeline;
+	/** the name of the function */
+	char* func;
+	/** number of bytes allocated */
+	uint64_t alloc;
+	/** number of bytes freed */
+	uint64_t free;
+};
+
+/**
+ * Other allocation stats
+ */
+struct alloc_misc {
+	/** number of region allocs */
+	uint64_t region_alloc;
+};
+
+/** print usage and exit */
+static void
+usage()
+{
+	printf("usage:	memstats <logfile>\n");
+	printf("statistics are printed on stdout.\n");
+	exit(1);
+}
+
+/** match logfile line to see if it needs accounting processing */
+static int
+match(char* line)
+{
+	/* f.e.:
+	 * [1187340064] unbound[24604:0] info: ul/rb.c:81 r_create malloc(12)
+	 * 0123456789 123456789 123456789 123456789
+	 */
+	if(strlen(line) < 36) /* up to 'info: ' */
+		return 0;
+	if(strncmp(line+30, "info: ", 6) != 0)
+		return 0;
+	if(strncmp(line+36, "stat ", 5) == 0)
+		return 0; /* skip the hex dumps */
+	if(strstr(line+36, "malloc("))
+		return 1;
+	else if(strstr(line+36, "calloc("))
+		return 1;
+	/* skip reallocs */
+	return 0;
+}
+
+/** read up the region stats */
+static void
+read_region_stat(char* line, struct alloc_misc* misc)
+{
+	long num = 0;
+	if(sscanf(line+50, "%ld", &num) != 1) {
+		printf("%s\n%s\n", line, line+50);
+		fatal_exit("unhandled region");
+	}
+	misc->region_alloc += num;
+}
+
+/** find or alloc codeline in tree */
+static struct codeline*
+get_codeline(rbtree_t* tree, char* key, char* func)
+{
+	struct codeline* cl = (struct codeline*)rbtree_search(tree, key);
+	if(!cl) {
+		cl = calloc(1, sizeof(*cl));
+		if(!cl) return 0;
+		cl->codeline = strdup(key);
+		if(!cl->codeline) return 0;
+		cl->func = strdup(func);
+		if(!cl->func) return 0;
+		cl->alloc = 0;
+		cl->node.key = cl->codeline;
+		(void)rbtree_insert(tree, &cl->node);
+	}
+	return cl;
+}
+
+/** read up the malloc stats */
+static void
+read_malloc_stat(char* line, rbtree_t* tree)
+{
+	char codeline[10240];
+	char name[10240];
+	int skip = 0;
+	long num = 0;
+	struct codeline* cl = 0;
+	if(sscanf(line+36, "%s %s %n", codeline, name, &skip) != 2) {
+		printf("%s\n%s\n", line, line+36);
+		fatal_exit("unhandled malloc");
+	}
+	if(sscanf(line+36+skip+7, "%ld", &num) != 1) {
+		printf("%s\n%s\n", line, line+36+skip+7);
+		fatal_exit("unhandled malloc");
+	}
+	cl = get_codeline(tree, codeline, name);
+	if(!cl)
+		fatal_exit("alloc failure");
+	cl->alloc += num;
+}
+
+/** read up the calloc stats */
+static void
+read_calloc_stat(char* line, rbtree_t* tree)
+{
+	char codeline[10240];
+	char name[10240];
+	int skip = 0;
+	long num = 0, sz = 0;
+	struct codeline* cl = 0;
+	if(sscanf(line+36, "%s %s %n", codeline, name, &skip) != 2) {
+		printf("%s\n%s\n", line, line+36);
+		fatal_exit("unhandled calloc");
+	}
+	if(sscanf(line+36+skip+7, "%ld, %ld", &num, &sz) != 2) {
+		printf("%s\n%s\n", line, line+36+skip+7);
+		fatal_exit("unhandled calloc");
+	}
+
+	cl = get_codeline(tree, codeline, name);
+	if(!cl)
+		fatal_exit("alloc failure");
+	cl->alloc += num*sz;
+}
+
+/** get size of file */
+static off_t
+get_file_size(const char* fname)
+{
+	struct stat s;
+	if(stat(fname, &s) < 0) {
+		fatal_exit("could not stat %s: %s", fname, strerror(errno));
+	}
+	return s.st_size;
+}
+
+/** read the logfile */
+static void
+readfile(rbtree_t* tree, const char* fname, struct alloc_misc* misc)
+{
+	off_t total = get_file_size(fname);
+	off_t done = (off_t)0;
+	int report = 0;
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	if(!in)
+		fatal_exit("could not open %s: %s", fname, strerror(errno));
+	printf("Reading %s of size %lld\n", fname, (long long)total);
+	while(fgets(buf, 102400, in)) {
+		buf[102400-1] = 0;
+		done += (off_t)strlen(buf);
+		/* progress count */
+		if((int)(((double)done / (double)total)*100.) > report) {
+			report = (int)(((double)done / (double)total)*100.);
+			fprintf(stderr, " %d%%", report);
+		}
+
+		if(!match(buf))
+			continue;
+		if(strncmp(buf+36, "region ", 7) == 0)
+			read_region_stat(buf, misc);
+		else if(strstr(buf+36, "malloc("))
+			read_malloc_stat(buf, tree);
+		else if(strstr(buf+36, "calloc("))
+			read_calloc_stat(buf, tree);
+		else {
+			printf("%s\n", buf);
+			fatal_exit("unhandled input");
+		}
+	}
+	fprintf(stderr, " done\n");
+	fclose(in);
+}
+
+/** print memory stats */
+static void
+printstats(rbtree_t* tree, struct alloc_misc* misc)
+{
+	struct codeline* cl;
+	uint64_t total = 0;
+	printf("%12lld in region alloc\n", (long long)misc->region_alloc);
+	total += misc->region_alloc;
+	RBTREE_FOR(cl, struct codeline*, tree) {
+		printf("%12lld in %s %s\n", (long long)cl->alloc, 
+			cl->codeline, cl->func);
+		total += cl->alloc;
+	}
+	printf("------------\n");
+	printf("%12lld total in %ld code lines\n", (long long)total, 
+		(long)tree->count);
+	printf("\n");
+}
+
+/** main program */
+int main(int argc, const char* argv[])
+{
+	rbtree_t* tree = 0;
+	struct alloc_misc misc;
+	if(argc != 2) {
+		usage();
+	}
+	tree = rbtree_create(codeline_cmp);
+	if(!tree)
+		fatal_exit("alloc failure");
+	memset(&misc, 0, sizeof(misc));
+	readfile(tree, argv[1], &misc);
+	printstats(tree, &misc);
+	return 0;
+}

testcode/mini_tpkg.sh

@@ -0,0 +1,111 @@
+# tpkg that only exes the files.
+args="../.."
+if test "$1" = "-a"; then
+	args=$2
+	shift
+	shift
+fi
+	
+if test "$1" = "clean"; then
+	echo "rm -f result.* .done* .tpkg.var.master .tpkg.var.test"
+	rm -f result.* .done* .tpkg.var.master .tpkg.var.test
+	exit 0
+fi
+if test "$1" = "fake"; then
+	echo "minitpkg fake $2"
+	echo "fake" > .done-`basename $2 .tpkg`
+	exit 0
+fi
+if test "$1" = "report" || test "$2" = "report"; then
+	echo "Minitpkg Report"
+	for result in result.*; do
+		name=`echo $result | sed -e 's/result\.//'`
+		if test -f ".done-$name"; then
+			if test "$1" != "-q"; then
+				echo "** PASSED ** : $name"
+			fi
+		else
+			echo "!! FAILED !! : $name"
+		fi
+	done
+	exit 0
+fi
+
+if test "$1" != 'exe'; then
+	# usage
+	echo "mini tpkg. Reduced functionality for old shells."
+	echo "	tpkg exe <file>"
+	echo "	tpkg fake <file>"
+	echo "	tpkg clean"
+	echo "	tpkg [-q] report"
+	exit 1
+fi
+shift
+
+# do not execute if the disk is too full
+#DISKLIMIT=100000
+# This check is not portable (to Solaris 10).
+#avail=`df . | tail -1 | awk '{print $4}'`
+#if test "$avail" -lt "$DISKLIMIT"; then
+	#echo "minitpkg: The disk is too full! Only $avail."
+	#exit 1
+#fi
+
+name=`basename $1 .tpkg`
+dir=$name.$$
+result=result.$name
+done=.done-$name
+success="no"
+shell="bash"
+
+# check already done
+if test -f .done-$name; then
+	echo "minitpkg .done-$name exists. skip test."
+	exit 0
+fi
+
+# Extract
+echo "minitpkg extract $1 to $dir"
+mkdir $dir
+gzip -cd $name.tpkg | (cd $dir; tar xf -)
+cd $dir
+mv $name.dir/* .
+
+# EXE
+echo "minitpkg exe $name"
+echo "minitpkg exe $name" > $result
+if test -f $name.pre; then
+	echo "minitpkg exe $name.pre"
+	echo "minitpkg exe $name.pre" >> $result
+	$shell $name.pre $args >> $result
+	if test $? -ne 0; then
+		echo "Warning: $name.pre did not exit successfully"
+	fi
+fi
+if test -f $name.test; then
+	echo "minitpkg exe $name.test"
+	echo "minitpkg exe $name.test" >> $result
+	$shell $name.test $args >>$result 2>&1
+	if test $? -ne 0; then
+		echo "$name: FAILED" >> $result
+		echo "$name: FAILED"
+		success="no"
+	else
+		echo "$name: PASSED" >> $result
+		echo "$name: PASSED" > ../.done-$name
+		echo "$name: PASSED"
+		success="yes"
+	fi
+fi
+if test -f $name.post; then
+	echo "minitpkg exe $name.post"
+	echo "minitpkg exe $name.post" >> $result
+	$shell $name.post $args >> $result
+	if test $? -ne 0; then
+		echo "Warning: $name.post did not exit successfully"
+	fi
+fi
+
+mv $result ..
+cd ..
+rm -rf $dir

testcode/pktview.c

@@ -0,0 +1,200 @@
+/*
+ * testcode/pktview.c - debug program to disassemble a DNS packet.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program shows a dns packet wire format.
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/data/dname.h"
+#include "util/data/msgparse.h"
+#include "testcode/unitmain.h"
+#include "testcode/readhex.h"
+
+/** usage information for pktview */
+void usage(char* argv[])
+{
+	printf("usage: %s\n", argv[0]);
+	printf("present hex packet on stdin.\n");
+	exit(1);
+}
+
+/** read hex input */
+void read_input(ldns_buffer* pkt, FILE* in)
+{
+	char buf[102400];
+	char* np = buf;
+	while(fgets(np, (int)sizeof(buf) - (np-buf), in)) {
+		if(buf[0] == ';') /* comment */
+			continue;
+		np = &np[strlen(np)];
+	}
+	hex_to_buf(pkt, buf);
+}
+
+/** analyze domain name in packet, possibly compressed */
+void analyze_dname(ldns_buffer* pkt)
+{
+	size_t oldpos = ldns_buffer_position(pkt);
+	size_t len;
+	printf("[pos %d] dname: ", (int)oldpos);
+	dname_print(stdout, pkt, ldns_buffer_current(pkt));
+	len = pkt_dname_len(pkt);
+	printf(" len=%d", (int)len);
+	if(ldns_buffer_position(pkt)-oldpos != len)
+		printf(" comprlen=%d\n", 
+			(int)(ldns_buffer_position(pkt)-oldpos));
+	else	printf("\n");
+}
+
+/** analyze rdata in packet */
+void analyze_rdata(ldns_buffer*pkt, const ldns_rr_descriptor* desc, 
+	uint16_t rdlen)
+{
+	int rdf = 0;
+	int count = (int)desc->_dname_count;
+	size_t len, oldpos;
+	while(rdlen > 0 && count) {
+		switch(desc->_wireformat[rdf]) {
+		case LDNS_RDF_TYPE_DNAME:
+			oldpos = ldns_buffer_position(pkt);
+			analyze_dname(pkt);
+			rdlen -= ldns_buffer_position(pkt)-oldpos;
+			count --;
+			len = 0;
+			break;
+		case LDNS_RDF_TYPE_STR:
+			len = ldns_buffer_current(pkt)[0] + 1;
+			break;
+		default:
+			len = get_rdf_size(desc->_wireformat[rdf]);
+		}
+		if(len) {
+			printf(" wf[%d]", (int)len);
+			ldns_buffer_skip(pkt, (ssize_t)len);
+			rdlen -= len;
+		}
+		rdf++;
+	}
+	if(rdlen) {
+		size_t i;
+		printf(" remain[%d]\n", (int)rdlen);
+		for(i=0; i<rdlen; i++)
+			printf(" %2.2X", (unsigned)ldns_buffer_current(pkt)[i]);
+		printf("\n");
+	}
+	else	printf("\n");
+	ldns_buffer_skip(pkt, (ssize_t)rdlen);
+}
+
+/** analyze rr in packet */
+void analyze_rr(ldns_buffer* pkt, int q)
+{
+	uint16_t type, dclass, len;
+	uint32_t ttl;
+	analyze_dname(pkt);
+	type = ldns_buffer_read_u16(pkt);
+	dclass = ldns_buffer_read_u16(pkt);
+	printf("type %s(%d)", ldns_rr_descript(type)?  
+		ldns_rr_descript(type)->_name: "??" , (int)type);
+	printf(" class %s(%d) ", ldns_lookup_by_id(ldns_rr_classes, 
+		(int)dclass)?ldns_lookup_by_id( ldns_rr_classes, 
+		(int)dclass)->name:"??", (int)dclass);
+	if(q) {
+		printf("\n");
+	} else {
+		ttl = ldns_buffer_read_u32(pkt);
+		printf(" ttl %d (0x%x)", (int)ttl, (unsigned)ttl);
+		len = ldns_buffer_read_u16(pkt);
+		printf(" rdata len %d:\n", (int)len);
+		if(ldns_rr_descript(type))
+			analyze_rdata(pkt, ldns_rr_descript(type), len);
+		else ldns_buffer_skip(pkt, (ssize_t)len);
+	}
+}
+
+/** analyse pkt */
+void analyze(ldns_buffer* pkt)
+{
+	uint16_t i, f, qd, an, ns, ar;
+	int rrnum = 0;
+	printf("packet length %d\n", (int)ldns_buffer_limit(pkt));
+	if(ldns_buffer_limit(pkt) < 12) return;
+
+	i = ldns_buffer_read_u16(pkt);
+	printf("id (hostorder): %d (0x%x)\n", (int)i, (unsigned)i);
+	f = ldns_buffer_read_u16(pkt);
+	printf("flags: 0x%x\n", (unsigned)f);
+	qd = ldns_buffer_read_u16(pkt);
+	printf("qdcount: %d\n", (int)qd);
+	an = ldns_buffer_read_u16(pkt);
+	printf("ancount: %d\n", (int)an);
+	ns = ldns_buffer_read_u16(pkt);
+	printf("nscount: %d\n", (int)ns);
+	ar = ldns_buffer_read_u16(pkt);
+	printf("arcount: %d\n", (int)ar);
+	
+	printf(";-- query section\n");
+	while(ldns_buffer_remaining(pkt) > 0) {
+		if(rrnum == (int)qd) 
+			printf(";-- answer section\n");
+		if(rrnum == (int)qd+(int)an) 
+			printf(";-- authority section\n");
+		if(rrnum == (int)qd+(int)an+(int)ns) 
+			printf(";-- additional section\n");
+		printf("rr %d ", rrnum);
+		analyze_rr(pkt, rrnum < (int)qd);
+		rrnum++;
+	}
+}
+
+/** main program for pktview */
+int main(int argc, char* argv[]) 
+{
+	ldns_buffer* pkt = ldns_buffer_new(65553);
+	if(argc != 1) {
+		usage(argv);
+	}
+	if(!pkt) fatal_exit("out of memory");
+
+	read_input(pkt, stdin);
+	analyze(pkt);
+
+	ldns_buffer_free(pkt);
+	return 0;
+}

testcode/readhex.c

@@ -0,0 +1,82 @@
+/*
+ * testcode/readhex.c - read hex data.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+#include "config.h"
+#include "testcode/readhex.h"
+#include "util/log.h"
+
+/** skip whitespace */
+static void
+skip_whites(const char** p)
+{
+	while(1) {
+		while(isspace((int)**p))
+			(*p)++;
+		if(**p == ';') {
+			/* comment, skip until newline */
+			while(**p && **p != '\n')
+				(*p)++;
+			if(**p == '\n')
+				(*p)++;
+		} else return;
+	}
+}
+
+/** takes a hex string and puts into buffer */
+void hex_to_buf(ldns_buffer* pkt, const char* hex)
+{
+	const char* p = hex;
+	int val;
+	ldns_buffer_clear(pkt);
+	while(*p) {
+		skip_whites(&p);
+		if(ldns_buffer_position(pkt) == ldns_buffer_limit(pkt))
+			fatal_exit("hex_to_buf: buffer too small");
+		if(!isalnum((int)*p))
+			break;
+		val = ldns_hexdigit_to_int(*p++) << 4;
+		skip_whites(&p);
+		log_assert(*p && isalnum((int)*p));
+		val |= ldns_hexdigit_to_int(*p++);
+		ldns_buffer_write_u8(pkt, (uint8_t)val);
+		skip_whites(&p);
+	}
+	ldns_buffer_flip(pkt);
+}
+

testcode/readhex.h

@@ -0,0 +1,47 @@
+/*
+ * testcode/readhex.h - read hex data.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+
+#ifndef TESTCODE_READHEX_H
+#define TESTCODE_READHEX_H
+
+/** helper to convert hex string to packet buffer */
+void hex_to_buf(ldns_buffer* pkt, const char* hex);
+
+#endif /* TESTCODE_READHEX_H */

testcode/replay.c

@@ -41,6 +41,7 @@
 
 #include "config.h"
 #include "util/log.h"
+#include "util/net_help.h"
 #include "testcode/replay.h"
 #include "testcode/ldns-testpkts.h"
 
@@ -85,6 +86,18 @@ replay_range_delete(struct replay_range* rng)
 	free(rng);
 }
 
+/** strip whitespace from end of string */
+static void
+strip_end_white(char* p)
+{
+	size_t i;
+	for(i = strlen(p); i > 0; i--) {
+		if(isspace((int)p[i-1]))
+			p[i-1] = 0;
+		else return;
+	}
+}
+
 /** 
  * Read a range from file. 
  * @param remain: Rest of line (after RANGE keyword).
@@ -99,7 +112,7 @@ replay_range_delete(struct replay_range* rng)
  */
 static struct replay_range*
 replay_range_read(char* remain, FILE* in, const char* name, int* lineno, 
-	char* line, uint16_t* ttl, ldns_rdf** or, ldns_rdf** prev)
+	char* line, uint32_t* ttl, ldns_rdf** or, ldns_rdf** prev)
 {
 	struct replay_range* rng = (struct replay_range*)malloc(
 		sizeof(struct replay_range));
@@ -120,10 +133,25 @@ replay_range_read(char* remain, FILE* in, const char* name, int* lineno,
 	while(fgets(line, MAX_LINE_LEN-1, in)) {
 		(*lineno)++;
 		parse = line;
-		while(isspace(*parse))
+		while(isspace((int)*parse))
 			parse++;
-		if(!*parse || *parse == ';')
+		if(!*parse || *parse == ';') {
+			pos = ftello(in);
 			continue;
+		}
+		if(parse_keyword(&parse, "ADDRESS")) {
+			while(isspace((int)*parse))
+				parse++;
+			strip_end_white(parse);
+			if(!extstrtoaddr(parse, &rng->addr, &rng->addrlen)) {
+				log_err("Line %d: could not read ADDRESS: %s", 
+					*lineno, parse);
+				free(rng);
+				return NULL;
+			}
+			pos = ftello(in);
+			continue;
+		}
 		if(parse_keyword(&parse, "RANGE_END")) {
 			return rng;
 		}
@@ -158,7 +186,7 @@ replay_range_read(char* remain, FILE* in, const char* name, int* lineno,
  */
 static struct replay_moment*
 replay_moment_read(char* remain, FILE* in, const char* name, int* lineno, 
-	uint16_t* ttl, ldns_rdf** or, ldns_rdf** prev)
+	uint32_t* ttl, ldns_rdf** or, ldns_rdf** prev)
 {
 	struct replay_moment* mom = (struct replay_moment*)malloc(
 		sizeof(struct replay_moment));
@@ -173,7 +201,7 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 		return NULL;
 	}
 	remain += skip;
-	while(isspace(*remain))
+	while(isspace((int)*remain))
 		remain++;
 	if(parse_keyword(&remain, "NOTHING")) {
 		mom->evt_type = repevt_nothing;
@@ -198,6 +226,16 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 		free(mom);
 		return NULL;
 	}
+	while(isspace((int)*remain))
+		remain++;
+	if(parse_keyword(&remain, "ADDRESS")) {
+		if(!extstrtoaddr(remain, &mom->addr, &mom->addrlen)) {
+			log_err("line %d: could not parse ADDRESS: %s", 
+				*lineno, remain);
+			free(mom);
+			return NULL;
+		}
+	}
 
 	if(readentry) {
 		mom->match = read_entry(in, name, lineno, ttl, or, prev);
@@ -210,12 +248,12 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 	return mom;
 }
 
-/** makes scenario with title on rest of line. */
+/** makes scenario with title on rest of line */
 static struct replay_scenario*
 make_scenario(char* line)
 {
 	struct replay_scenario* scen;
-	while(isspace(*line))
+	while(isspace((int)*line))
 		line++;
 	if(!*line) {
 		log_err("scenario: no title given");
@@ -234,21 +272,20 @@ make_scenario(char* line)
 }
 
 struct replay_scenario* 
-replay_scenario_read(FILE* in, const char* name)
+replay_scenario_read(FILE* in, const char* name, int* lineno)
 {
 	char line[MAX_LINE_LEN];
 	char *parse;
-	int lineno = 0;
 	struct replay_scenario* scen = NULL;
-	uint16_t ttl = 3600;
+	uint32_t ttl = 3600;
 	ldns_rdf* or = NULL;
 	ldns_rdf* prev = NULL;
 	line[MAX_LINE_LEN-1]=0;
 
 	while(fgets(line, MAX_LINE_LEN-1, in)) {
 		parse=line;
-		lineno++;
-		while(isspace(*parse))
+		(*lineno)++;
+		while(isspace((int)*parse))
 			parse++;
 		if(!*parse) 
 			continue; /* empty line */
@@ -257,26 +294,26 @@ replay_scenario_read(FILE* in, const char* name)
 		if(parse_keyword(&parse, "SCENARIO_BEGIN")) {
 			scen = make_scenario(parse);
 			if(!scen)
-				fatal_exit("%d: could not make scen", lineno);
+				fatal_exit("%d: could not make scen", *lineno);
 			continue;
 		} 
 		if(!scen)
-			fatal_exit("%d: expected SCENARIO", lineno);
+			fatal_exit("%d: expected SCENARIO", *lineno);
 		if(parse_keyword(&parse, "RANGE_BEGIN")) {
 			struct replay_range* newr = replay_range_read(parse, 
-				in, name, &lineno, line, &ttl, &or, &prev);
+				in, name, lineno, line, &ttl, &or, &prev);
 			if(!newr)
-				fatal_exit("%d: bad range", lineno);
+				fatal_exit("%d: bad range", *lineno);
 			newr->next_range = scen->range_list;
 			scen->range_list = newr;
 		} else if(parse_keyword(&parse, "STEP")) {
 			struct replay_moment* mom = replay_moment_read(parse, 
-				in, name, &lineno, &ttl, &or, &prev);
+				in, name, lineno, &ttl, &or, &prev);
 			if(!mom)
-				fatal_exit("%d: bad moment", lineno);
+				fatal_exit("%d: bad moment", *lineno);
 			if(scen->mom_last && 
 				scen->mom_last->time_step >= mom->time_step)
-				fatal_exit("%d: time goes backwards", lineno);
+				fatal_exit("%d: time goes backwards", *lineno);
 			if(scen->mom_last)
 				scen->mom_last->mom_next = mom;
 			else	scen->mom_first = mom;

testcode/replay.h

@@ -44,11 +44,13 @@
  * ; comment line.
  * SCENARIO_BEGIN name_of_scenario
  * RANGE_BEGIN start_time end_time
+ *    ; give ip of the virtual server, it matches any ip if not present.
+ *    ADDRESS ip_address 
  *    match_entries
  * RANGE_END
  * ; more RANGE items.
  * ; go to the next moment
- * STEP time_step event_type
+ * STEP time_step event_type [ADDRESS ip_address]
  * ; event_type can be:
  *	o NOTHING - nothing
  *	o QUERY - followed by entry
@@ -158,6 +160,11 @@ struct replay_moment {
 	/** The sent packet must match this. Incoming events, the data. */
 	struct entry* match;
 
+	/** address that must be matched, or packet remote host address. */
+	struct sockaddr_storage addr;
+	/** length of addr, if 0, then any address will do */
+	socklen_t addrlen;
+
 	/** what pending query should timeout or is answered. or 
 	 * NULL for last sent query. 
 	 * Unused at this time.
@@ -173,6 +180,10 @@ struct replay_range {
 	int start_step;
 	/** end step of time range. */
 	int end_step;
+	/** address of where this range is served. */
+	struct sockaddr_storage addr;
+	/** length of addr, if 0, then any address will do */
+	socklen_t addrlen;
 
 	/** Matching list */
 	struct entry* match;
@@ -217,6 +228,8 @@ struct replay_runtime {
 	void (*sig_cb)(int, void*);
 	/** signal handler user arg */
 	void *sig_cb_arg;
+	/** time to exit cleanly */
+	int exit_cleanly;
 
 	/** size of buffers */
 	size_t bufsize;
@@ -245,6 +258,10 @@ struct fake_pending {
 	ldns_pkt* pkt;
 	/** by what transport was the query sent out */
 	enum transport_type transport;
+	/** if this is a serviced query */
+	int serviced;
+	/** the runtime structure this is part of */
+	struct replay_runtime* runtime;
 };
 
 /**
@@ -263,9 +280,11 @@ struct replay_answer {
  * Read a replay scenario from the file.
  * @param in: file to read from.
  * @param name: name to print in errors.
+ * @param lineno: incremented for every line read.
  * @return: Scenario. NULL if no scenario read.
  */
-struct replay_scenario* replay_scenario_read(FILE* in, const char* name);
+struct replay_scenario* replay_scenario_read(FILE* in, const char* name, 
+	int* lineno);
 
 /**
  * Delete scenario.

testcode/signit.c

@@ -0,0 +1,249 @@
+/*
+ * testcode/signit.c - debug tool to sign rrsets with given keys.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program signs rrsets with the given keys. It can be used to 
+ * construct input to test the validator with.
+ */
+#include "config.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+
+/**
+ * Key settings
+ */
+struct keysets {
+	/** signature inception */
+	uint32_t incep;
+	/** signature expiration */
+	uint32_t expi;
+	/** owner name */
+	char* owner;
+	/** keytag */
+	uint16_t keytag;
+	/** DNSKEY flags */
+	uint16_t flags;
+};
+
+/** print usage and exit */
+static void
+usage()
+{
+	printf("usage:	signit expi ince keytag owner keyfile\n");
+	printf("present rrset data on stdin.\n");
+	printf("signed data is printed to stdout.\n");
+	printf("\n");
+	printf("Or use:	signit NSEC3PARAM hash flags iter salt\n");
+	printf("present names on stdin, hashed names are printed to stdout.\n");
+	exit(1);
+}
+
+/** read expi ince keytag owner from cmdline */
+static void
+parse_cmdline(char *argv[], struct keysets* s)
+{
+	s->expi = cfg_convert_timeval(argv[1]);
+	s->incep = cfg_convert_timeval(argv[2]);
+	s->keytag = (uint16_t)atoi(argv[3]);
+	s->owner = argv[4];
+	s->flags = DNSKEY_BIT_ZSK; /* to enforce signing */
+}
+
+/** read all key files, exit on error */
+static ldns_key_list*
+read_keys(int num, char* names[], struct keysets* set)
+{
+	int i;
+	ldns_key_list* keys = ldns_key_list_new();
+	ldns_key* k;
+	ldns_rdf* rdf;
+	ldns_status s;
+	int b;
+	FILE* in;
+
+	if(!keys) fatal_exit("alloc failure");
+	for(i=0; i<num; i++) {
+		printf("read keyfile %s\n", names[i]);
+		in = fopen(names[i], "r");
+		if(!in) fatal_exit("could not open %s: %s", names[i],
+				strerror(errno));
+		s = ldns_key_new_frm_fp(&k, in);
+		fclose(in);
+		if(s != LDNS_STATUS_OK)
+			fatal_exit("bad keyfile %s: %s", names[i],
+				ldns_get_errorstr_by_id(s));
+		ldns_key_set_expiration(k, set->expi);
+		ldns_key_set_inception(k, set->incep);
+		s = ldns_str2rdf_dname(&rdf, set->owner);
+		if(s != LDNS_STATUS_OK)
+			fatal_exit("bad owner name %s: %s", set->owner,
+				ldns_get_errorstr_by_id(s));
+		ldns_key_set_pubkey_owner(k, rdf);
+		ldns_key_set_flags(k, set->flags);
+		ldns_key_set_keytag(k, set->keytag);
+		b = ldns_key_list_push_key(keys, k);
+		log_assert(b);
+	}
+	return keys;
+}
+
+/** read list of rrs from the file */
+static ldns_rr_list*
+read_rrs(FILE* in)
+{
+	uint32_t my_ttl = 3600;
+	ldns_rdf *my_origin = NULL;
+	ldns_rdf *my_prev = NULL;
+	ldns_status s;
+	int line_nr = 1;
+	int b;
+
+	ldns_rr_list* list;
+	ldns_rr *rr;
+
+	list = ldns_rr_list_new();
+	if(!list) fatal_exit("alloc error");
+
+	while(!feof(in)) {
+		s = ldns_rr_new_frm_fp_l(&rr, in, &my_ttl, &my_origin,
+			&my_prev, &line_nr);
+		if(s == LDNS_STATUS_SYNTAX_TTL || 
+			s == LDNS_STATUS_SYNTAX_ORIGIN ||
+			s == LDNS_STATUS_SYNTAX_EMPTY)
+			continue;
+		else if(s != LDNS_STATUS_OK)
+			fatal_exit("parse error in line %d: %s", line_nr,
+				ldns_get_errorstr_by_id(s));
+		b = ldns_rr_list_push_rr(list, rr);
+		log_assert(b);
+	}
+	printf("read %d lines\n", line_nr);
+
+	return list;
+}
+
+/** sign the rrs with the keys */
+static void
+signit(ldns_rr_list* rrs, ldns_key_list* keys)
+{
+	ldns_rr_list* rrset;
+	ldns_rr_list* sigs;
+	
+	while(ldns_rr_list_rr_count(rrs) > 0) {
+		rrset = ldns_rr_list_pop_rrset(rrs);
+		if(!rrset) fatal_exit("copy alloc failure");
+		sigs = ldns_sign_public(rrset, keys);
+		if(!sigs) fatal_exit("failed to sign");
+		ldns_rr_list_print(stdout, rrset);
+		ldns_rr_list_print(stdout, sigs);
+		printf("\n");
+		ldns_rr_list_free(rrset);
+		ldns_rr_list_free(sigs);
+	}
+}
+
+/** process keys and signit */
+static void
+process_keys(int argc, char* argv[])
+{
+	ldns_rr_list* rrs;
+	ldns_key_list* keys;
+	struct keysets settings;
+	log_assert(argc == 6);
+
+	parse_cmdline(argv, &settings);
+	keys = read_keys(1, argv+5, &settings);
+	rrs = read_rrs(stdin);
+	signit(rrs, keys);
+
+	ldns_rr_list_deep_free(rrs);
+	ldns_key_list_free(keys);
+}
+
+/** process nsec3 params and perform hashing */
+static void
+process_nsec3(int argc, char* argv[])
+{
+	char line[10240];
+	ldns_rdf* salt;
+	ldns_rdf* in, *out;
+	ldns_status status;
+	status = ldns_str2rdf_nsec3_salt(&salt, argv[5]);
+	if(status != LDNS_STATUS_OK)
+		fatal_exit("Could not parse salt %s: %s", argv[5],
+			ldns_get_errorstr_by_id(status));
+	log_assert(argc == 6);
+	while(fgets(line, (int)sizeof(line), stdin)) {
+		if(strlen(line) > 0)
+			line[strlen(line)-1] = 0; /* remove trailing newline */
+		if(line[0]==0)
+			continue;
+		status = ldns_str2rdf_dname(&in, line);
+		if(status != LDNS_STATUS_OK)
+			fatal_exit("Could not parse name %s: %s", line,
+				ldns_get_errorstr_by_id(status));
+		ldns_rdf_print(stdout, in);
+		printf(" -> ");
+		/* arg 3 is flags, unused */
+		out = ldns_nsec3_hash_name(in, (uint8_t)atoi(argv[2]), 
+			(uint16_t)atoi(argv[4]),
+			ldns_rdf_data(salt)[0], ldns_rdf_data(salt)+1);
+		if(!out)
+			fatal_exit("Could not hash %s", line);
+		ldns_rdf_print(stdout, out);
+		printf("\n");
+		ldns_rdf_deep_free(in);
+		ldns_rdf_deep_free(out);
+	}
+	ldns_rdf_deep_free(salt);
+}
+
+/** main program */
+int main(int argc, char* argv[])
+{
+	log_init(NULL, 0, NULL);
+	if(argc != 6) {
+		usage();
+	}
+	if(strcmp(argv[1], "NSEC3PARAM") == 0) {
+		process_nsec3(argc, argv);
+		return 0;
+	}
+	process_keys(argc, argv);
+	return 0;
+}

testcode/testbed.sh

@@ -43,10 +43,10 @@ function dotest()
 	fi
 
 	cat >makeconf.mak.$$ << EOF 
-configure:	configure.ac
-	$AC_CMD
-	touch configure
-Makefile:	configure
+#configure:	configure.ac
+#	$AC_CMD
+#	touch configure
+Makefile:	Makefile.in #configure
 	./configure $CONFIGURE_FLAGS $DISABLE
 	touch Makefile 
 EOF
@@ -77,12 +77,11 @@ EOF
 		# echossh $1 "gtar xzf unbound_ttt.tar.gz && rm unbound_ttt.tar.gz"
 	fi
 	echossh $1 "cd $2; $MAKE_CMD -f makeconf.mak.$$ Makefile"
-	echossh $1 "cd $2; $MAKE_CMD"
+	echossh $1 "cd $2; $MAKE_CMD all tests"
 	echossh $1 "cd $2; $MAKE_CMD doc"
 	if test $RUN_TEST = yes; then
-	echossh $1 "cd $2/testdata; tpkg clean"
 	echossh $1 "cd $2; bash testcode/do-tests.sh"
-	echossh $1 "cd $2/testdata; tpkg -q report" | tee -a $REPORT_FILE
+	echossh $1 "cd $2/testdata; sh ../testcode/mini_tpkg.sh -q report" | tee -a $REPORT_FILE
 	fi
 	echossh $1 "cd $2; rm -f makeconf.mak.$$"
 	rm -f makeconf.mak.$$

testcode/testbound.c

@@ -51,6 +51,11 @@
 #include "daemon/unbound.c"
 #undef main
 
+/** maximum line length for lines in the replay file. */
+#define MAX_LINE_LEN 1024
+/** the config file (removed at exit) */
+static char cfgfile[MAX_LINE_LEN];
+
 /** give commandline usage for testbound. */
 static void
 testbound_usage()
@@ -79,7 +84,7 @@ add_opts(char* optarg, int* pass_argc, char* pass_argv[])
 {
 	char *p = optarg, *np;
 	size_t len;
-	while(p && isspace(*p)) 
+	while(p && isspace((int)*p)) 
 		p++;
 	while(p && *p) {
 		/* find location of next string and length of this one */
@@ -97,35 +102,83 @@ add_opts(char* optarg, int* pass_argc, char* pass_argv[])
 		(*pass_argc)++;
 		/* go to next option */
 	        p = np;
-		while(p && isspace(*p)) 
+		while(p && isspace((int)*p)) 
 			p++;
 	}
 }
 
-/** pretty print commandline for unbound in this test. */
+/** pretty print commandline for unbound in this test */
 static void
 echo_cmdline(int argc, char* argv[])
 {
 	int i;
-	printf("testbound is starting:");
+	fprintf(stderr, "testbound is starting:");
 	for(i=0; i<argc; i++) {
-		printf(" [%s]", argv[i]);
+		fprintf(stderr, " [%s]", argv[i]);
 	}
-	printf("\n");
+	fprintf(stderr, "\n");
+}
+
+/** process config elements */
+static void
+setup_config(FILE* in, char* configfile, int* lineno,
+	int* pass_argc, char* pass_argv[])
+{
+	char line[MAX_LINE_LEN];
+	char* parse;
+	FILE* cfg;
+	snprintf(configfile, MAX_LINE_LEN, "/tmp/testbound_cfg_%u.tmp", 
+		(unsigned)getpid());
+	add_opts("-c", pass_argc, pass_argv);
+	add_opts(configfile, pass_argc, pass_argv);
+	cfg = fopen(configfile, "w");
+	if(!cfg) fatal_exit("could not open %s: %s", 
+			configfile, strerror(errno));
+	line[MAX_LINE_LEN-1] = 0;
+	/* some basic settings to not pollute the host system */
+	fprintf(cfg, "server:	use-syslog: no\n");
+	fprintf(cfg, "		directory: \"\"\n");
+	fprintf(cfg, "		chroot: \"\"\n");
+	fprintf(cfg, "		username: \"\"\n");
+	fprintf(cfg, "		pidfile: \"\"\n");
+	while(fgets(line, MAX_LINE_LEN-1, in)) {
+		parse = line;
+		(*lineno)++;
+		while(isspace((int)*parse))
+			parse++;
+		if(!*parse || parse[0] == ';')
+			continue;
+		if(strncmp(parse, "COMMANDLINE", 11) == 0) {
+			parse[strlen(parse)-1] = 0; /* strip off \n */
+			add_opts(parse+11, pass_argc, pass_argv);
+			continue;
+		}
+		if(strncmp(parse, "CONFIG_END", 10) == 0) {
+			fclose(cfg);
+			return;
+		}
+		fputs(line, cfg);
+	}
+	fatal_exit("No CONFIG_END in input file");
+
 }
 
 /** read playback file */
 static struct replay_scenario* 
-setup_playback(const char* filename)
+setup_playback(const char* filename, char* configfile,
+	int* pass_argc, char* pass_argv[])
 {
 	struct replay_scenario* scen = NULL;
+	int lineno = 0;
+
 	if(filename) {
 		FILE *in = fopen(filename, "r");
 		if(!in) {
 			perror(filename);
 			exit(1);
 		}
-		scen = replay_scenario_read(in, filename);
+		setup_config(in, configfile, &lineno, pass_argc, pass_argv);
+		scen = replay_scenario_read(in, filename, &lineno);
 		fclose(in);
 		if(!scen)
 			fatal_exit("Could not read: %s", filename);
@@ -134,6 +187,12 @@ setup_playback(const char* filename)
 	log_info("Scenario: %s", scen->title);
 	return scen;
 }
+
+/** remove config file at exit */
+void remove_configfile(void)
+{
+	unlink(cfgfile);
+}
 	
 /**
  * Main fake event test program. Setup, teardown and report errors.
@@ -151,10 +210,13 @@ main(int argc, char* argv[])
 	char* init_optarg = optarg;
 	struct replay_scenario* scen = NULL;
 
+	log_init(NULL, 0, NULL);
 	log_info("Start of %s testbound program.", PACKAGE_STRING);
 	/* determine commandline options for the daemon */
+	cfgfile[0] = 0;
 	pass_argc = 1;
 	pass_argv[0] = "unbound";
+	add_opts("-d", &pass_argc, pass_argv);
 	while( (c=getopt(argc, argv, "ho:p:")) != -1) {
 		switch(c) {
 		case 'p':
@@ -176,9 +238,11 @@ main(int argc, char* argv[])
 		testbound_usage();
 		return 1;
 	}
+	if(atexit(&remove_configfile) != 0)
+		fatal_exit("atexit() failed: %s", strerror(errno));
 
 	/* setup test environment */
-	scen = setup_playback(playback_file);
+	scen = setup_playback(playback_file, cfgfile, &pass_argc, pass_argv);
 	/* init fake event backend */
 	fake_event_init(scen);
 
@@ -195,5 +259,7 @@ main(int argc, char* argv[])
 	fake_event_cleanup();
 	for(c=1; c<pass_argc; c++)
 		free(pass_argv[c]);
+	if(res == 0)
+		log_info("Testbound Exit Success");
 	return res;
 }

testcode/unitanchor.c

@@ -0,0 +1,118 @@
+/*
+ * testcode/unitanchor.c - unit test for trust anchor storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls trust anchor unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/data/dname.h"
+#include "testcode/unitmain.h"
+#include "validator/val_anchor.h"
+
+/** test empty set */
+static void
+test_anchor_empty(struct val_anchors* a)
+{
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c) == NULL);
+}
+
+/** test set of one anchor */
+static void
+test_anchor_one(ldns_buffer* buff, struct val_anchors* a)
+{
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchor_store_str(a, buff, 
+		"nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"));
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c) != NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c) != NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c) != NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002oo\000", 4, c) == NULL);
+}
+
+/** test with several anchors */
+static void
+test_anchors(ldns_buffer* buff, struct val_anchors* a)
+{
+	struct trust_anchor* ta;
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchor_store_str(a, buff, 
+	"labs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"));
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(ta = anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c));
+	unit_assert(query_dname_compare(ta->name, (uint8_t*)"\002nl\000")==0);
+	unit_assert(ta = anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c));
+	unit_assert(query_dname_compare(ta->name, 
+		(uint8_t*)"\004labs\002nl\000") == 0);
+	unit_assert(ta = anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c));
+	unit_assert(query_dname_compare(ta->name, 
+		(uint8_t*)"\002nl\000") == 0);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002oo\000", 4, c) == NULL);
+}
+
+void anchors_test()
+{
+	ldns_buffer* buff = ldns_buffer_new(65800);
+	struct val_anchors* a;
+	unit_assert(a = anchors_create());
+	ldns_buffer_flip(buff);
+	test_anchor_empty(a);
+	test_anchor_one(buff, a);
+	test_anchors(buff, a);
+	anchors_delete(a);
+	ldns_buffer_free(buff);
+}

testcode/unitdname.c

@@ -0,0 +1,755 @@
+/*
+ * testcode/unitdname.c - unit test for dname routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls dname unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "util/data/dname.h"
+
+/** put dname into buffer */
+static ldns_buffer*
+dname_to_buf(ldns_buffer* b, const char* str)
+{
+	ldns_rdf* rdf;
+	ldns_status status;
+	ldns_buffer_clear(b);
+	rdf = ldns_dname_new_frm_str(str);	
+	status = ldns_dname2buffer_wire(b, rdf);
+	if(status != LDNS_STATUS_OK)
+		fatal_exit("%s ldns: %s", __func__, 
+			ldns_get_errorstr_by_id(status));
+	ldns_rdf_deep_free(rdf);
+	ldns_buffer_flip(b);
+	return b;
+}
+
+/** test query_dname_len function */
+static void 
+dname_test_qdl(ldns_buffer* buff)
+{
+	unit_assert( query_dname_len(buff) == 0);
+	unit_assert( query_dname_len(dname_to_buf(buff, ".")) == 1 );
+	unit_assert( query_dname_len(dname_to_buf(buff, "bla.foo.")) == 9 );
+	unit_assert( query_dname_len(dname_to_buf(buff, "x.y.z.example.com."
+		)) == 19 );
+}
+
+/** test query_dname_tolower */
+static void
+dname_test_qdtl(ldns_buffer* buff)
+{
+	ldns_buffer_write_at(buff, 0, "\012abCDeaBCde\003cOm\000", 16);
+	query_dname_tolower(ldns_buffer_begin(buff));
+	unit_assert( memcmp(ldns_buffer_begin(buff), 
+		"\012abcdeabcde\003com\000", 16) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\001+\012abC{e-ZYXe\003NET\000", 18);
+	query_dname_tolower(ldns_buffer_begin(buff));
+	unit_assert( memcmp(ldns_buffer_begin(buff), 
+		"\001+\012abc{e-zyxe\003net\000", 18) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\000", 1);
+	query_dname_tolower(ldns_buffer_begin(buff));
+	unit_assert( memcmp(ldns_buffer_begin(buff), "\000", 1) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\002NL\000", 4);
+	query_dname_tolower(ldns_buffer_begin(buff));
+	unit_assert( memcmp(ldns_buffer_begin(buff), "\002nl\000", 4) == 0);
+}
+
+/** test query_dname_compare */
+static void
+dname_test_query_dname_compare()
+{
+	unit_assert(query_dname_compare((uint8_t*)"", (uint8_t*)"") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\001a", 
+					(uint8_t*)"\001a") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc\001a") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003aBc\001a", 
+					(uint8_t*)"\003AbC\001A") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc", 
+					(uint8_t*)"\003abc\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc") == +1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"") == +1);
+	unit_assert(query_dname_compare((uint8_t*)"", 
+					(uint8_t*)"\003abc\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003xxx\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003axx\001a", 
+					(uint8_t*)"\003abc\001a") == 1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc\001Z") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001Z", 
+					(uint8_t*)"\003abc\001a") == 1);
+}
+
+/** test dname_count_labels */
+static void
+dname_test_count_labels()
+{
+	unit_assert(dname_count_labels((uint8_t*)"") == 1);
+	unit_assert(dname_count_labels((uint8_t*)"\003com") == 2);
+	unit_assert(dname_count_labels((uint8_t*)"\003org") == 2);
+	unit_assert(dname_count_labels((uint8_t*)"\007example\003com") == 3);
+	unit_assert(dname_count_labels((uint8_t*)"\003bla\007example\003com") 
+		== 4);
+}
+
+/** test dname_count_size_labels */
+static void
+dname_test_count_size_labels()
+{
+	size_t sz = 0;
+	unit_assert(dname_count_size_labels((uint8_t*)"", &sz) == 1);
+	unit_assert(sz == 1);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003com", &sz) == 2);
+	unit_assert(sz == 5);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003org", &sz) == 2);
+	unit_assert(sz == 5);
+	unit_assert(dname_count_size_labels((uint8_t*)"\007example\003com", 
+		&sz) == 3);
+	unit_assert(sz == 13);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003bla\007example"
+		"\003com", &sz) == 4);
+	unit_assert(sz == 17);
+}
+
+
+/** test pkt_dname_len */
+static void
+dname_test_pkt_dname_len(ldns_buffer* buff)
+{
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\000", 1);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 1 );
+	unit_assert( ldns_buffer_position(buff) == 1);
+
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\003org\000", 5);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 5 );
+	unit_assert( ldns_buffer_position(buff) == 5);
+
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\007example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 16 );
+	unit_assert( ldns_buffer_position(buff) == 16);
+
+	/* invalid compression pointer: to self */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\300\000os\007example\003org\000", 17);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* valid compression pointer */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\003com\000\040\300\000", 8);
+	ldns_buffer_flip(buff);
+	ldns_buffer_set_position(buff, 6);
+	unit_assert( pkt_dname_len(buff) == 5 );
+	unit_assert( ldns_buffer_position(buff) == 8);
+
+	/* unknown label type */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\107example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* label too long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\047example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* label exceeds packet */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\007example\007org\004", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* name very long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, 
+		"\020a1cdef5555544444"
+		"\020a2cdef5555544444"
+		"\020a3cdef5555544444"
+		"\020a4cdef5555544444"
+		"\020a5cdef5555544444"
+		"\020a6cdef5555544444"
+		"\020a7cdef5555544444"
+		"\020a8cdef5555544444"
+		"\020a9cdef5555544444"
+		"\020aAcdef5555544444"
+		"\020aBcdef5555544444"
+		"\020aCcdef5555544444"
+		"\020aDcdef5555544444"
+		"\020aEcdef5555544444"	/* 238 up to here */
+		"\007aabbccd"		/* 246 up to here */
+		"\007example\000"	/* 255 to here */
+		, 255);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 255 );
+	unit_assert( ldns_buffer_position(buff) == 255);
+
+	/* name too long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, 
+		"\020a1cdef5555544444"
+		"\020a2cdef5555544444"
+		"\020a3cdef5555544444"
+		"\020a4cdef5555544444"
+		"\020a5cdef5555544444"
+		"\020a6cdef5555544444"
+		"\020a7cdef5555544444"
+		"\020a8cdef5555544444"
+		"\020a9cdef5555544444"
+		"\020aAcdef5555544444"
+		"\020aBcdef5555544444"
+		"\020aCcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aDcdef5555544444"
+		"\020aEcdef5555544444"	/* 238 up to here */
+		"\007aabbccd"		/* 246 up to here */
+		"\007example\000"	/* 255 to here */
+		, 255);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+}
+
+/** test dname_lab_cmp */
+static void
+dname_test_dname_lab_cmp()
+{
+	int ml = 0; /* number of labels that matched exactly */
+
+	/* test for equality succeeds */
+	unit_assert(dname_lab_cmp((uint8_t*)"", 1, (uint8_t*)"", 1, &ml) == 0);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003net", 2, 
+		(uint8_t*)"\003net", 2, 
+		&ml) == 0);
+	unit_assert(ml == 2);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == 0);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\004test\007example\003net", 4, 
+		(uint8_t*)"\004test\007example\003net", 4, 
+		&ml) == 0);
+	unit_assert(ml == 4);
+
+	/* root is smaller than anything else */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"", 1, 
+		(uint8_t*)"\003net", 2, 
+		&ml) == -1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003net", 2, 
+		(uint8_t*)"", 1, 
+		&ml) == 1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"", 1, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == -1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"", 1, 
+		&ml) == 1);
+	unit_assert(ml == 1);
+
+	/* label length makes a difference */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\004neta", 2, 
+		(uint8_t*)"\003net", 2, 
+		&ml) != 0);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\002ne", 2, 
+		(uint8_t*)"\004neta", 2, 
+		&ml) != 0);
+	unit_assert(ml == 1);
+
+	/* contents follow the zone apex */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == 1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+
+	/* label content makes a difference */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003aag\007example\003net", 4, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003aag\007example\003net", 4, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bla\003aag\007example\003net", 5, 
+		(uint8_t*)"\003aag\003bla\007example\003net", 5, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\02sn\003opt\003aag\007example\003net", 6, 
+		(uint8_t*)"\02sn\003opt\003bla\007example\003net", 6, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+
+	/* but lowercase/uppercase does not make a difference. */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bLa\007examPLe\003net", 4, 
+		(uint8_t*)"\003bla\007eXAmple\003nET", 4, 
+		&ml) == 0);
+	unit_assert(ml == 4);
+}
+
+/** test dname_subdomain_c */
+static void
+dname_test_subdomain()
+{
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"",
+		(uint8_t*)""));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\003com",
+		(uint8_t*)""));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"",
+		(uint8_t*)"\003com"));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\003com"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\007example\003com"));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)""));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003com"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003org"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\007example\003net",
+		(uint8_t*)"\003org"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\007example\003org"));
+}
+
+/** test dname_strict_subdomain */
+static void
+dname_test_strict_subdomain()
+{
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"", 1,
+		(uint8_t*)"", 1));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\003com", 2,
+		(uint8_t*)"", 1));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"", 1,
+		(uint8_t*)"\003com", 2));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\007example\003com", 3,
+		(uint8_t*)"\003com", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003com", 2,
+		(uint8_t*)"\007example\003com", 3));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\007example\003com", 3,
+		(uint8_t*)"", 1));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\003com", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\003org", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\007example\003net", 3,
+		(uint8_t*)"\003org", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\007example\003org", 3));
+}
+
+/** test dname_is_root */
+static void
+dname_test_isroot()
+{
+	unit_assert(dname_is_root((uint8_t*)"\000"));
+	unit_assert(!dname_is_root((uint8_t*)"\001a\000"));
+	unit_assert(!dname_is_root((uint8_t*)"\005abvcd\003com\000"));
+	/* malformed dname in this test, but should work */
+	unit_assert(!dname_is_root((uint8_t*)"\077a\000"));
+	unit_assert(dname_is_root((uint8_t*)"\000"));
+}
+
+/** test dname_remove_label */
+static void
+dname_test_removelabel()
+{
+	uint8_t* orig = (uint8_t*)"\007example\003com\000";
+	uint8_t* n = orig;
+	size_t l = 13;
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+8 );
+	unit_assert( l == 5 );
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+12 );
+	unit_assert( l == 1 );
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+12 );
+	unit_assert( l == 1 );
+}
+
+/** test dname_signame_label_count */
+static void
+dname_test_sigcount()
+{
+	unit_assert(dname_signame_label_count((uint8_t*)"\000") == 0);
+	unit_assert(dname_signame_label_count((uint8_t*)"\001*\000") == 0);
+	unit_assert(dname_signame_label_count((uint8_t*)"\003xom\000") == 1);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\003xom\000") == 1);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\007example\003xom\000") == 2);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\007example\003xom\000") == 2);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\003www\007example\003xom\000") == 3);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\003www\007example\003xom\000") == 3);
+}
+
+/** test dname_is_wild routine */
+static void
+dname_test_iswild()
+{
+	unit_assert( !dname_is_wild((uint8_t*)"\000") );
+	unit_assert( dname_is_wild((uint8_t*)"\001*\000") );
+	unit_assert( !dname_is_wild((uint8_t*)"\003net\000") );
+	unit_assert( dname_is_wild((uint8_t*)"\001*\003net\000") );
+}
+
+/** test dname_canonical_compare */
+static void
+dname_test_canoncmp()
+{
+	/* equality */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net\000",
+		(uint8_t*)"\003net\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003net\000",
+		(uint8_t*)"\007example\003net\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004test\007example\003net\000",
+		(uint8_t*)"\004test\007example\003net\000"
+		) == 0);
+
+	/* subdomains */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\000"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\003com"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\003com"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\007example\003com"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\000"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\007example\003com"
+		) == -1);
+
+	/* compare rightmost label */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003com"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003org"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003net",
+		(uint8_t*)"\003org"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003org",
+		(uint8_t*)"\007example\003net"
+		) == 1);
+
+	/* label length makes a difference; but only if rest is equal */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004neta",
+		(uint8_t*)"\003net"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\002ne",
+		(uint8_t*)"\004neta"
+		) == -1);
+
+	/* label content */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003aag\007example\003net",
+		(uint8_t*)"\003bla\007example\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bla\007example\003net",
+		(uint8_t*)"\003aag\007example\003net"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bla\003aag\007example\003net",
+		(uint8_t*)"\003aag\003bla\007example\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\02sn\003opt\003aag\007example\003net",
+		(uint8_t*)"\02sn\003opt\003bla\007example\003net"
+		) == -1);
+
+	/* lowercase during compare */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bLa\007examPLe\003net",
+		(uint8_t*)"\003bla\007eXAmple\003nET"
+		) == 0);
+
+	/* example from 4034 */
+	/* example a.example yljkjljk.a.example Z.a.example zABC.a.EXAMPLE
+	 z.example \001.z.example *.z.example \200.z.example */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"",
+		(uint8_t*)"\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)"\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\001Z\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001\200\001z\007example"
+		) == -1);
+	/* same example in reverse */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)""
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\001Z\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001z\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\200\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == 1);
+	/* same example for equality */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)"\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\001Z\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\200\001z\007example",
+		(uint8_t*)"\001\200\001z\007example"
+		) == 0);
+}
+
+/** Test dname_get_shared_topdomain */
+static void
+dname_test_topdomain()
+{
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"",
+			(uint8_t*)""), 
+		(uint8_t*)"") == 0);
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"\003www\007example\003com",
+			(uint8_t*)"\003www\007example\003com"), 
+		(uint8_t*)"\003www\007example\003com") == 0);
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"\003www\007example\003com",
+			(uint8_t*)"\003bla\007example\003com"), 
+		(uint8_t*)"\007example\003com") == 0);
+}
+
+void dname_test()
+{
+	ldns_buffer* buff = ldns_buffer_new(65800);
+	ldns_buffer_flip(buff);
+	dname_test_qdl(buff);
+	dname_test_qdtl(buff);
+	dname_test_query_dname_compare();
+	dname_test_count_labels();
+	dname_test_count_size_labels();
+	dname_test_dname_lab_cmp();
+	dname_test_pkt_dname_len(buff);
+	dname_test_strict_subdomain();
+	dname_test_subdomain();
+	dname_test_isroot();
+	dname_test_removelabel();
+	dname_test_sigcount();
+	dname_test_iswild();
+	dname_test_canoncmp();
+	dname_test_topdomain();
+	ldns_buffer_free(buff);
+}

testcode/unitlruhash.c

@@ -0,0 +1,499 @@
+/*
+ * testcode/unitlruhash.c - unit test for lruhash table.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Tests the locking LRU keeping hash table implementation.
+ */
+
+#include "config.h"
+#include "testcode/unitmain.h"
+#include "util/log.h"
+#include "util/storage/lruhash.h"
+#include "util/storage/slabhash.h" /* for the test structures */
+
+/** use this type for the lruhash test key */
+typedef struct slabhash_testkey testkey_t;
+/** use this type for the lruhash test data */
+typedef struct slabhash_testdata testdata_t;
+
+/** delete key */
+static void delkey(struct slabhash_testkey* k) {
+	lock_rw_destroy(&k->entry.lock); free(k);}
+/** delete data */
+static void deldata(struct slabhash_testdata* d) {free(d);}
+
+/** hash func, very bad to improve collisions */
+static hashvalue_t myhash(int id) {return (hashvalue_t)id & 0x0f;}
+/** allocate new key, fill in hash */
+static testkey_t* newkey(int id) {
+	testkey_t* k = (testkey_t*)calloc(1, sizeof(testkey_t));
+	if(!k) fatal_exit("out of memory");
+	k->id = id;
+	k->entry.hash = myhash(id);
+	k->entry.key = k;
+	lock_rw_init(&k->entry.lock);
+	return k;
+}
+/** new data el */
+static testdata_t* newdata(int val) {
+	testdata_t* d = (testdata_t*)calloc(1, 
+		sizeof(testdata_t));
+	if(!d) fatal_exit("out of memory");
+	d->data = val;
+	return d;
+}
+
+/** test bin_find_entry function and bin_overflow_remove */
+static void
+test_bin_find_entry(struct lruhash* table)
+{
+	testkey_t* k = newkey(12);
+	testdata_t* d = newdata(128);
+	testkey_t* k2 = newkey(12 + 1024);
+	testkey_t* k3 = newkey(14);
+	testkey_t* k4 = newkey(12 + 1024*2);
+	hashvalue_t h = myhash(12);
+	struct lruhash_bin bin;
+	memset(&bin, 0, sizeof(bin));
+	bin_init(&bin, 1);
+
+	/* remove from empty list */
+	bin_overflow_remove(&bin, &k->entry);
+
+	/* find in empty list */
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	/* insert */
+	lock_quick_lock(&bin.lock);
+	bin.overflow_list = &k->entry;
+	lock_quick_unlock(&bin.lock);
+
+	/* find, hash not OK. */
+	unit_assert( bin_find_entry(table, &bin, myhash(13), k) == NULL );
+
+	/* find, hash OK, but cmp not */
+	unit_assert( k->entry.hash == k2->entry.hash );
+	unit_assert( bin_find_entry(table, &bin, h, k2) == NULL );
+
+	/* find, hash OK, and cmp too */
+	unit_assert( bin_find_entry(table, &bin, h, k) == &k->entry );
+
+	/* remove the element */
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	/* prepend two different elements; so the list is long */
+	/* one has the same hash, but different cmp */
+	lock_quick_lock(&bin.lock);
+	unit_assert( k->entry.hash == k4->entry.hash );
+	k4->entry.overflow_next = &k->entry;
+	k3->entry.overflow_next = &k4->entry;
+	bin.overflow_list = &k3->entry;
+	lock_quick_unlock(&bin.lock);
+
+	/* find, hash not OK. */
+	unit_assert( bin_find_entry(table, &bin, myhash(13), k) == NULL );
+
+	/* find, hash OK, but cmp not */
+	unit_assert( k->entry.hash == k2->entry.hash );
+	unit_assert( bin_find_entry(table, &bin, h, k2) == NULL );
+
+	/* find, hash OK, and cmp too */
+	unit_assert( bin_find_entry(table, &bin, h, k) == &k->entry );
+
+	/* remove middle element */
+	unit_assert( bin_find_entry(table, &bin, k4->entry.hash, k4) 
+		== &k4->entry );
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k4->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, k4->entry.hash, k4) == NULL);
+
+	/* remove last element */
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	lock_quick_destroy(&bin.lock);
+	delkey(k);
+	delkey(k2);
+	delkey(k3);
+	delkey(k4);
+	deldata(d);
+}
+
+/** test lru_front lru_remove */
+static void test_lru(struct lruhash* table)
+{
+	testkey_t* k = newkey(12);
+	testkey_t* k2 = newkey(14);
+	lock_quick_lock(&table->lock);
+
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+
+	/* add one */
+	lru_front(table, &k->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	/* remove it */
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+
+	/* add two */
+	lru_front(table, &k->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	lru_front(table, &k2->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k->entry);
+	/* remove first in list */
+	lru_remove(table, &k2->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	lru_front(table, &k2->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k->entry);
+	/* remove last in list */
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k2->entry);
+
+	/* empty the list */
+	lru_remove(table, &k2->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+	lock_quick_unlock(&table->lock);
+	delkey(k);
+	delkey(k2);
+}
+
+/** test hashtable using short sequence */
+static void
+test_short_table(struct lruhash* table) 
+{
+	testkey_t* k = newkey(12);
+	testkey_t* k2 = newkey(14);
+	testdata_t* d = newdata(128);
+	testdata_t* d2 = newdata(129);
+	
+	k->entry.data = d;
+	k2->entry.data = d2;
+
+	lruhash_insert(table, myhash(12), &k->entry, d, NULL);
+	lruhash_insert(table, myhash(14), &k2->entry, d2, NULL);
+	
+	unit_assert( lruhash_lookup(table, myhash(12), k, 0) == &k->entry);
+	lock_rw_unlock( &k->entry.lock );
+	unit_assert( lruhash_lookup(table, myhash(14), k2, 0) == &k2->entry);
+	lock_rw_unlock( &k2->entry.lock );
+	lruhash_remove(table, myhash(12), k);
+	lruhash_remove(table, myhash(14), k2);
+}
+
+/** number of hash test max */
+#define HASHTESTMAX 25
+
+/** test adding a random element */
+static void
+testadd(struct lruhash* table, testdata_t* ref[])
+{
+	int numtoadd = random() % HASHTESTMAX;
+	testdata_t* data = newdata(numtoadd);
+	testkey_t* key = newkey(numtoadd);
+	key->entry.data = data;
+	lruhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	ref[numtoadd] = data;
+}
+
+/** test adding a random element */
+static void
+testremove(struct lruhash* table, testdata_t* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	testkey_t* key = newkey(num);
+	lruhash_remove(table, myhash(num), key);
+	ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element */
+static void
+testlookup(struct lruhash* table, testdata_t* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	testkey_t* key = newkey(num);
+	struct lruhash_entry* en = lruhash_lookup(table, myhash(num), key, 0);
+	testdata_t* data = en? (testdata_t*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0) log_info("lookup %d got %d, expect %d", num, en? data->data :-1,
+		ref[num]? ref[num]->data : -1);
+	unit_assert( data == ref[num] );
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** check integrity of hash table */
+static void
+check_table(struct lruhash* table)
+{
+	struct lruhash_entry* p;
+	size_t c = 0;
+	lock_quick_lock(&table->lock);
+	unit_assert( table->num <= table->size);
+	unit_assert( table->size_mask == (int)table->size-1 );
+	unit_assert( (table->lru_start && table->lru_end) ||
+		(!table->lru_start && !table->lru_end) );
+	unit_assert( table->space_used <= table->space_max );
+	/* check lru list integrity */
+	if(table->lru_start)
+		unit_assert(table->lru_start->lru_prev == NULL);
+	if(table->lru_end)
+		unit_assert(table->lru_end->lru_next == NULL);
+	p = table->lru_start;
+	while(p) {
+		if(p->lru_prev) {
+			unit_assert(p->lru_prev->lru_next == p);
+		}
+		if(p->lru_next) {
+			unit_assert(p->lru_next->lru_prev == p);
+		}
+		c++;
+		p = p->lru_next;
+	}
+	unit_assert(c == table->num);
+
+	/* this assertion is specific to the unit test */
+	unit_assert( table->space_used == 
+		table->num * test_slabhash_sizefunc(NULL, NULL) );
+	lock_quick_unlock(&table->lock);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testadd_unlim(struct lruhash* table, testdata_t** ref)
+{
+	int numtoadd = random() % (HASHTESTMAX * 10);
+	testdata_t* data = newdata(numtoadd);
+	testkey_t* key = newkey(numtoadd);
+	key->entry.data = data;
+	lruhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	if(ref)
+		ref[numtoadd] = data;
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testremove_unlim(struct lruhash* table, testdata_t** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	testkey_t* key = newkey(num);
+	lruhash_remove(table, myhash(num), key);
+	if(ref)
+		ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testlookup_unlim(struct lruhash* table, testdata_t** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	testkey_t* key = newkey(num);
+	struct lruhash_entry* en = lruhash_lookup(table, myhash(num), key, 0);
+	testdata_t* data = en? (testdata_t*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0 && ref) log_info("lookup unlim %d got %d, expect %d", num, en ? 
+		data->data :-1, ref[num] ? ref[num]->data : -1);
+	if(data && ref) {
+		/* its okay for !data, it fell off the lru */
+		unit_assert( data == ref[num] );
+	}
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** test with long sequence of adds, removes and updates, and lookups */
+static void
+test_long_table(struct lruhash* table) 
+{
+	/* assuming it all fits in the hastable, this check will work */
+	testdata_t* ref[HASHTESTMAX * 100];
+	size_t i;
+	memset(ref, 0, sizeof(ref));
+	/* test assumption */
+	if(0) log_info(" size %d x %d < %d", (int)test_slabhash_sizefunc(NULL, NULL), 
+		(int)HASHTESTMAX, (int)table->space_max);
+	unit_assert( test_slabhash_sizefunc(NULL, NULL)*HASHTESTMAX < table->space_max);
+	if(0) lruhash_status(table, "unit test", 1);
+	srandom(48);
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		if(i == 500) {
+			lruhash_clear(table);
+			memset(ref, 0, sizeof(ref));
+			continue;
+		}
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd(table, ref);
+				break;
+			case 1:
+				testremove(table, ref);
+				break;
+			case 2:
+				testlookup(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(table, "unit test", 1);
+		check_table(table);
+		unit_assert( table->num <= HASHTESTMAX );
+	}
+
+	/* test more, but 'ref' assumption does not hold anymore */
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(table, ref);
+				break;
+			case 1:
+				testremove_unlim(table, ref);
+				break;
+			case 2:
+				testlookup_unlim(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(table, "unlim", 1);
+		check_table(table);
+	}
+}
+
+/** structure to threaded test the lru hash table */
+struct test_thr {
+	/** thread num, first entry. */
+	int num;
+	/** id */
+	ub_thread_t id;
+	/** hash table */
+	struct lruhash* table;
+};
+
+/** main routine for threaded hash table test */
+static void*
+test_thr_main(void* arg) 
+{
+	struct test_thr* t = (struct test_thr*)arg;
+	int i;
+	log_thread_set(&t->num);
+	for(i=0; i<1000; i++) {
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(t->table, NULL);
+				break;
+			case 1:
+				testremove_unlim(t->table, NULL);
+				break;
+			case 2:
+				testlookup_unlim(t->table, NULL);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(t->table, "hashtest", 1);
+		if(i % 100 == 0) /* because of locking, not all the time */
+			check_table(t->table);
+	}
+	check_table(t->table);
+	return NULL;
+}
+
+/** test hash table access by multiple threads */
+static void
+test_threaded_table(struct lruhash* table)
+{
+	int numth = 10;
+	struct test_thr t[100];
+	int i;
+
+	for(i=1; i<numth; i++) {
+		t[i].num = i;
+		t[i].table = table;
+		ub_thread_create(&t[i].id, test_thr_main, &t[i]);
+	}
+
+	for(i=1; i<numth; i++) {
+		ub_thread_join(t[i].id);
+	}
+	if(0) lruhash_status(table, "hashtest", 1);
+}
+
+void lruhash_test()
+{
+	/* start very very small array, so it can do lots of table_grow() */
+	/* also small in size so that reclaim has to be done quickly. */
+	struct lruhash* table ;
+	printf("lruhash test\n");
+	table = lruhash_create(2, 4096, 
+		test_slabhash_sizefunc, test_slabhash_compfunc, 
+		test_slabhash_delkey, test_slabhash_deldata, NULL);
+	test_bin_find_entry(table);
+	test_lru(table);
+	test_short_table(table);
+	test_long_table(table);
+	lruhash_delete(table);
+	table = lruhash_create(2, 4096, 
+		test_slabhash_sizefunc, test_slabhash_compfunc, 
+		test_slabhash_delkey, test_slabhash_deldata, NULL);
+	test_threaded_table(table);
+	lruhash_delete(table);
+}

testcode/unitmain.c

@@ -41,11 +41,48 @@
 
 #include "config.h"
 #include "util/log.h"
+#include "testcode/unitmain.h"
 
 /** number of tests done */
 int testcount = 0;
-/** test bool x, exits on failure, increases testcount. */
-#define unit_assert(x) testcount++; log_assert(x);
+
+#include "util/alloc.h"
+/** test alloc code */
+static void
+alloc_test() {
+	alloc_special_t *t1, *t2;
+	struct alloc_cache major, minor1, minor2;
+	int i;
+
+	alloc_init(&major, NULL, 0);
+	alloc_init(&minor1, &major, 0);
+	alloc_init(&minor2, &major, 1);
+
+	t1 = alloc_special_obtain(&minor1);
+	alloc_clear(&minor1);
+
+	alloc_special_release(&minor2, t1);
+	t2 = alloc_special_obtain(&minor2);
+	unit_assert( t1 == t2 ); /* reused */
+	alloc_special_release(&minor2, t1);
+
+	for(i=0; i<100; i++) {
+		t1 = alloc_special_obtain(&minor1);
+		alloc_special_release(&minor2, t1);
+	}
+	if(0) {
+		alloc_stats(&minor1);
+		alloc_stats(&minor2);
+		alloc_stats(&major);
+	}
+	/* reuse happened */
+	unit_assert(minor1.num_quar + minor2.num_quar + major.num_quar == 11);
+
+	alloc_clear(&minor1);
+	alloc_clear(&minor2);
+	unit_assert(major.num_quar == 11);
+	alloc_clear(&major);
+}
 
 #include "util/net_help.h"
 /** test net code */
@@ -60,6 +97,127 @@ net_test()
 	unit_assert( !str_is_ip6("213.154.224.12") );
 	unit_assert( !str_is_ip6("213.154.224.255") );
 	unit_assert( !str_is_ip6("255.255.255.0") );
+	unit_assert( is_pow2(0) );
+	unit_assert( is_pow2(1) );
+	unit_assert( is_pow2(2) );
+	unit_assert( is_pow2(4) );
+	unit_assert( is_pow2(8) );
+	unit_assert( is_pow2(16) );
+	unit_assert( is_pow2(1024) );
+	unit_assert( is_pow2(1024*1024) );
+	unit_assert( is_pow2(1024*1024*1024) );
+	unit_assert( !is_pow2(3) );
+	unit_assert( !is_pow2(5) );
+	unit_assert( !is_pow2(6) );
+	unit_assert( !is_pow2(7) );
+	unit_assert( !is_pow2(9) );
+	unit_assert( !is_pow2(10) );
+	unit_assert( !is_pow2(11) );
+	unit_assert( !is_pow2(17) );
+	unit_assert( !is_pow2(23) );
+	unit_assert( !is_pow2(257) );
+	unit_assert( !is_pow2(259) );
+}
+
+#include "util/rtt.h"
+/** test RTT code */
+static void
+rtt_test()
+{
+	int init = 376;
+	int i;
+	struct rtt_info r;
+	rtt_init(&r);
+	/* initial value sensible */
+	unit_assert( rtt_timeout(&r) == init );
+	rtt_lost(&r);
+	unit_assert( rtt_timeout(&r) == init*2 );
+	rtt_lost(&r);
+	unit_assert( rtt_timeout(&r) == init*4 );
+	rtt_update(&r, 4000);
+	unit_assert( rtt_timeout(&r) >= 2000 );
+	rtt_lost(&r);
+	for(i=0; i<100; i++) {
+		rtt_lost(&r); 
+		unit_assert( rtt_timeout(&r) > RTT_MIN_TIMEOUT-1);
+		unit_assert( rtt_timeout(&r) < RTT_MAX_TIMEOUT+1);
+	}
+}
+
+#include "services/cache/infra.h"
+#include "util/config_file.h"
+/** test host cache */
+static void
+infra_test()
+{
+	int one = 1;
+	uint8_t* zone = (uint8_t*)"\007example\003com\000";
+	size_t zonelen = 13;
+	struct infra_cache* slab;
+	struct config_file* cfg = config_create();
+	time_t now = 0;
+	int vs, to;
+	struct infra_host_key* k;
+	struct infra_host_data* d;
+	int init = 376;
+
+	slab = infra_create(cfg);
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == init );
+
+	unit_assert( infra_rtt_update(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), -1, now) );
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == init*2 );
+
+	unit_assert( infra_edns_update(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), -1, now) );
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == -1 && to == init*2 );
+
+	now += cfg->host_ttl + 10;
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == init );
+	
+	unit_assert( infra_set_lame(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), zone, zonelen, now, 0) );
+	unit_assert( (d=infra_lookup_host(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), 0, now, &k)) );
+	unit_assert( d->ttl == now+cfg->host_ttl );
+	unit_assert( d->edns_version == 0 );
+	unit_assert( infra_lookup_lame(d, zone, zonelen, now)==1 );
+	unit_assert( !infra_lookup_lame(d, zone, zonelen, 
+		now+cfg->lame_ttl+10) );
+	unit_assert( !infra_lookup_lame(d, (uint8_t*)"\000", 1, now) );
+	lock_rw_unlock(&k->entry.lock);
+
+	infra_delete(slab);
+	config_delete(cfg);
+}
+
+#include "util/random.h"
+/** test randomness */
+static void
+rnd_test()
+{
+	struct ub_randstate r;
+	int num = 100, i;
+	long int a[100];
+	unit_assert( ub_initstate((unsigned)time(NULL), &r, 256) );
+	for(i=0; i<num; i++) {
+		a[i] = ub_random(&r);
+		unit_assert(a[i] >= 0);
+		unit_assert((size_t)a[i] <= (size_t)RAND_MAX);
+		if(i > 5)
+			unit_assert(a[i] != a[i-1] || a[i] != a[i-2] ||
+				a[i] != a[i-3] || a[i] != a[i-4] ||
+				a[i] != a[i-5] || a[i] != a[i-6]);
+	}
+	ub_randfree(&r);
 }
 
 /**
@@ -70,13 +228,26 @@ net_test()
 int 
 main(int argc, char* argv[])
 {
+	log_init(NULL, 0, NULL);
 	if(argc != 1) {
 		printf("usage: %s\n", argv[0]);
 		printf("\tperforms unit tests.\n");
 		return 1;
 	}
 	printf("Start of %s unit test.\n", PACKAGE_STRING);
+	checklock_start();
+	rnd_test();
+	verify_test();
 	net_test();
-	printf("%d tests succeeded\n", testcount);
+	dname_test();
+	anchors_test();
+	rtt_test();
+	alloc_test();
+	lruhash_test();
+	slabhash_test();
+	infra_test();
+	msgparse_test();
+	checklock_stop();
+	printf("%d checks ok.\n", testcount);
 	return 0;
 }

testcode/unitmain.h

@@ -0,0 +1,63 @@
+/*
+ * testcode/unitmain.h - unit test main program for unbound.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+
+#ifndef TESTCODE_UNITMAIN_H
+#define TESTCODE_UNITMAIN_H
+#include "util/log.h"
+
+/** number of tests done */
+extern int testcount;
+/** test bool x, exits on failure, increases testcount. */
+#define unit_assert(x) do {testcount++; log_assert(x);} while(0)
+
+/** unit test lruhashtable implementation */
+void lruhash_test();
+/** unit test slabhashtable implementation */
+void slabhash_test();
+/** unit test for msgreply and msgparse */
+void msgparse_test();
+/** unit test dname handling functions */
+void dname_test();
+/** unit test trust anchor storage functions */
+void anchors_test();
+/** unit test for verification functions */
+void verify_test();
+
+#endif /* TESTCODE_UNITMAIN_H */

testcode/unitmsgparse.c

@@ -0,0 +1,452 @@
+/*
+ * testcode/unitmsgparse.c - unit test for msg parse routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls msg parse unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "util/data/msgparse.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgencode.h"
+#include "util/alloc.h"
+#include "util/regional.h"
+#include "util/net_help.h"
+#include "testcode/readhex.h"
+
+/** verbose message parse unit test */
+static int vbmp = 0;
+/** if matching within a section should disregard the order of RRs. */
+static int matches_nolocation = 0;
+
+/** match two rr lists */
+static int
+match_list(ldns_rr_list* q, ldns_rr_list *p)
+{
+	size_t i;
+	if(ldns_rr_list_rr_count(q) != ldns_rr_list_rr_count(p)) {
+		verbose(3, "rrlistcount different %d %d", 
+			(int)ldns_rr_list_rr_count(q), 
+			(int)ldns_rr_list_rr_count(p));
+		return 0;
+	}
+	for(i=0; i<ldns_rr_list_rr_count(q); i++)
+	{
+		if(matches_nolocation) {
+			if(!ldns_rr_list_contains_rr(p, ldns_rr_list_rr(q, i)))
+			{
+				verbose(3, "rr %u not found", (unsigned)i);
+				return 0;
+			}
+		} else {
+			if(ldns_rr_compare(ldns_rr_list_rr(q, i),
+				ldns_rr_list_rr(p, i)) != 0) {
+				verbose(3, "rr %u different", (unsigned)i);
+				return 0;
+			}
+			/* and check the ttl */
+			if(ldns_rr_ttl(ldns_rr_list_rr(q, i)) !=
+				ldns_rr_ttl(ldns_rr_list_rr(p, i))) {
+				verbose(3, "rr %u ttl different", (unsigned)i);
+				return 0;
+			}
+		}
+	}
+	return 1;
+}
+
+/** match edns sections */
+static int
+match_edns(ldns_pkt* q, ldns_pkt* p)
+{
+	if(ldns_pkt_edns_udp_size(q) != ldns_pkt_edns_udp_size(p))
+		return 0;
+	if(ldns_pkt_edns_extended_rcode(q) != ldns_pkt_edns_extended_rcode(p))
+		return 0;
+	if(ldns_pkt_edns_version(q) != ldns_pkt_edns_version(p))
+		return 0;
+	if(ldns_pkt_edns_do(q) != ldns_pkt_edns_do(p))
+		return 0;
+	if(ldns_pkt_edns_z(q) != ldns_pkt_edns_z(p))
+		return 0;
+	if(ldns_rdf_compare(ldns_pkt_edns_data(q), ldns_pkt_edns_data(p)) != 0)
+		return 0;
+	return 1;
+}
+
+/** compare two booleans */
+static int
+cmp_bool(int x, int y)
+{
+	if(!x && !y) return 0;
+	if(x && y) return 0;
+	if(!x) return -1;
+	return 1;
+}
+
+/** match all of the packet */
+static int
+match_all(ldns_pkt* q, ldns_pkt* p)
+{
+	if(ldns_pkt_get_opcode(q) != ldns_pkt_get_opcode(p))
+	{ verbose(3, "allmatch: opcode different"); return 0;}
+	if(ldns_pkt_get_rcode(q) != ldns_pkt_get_rcode(p))
+	{ verbose(3, "allmatch: rcode different"); return 0;}
+	if(ldns_pkt_id(q) != ldns_pkt_id(p))
+	{ verbose(3, "allmatch: id different"); return 0;}
+	if(cmp_bool(ldns_pkt_qr(q), ldns_pkt_qr(p)) != 0)
+	{ verbose(3, "allmatch: qr different"); return 0;}
+	if(cmp_bool(ldns_pkt_aa(q), ldns_pkt_aa(p)) != 0)
+	{ verbose(3, "allmatch: aa different"); return 0;}
+	if(cmp_bool(ldns_pkt_tc(q), ldns_pkt_tc(p)) != 0)
+	{ verbose(3, "allmatch: tc different"); return 0;}
+	if(cmp_bool(ldns_pkt_rd(q), ldns_pkt_rd(p)) != 0)
+	{ verbose(3, "allmatch: rd different"); return 0;}
+	if(cmp_bool(ldns_pkt_cd(q), ldns_pkt_cd(p)) != 0)
+	{ verbose(3, "allmatch: cd different"); return 0;}
+	if(cmp_bool(ldns_pkt_ra(q), ldns_pkt_ra(p)) != 0)
+	{ verbose(3, "allmatch: ra different"); return 0;}
+	if(cmp_bool(ldns_pkt_ad(q), ldns_pkt_ad(p)) != 0)
+	{ verbose(3, "allmatch: ad different"); return 0;}
+	if(ldns_pkt_qdcount(q) != ldns_pkt_qdcount(p))
+	{ verbose(3, "allmatch: qdcount different"); return 0;}
+	if(ldns_pkt_ancount(q) != ldns_pkt_ancount(p))
+	{ verbose(3, "allmatch: ancount different"); return 0;}
+	if(ldns_pkt_nscount(q) != ldns_pkt_nscount(p))
+	{ verbose(3, "allmatch: nscount different"); return 0;}
+	if(ldns_pkt_arcount(q) != ldns_pkt_arcount(p))
+	{ verbose(3, "allmatch: arcount different"); return 0;}
+	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p)))
+	{ verbose(3, "allmatch: qd section different"); return 0;}
+	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p)))
+	{ verbose(3, "allmatch: an section different"); return 0;}
+	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p)))
+	{ verbose(3, "allmatch: ns section different"); return 0;}
+	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p)))
+	{ verbose(3, "allmatch: ar section different"); return 0;}
+	if(!match_edns(q, p))
+	{ verbose(3, "edns different."); return 0;}
+	return 1;
+}
+
+/** see if buffers contain the same packet */
+static int
+test_buffers(ldns_buffer* pkt, ldns_buffer* out)
+{
+	ldns_pkt* p1=0, *p2=0;
+	ldns_status s1, s2;
+	/* check binary same */
+	if(ldns_buffer_limit(pkt) == ldns_buffer_limit(out) &&
+		memcmp(ldns_buffer_begin(pkt), ldns_buffer_begin(out),
+			ldns_buffer_limit(pkt)) == 0) {
+		if(vbmp) printf("binary the same (length=%u)\n",
+				(unsigned)ldns_buffer_limit(pkt));
+		return 1;
+	}
+
+	if(vbmp) {
+		size_t sz = 16;
+		size_t count;
+		size_t lim = ldns_buffer_limit(out);
+		if(ldns_buffer_limit(pkt) < lim)
+			lim = ldns_buffer_limit(pkt);
+		for(count=0; count<lim; count+=sz) {
+			size_t rem = sz;
+			if(lim-count < sz) rem = lim-count;
+			if(memcmp(ldns_buffer_at(pkt, count), 
+				ldns_buffer_at(out, count), rem) == 0) {
+				log_info("same %d %d", (int)count, (int)rem);
+				log_hex("same: ", ldns_buffer_at(pkt, count),
+					rem);
+			} else {
+				log_info("diff %d %d", (int)count, (int)rem);
+				log_hex("difp: ", ldns_buffer_at(pkt, count),
+					rem);
+				log_hex("difo: ", ldns_buffer_at(out, count),
+					rem);
+			}
+		}
+	}
+	/* check if it 'means the same' */
+	s1 = ldns_buffer2pkt_wire(&p1, pkt);
+	s2 = ldns_buffer2pkt_wire(&p2, out);
+	if(vbmp) {
+		log_buf(0, "orig in hex", pkt);
+		log_buf(0, "unbound out in hex", out);
+		printf("\npacket from unbound (%d):\n", 
+			(int)ldns_buffer_limit(out));
+		ldns_pkt_print(stdout, p2);
+
+		printf("\npacket original (%d):\n", 
+			(int)ldns_buffer_limit(pkt));
+		ldns_pkt_print(stdout, p1);
+		printf("\n");
+	}
+	if(s1 != s2) {
+		/* oops! */
+		printf("input ldns parse: %s, output ldns parse: %s.\n",
+			ldns_get_errorstr_by_id(s1), 
+			ldns_get_errorstr_by_id(s2));
+		unit_assert(0);
+	}
+	/* compare packets */
+	unit_assert(match_all(p1, p2));
+	ldns_pkt_free(p1);
+	ldns_pkt_free(p2);
+	return 0;
+}
+
+/** check if unbound formerr equals ldns formerr */
+static void
+checkformerr(ldns_buffer* pkt)
+{
+	ldns_pkt* p;
+	ldns_status status = ldns_buffer2pkt_wire(&p, pkt);
+	if(vbmp) printf("formerr, ldns parse is: %s\n",
+			ldns_get_errorstr_by_id(status));
+	if(status == LDNS_STATUS_OK) {
+		printf("Formerr, but ldns gives packet:\n");
+		ldns_pkt_print(stdout, p);
+		exit(1);
+	}
+	unit_assert(status != LDNS_STATUS_OK);
+}
+
+/** test a packet */
+static void
+testpkt(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out, 
+	const char* hex)
+{
+	struct query_info qi;
+	struct reply_info* rep = 0;
+	int ret;
+	uint16_t id;
+	uint16_t flags;
+	uint32_t timenow = 0;
+	struct regional* region = regional_create();
+	struct edns_data edns;
+
+	hex_to_buf(pkt, hex);
+	memmove(&id, ldns_buffer_begin(pkt), sizeof(id));
+	if(ldns_buffer_limit(pkt) < 2)
+		flags = 0;
+	else	memmove(&flags, ldns_buffer_at(pkt, 2), sizeof(flags));
+	flags = ntohs(flags);
+	ret = reply_info_parse(pkt, alloc, &qi, &rep, region, &edns);
+	if(ret != 0) {
+		if(vbmp) printf("parse code %d: %s\n", ret, 
+			ldns_lookup_by_id(ldns_rcodes, ret)->name);
+		if(ret == LDNS_RCODE_FORMERR)
+			checkformerr(pkt);
+		unit_assert(ret != LDNS_RCODE_SERVFAIL);
+	} else {
+		const size_t lim = 512;
+		ret = reply_info_encode(&qi, rep, id, flags, out, timenow,
+			region, 65535, (int)(edns.bits & EDNS_DO) );
+		unit_assert(ret != 0); /* udp packets should fit */
+		attach_edns_record(out, &edns);
+		if(vbmp) printf("inlen %u outlen %u\n", 
+			(unsigned)ldns_buffer_limit(pkt),
+			(unsigned)ldns_buffer_limit(out));
+		test_buffers(pkt, out);
+
+		if(ldns_buffer_limit(out) > lim) {
+			ret = reply_info_encode(&qi, rep, id, flags, out, 
+				timenow, region, 
+				lim - calc_edns_field_size(&edns),
+				(int)(edns.bits & EDNS_DO));
+			unit_assert(ret != 0); /* should fit, but with TC */
+			attach_edns_record(out, &edns);
+			if( LDNS_QDCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_QDCOUNT(ldns_buffer_begin(pkt)) ||
+				LDNS_ANCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_ANCOUNT(ldns_buffer_begin(pkt)) ||
+				LDNS_NSCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_NSCOUNT(ldns_buffer_begin(pkt)))
+				unit_assert(
+				LDNS_TC_WIRE(ldns_buffer_begin(out)));
+				/* must set TC bit if shortened */
+			unit_assert(ldns_buffer_limit(out) <= lim);
+		}
+	} 
+
+	query_info_clear(&qi);
+	reply_info_parsedelete(rep, alloc);
+	regional_destroy(region);
+}
+
+/** simple test of parsing */
+static void
+simpletest(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out)
+{
+	/* a root query  drill -q - */
+	testpkt(pkt, alloc, out, 
+		" c5 40 01 00 00 01 00 00 00 00 00 00 00 00 02 00 01 ");
+
+	/* very small packet */
+	testpkt(pkt, alloc, out, 
+"; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19\n"
+";-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n"
+"74 0c 85 83 00 01 00 00 00 01 00 00 03 62 6c 61 09 6e 6c 6e    ;          1-  20\n"
+"65 74 6c 61 62 73 02 6e 6c 00 00 0f 00 01 09 6e 6c 6e 65 74    ;         21-  40\n"
+"6c 61 62 73 02 6e 6c 00 00 06 00 01 00 00 46 50 00 40 04 6f    ;         41-  60\n"
+"70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 0a 68 6f    ;         61-  80\n"
+"73 74 6d 61 73 74 65 72 09 6e 6c 6e 65 74 6c 61 62 73 02 6e    ;         81- 100\n"
+"6c 00 77 a1 02 58 00 00 70 80 00 00 1c 20 00 09 3a 80 00 00    ;        101- 120\n"
+"46 50\n");
+	
+	/* a root reply  drill -w - */
+	testpkt(pkt, alloc, out, 
+	" ; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19\n"
+	" ;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n"
+	" 97 3f 81 80 00 01 00 0d 00 00 00 02 00 00 02 00 01 00 00 02    ;          1-  20\n"
+	" 00 01 00 06 6d 38 00 14 01 49 0c 52 4f 4f 54 2d 53 45 52 56    ;         21-  40\n"
+	" 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01    ;         41-  60\n"
+	" 4a 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00    ;         61-  80\n"
+	" 00 02 00 01 00 06 6d 38 00 14 01 4b 0c 52 4f 4f 54 2d 53 45    ;         81- 100\n"
+	" 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00    ;        101- 120\n"
+	" 14 01 4c 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54    ;        121- 140\n"
+	" 00 00 00 02 00 01 00 06 6d 38 00 14 01 4d 0c 52 4f 4f 54 2d    ;        141- 160\n"
+	" 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d    ;        161- 180\n"
+	" 38 00 14 01 41 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e    ;        181- 200\n"
+	" 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 42 0c 52 4f 4f    ;        201- 220\n"
+	" 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00    ;        221- 240\n"
+	" 06 6d 38 00 14 01 43 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53    ;        241- 260\n"
+	" 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 44 0c 52    ;        261- 280\n"
+	" 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00    ;        281- 300\n"
+	" 01 00 06 6d 38 00 14 01 45 0c 52 4f 4f 54 2d 53 45 52 56 45    ;        301- 320\n"
+	" 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 46    ;        321- 340\n"
+	" 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00    ;        341- 360\n"
+	" 02 00 01 00 06 6d 38 00 14 01 47 0c 52 4f 4f 54 2d 53 45 52    ;        361- 380\n"
+	" 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14    ;        381- 400\n"
+	" 01 48 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00    ;        401- 420\n"
+	" 01 41 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00    ;        421- 440\n"
+	" 00 01 00 01 00 02 64 b9 00 04 c6 29 00 04 01 4a 0c 52 4f 4f    ;        441- 460\n"
+	" 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 01 00 01 00 02    ;        461- 480\n"
+	" 64 b9 00 04 c0 3a 80 1e  ");
+}
+
+/** simple test of parsing, pcat file */
+static void
+testfromfile(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
+	const char* fname)
+{
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	int no=0;
+	if(!in) {
+		perror("fname");
+		return;
+	}
+	while(fgets(buf, (int)sizeof(buf), in)) {
+		if(buf[0] == ';') /* comment */
+			continue;
+		if(strlen(buf) < 10) /* skip pcat line numbers. */
+			continue;
+		if(vbmp) {
+			printf("test no %d: %s", no, buf);
+			fflush(stdout);
+		}
+		testpkt(pkt, alloc, out, buf);
+		no++;
+	}
+	fclose(in);
+}
+
+/** simple test of parsing, drill file */
+static void
+testfromdrillfile(ldns_buffer* pkt, struct alloc_cache* alloc, 
+	ldns_buffer* out, const char* fname)
+{
+	/*  ;-- is used to indicate a new message */
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	char* np = buf;
+	buf[0]=0;
+	if(!in) {
+		perror("fname");
+		return;
+	}
+	while(fgets(np, (int)sizeof(buf) - (np-buf), in)) {
+		if(strncmp(np, ";--", 3) == 0) {
+			/* new entry */
+			/* test previous */
+			if(np != buf)
+				testpkt(pkt, alloc, out, buf);
+			/* set for new entry */
+			np = buf;
+			buf[0]=0;
+			continue;
+		}
+		if(np[0] == ';') /* comment */
+			continue;
+		np = &np[strlen(np)];
+	}
+	testpkt(pkt, alloc, out, buf);
+	fclose(in);
+}
+
+void msgparse_test()
+{
+	ldns_buffer* pkt = ldns_buffer_new(65553);
+	ldns_buffer* out = ldns_buffer_new(65553);
+	struct alloc_cache super_a, alloc;
+	/* init */
+	alloc_init(&super_a, NULL, 0);
+	alloc_init(&alloc, &super_a, 2);
+
+	printf("testmsgparse\n");
+	simpletest(pkt, &alloc, out);
+	/* plain hex dumps, like pcat */
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.1");
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.2");
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.3");
+	/* like from drill -w - */
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.4");
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.5");
+
+	matches_nolocation = 1; /* RR order not important for the next test */
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.6");
+	matches_nolocation = 0; 
+
+	/* cleanup */
+	alloc_clear(&alloc);
+	alloc_clear(&super_a);
+	ldns_buffer_free(pkt);
+	ldns_buffer_free(out);
+}

testcode/unitslabhash.c

@@ -0,0 +1,376 @@
+/*
+ * testcode/unitslabhash.c - unit test for slabhash table.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Tests the locking LRU keeping hash table implementation.
+ */
+
+#include "config.h"
+#include "testcode/unitmain.h"
+#include "util/log.h"
+#include "util/storage/slabhash.h"
+
+/** use this type for the slabhash test key */
+typedef struct slabhash_testkey testkey_t;
+/** use this type for the slabhash test data */
+typedef struct slabhash_testdata testdata_t;
+
+/** delete key */
+static void delkey(struct slabhash_testkey* k) {
+	lock_rw_destroy(&k->entry.lock); free(k);}
+
+/** hash func, very bad to improve collisions, both high and low bits */
+static hashvalue_t myhash(int id) {
+	hashvalue_t h = (hashvalue_t)id & 0x0f;
+	h |= (h << 28);
+	return h;
+}
+
+/** allocate new key, fill in hash */
+static testkey_t* newkey(int id) {
+	testkey_t* k = (testkey_t*)calloc(1, sizeof(testkey_t));
+	if(!k) fatal_exit("out of memory");
+	k->id = id;
+	k->entry.hash = myhash(id);
+	k->entry.key = k;
+	lock_rw_init(&k->entry.lock);
+	return k;
+}
+/** new data el */
+static testdata_t* newdata(int val) {
+	testdata_t* d = (testdata_t*)calloc(1, 
+		sizeof(testdata_t));
+	if(!d) fatal_exit("out of memory");
+	d->data = val;
+	return d;
+}
+
+/** test hashtable using short sequence */
+static void
+test_short_table(struct slabhash* table) 
+{
+	testkey_t* k = newkey(12);
+	testkey_t* k2 = newkey(14);
+	testdata_t* d = newdata(128);
+	testdata_t* d2 = newdata(129);
+	
+	k->entry.data = d;
+	k2->entry.data = d2;
+
+	slabhash_insert(table, myhash(12), &k->entry, d, NULL);
+	slabhash_insert(table, myhash(14), &k2->entry, d2, NULL);
+	
+	unit_assert( slabhash_lookup(table, myhash(12), k, 0) == &k->entry);
+	lock_rw_unlock( &k->entry.lock );
+	unit_assert( slabhash_lookup(table, myhash(14), k2, 0) == &k2->entry);
+	lock_rw_unlock( &k2->entry.lock );
+	slabhash_remove(table, myhash(12), k);
+	slabhash_remove(table, myhash(14), k2);
+}
+
+/** number of hash test max */
+#define HASHTESTMAX 32
+
+/** test adding a random element */
+static void
+testadd(struct slabhash* table, testdata_t* ref[])
+{
+	int numtoadd = random() % HASHTESTMAX;
+	testdata_t* data = newdata(numtoadd);
+	testkey_t* key = newkey(numtoadd);
+	key->entry.data = data;
+	slabhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	ref[numtoadd] = data;
+}
+
+/** test adding a random element */
+static void
+testremove(struct slabhash* table, testdata_t* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	testkey_t* key = newkey(num);
+	slabhash_remove(table, myhash(num), key);
+	ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element */
+static void
+testlookup(struct slabhash* table, testdata_t* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	testkey_t* key = newkey(num);
+	struct lruhash_entry* en = slabhash_lookup(table, myhash(num), key, 0);
+	testdata_t* data = en? (testdata_t*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0) log_info("lookup %d got %d, expect %d", num, en? data->data :-1,
+		ref[num]? ref[num]->data : -1);
+	unit_assert( data == ref[num] );
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** check integrity of hash table */
+static void
+check_lru_table(struct lruhash* table)
+{
+	struct lruhash_entry* p;
+	size_t c = 0;
+	lock_quick_lock(&table->lock);
+	unit_assert( table->num <= table->size);
+	unit_assert( table->size_mask == (int)table->size-1 );
+	unit_assert( (table->lru_start && table->lru_end) ||
+		(!table->lru_start && !table->lru_end) );
+	unit_assert( table->space_used <= table->space_max );
+	/* check lru list integrity */
+	if(table->lru_start)
+		unit_assert(table->lru_start->lru_prev == NULL);
+	if(table->lru_end)
+		unit_assert(table->lru_end->lru_next == NULL);
+	p = table->lru_start;
+	while(p) {
+		if(p->lru_prev) {
+			unit_assert(p->lru_prev->lru_next == p);
+		}
+		if(p->lru_next) {
+			unit_assert(p->lru_next->lru_prev == p);
+		}
+		c++;
+		p = p->lru_next;
+	}
+	unit_assert(c == table->num);
+
+	/* this assertion is specific to the unit test */
+	unit_assert( table->space_used == 
+		table->num * test_slabhash_sizefunc(NULL, NULL) );
+	lock_quick_unlock(&table->lock);
+}
+
+/** check integrity of hash table */
+static void
+check_table(struct slabhash* table)
+{
+	size_t i;
+	for(i=0; i<table->size; i++)
+		check_lru_table(table->array[i]);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testadd_unlim(struct slabhash* table, testdata_t** ref)
+{
+	int numtoadd = random() % (HASHTESTMAX * 10);
+	testdata_t* data = newdata(numtoadd);
+	testkey_t* key = newkey(numtoadd);
+	key->entry.data = data;
+	slabhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	if(ref)
+		ref[numtoadd] = data;
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testremove_unlim(struct slabhash* table, testdata_t** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	testkey_t* key = newkey(num);
+	slabhash_remove(table, myhash(num), key);
+	if(ref)
+		ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testlookup_unlim(struct slabhash* table, testdata_t** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	testkey_t* key = newkey(num);
+	struct lruhash_entry* en = slabhash_lookup(table, myhash(num), key, 0);
+	testdata_t* data = en? (testdata_t*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0 && ref) log_info("lookup unlim %d got %d, expect %d", num, en ? 
+		data->data :-1, ref[num] ? ref[num]->data : -1);
+	if(data && ref) {
+		/* its okay for !data, it fell off the lru */
+		unit_assert( data == ref[num] );
+	}
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** test with long sequence of adds, removes and updates, and lookups */
+static void
+test_long_table(struct slabhash* table) 
+{
+	/* assuming it all fits in the hastable, this check will work */
+	testdata_t* ref[HASHTESTMAX * 100];
+	size_t i;
+	memset(ref, 0, sizeof(ref));
+	/* test assumption */
+	if(0) slabhash_status(table, "unit test", 1);
+	srandom(48);
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		if(i == 500) {
+			slabhash_clear(table);
+			memset(ref, 0, sizeof(ref));
+			continue;
+		}
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd(table, ref);
+				break;
+			case 1:
+				testremove(table, ref);
+				break;
+			case 2:
+				testlookup(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(table, "unit test", 1);
+		check_table(table);
+	}
+
+	/* test more, but 'ref' assumption does not hold anymore */
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(table, ref);
+				break;
+			case 1:
+				testremove_unlim(table, ref);
+				break;
+			case 2:
+				testlookup_unlim(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(table, "unlim", 1);
+		check_table(table);
+	}
+}
+
+/** structure to threaded test the lru hash table */
+struct slab_test_thr {
+	/** thread num, first entry. */
+	int num;
+	/** id */
+	ub_thread_t id;
+	/** hash table */
+	struct slabhash* table;
+};
+
+/** main routine for threaded hash table test */
+static void*
+test_thr_main(void* arg) 
+{
+	struct slab_test_thr* t = (struct slab_test_thr*)arg;
+	int i;
+	log_thread_set(&t->num);
+	for(i=0; i<1000; i++) {
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(t->table, NULL);
+				break;
+			case 1:
+				testremove_unlim(t->table, NULL);
+				break;
+			case 2:
+				testlookup_unlim(t->table, NULL);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(t->table, "hashtest", 1);
+		if(i % 100 == 0) /* because of locking, not all the time */
+			check_table(t->table);
+	}
+	check_table(t->table);
+	return NULL;
+}
+
+/** test hash table access by multiple threads */
+static void
+test_threaded_table(struct slabhash* table)
+{
+	int numth = 10;
+	struct slab_test_thr t[100];
+	int i;
+
+	for(i=1; i<numth; i++) {
+		t[i].num = i;
+		t[i].table = table;
+		ub_thread_create(&t[i].id, test_thr_main, &t[i]);
+	}
+
+	for(i=1; i<numth; i++) {
+		ub_thread_join(t[i].id);
+	}
+	if(0) slabhash_status(table, "hashtest", 1);
+}
+
+void slabhash_test()
+{
+	/* start very very small array, so it can do lots of table_grow() */
+	/* also small in size so that reclaim has to be done quickly. */
+	struct slabhash* table;
+	printf("slabhash test\n");
+	table = slabhash_create(4, 2, 5200, 
+		test_slabhash_sizefunc, test_slabhash_compfunc, 
+		test_slabhash_delkey, test_slabhash_deldata, NULL);
+	test_short_table(table);
+	test_long_table(table);
+	slabhash_delete(table);
+	table = slabhash_create(4, 2, 5200, 
+		test_slabhash_sizefunc, test_slabhash_compfunc, 
+		test_slabhash_delkey, test_slabhash_deldata, NULL);
+	test_threaded_table(table);
+	slabhash_delete(table);
+}

testcode/unitverify.c

@@ -0,0 +1,465 @@
+/*
+ * testcode/unitverify.c - unit test for signature verification routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls verification unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "validator/val_sigcrypt.h"
+#include "validator/val_nsec.h"
+#include "validator/val_nsec3.h"
+#include "validator/validator.h"
+#include "testcode/ldns-testpkts.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/regional.h"
+#include "util/alloc.h"
+#include "util/rbtree.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/config_file.h"
+
+/** verbose signature test */
+static int vsig = 0;
+
+/** entry to packet buffer with wireformat */
+static void
+entry_to_buf(struct entry* e, ldns_buffer* pkt)
+{
+	unit_assert(e->reply_list);
+	if(e->reply_list->reply_from_hex) {
+		ldns_buffer_copy(pkt, e->reply_list->reply_from_hex);
+	} else {
+		ldns_status status;
+		size_t answer_size;
+		uint8_t* ans = NULL;
+		status = ldns_pkt2wire(&ans, e->reply_list->reply, 
+			&answer_size);
+		if(status != LDNS_STATUS_OK) {
+			log_err("could not create reply: %s",
+				ldns_get_errorstr_by_id(status));
+			fatal_exit("error in test");
+		}
+		ldns_buffer_clear(pkt);
+		ldns_buffer_write(pkt, ans, answer_size);
+		ldns_buffer_flip(pkt);
+		free(ans);
+	}
+}
+
+/** entry to reply info conversion */
+static void
+entry_to_repinfo(struct entry* e, struct alloc_cache* alloc, 
+	struct regional* region, ldns_buffer* pkt, struct query_info* qi, 
+	struct reply_info** rep)
+{
+	int ret;
+	struct edns_data edns;
+	entry_to_buf(e, pkt);
+	/* lock alloc lock to please lock checking software. 
+	 * alloc_special_obtain assumes it is talking to a ub-alloc,
+	 * and does not need to perform locking. Here the alloc is
+	 * the only one, so we lock it here */
+	lock_quick_lock(&alloc->lock);
+	ret = reply_info_parse(pkt, alloc, qi, rep, region, &edns);
+	lock_quick_unlock(&alloc->lock);
+	if(ret != 0) {
+		printf("parse code %d: %s\n", ret,
+			ldns_lookup_by_id(ldns_rcodes, ret)->name);
+		unit_assert(ret != 0);
+	}
+}
+
+/** extract DNSKEY rrset from answer and convert it */
+static struct ub_packed_rrset_key* 
+extract_keys(struct entry* e, struct alloc_cache* alloc, 
+	struct regional* region, ldns_buffer* pkt)
+{
+	struct ub_packed_rrset_key* dnskey = NULL;
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	size_t i;
+
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+	for(i=0; i<rep->an_numrrsets; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == LDNS_RR_TYPE_DNSKEY) {
+			dnskey = rep->rrsets[i];
+			rep->rrsets[i] = NULL;
+			break;
+		}
+	}
+	unit_assert(dnskey);
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+	return dnskey;
+}
+
+/** return true if answer should be bogus */
+static int
+should_be_bogus(struct ub_packed_rrset_key* rrset)
+{
+	struct packed_rrset_data* d = (struct packed_rrset_data*)rrset->
+		entry.data;
+	if(d->rrsig_count == 0)
+		return 1;
+	return 0;
+}
+
+/** verify and test one rrset against the key rrset */
+static void
+verifytest_rrset(struct module_env* env, struct val_env* ve, 
+	struct ub_packed_rrset_key* rrset, struct ub_packed_rrset_key* dnskey)
+{
+	enum sec_status sec;
+	if(vsig) {
+		log_nametypeclass(VERB_DETAIL, "verify of rrset",
+			rrset->rk.dname, ntohs(rrset->rk.type),
+			ntohs(rrset->rk.rrset_class));
+	}
+	sec = dnskeyset_verify_rrset(env, ve, rrset, dnskey);
+	if(vsig) {
+		printf("verify outcome is: %s\n", sec_status_to_string(sec));
+	}
+	if(should_be_bogus(rrset)) {
+		unit_assert(sec == sec_status_bogus);
+	} else {
+		unit_assert(sec == sec_status_secure);
+	}
+}
+
+/** verify and test an entry - every rr in the message */
+static void
+verifytest_entry(struct entry* e, struct alloc_cache* alloc, 
+	struct regional* region, ldns_buffer* pkt, 
+	struct ub_packed_rrset_key* dnskey, struct module_env* env, 
+	struct val_env* ve)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	size_t i;
+
+	regional_free_all(region);
+	if(vsig) {
+		printf("verifying pkt:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+
+	for(i=0; i<rep->rrset_count; i++) {
+		verifytest_rrset(env, ve, rep->rrsets[i], dnskey);
+	}
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+/** find RRset in reply by type */
+static struct ub_packed_rrset_key*
+find_rrset_type(struct reply_info* rep, uint16_t type)
+{
+	size_t i;
+	for(i=0; i<rep->rrset_count; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == type)
+			return rep->rrsets[i];
+	}
+	return NULL;
+}
+
+/** DS sig test an entry - get DNSKEY and DS in entry and verify */
+static void
+dstest_entry(struct entry* e, struct alloc_cache* alloc, 
+	struct regional* region, ldns_buffer* pkt, struct module_env* env)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	struct ub_packed_rrset_key* ds, *dnskey;
+	int ret;
+
+	regional_free_all(region);
+	if(vsig) {
+		printf("verifying DS-DNSKEY match:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+	ds = find_rrset_type(rep, LDNS_RR_TYPE_DS);
+	dnskey = find_rrset_type(rep, LDNS_RR_TYPE_DNSKEY);
+	/* check test is OK */
+	unit_assert(ds && dnskey);
+
+	ret = ds_digest_match_dnskey(env, dnskey, 0, ds, 0);
+	if(strncmp((char*)qinfo.qname, "\003yes", 4) == 0) {
+		if(vsig) {
+			printf("result(yes)= %s\n", ret?"yes":"no");
+		}
+		unit_assert(ret);
+	} else if (strncmp((char*)qinfo.qname, "\002no", 3) == 0) {
+		if(vsig) {
+			printf("result(no)= %s\n", ret?"yes":"no");
+		}
+		unit_assert(!ret);
+		verbose(VERB_DETAIL, "DS fail: OK; matched unit test");
+	} else {
+		fatal_exit("Bad qname in DS unit test, yes or no");
+	}
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+/** verify from a file */
+static void
+verifytest_file(const char* fname, const char* at_date)
+{
+	/* 
+	 * The file contains a list of ldns-testpkts entries.
+	 * The first entry must be a query for DNSKEY.
+	 * The answer rrset is the keyset that will be used for verification
+	 */
+	struct ub_packed_rrset_key* dnskey;
+	struct regional* region = regional_create();
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+	struct module_env env;
+	struct val_env ve;
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	alloc_init(&alloc, NULL, 1);
+	memset(&env, 0, sizeof(env));
+	memset(&ve, 0, sizeof(ve));
+	env.scratch = region;
+	env.scratch_buffer = buf;
+	ve.date_override = cfg_convert_timeval(at_date);
+	unit_assert(region && buf);
+	dnskey = extract_keys(list, &alloc, region, buf);
+	if(vsig) log_nametypeclass(VERB_DETAIL, "test dnskey",
+			dnskey->rk.dname, ntohs(dnskey->rk.type), 
+			ntohs(dnskey->rk.rrset_class));
+	/* ready to go! */
+	for(e = list->next; e; e = e->next) {
+		verifytest_entry(e, &alloc, region, buf, dnskey, &env, &ve);
+	}
+
+	ub_packed_rrset_parsedelete(dnskey, &alloc);
+	delete_entry(list);
+	regional_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+/** verify DS matches DNSKEY from a file */
+static void
+dstest_file(const char* fname)
+{
+	/* 
+	 * The file contains a list of ldns-testpkts entries.
+	 * The first entry must be a query for DNSKEY.
+	 * The answer rrset is the keyset that will be used for verification
+	 */
+	struct regional* region = regional_create();
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+	struct module_env env;
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	alloc_init(&alloc, NULL, 1);
+	memset(&env, 0, sizeof(env));
+	env.scratch = region;
+	env.scratch_buffer = buf;
+	unit_assert(region && buf);
+
+	/* ready to go! */
+	for(e = list; e; e = e->next) {
+		dstest_entry(e, &alloc, region, buf, &env);
+	}
+
+	delete_entry(list);
+	regional_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+/** helper for unittest of NSEC routines */
+static int
+unitest_nsec_has_type_rdata(char* bitmap, size_t len, uint16_t type)
+{
+	return nsecbitmap_has_type_rdata((uint8_t*)bitmap, len, type);
+}
+
+/** Test NSEC type bitmap routine */
+static void
+nsectest()
+{
+	/* bitmap starts at type bitmap rdata field */
+	/* from rfc 4034 example */
+	char* bitmap = "\000\006\100\001\000\000\000\003"
+		"\004\033\000\000\000\000\000\000"
+		"\000\000\000\000\000\000\000\000"
+		"\000\000\000\000\000\000\000\000"
+		"\000\000\000\000\040";
+	size_t len = 37;
+
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 0));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_A));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 2));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 3));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 4));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 5));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 6));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 7));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 8));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 9));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 10));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 11));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 12));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 13));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 14));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_MX));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_RRSIG));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_NSEC));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, 1234));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1233));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1235));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1236));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1237));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1238));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1239));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1240));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 2230));
+}
+
+/** Test hash algo - NSEC3 hash it and compare result */
+static void
+nsec3_hash_test_entry(struct entry* e, rbtree_t* ct,
+	struct alloc_cache* alloc, struct regional* region, 
+	ldns_buffer* buf)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	struct ub_packed_rrset_key* answer, *nsec3;
+	struct nsec3_cached_hash* hash;
+	int ret;
+	uint8_t* qname;
+
+	if(vsig) {
+		printf("verifying NSEC3 hash:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, buf, &qinfo, &rep);
+	nsec3 = find_rrset_type(rep, LDNS_RR_TYPE_NSEC3);
+	answer = find_rrset_type(rep, LDNS_RR_TYPE_AAAA);
+	qname = regional_alloc_init(region, qinfo.qname, qinfo.qname_len);
+	/* check test is OK */
+	unit_assert(nsec3 && answer && qname);
+
+	ret = nsec3_hash_name(ct, region, buf, nsec3, 0, qname,
+		qinfo.qname_len, &hash);
+	if(ret != 1) {
+		printf("Bad nsec3_hash_name retcode %d\n", ret);
+		unit_assert(ret == 1);
+	}
+	unit_assert(hash->dname && hash->hash && hash->hash_len &&
+		hash->b32 && hash->b32_len);
+	unit_assert(hash->b32_len == (size_t)answer->rk.dname[0]);
+	/* does not do lowercasing. */
+	unit_assert(memcmp(hash->b32, answer->rk.dname+1, hash->b32_len) 
+		== 0);
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+
+/** Read file to test NSEC3 hash algo */
+static void
+nsec3_hash_test(const char* fname)
+{
+	/* 
+	 * The list contains a list of ldns-testpkts entries.
+	 * Every entry is a test.
+	 * 	The qname is hashed.
+	 * 	The answer section AAAA RR name is the required result.
+	 * 	The auth section NSEC3 is used to get hash parameters.
+	 * The hash cache is maintained per file.
+	 *
+	 * The test does not perform canonicalization during the compare.
+	 */
+	rbtree_t ct;
+	struct regional* region = regional_create();
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	rbtree_init(&ct, &nsec3_hash_cmp);
+	alloc_init(&alloc, NULL, 1);
+	unit_assert(region && buf);
+
+	/* ready to go! */
+	for(e = list; e; e = e->next) {
+		nsec3_hash_test_entry(e, &ct, &alloc, region, buf);
+	}
+
+	delete_entry(list);
+	regional_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+void 
+verify_test()
+{
+	printf("verify test\n");
+	verifytest_file("testdata/test_signatures.1", "20070818005004");
+	dstest_file("testdata/test_ds_sig.1");
+	nsectest();
+	nsec3_hash_test("testdata/test_nsec3_hash.1");
+}

testdata/Kexample.com.+003+02854.ds

@@ -0,0 +1 @@
+example.com.	3600	IN	DS	2854 3 1 46e4ffc6e9a4793b488954bd3f0cc6af0dfb201b

testdata/Kexample.com.+003+02854.key

@@ -0,0 +1 @@
+example.com.	3600	IN	DNSKEY	256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}

testdata/Kexample.com.+003+02854.private

@@ -0,0 +1,7 @@
+Private-key-format: v1.2
+Algorithm: 3 (DSA)
+Prime(p): +sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKQ==
+Subprime(q): tctSxapStjclgRHdPhMEkgizvSM=
+Base(g): LGE7N8tUTWCoDQ/B5lHx21jdo1BJGfew+nAmvoyL8+pAoAwytn1yQml/X5tAw46/GDPPZTUZLXxvfJkpyBMwRg==
+Private_value(x): XMs4XYi1oNckzTPvGMkgG5IiuzY=
+Public_value(y): ic3fxmWM4vwTdg80NDLK0sGF37DSxTgW7PDyTFuC2CMXnjnuq1IJFjhewgvQ4f3XbkNUK7CYZmQBBX3egcmFfg==

testdata/Kexample.com.+005+30899.ds

@@ -0,0 +1 @@
+example.com.	3600	IN	DS	30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512

testdata/Kexample.com.+005+30899.key

@@ -0,0 +1 @@
+example.com.	3600	IN	DNSKEY	256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b}

testdata/Kexample.com.+005+30899.private

@@ -0,0 +1,10 @@
+Private-key-format: v1.2
+Algorithm: 5 (RSASHA1)
+Modulus: 0ONXIUfQxB7f2iMyBQKmp2w5UX5SaEbOcs9YxbKESE+3Vn9K/j3g7nsHcZLMzR+sJ1OEC/KXyhUYeTyGFQSytw==
+PublicExponent: Aw==
+PrivateExponent: i0I6Fi/ggr8/5sIhWKxvGkgmNlQ28C80TIo7LncC2t6ar2Q5rpyiDxEHvFLfphRh108ZOqf2tQdHx7tXTx5Gqw==
+Prime1: 9WS85Q92ilTAuGiVi+KesKzrFqF98l2Gpu4003hfmbc=
+Prime2: 2eqsD2jcY4Mgw26A8XFiaLdxx5J4s10Dhd9ur6X3rwE=
+Exponent1: o5h97gpPBuMrJZsOXUG/IHNHZGupTD5ZxJ7N4lA/u88=
+Exponent2: kUcdX5s9l6zAgkmrS6DsRc+hL7b7Ij4CWT+fH8P6dKs=
+Coefficient: PVZrElFmz9tWa4kwu9jArjcocycYu0eBycgguQ03J7w=

testdata/fetch_glue.rpl

@@ -0,0 +1,149 @@
+; config options
+stub-zone:
+	name: "."
+	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
+CONFIG_END
+
+SCENARIO_BEGIN Fetch glue when missing or timed out.
+
+; K.ROOT-SERVERS.NET.
+RANGE_BEGIN 0 100
+	ADDRESS 193.0.14.129 
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+. IN NS
+SECTION ANSWER
+. IN NS	K.ROOT-SERVERS.NET.
+SECTION ADDITIONAL
+K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION AUTHORITY
+com.	IN NS	a.gtld-servers.net.
+SECTION ADDITIONAL
+a.gtld-servers.net.	IN 	A	192.5.6.30
+ENTRY_END
+RANGE_END
+
+; a.gtld-servers.net.
+RANGE_BEGIN 0 100
+	ADDRESS 192.5.6.30
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+; is missing the glue!
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. 	IN 	A
+SECTION ANSWER
+ns.example.com.		IN 	A	1.2.3.4
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. 	IN 	AAAA
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+RANGE_END
+
+; ns.example.com.
+RANGE_BEGIN 0 100
+	ADDRESS 1.2.3.4
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com.		IN 	A	1.2.3.4
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. IN A
+SECTION ANSWER
+ns.example.com. IN A	1.2.3.4
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. IN AAAA
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com. IN A	1.2.3.4
+ENTRY_END
+
+RANGE_END
+
+STEP 1 QUERY
+ENTRY_BEGIN
+REPLY RD
+SECTION QUESTION
+www.example.com. IN A
+ENTRY_END
+
+; recursion happens here.
+STEP 10 CHECK_ANSWER
+ENTRY_BEGIN
+MATCH all
+REPLY QR RD RA NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com.		IN 	A	1.2.3.4
+ENTRY_END
+
+; due to ordering of answer packets, this is still outstanding, remove it
+STEP 21 CHECK_OUT_QUERY
+ENTRY_BEGIN
+ADJUST copy_id
+MATCH qname qtype
+REPLY QR
+SECTION QUESTION
+ns.example.com IN AAAA
+ENTRY_END
+
+SCENARIO_END