fixup exit status post test.

git-svn-id: file:///svn/unbound/trunk@641 be551aaa-1e26-0410-a405-d3ace91eadb9

Makefile.in

@@ -20,12 +20,14 @@ exec_prefix=@exec_prefix@
 bindir=@bindir@
 mandir=@mandir@
 libdir=@libdir@
+# datarootdir is here to please some checkers, use datadir.
 datarootdir=@datarootdir@
 datadir=@datadir@
 includedir=@includedir@
 doxygen=@doxygen@
 libtool=@libtool@
 ldnsdir=@ldnsdir@
+staticexe=@staticexe@
 
 YACC=@YACC@
 LEX=@LEX@
@@ -44,26 +46,43 @@ endif
 BUILD=build/
 
 LINT=splint
-LINTFLAGS=+quiet -weak -warnposix -unrecog -Din_addr_t=uint32_t -Du_int=unsigned -Du_char=uint8_t -preproc
+LINTFLAGS=+quiet -weak -warnposix -unrecog -Din_addr_t=uint32_t -Du_int=unsigned -Du_char=uint8_t -preproc -Drlimit=rlimit64
 # compat with openssl linux edition.
 LINTFLAGS+="-DBN_ULONG=unsigned long" -Dkrb5_int32=int "-Dkrb5_ui_4=unsigned int" -DPQ_64BIT=uint64_t
 
-INSTALL=$(srcdir)/install-sh 
+INSTALL=$(srcdir)/install-sh
 
-COMMON_SRC=$(wildcard services/*.c util/*.c) util/configparser.c util/configlexer.c testcode/checklocks.c
+COMMON_SRC=$(wildcard services/*.c services/cache/*.c util/*.c \
+	util/data/*.c util/storage/*.c iterator/*.c validator/*.c) \
+	util/configparser.c util/configlexer.c testcode/checklocks.c
 COMMON_OBJ=$(addprefix $(BUILD),$(COMMON_SRC:.c=.o))
 COMPAT_OBJ=$(addprefix $(BUILD)compat/,$(LIBOBJS))
-UNITTEST_SRC=testcode/unitmain.c $(COMMON_SRC)
+UNITTEST_SRC=$(wildcard testcode/unit*.c) testcode/readhex.c \
+	testcode/ldns-testpkts.c $(COMMON_SRC)
 UNITTEST_OBJ=$(addprefix $(BUILD),$(UNITTEST_SRC:.c=.o)) $(COMPAT_OBJ)
 DAEMON_SRC=$(wildcard daemon/*.c) $(COMMON_SRC)
 DAEMON_OBJ=$(addprefix $(BUILD),$(DAEMON_SRC:.c=.o)) $(COMPAT_OBJ)
-TESTBOUND_SRC=testcode/testbound.c testcode/ldns-testpkts.c daemon/worker.c daemon/daemon.c testcode/replay.c testcode/fake_event.c $(filter-out util/netevent.c services/listen_dnsport.c services/outside_network.c, $(COMMON_SRC))
+TESTBOUND_SRC=testcode/testbound.c testcode/ldns-testpkts.c \
+	daemon/worker.c daemon/daemon.c daemon/stats.c testcode/replay.c \
+	testcode/fake_event.c $(filter-out util/netevent.c \
+	services/listen_dnsport.c services/outside_network.c, $(COMMON_SRC))
 TESTBOUND_OBJ=$(addprefix $(BUILD),$(TESTBOUND_SRC:.c=.o)) $(COMPAT_OBJ)
-ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) $(TESTBOUND_SRC)
-ALL_OBJ=$(addprefix $(BUILD),$(ALL_SRC:.c=.o) $(addprefix compat/,$(LIBOBJS))) $(COMPAT_OBJ)
+LOCKVERIFY_SRC=testcode/lock_verify.c $(COMMON_SRC)
+LOCKVERIFY_OBJ=$(addprefix $(BUILD),$(LOCKVERIFY_SRC:.c=.o)) $(COMPAT_OBJ)
+PKTVIEW_SRC=testcode/pktview.c testcode/readhex.c $(COMMON_SRC)
+PKTVIEW_OBJ=$(addprefix $(BUILD),$(PKTVIEW_SRC:.c=.o)) $(COMPAT_OBJ)
+SIGNIT_SRC=testcode/signit.c $(COMMON_SRC)
+SIGNIT_OBJ=$(addprefix $(BUILD),$(SIGNIT_SRC:.c=.o)) $(COMPAT_OBJ)
+MEMSTATS_SRC=testcode/memstats.c $(COMMON_SRC)
+MEMSTATS_OBJ=$(addprefix $(BUILD),$(MEMSTATS_SRC:.c=.o)) $(COMPAT_OBJ)
+ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) \
+	$(TESTBOUND_SRC) $(LOCKVERIFY_SRC) $(PKTVIEW_SRC) $(SIGNIT_SRC) \
+	$(MEMSTATS_SRC)
+ALL_OBJ=$(addprefix $(BUILD),$(ALL_SRC:.c=.o) \
+	$(addprefix compat/,$(LIBOBJS))) $(COMPAT_OBJ)
 
 COMPILE=$(LIBTOOL) --tag=CC --mode=compile $(CC) $(CPPFLAGS) $(CFLAGS)
-LINK=$(LIBTOOL) --tag=CC --mode=link $(CC) $(CFLAGS) $(LDFLAGS)
+LINK=$(LIBTOOL) --tag=CC --mode=link $(CC) $(staticexe) $(RUNTIME_PATH) $(CFLAGS) $(LDFLAGS)
 LINK_LIB=$(LIBTOOL) --tag=CC --mode=link $(CC) $(CFLAGS) $(LDFLAGS) -release $(VERSION)
 
 $(BUILD)%.o:    $(srcdir)/%.c 
@@ -71,9 +90,14 @@ $(BUILD)%.o:    $(srcdir)/%.c
 	@if test ! -d $(dir $@); then $(INSTALL) -d $(patsubst %/,%,$(dir $@)); fi
 	$Q$(COMPILE) -c $< -o $@
 
-.PHONY:	clean realclean doc lint all 
+.PHONY:	clean realclean doc lint all install uninstall tests test
 
-all:	$(COMMON_OBJ) unbound unittest testbound
+all:	$(COMMON_OBJ) unbound 
+
+tests:	unittest testbound lock-verify pktview signit memstats
+
+test:	tests
+	bash testcode/do-tests.sh
 
 unbound:	$(DAEMON_OBJ)
 	$(INFO) Link $@
@@ -87,17 +111,38 @@ testbound:	$(TESTBOUND_OBJ)
 	$(INFO) Link $@
 	$Q$(LINK) -o $@ $^ $(LIBS)
 
-testcode/ldns-testpkts.c:	$(ldnsdir)/examples/ldns-testpkts.c \
-			$(ldnsdir)/examples/ldns-testpkts.h
-	cp $(ldnsdir)/examples/ldns-testpkts.c testcode/ldns-testpkts.c
-	cp $(ldnsdir)/examples/ldns-testpkts.h testcode/ldns-testpkts.h
+lock-verify:	$(LOCKVERIFY_OBJ)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $^ $(LIBS)
+
+pktview:	$(PKTVIEW_OBJ)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $^ $(LIBS)
+
+signit:	$(SIGNIT_OBJ)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $^ $(LIBS)
+
+memstats:	$(MEMSTATS_OBJ)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $^ $(LIBS)
+
+#testcode/ldns-testpkts.c:	$(ldnsdir)/examples/ldns-testpkts.c \
+#			$(ldnsdir)/examples/ldns-testpkts.h
+#	cp $(ldnsdir)/examples/ldns-testpkts.c testcode/ldns-testpkts.c
+#	cp $(ldnsdir)/examples/ldns-testpkts.h testcode/ldns-testpkts.h
 
 util/config_file.c:	util/configparser.h
 util/configlexer.c:  $(srcdir)/util/configlexer.lex util/configparser.h
 	$(INFO) Lex $<
+ifeq "$(strip $(LEX))" ":"
+	$Qecho "Need to rebuild lexer, but no lex program"
+	exit 1
+else
 	@if test ! -d util; then $(INSTALL) -d util; fi
 	$Qecho "#include \"util/configyyrename.h\"" > $@
-	$Q$(LEX) -i -t $< >> $@
+	$Q$(LEX) -t $< >> $@
+endif
 
 util/configparser.c util/configparser.h:  $(srcdir)/util/configparser.y
 	$(INFO) Yacc $<
@@ -114,12 +159,13 @@ realclean: clean
 	rm -f util/configlexer.c util/configparser.c util/configparser.h
 	rm -f Makefile 
 
-lint:
-	$Qfor i in $(filter-out util/configparser.c,$(filter-out util/configlexer.c,$(sort $(ALL_SRC)))); do \
-		echo lint $$i; \
-		$(LINT) $(LINTFLAGS) -I. -I$(srcdir) -I$(ldnsdir)/include $(srcdir)/$$i ; \
-		if [ $$? -ne 0 ] ; then exit 1 ; fi ; \
-	done
+$(BUILD)%.lint:    $(srcdir)/%.c 
+	$(INFO) Lint $<
+	@if test ! -d $(dir $@); then $(INSTALL) -d $(patsubst %/,%,$(dir $@)); fi
+	$Q$(LINT) $(LINTFLAGS) -I. -I$(srcdir) -I$(ldnsdir)/include $<
+	$Qtouch $@
+
+lint:	$(addprefix $(BUILD),$(filter-out util/configparser.lint,$(filter-out util/configlexer.lint,$(sort $(ALL_SRC:.c=.lint)))))
 
 tags:	$(srcdir)/*.[ch] $(srcdir)/*/*.[ch]
 	ctags  -f $(srcdir)/tags $(srcdir)/*.[ch] $(srcdir)/*/*.[ch]
@@ -129,6 +175,19 @@ ifdef doxygen
 	$(doxygen) $(srcdir)/doc/unbound.doxygen
 endif
 
+install:
+	$(INSTALL) -d $(bindir)
+	$(INSTALL) -d $(mandir)
+	$(INSTALL) -d $(mandir)/man8
+	$(INSTALL) -d $(mandir)/man5
+	$(LIBTOOL) --mode=install cp unbound $(bindir)/unbound
+	$(INSTALL) -c -m 644 $(srcdir)/doc/unbound.8 $(mandir)/man8
+	$(INSTALL) -c -m 644 $(srcdir)/doc/unbound.conf.5 $(mandir)/man5
+
+uninstall:
+	rm -f -- $(bindir)/unbound 
+	rm -f -- $(mandir)/man8/unbound.8 $(mandir)/man5/unbound.conf.5
+
 # Automatic dependencies.
 $(BUILD)%.d: $(srcdir)/%.c
 	$(INFO) Depend $<
@@ -137,4 +196,8 @@ $(BUILD)%.d: $(srcdir)/%.c
 	              | sed '\''s!\(.*\)\.o[ :]*!$(dir $@)\1.o $@ : !g'\'' > $@; \
 	              [ -s $@ ] || rm -f $@'
 
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(MAKECMDGOALS),realclean)
 -include $(addprefix $(BUILD),$(ALL_SRC:.c=.d))
+endif
+endif

aclocal.m4

@@ -1,7 +1,7 @@
-# generated automatically by aclocal 1.9.6 -*- Autoconf -*-
+# generated automatically by aclocal 1.10 -*- Autoconf -*-
 
 # Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-# 2005  Free Software Foundation, Inc.
+# 2005, 2006  Free Software Foundation, Inc.
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
 # with or without modifications, as long as this notice is preserved.
@@ -1597,7 +1597,7 @@ linux*)
 
   # Append ld.so.conf contents to the search path
   if test -f /etc/ld.so.conf; then
-    lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;s/[:,	]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;/^$/d' | tr '\n' ' '`
+    lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/^[	 ]*//;s/#.*//;/^[^\/]/d;s/[:,	]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;/^$/d' | tr '\n' ' '`
     sys_lib_dlsearch_path_spec="/lib${libsuff} /usr/lib${libsuff} $lt_ld_extra"
   fi
 
@@ -6370,6 +6370,7 @@ do
     done
   done
 done
+IFS=$as_save_IFS
 lt_ac_max=0
 lt_ac_count=0
 # Add /usr/xpg4/bin/sed as it is typically found on Solaris
@@ -6402,6 +6403,7 @@ for lt_ac_sed in $lt_ac_sed_list /usr/xpg4/bin/sed; do
 done
 ])
 SED=$lt_cv_path_SED
+AC_SUBST([SED])
 AC_MSG_RESULT([$SED])
 ])
 

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 AC_PREREQ(2.57)
 
-AC_INIT(unbound, 0.1, wouter@nlnetlabs.nl, unbound)
+AC_INIT(unbound, 0.5, wouter@nlnetlabs.nl, unbound)
 
 CFLAGS=
 AC_AIX
@@ -90,7 +90,7 @@ CHECK_COMPILER_FLAG(O2, [CFLAGS="$CFLAGS -O2"])
 
 AC_CHECK_HEADERS([getopt.h time.h],,, [AC_INCLUDES_DEFAULT])
 
-CHECK_COMPILER_FLAG_NEEDED(-std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600,
+CHECK_COMPILER_FLAG_NEEDED(-std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600 -D_ALL_SOURCE,
 [
 #include "confdefs.h"
 #include <stdlib.h>
@@ -121,7 +121,7 @@ int test() {
 	str = gai_strerror(0);
 	return a;
 }
-], [CFLAGS="$CFLAGS -std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600"])
+], [CFLAGS="$CFLAGS -std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600 -D_ALL_SOURCE"])
 
 CHECK_COMPILER_FLAG_NEEDED(-std=c99,
 [
@@ -269,7 +269,7 @@ AC_PROG_LIBTOOL
 
 # Checks for header files.
 AC_HEADER_STDC
-AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h arpa/inet.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/uio.h arpa/inet.h],,, [AC_INCLUDES_DEFAULT])
 
 # check for types
 AC_CHECK_TYPE(int8_t, char)
@@ -313,17 +313,60 @@ AC_INCLUDES_DEFAULT
 #endif
 ])
 
+# Check the type of struct msghdr msg_iovlen.
+AC_MSG_CHECKING([sign of struct msghdr.msg_iovlen])
+AC_RUN_IFELSE([
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/socket.h>
+int main()
+{
+	struct msghdr h;
+	h.msg_iovlen = -1;
+	if(10/h.msg_iovlen == 0) return 0; /* unsigned */
+	return 1; /* signed */
+}
+], [ 
+# if compiled and exit status 0
+cv_msgiovlen_type="unsigned" ], [ 
+# not compiled or bad exit status
+cv_msgiovlen_type="int" ], 
+# when cross compiling
+[ cv_msgiovlen_type="int" ])
+AC_MSG_RESULT($cv_msgiovlen_type)
+AC_DEFINE_UNQUOTED([TYPE_MSGIOVLEN], $cv_msgiovlen_type, [Define to 'int' or type of struct msghdr.msg_iovlen.])
+
+# Check for yyunput
+AC_DEFUN([CHECK_YYUNPUT],
+[AC_MSG_CHECKING(whether lex can not generate yyunput)
+AC_CACHE_VAL(ac_cv_c_yy_unput,
+[ac_cv_c_yy_unput=no
+if $LEX --nounput -h 2>&1 | grep "do not generate" 2>&1 >/dev/null; then
+   ac_cv_c_yy_unput="yes"
+else
+   ac_cv_c_yy_unput="no"
+fi
+])
+
+AC_MSG_RESULT($ac_cv_c_yy_unput)
+if test $ac_cv_c_yy_unput = yes; then
+	LEX="$LEX --nounput"
+fi
+])dnl
+
+CHECK_YYUNPUT
+
 # Checks for libraries.
 AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
                                     [enable SSL (will check /usr/local/ssl
-                            /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr/sfw /usr)]),[
+                            /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /opt/local /usr/sfw /usr)]),[
         ],[
             withval="yes"
         ])
     if test x_$withval != x_no; then
         AC_MSG_CHECKING(for SSL)
         if test x_$withval = x_ -o x_$withval = x_yes; then
-            withval="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr/sfw /usr"
+            withval="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /opt/local /usr/sfw /usr"
         fi
         for dir in $withval; do
             ssldir="$dir"
@@ -349,10 +392,11 @@ AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
 	AC_SUBST(RUNTIME_PATH)
     fi
 AC_CHECK_HEADERS([openssl/ssl.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([openssl/err.h],,, [AC_INCLUDES_DEFAULT])
 
 # check for thread library.
 AC_ARG_WITH(pthreads, AC_HELP_STRING([--with-pthreads], 
- [use pthreads library, or --without--pthreads to disable threading support.]), 
+ [use pthreads library, or --without-pthreads to disable threading support.]), 
  [ ],[ withval="yes" ])
 ub_have_pthreads=no
 if test x_$withval != x_no; then
@@ -363,7 +407,7 @@ if test x_$withval != x_no; then
 		CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
 		CC="$PTHREAD_CC"
 		ub_have_pthreads=yes
-		AC_CHECK_TYPES(pthread_spinlock_t,,,[#include <pthread.h>])
+		AC_CHECK_TYPES([pthread_spinlock_t, pthread_rwlock_t],,,[#include <pthread.h>])
 		])
 fi
 
@@ -386,14 +430,16 @@ if test x_$withval != x_no; then
 	fi
 fi
 
+# check for library used by libevent after 1.3c
+AC_CHECK_LIB(rt, clock_gettime)
 # check for libevent
 AC_ARG_WITH(libevent, AC_HELP_STRING([--with-libevent=pathname],
-    [set path to libevent  (will check /usr/local /usr/lib /usr/pkg /usr/sfw /usr)]),
+    [set path to libevent  (will check /usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr)]),
     [ ],[ withval="yes" ])
 if test x_$withval != x_no; then
         AC_MSG_CHECKING(for libevent)
         if test x_$withval = x_ -o x_$withval = x_yes; then
-            withval="/usr/local /usr/lib /usr/pkg /usr/sfw /usr";
+            withval="/usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr";
         fi
         for dir in $withval; do
             thedir="$dir"
@@ -410,7 +456,10 @@ if test x_$withval != x_no; then
                 	CPPFLAGS="$CPPFLAGS -I$thedir";
             		LDFLAGS="$thedir/.libs/*.o $LDFLAGS";
 		else
-            		AC_MSG_ERROR(Cannot find the libevent library in $withval)
+            		AC_MSG_ERROR([Cannot find the libevent library in $withval
+You can restart configure with --with-libevent=no to use a builtin alternative.
+Please note that this alternative is not as capable as libevent when using
+many outgoing ports.  ])
 		fi
         else
             AC_MSG_RESULT(found in $thedir)
@@ -426,13 +475,40 @@ else
 	AC_DEFINE(USE_MINI_EVENT, 1, [Define if you want to use internal select based events])
 fi
 
+# set static linking if requested
+AC_SUBST(staticexe)
+staticexe=""
+AC_ARG_ENABLE(staticexe, AC_HELP_STRING([--enable-static-exe],
+	[ enable to compile executables statically against event, ldns libs, for debug purposes ]), 
+	, )
+if test x_$enable_static_exe = x_yes; then
+	staticexe="-static"
+fi
+
+# set lock checking if requested
+AC_ARG_ENABLE(lock_checks, AC_HELP_STRING([--enable-lock-checks],
+	[ enable to check lock and unlock calls, for debug purposes ]), 
+	, )
+if test x_$enable_lock_checks = x_yes; then
+	AC_DEFINE(ENABLE_LOCK_CHECKS, 1, [Define if you want to use debug lock checking (slow).])
+fi
+
+# set memory allocation checking if requested
+AC_ARG_ENABLE(alloc-checks, AC_HELP_STRING([--enable-alloc-checks],
+	[ enable to check memory allocation, for debug purposes ]), 
+	, )
+if test x_$enable_alloc_checks = x_yes; then
+	AC_DEFINE(UNBOUND_ALLOC_STATS, 1, [use statistics for allocs and frees, for debug use])
+else
+	AC_FUNC_MALLOC
+fi
+
 # check to see if libraries are needed for these functions.
 AC_CHECK_LIB(socket, socket)
 AC_CHECK_LIB(nsl, inet_pton)
 
 AC_FUNC_CHOWN
 AC_FUNC_FORK
-AC_FUNC_MALLOC
 AC_TYPE_SIGNAL
 AC_FUNC_FSEEKO
 AC_SYS_LARGEFILE
@@ -489,6 +565,10 @@ AH_BOTTOM([
 #include <sys/socket.h>
 #endif
 
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+
 #ifdef HAVE_NETINET_IN_H
 #include <netinet/in.h>
 #endif
@@ -501,6 +581,10 @@ AH_BOTTOM([
 #include <getopt.h>
 #endif
 
+#ifdef HAVE_OPENSSL_ERR_H
+#include <openssl/err.h>
+#endif
+
 #ifdef HAVE_ATTR_FORMAT
 #  define ATTR_FORMAT(archetype, string_index, first_to_check) \
     __attribute__ ((format (archetype, string_index, first_to_check)))
@@ -572,6 +656,25 @@ struct sockaddr_storage;
 
 #include "ldns/ldns.h"
 
+#ifdef UNBOUND_ALLOC_STATS
+#  define malloc(s) unbound_stat_malloc_log(s, __FILE__, __LINE__, __func__)
+#  define calloc(n,s) unbound_stat_calloc_log(n, s, __FILE__, __LINE__, __func__)
+#  define free(p) unbound_stat_free_log(p, __FILE__, __LINE__, __func__)
+#  define realloc(p,s) unbound_stat_realloc_log(p, s, __FILE__, __LINE__, __func__)
+void *unbound_stat_malloc(size_t size);
+void *unbound_stat_calloc(size_t nmemb, size_t size);
+void unbound_stat_free(void *ptr);
+void *unbound_stat_realloc(void *ptr, size_t size);
+void *unbound_stat_malloc_log(size_t size, const char* file, int line,
+	const char* func);
+void *unbound_stat_calloc_log(size_t nmemb, size_t size, const char* file,
+	int line, const char* func);
+void unbound_stat_free_log(void *ptr, const char* file, int line,
+	const char* func);
+void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
+	int line, const char* func);
+#endif /* UNBOUND_ALLOC_STATS */
+
 /** default port for DNS traffic. */
 #define UNBOUND_DNS_PORT 53
 ])

daemon/daemon.c

@@ -39,15 +39,19 @@
  * The daemon consists of global settings and a number of workers.
  */
 
-/** buffer size for network connections */
-#define BUFSZ 65552
-
 #include "config.h"
 #include "daemon/daemon.h"
 #include "daemon/worker.h"
 #include "util/log.h"
 #include "util/config_file.h"
+#include "util/data/msgreply.h"
+#include "util/storage/slabhash.h"
 #include "services/listen_dnsport.h"
+#include "services/cache/rrset.h"
+#include "services/cache/infra.h"
+#include "util/module.h"
+#include "iterator/iterator.h"
+#include "validator/validator.h"
 #include <signal.h>
 
 /** How many quit requests happened. */
@@ -56,7 +60,10 @@ static int sig_record_quit = 0;
 static int sig_record_reload = 0;
 
 /** used when no other sighandling happens, so we don't die
-  * when multiple signals in quick succession are sent to us. */
+  * when multiple signals in quick succession are sent to us. 
+  * @param sig: signal number.
+  * @return signal handler return type (void or int).
+  */
 static RETSIGTYPE record_sigh(int sig)
 {
 #ifdef LIBEVENT_SIGNAL_PROBLEM
@@ -115,8 +122,15 @@ daemon_init()
 		return NULL;
 	signal_handling_record();
 	checklock_start();
+	ERR_load_crypto_strings();
 	daemon->need_to_exit = 0;
-	alloc_init(&daemon->superalloc, NULL);
+	daemon->num_modules = 0;
+	if(!(daemon->env = (struct module_env*)calloc(1, 
+		sizeof(*daemon->env)))) {
+		free(daemon);
+		return NULL;
+	}
+	alloc_init(&daemon->superalloc, NULL, 0);
 	return daemon;	
 }
 
@@ -133,6 +147,137 @@ daemon_open_shared_ports(struct daemon* daemon)
 	return 1;
 }
 
+/** count number of modules (words) in the string */
+static int
+count_modules(const char* s)
+{
+	int num = 0;
+	if(!s)
+		return 0;
+	while(*s) {
+		/* skip whitespace */
+		while(*s && isspace((int)*s))
+			s++;
+		if(*s && !isspace((int)*s)) {
+			/* skip identifier */
+			num++;
+			while(*s && !isspace((int)*s))
+				s++;
+		}
+	}
+	return num;
+}
+
+/**
+ * Get funcblock for module name
+ * @param str: string with module name. Advanced to next value on success.
+ * @return funcblock or NULL on error.
+ */
+static struct module_func_block*
+daemon_module_factory(const char** str)
+{
+	/* these are the modules available */
+	int num = 2;
+	const char* names[] = {"iterator", "validator", NULL};
+	struct module_func_block* (*fb[])(void) = 
+		{&iter_get_funcblock, &val_get_funcblock, NULL};
+
+	int i;
+	const char* s = *str;
+	while(*s && isspace((int)*s))
+		s++;
+	for(i=0; i<num; i++) {
+		if(strncmp(names[i], s, strlen(names[i])) == 0) {
+			s += strlen(names[i]);
+			*str = s;
+			return (*fb[i])();
+		}
+	}
+	return NULL;
+}
+
+/**
+ * Read config file module settings and set up the modfunc block
+ * @param daemon: the daemon.
+ * @return false on error
+ */
+static int
+daemon_config_modules(struct daemon* daemon)
+{
+	const char* str = daemon->cfg->module_conf;
+	int i;
+	verbose(VERB_DETAIL, "module config: \"%s\"", str);
+	daemon->num_modules = count_modules(str);
+	if(daemon->num_modules == 0) {
+		log_err("error: no modules specified");
+		return 0;
+	}
+	if(daemon->num_modules > MAX_MODULE) {
+		log_err("error: too many modules (%d max %d)",
+			daemon->num_modules, MAX_MODULE);
+		return 0;
+	}
+	daemon->modfunc = (struct module_func_block**)calloc((size_t)
+		daemon->num_modules, sizeof(struct module_func_block*));
+	if(!daemon->modfunc) {
+		log_err("out of memory");
+		return 0;
+	}
+	for(i=0; i<daemon->num_modules; i++) {
+		daemon->modfunc[i] = daemon_module_factory(&str);
+		if(!daemon->modfunc[i]) {
+			log_err("Unknown value for first module in: '%s'",
+				str);
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/**
+ * Desetup the modules, deinit, delete.
+ * @param daemon: the daemon.
+ */
+static void
+daemon_desetup_modules(struct daemon* daemon)
+{
+	int i;
+	for(i=0; i<daemon->num_modules; i++) {
+		(*daemon->modfunc[i]->deinit)(daemon->env, i);
+	}
+	daemon->num_modules = 0;
+	free(daemon->modfunc);
+	daemon->modfunc = 0;
+}
+
+/**
+ * Setup modules. Assigns ids and calls module_init.
+ * @param daemon: the daemon
+ */
+static void daemon_setup_modules(struct daemon* daemon)
+{
+	int i;
+	if(daemon->num_modules != 0)
+		daemon_desetup_modules(daemon);
+	/* fixed setup of the modules */
+	if(!daemon_config_modules(daemon)) {
+		fatal_exit("failed to setup modules");
+	}
+	daemon->env->cfg = daemon->cfg;
+	daemon->env->alloc = &daemon->superalloc;
+	daemon->env->worker = NULL;
+	daemon->env->send_packet = &worker_send_packet;
+	daemon->env->send_query = &worker_send_query;
+	daemon->env->need_to_validate = 0; /* set by module init below */
+	for(i=0; i<daemon->num_modules; i++) {
+		log_info("init module %d: %s", i, daemon->modfunc[i]->name);
+		if(!(*daemon->modfunc[i]->init)(daemon->env, i)) {
+			fatal_exit("module init for module %s failed",
+				daemon->modfunc[i]->name);
+		}
+	}
+}
+
 /**
  * Allocate empty worker structures. With backptr and thread-number,
  * from 0..numthread initialised. Used as user arguments to new threads.
@@ -148,7 +293,7 @@ daemon_create_workers(struct daemon* daemon)
 		sizeof(struct worker*));
 	for(i=0; i<daemon->num; i++) {
 		if(!(daemon->workers[i] = worker_create(daemon, i)))
-			fatal_exit("malloc failure");
+			fatal_exit("could not create worker");
 	}
 }
 
@@ -182,7 +327,6 @@ static void*
 thread_start(void* arg)
 {
 	struct worker* worker = (struct worker*)arg;
-	int num = worker->thread_num;
 	log_thread_set(&worker->thread_num);
 	ub_thread_blocksigs();
 #if !defined(HAVE_PTHREAD) && !defined(HAVE_SOLARIS_THREADS)
@@ -190,10 +334,9 @@ thread_start(void* arg)
 	close(worker->cmd_send_fd);
 	worker->cmd_send_fd = -1;
 	close_other_pipes(worker->daemon, worker->thread_num);
-#endif /* no threads */
-	if(!worker_init(worker, worker->daemon->cfg, worker->daemon->ports,
-		BUFSZ, 0))
-		fatal_exit("Could not initialize thread #%d", num);
+#endif
+	if(!worker_init(worker, worker->daemon->cfg, worker->daemon->ports, 0))
+		fatal_exit("Could not initialize thread");
 
 	worker_work(worker);
 	return NULL;
@@ -250,6 +393,10 @@ void
 daemon_fork(struct daemon* daemon)
 {
 	log_assert(daemon);
+
+	/* setup modules */
+	daemon_setup_modules(daemon);
+
 	/* first create all the worker structures, so we can pass
 	 * them to the newly created threads. 
 	 */
@@ -263,8 +410,7 @@ daemon_fork(struct daemon* daemon)
 	/* Special handling for the main thread. This is the thread
 	 * that handles signals.
 	 */
-	if(!worker_init(daemon->workers[0], daemon->cfg, daemon->ports,
-		BUFSZ, 1))
+	if(!worker_init(daemon->workers[0], daemon->cfg, daemon->ports, 1))
 		fatal_exit("Could not initialize main thread");
 	signal_handling_playback(daemon->workers[0]);
 
@@ -302,10 +448,24 @@ daemon_delete(struct daemon* daemon)
 {
 	if(!daemon)
 		return;
+	daemon_desetup_modules(daemon);
 	listening_ports_free(daemon->ports);
+	if(daemon->env) {
+		slabhash_delete(daemon->env->msg_cache);
+		rrset_cache_delete(daemon->env->rrset_cache);
+		infra_delete(daemon->env->infra_cache);
+	}
 	alloc_clear(&daemon->superalloc);
 	free(daemon->cwd);
 	free(daemon->pidfile);
+	free(daemon->env);
 	free(daemon);
+	/* libcrypto cleanup */
+	/* CONF_modules_unload(1); */
+	EVP_cleanup();
+	/* ENGINE_cleanup(); */
+	CRYPTO_cleanup_all_ex_data(); /* safe, no more threads right now */
+	ERR_remove_state(0);
+	ERR_free_strings();
 	checklock_stop();
 }

daemon/daemon.h

@@ -47,6 +47,9 @@
 struct config_file;
 struct worker;
 struct listen_port;
+struct slabhash;
+struct module_env;
+struct rrset_cache;
 
 /**
  * Structure holding worker list.
@@ -71,6 +74,12 @@ struct daemon {
 	int need_to_exit;
 	/** master allocation cache */
 	struct alloc_cache superalloc;
+	/** the module environment master value, copied and changed by threads*/
+	struct module_env* env;
+	/** number of modules active, ids from 0 to num-1. */
+	int num_modules;
+	/** the module callbacks, array of num_modules length */
+	struct module_func_block** modfunc;
 };
 
 /**

daemon/stats.c

@@ -0,0 +1,72 @@
+/*
+ * daemon/stats.c - collect runtime performance indicators.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file describes the data structure used to collect runtime performance
+ * numbers. These 'statistics' may be of interest to the operator.
+ */
+#include "config.h"
+#include "daemon/stats.h"
+#include "daemon/worker.h"
+#include "services/mesh.h"
+
+void server_stats_init(struct server_stats* stats)
+{
+	memset(stats, 0, sizeof(stats));
+}
+
+void server_stats_querymiss(struct server_stats* stats, struct worker* worker)
+{
+	stats->num_queries_missed_cache++;
+	stats->sum_query_list_size += worker->env.mesh->all.count;
+	if(worker->env.mesh->all.count > stats->max_query_list_size)
+		stats->max_query_list_size = worker->env.mesh->all.count;
+}
+
+void server_stats_log(struct server_stats* stats, int threadnum)
+{
+	log_info("server stats for thread %d: %u queries, %u from cache", 
+		threadnum, (unsigned)stats->num_queries, 
+		(unsigned)(stats->num_queries - 
+			stats->num_queries_missed_cache));
+	log_info("server stats for thread %d: requestlist max %u avg %g "
+		"exceeded %u", threadnum, (unsigned)stats->max_query_list_size,
+		stats->num_queries_missed_cache?
+			(double)stats->sum_query_list_size/
+			stats->num_queries_missed_cache : 0.0,
+		(unsigned)stats->num_query_list_exceeded);
+}

daemon/stats.h

@@ -0,0 +1,77 @@
+/*
+ * daemon/stats.h - collect runtime performance indicators.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file describes the data structure used to collect runtime performance
+ * numbers. These 'statistics' may be of interest to the operator.
+ */
+
+#ifndef DAEMON_STATS_H
+#define DAEMON_STATS_H
+struct worker;
+
+/** per worker statistics */
+struct server_stats {
+	/** number of queries from clients received. */
+	size_t num_queries;
+	/** number of queries that had a cache-miss. */
+	size_t num_queries_missed_cache;
+
+	/**
+	 * Sum of the querylistsize of the worker for 
+	 * every query that missed cache. To calculate average.
+	 */
+	size_t sum_query_list_size;
+	/** max value of query list size reached. */
+	size_t max_query_list_size;
+	/** number of times that the query_list_size was insufficient */
+	size_t num_query_list_exceeded;
+};
+
+/** 
+ * Initialize server stats to 0.
+ * @param stats: what to init (this is alloced by the caller).
+ */
+void server_stats_init(struct server_stats* stats);
+
+/** add query if it missed the cache */
+void server_stats_querymiss(struct server_stats* stats, struct worker* worker);
+
+/** display the stats to the log */
+void server_stats_log(struct server_stats* stats, int threadnum);
+
+#endif /* DAEMON_STATS_H */

daemon/unbound.c

@@ -44,9 +44,19 @@
 #include "util/log.h"
 #include "daemon/daemon.h"
 #include "util/config_file.h"
+#include "util/storage/slabhash.h"
+#include "services/listen_dnsport.h"
+#include "services/cache/rrset.h"
+#include "services/cache/infra.h"
+#include "util/data/msgreply.h"
+#include "util/module.h"
 #include <signal.h>
 #include <fcntl.h>
 #include <pwd.h>
+#include <sys/resource.h>
+
+/** global debug value to keep track of heap memory allocation */
+void* unbound_start_brk = 0;
 
 /** print usage. */
 static void usage()
@@ -62,11 +72,48 @@ static void usage()
 	printf("Report bugs to %s\n", PACKAGE_BUGREPORT);
 }
 
+/** check file descriptor count */
+static void
+checkrlimits(struct config_file* cfg)
+{
+	int list = ((cfg->do_ip4?1:0) + (cfg->do_ip6?1:0)) * 
+		((cfg->do_udp?1:0) + (cfg->do_tcp?1 + 
+			(int)cfg->incoming_num_tcp:0));
+	size_t ifs = (size_t)(cfg->num_ifs==0?1:cfg->num_ifs);
+	size_t listen_num = list*ifs;
+	size_t outnum = cfg->outgoing_num_ports*ifs + cfg->outgoing_num_tcp;
+	size_t misc = 4; /* logfile, pidfile, stdout... */
+	size_t perthread = listen_num + outnum + 2/*cmdpipe*/ + 2/*libevent*/
+		+ misc; 
+#if !defined(HAVE_PTHREAD) && !defined(HAVE_SOLARIS_THREADS)
+	int numthread = 1; /* it forks */
+#else
+	int numthread = cfg->num_threads;
+#endif
+	size_t total = numthread * perthread + misc;
+	struct rlimit rlim;
+	if(getrlimit(RLIMIT_NOFILE, &rlim) < 0) {
+		log_warn("getrlimit: %s", strerror(errno));
+		return;
+	}
+	if(rlim.rlim_cur == (rlim_t)RLIM_INFINITY)
+		return;
+	if((size_t)rlim.rlim_cur < total) {
+		log_err("Not enough sockets available. Increase "
+			"ulimit(open files).");
+		log_err("or decrease number of threads, outgoing num ports, "
+			"outgoing num tcp or number of interfaces");
+		log_err("estimate %u fds high mark, %u available", 
+			(unsigned)total, (unsigned)rlim.rlim_cur);
+		fatal_exit("Not enough file descriptors available");
+	}
+}
+
 /** to changedir, logfile */
 static void
 apply_dir(struct daemon* daemon, struct config_file* cfg, int cmdline_verbose)
 {
-	/* apply changes if they have changed */
+	/* apply if they have changed */
 	daemon->cfg = cfg;
 	verbosity = cmdline_verbose + cfg->verbosity;
 	if(cfg->directory && cfg->directory[0]) {
@@ -77,12 +124,34 @@ apply_dir(struct daemon* daemon, struct config_file* cfg, int cmdline_verbose)
 			}
 			free(daemon->cwd);
 			if(!(daemon->cwd = strdup(cfg->directory)))
-				fatal_exit("malloc failed");
+				log_err("cwd: malloc failed");
 		}
 	}
+	if(!daemon->env->msg_cache ||
+	   cfg->msg_cache_size != slabhash_get_size(daemon->env->msg_cache) ||
+	   cfg->msg_cache_slabs != daemon->env->msg_cache->size) {
+		slabhash_delete(daemon->env->msg_cache);
+		daemon->env->msg_cache = slabhash_create(cfg->msg_cache_slabs, 
+			HASH_DEFAULT_STARTARRAY, cfg->msg_cache_size, 
+			msgreply_sizefunc, query_info_compare,
+			query_entry_delete, reply_info_delete, NULL);
+		if(!daemon->env->msg_cache) {
+			fatal_exit("malloc failure updating config settings");
+		}
+	}
+	if((daemon->env->rrset_cache = rrset_cache_adjust(
+		daemon->env->rrset_cache, cfg, &daemon->superalloc)) == 0)
+		fatal_exit("malloc failure updating config settings");
+	if((daemon->env->infra_cache = infra_adjust(daemon->env->infra_cache, 
+		cfg))==0)
+		fatal_exit("malloc failure updating config settings");
+	checkrlimits(cfg);
 }
 
-/** Read existing pid from pidfile. */
+/** Read existing pid from pidfile. 
+ * @param file: file name of pid file.
+ * @return: the pid from the file or -1 if none.
+ */
 static pid_t
 readpid (const char* file)
 {
@@ -93,10 +162,16 @@ readpid (const char* file)
 	ssize_t l;
 
 	if ((fd = open(file, O_RDONLY)) == -1) {
+		if(errno != ENOENT)
+			log_err("Could not read pidfile %s: %s",
+				file, strerror(errno));
 		return -1;
 	}
 
 	if (((l = read(fd, pidbuf, sizeof(pidbuf)))) == -1) {
+		if(errno != ENOENT)
+			log_err("Could not read pidfile %s: %s",
+				file, strerror(errno));
 		close(fd);
 		return -1;
 	}
@@ -105,7 +180,6 @@ readpid (const char* file)
 
 	/* Empty pidfile means no pidfile... */
 	if (l == 0) {
-		errno = ENOENT;
 		return -1;
 	}
 
@@ -117,7 +191,10 @@ readpid (const char* file)
 	return pid;
 }
 
-/** write pid to file. */
+/** write pid to file. 
+ * @param pidfile: file name of pid file.
+ * @param pid: pid to write to file.
+ */
 static void
 writepid (const char* pidfile, pid_t pid)
 {
@@ -143,13 +220,8 @@ static void
 checkoldpid(struct config_file* cfg)
 {
 	pid_t old;
-	if((old = readpid(cfg->pidfile)) == -1) {
-		if(errno != ENOENT) {
-			log_err("Could not read pidfile %s: %s",
-				cfg->pidfile, strerror(errno));
-		}
-	} else {
-		/** see if it is still alive */
+	if((old = readpid(cfg->pidfile)) != -1) {
+		/* see if it is still alive */
 		if(kill(old, 0) == 0 || errno == EPERM)
 			log_warn("unbound is already running as pid %u.", 
 				(unsigned)old);
@@ -158,6 +230,34 @@ checkoldpid(struct config_file* cfg)
 	}
 }
 
+/** detach from command line */
+static void
+detach(struct config_file* cfg)
+{
+	int fd;
+	/* Take off... */
+	switch (fork()) {
+		case 0:
+			break;
+		case -1:
+			unlink(cfg->pidfile);
+			fatal_exit("fork failed: %s", strerror(errno));
+		default:
+			/* exit interactive session */
+			exit(0);
+	}
+	/* detach */
+	if(setsid() == -1)
+		fatal_exit("setsid() failed: %s", strerror(errno));
+	if ((fd = open("/dev/null", O_RDWR, 0)) != -1) {
+		(void)dup2(fd, STDIN_FILENO);
+		(void)dup2(fd, STDOUT_FILENO);
+		(void)dup2(fd, STDERR_FILENO);
+		if (fd > 2)
+			(void)close(fd);
+	}
+}
+
 /** daemonize, drop user priviliges and chroot if needed */
 static void
 do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
@@ -186,32 +286,12 @@ do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
 	/* init logfile just before fork */
 	log_init(cfg->logfile);
 	if(!debug_mode && cfg->do_daemonize) {
-		int fd;
-		/* Take off... */
-		switch (fork()) {
-			case 0:
-				break;
-			case -1:
-				unlink(cfg->pidfile);
-				fatal_exit("fork failed: %s", strerror(errno));
-			default:
-				/* exit interactive session */
-				exit(0);
-		}
-		/* detach */
-		if(setsid() == -1)
-			fatal_exit("setsid() failed: %s", strerror(errno));
-		if ((fd = open("/dev/null", O_RDWR, 0)) != -1) {
-			(void)dup2(fd, STDIN_FILENO);
-			(void)dup2(fd, STDOUT_FILENO);
-			(void)dup2(fd, STDERR_FILENO);
-			if (fd > 2)
-				(void)close(fd);
-		}
+		detach(cfg);
 	}
 	if(cfg->pidfile && cfg->pidfile[0]) {
 		writepid(cfg->pidfile, getpid());
-		daemon->pidfile = strdup(cfg->pidfile);
+		if(!(daemon->pidfile = strdup(cfg->pidfile)))
+			log_err("pidf: malloc failed");
 	}
 }
 
@@ -222,7 +302,8 @@ do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
  *    These increase verbosity as specified in the config file.
  * @param debug_mode: if set, do not daemonize.
  */
-static void run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode)
+static void 
+run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode)
 {
 	struct config_file* cfg = NULL;
 	struct daemon* daemon = NULL;
@@ -282,6 +363,9 @@ main(int argc, char* argv[])
 	int cmdline_verbose = 0;
 	int debug_mode = 0;
 
+	/* take debug snapshot of heap */
+	unbound_start_brk = sbrk(0);
+
 	log_init(NULL);
 	/* parse the options */
 	while( (c=getopt(argc, argv, "c:dhv")) != -1) {
@@ -312,5 +396,6 @@ main(int argc, char* argv[])
 	}
 
 	run_daemon(cfgfile, cmdline_verbose, debug_mode);
+	log_init(NULL); /* close logfile */
 	return 0;
 }

daemon/worker.c

@@ -47,8 +47,19 @@
 #include "daemon/daemon.h"
 #include "util/netevent.h"
 #include "util/config_file.h"
+#include "util/module.h"
+#include "util/region-allocator.h"
+#include "util/storage/slabhash.h"
 #include "services/listen_dnsport.h"
 #include "services/outside_network.h"
+#include "services/outbound_list.h"
+#include "services/cache/rrset.h"
+#include "services/cache/infra.h"
+#include "services/cache/dns.h"
+#include "services/mesh.h"
+#include "util/data/msgparse.h"
+#include "util/data/msgencode.h"
+#include "util/data/dname.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #  include <sys/types.h>
@@ -56,10 +67,98 @@
 #include <netdb.h>
 #include <signal.h>
 
-/** timeout in seconds for UDP queries to auth servers. TODO: proper rtt */
-#define UDP_QUERY_TIMEOUT 5
-/** the size of ID and flags, opcode, rcode in dns packet */
-#define ID_AND_FLAGS 4
+/** Size of an UDP datagram */
+#define NORMAL_UDP_SIZE	512 /* bytes */
+
+#ifdef UNBOUND_ALLOC_STATS
+/** measure memory leakage */
+static void
+debug_memleak(size_t accounted, size_t heap, 
+	size_t total_alloc, size_t total_free)
+{
+	static int init = 0;
+	static size_t base_heap, base_accounted, base_alloc, base_free;
+	size_t base_af, cur_af, grow_af, grow_acc;
+	if(!init) {
+		init = 1;
+		base_heap = heap;
+		base_accounted = accounted;
+		base_alloc = total_alloc;
+		base_free = total_free;
+	}
+	base_af = base_alloc - base_free;
+	cur_af = total_alloc - total_free;
+	grow_af = cur_af - base_af;
+	grow_acc = accounted - base_accounted;
+	log_info("Leakage: %d leaked. growth: %u use, %u acc, %u heap",
+		(int)(grow_af - grow_acc), (unsigned)grow_af, 
+		(unsigned)grow_acc, (unsigned)(heap - base_heap));
+}
+
+/** give debug heap size indication */
+static void
+debug_total_mem(size_t calctotal)
+{
+	extern void* unbound_start_brk;
+	extern size_t unbound_mem_alloc, unbound_mem_freed;
+	void* cur = sbrk(0);
+	int total = cur-unbound_start_brk;
+	log_info("Total heap memory estimate: %u  total-alloc: %u  "
+		"total-free: %u", (unsigned)total, 
+		(unsigned)unbound_mem_alloc, (unsigned)unbound_mem_freed);
+	debug_memleak(calctotal, (size_t)total, 
+		unbound_mem_alloc, unbound_mem_freed);
+}
+#endif /* UNBOUND_ALLOC_STATS */
+
+/** Report on memory usage by this thread and global */
+void
+worker_mem_report(struct worker* ATTR_UNUSED(worker), 
+	struct serviced_query* ATTR_UNUSED(cur_serv))
+{
+#ifdef UNBOUND_ALLOC_STATS
+	/* debug func in validator module */
+	size_t total, front, back, mesh, msg, rrset, infra, ac, superac;
+	size_t me, iter, val;
+	int i;
+	if(verbosity < VERB_ALGO) 
+		return;
+	front = listen_get_mem(worker->front);
+	back = outnet_get_mem(worker->back);
+	msg = slabhash_get_mem(worker->env.msg_cache);
+	rrset = slabhash_get_mem(&worker->env.rrset_cache->table);
+	infra = infra_get_mem(worker->env.infra_cache);
+	mesh = mesh_get_mem(worker->env.mesh);
+	ac = alloc_get_mem(&worker->alloc);
+	superac = alloc_get_mem(&worker->daemon->superalloc);
+	iter = 0;
+	val = 0;
+	for(i=0; i<worker->env.mesh->num_modules; i++) {
+		if(strcmp(worker->env.mesh->modfunc[i]->name, "validator")==0)
+			val += (*worker->env.mesh->modfunc[i]->get_mem)
+				(&worker->env, i);
+		else	iter += (*worker->env.mesh->modfunc[i]->get_mem)
+				(&worker->env, i);
+	}
+	me = sizeof(*worker) + sizeof(*worker->base) + sizeof(*worker->comsig)
+		+ comm_point_get_mem(worker->cmd_com) + 
+		sizeof(worker->rndstate) + region_get_mem(worker->scratchpad)+
+		sizeof(*worker->env.scratch_buffer) + 
+		ldns_buffer_capacity(worker->env.scratch_buffer);
+	if(cur_serv) {
+		me += serviced_get_mem(cur_serv);
+	}
+	total = front+back+mesh+msg+rrset+infra+iter+val+ac+superac+me;
+	log_info("Memory conditions: %u front=%u back=%u mesh=%u msg=%u "
+		"rrset=%u infra=%u iter=%u val=%u "
+		"alloccache=%u globalalloccache=%u me=%u",
+		(unsigned)total, (unsigned)front, (unsigned)back, 
+		(unsigned)mesh, (unsigned)msg, (unsigned)rrset, 
+		(unsigned)infra, (unsigned)iter, (unsigned)val, (unsigned)ac, 
+		(unsigned)superac, (unsigned)me);
+	debug_total_mem(total);
+#endif /* UNBOUND_ALLOC_STATS */
+}
 
 void 
 worker_send_cmd(struct worker* worker, ldns_buffer* buffer,
@@ -75,74 +174,86 @@ worker_send_cmd(struct worker* worker, ldns_buffer* buffer,
 		log_err("write socket: %s", strerror(errno));
 }
 
-/** reply to query with given error code */
-static void 
-replyerror(int r, struct worker* worker)
-{
-	LDNS_QR_SET(ldns_buffer_begin(worker->query_reply.c->buffer));
-	LDNS_RCODE_SET(ldns_buffer_begin(worker->query_reply.c->buffer), r);
-	comm_point_send_reply(&worker->query_reply);
-	if(worker->num_requests == 1)  {
-		/* no longer at max, start accepting again. */
-		listen_resume(worker->front);
-	}
-	worker->num_requests --;
-}
-
 /** process incoming replies from the network */
 static int 
 worker_handle_reply(struct comm_point* c, void* arg, int error, 
-	struct comm_reply* ATTR_UNUSED(reply_info))
+	struct comm_reply* reply_info)
 {
-	struct worker* worker = (struct worker*)arg;
-	verbose(VERB_DETAIL, "reply to query with stored ID %d", 
-		worker->query_id);
-	LDNS_ID_SET(ldns_buffer_begin(worker->query_reply.c->buffer),
-		worker->query_id);
+	struct module_qstate* q = (struct module_qstate*)arg;
+	struct worker* worker = q->env->worker;
+	struct outbound_entry e;
+	e.qstate = q;
+	e.qsent = NULL;
+
 	if(error != 0) {
-		replyerror(LDNS_RCODE_SERVFAIL, worker);
+		mesh_report_reply(worker->env.mesh, &e, 0, reply_info);
+		worker_mem_report(worker, NULL);
 		return 0;
 	}
-	/* woohoo a reply! */
-	ldns_buffer_clear(worker->query_reply.c->buffer);
-	ldns_buffer_skip(worker->query_reply.c->buffer, ID_AND_FLAGS);
-	ldns_buffer_write(worker->query_reply.c->buffer, 
-		ldns_buffer_at(c->buffer, ID_AND_FLAGS), 
-		ldns_buffer_limit(c->buffer) - ID_AND_FLAGS);
-	LDNS_QR_SET(ldns_buffer_begin(worker->query_reply.c->buffer));
-	ldns_buffer_flip(worker->query_reply.c->buffer);
-	comm_point_send_reply(&worker->query_reply);
-	if(worker->num_requests == 1)  {
-		/* no longer at max, start accepting again. */
-		listen_resume(worker->front);
+	/* sanity check. */
+	if(!LDNS_QR_WIRE(ldns_buffer_begin(c->buffer))
+		|| LDNS_OPCODE_WIRE(ldns_buffer_begin(c->buffer)) != 
+			LDNS_PACKET_QUERY
+		|| LDNS_QDCOUNT(ldns_buffer_begin(c->buffer)) > 1) {
+		/* error becomes timeout for the module as if this reply
+		 * never arrived. */
+		mesh_report_reply(worker->env.mesh, &e, 0, reply_info);
+		worker_mem_report(worker, NULL);
+		return 0;
 	}
-	worker->num_requests --;
+	mesh_report_reply(worker->env.mesh, &e, 1, reply_info);
+	worker_mem_report(worker, NULL);
 	return 0;
 }
 
-/** process incoming request */
-static void 
-worker_process_query(struct worker* worker) 
+/** process incoming serviced query replies from the network */
+static int 
+worker_handle_service_reply(struct comm_point* c, void* arg, int error, 
+	struct comm_reply* reply_info)
 {
-	/* query the forwarding address */
-	worker->query_id = LDNS_ID_WIRE(ldns_buffer_begin(
-		worker->query_reply.c->buffer));
-	verbose(VERB_DETAIL, "process_query ID %d", worker->query_id);
-	pending_udp_query(worker->back, worker->query_reply.c->buffer, 
-		&worker->fwd_addr, worker->fwd_addrlen, UDP_QUERY_TIMEOUT,
-		worker_handle_reply, worker, worker->rndstate);
+	struct outbound_entry* e = (struct outbound_entry*)arg;
+	struct worker* worker = e->qstate->env->worker;
+	struct serviced_query *sq = e->qsent;
+
+	verbose(VERB_ALGO, "worker svcd callback for qstate %p", e->qstate);
+	if(error != 0) {
+		mesh_report_reply(worker->env.mesh, e, 0, reply_info);
+		worker_mem_report(worker, sq);
+		return 0;
+	}
+	/* sanity check. */
+	if(!LDNS_QR_WIRE(ldns_buffer_begin(c->buffer))
+		|| LDNS_OPCODE_WIRE(ldns_buffer_begin(c->buffer)) != 
+			LDNS_PACKET_QUERY
+		|| LDNS_QDCOUNT(ldns_buffer_begin(c->buffer)) > 1) {
+		/* error becomes timeout for the module as if this reply
+		 * never arrived. */
+		verbose(VERB_ALGO, "worker: bad reply handled as timeout");
+		mesh_report_reply(worker->env.mesh, e, 0, reply_info);
+		worker_mem_report(worker, sq);
+		return 0;
+	}
+	mesh_report_reply(worker->env.mesh, e, 1, reply_info);
+	worker_mem_report(worker, sq);
+	return 0;
 }
 
 /** check request sanity. Returns error code, 0 OK, or -1 discard. 
  * @param pkt: the wire packet to examine for sanity.
+ * @param worker: parameters for checking.
 */
 static int 
-worker_check_request(ldns_buffer* pkt)
+worker_check_request(ldns_buffer* pkt, struct worker* worker)
 {
 	if(ldns_buffer_limit(pkt) < LDNS_HEADER_SIZE) {
 		verbose(VERB_DETAIL, "request too short, discarded");
 		return -1;
 	}
+	if(ldns_buffer_limit(pkt) > NORMAL_UDP_SIZE && 
+		worker->daemon->cfg->harden_large_queries) {
+		verbose(VERB_DETAIL, "request too large, discarded");
+		return -1;
+	}
 	if(LDNS_QR_WIRE(ldns_buffer_begin(pkt))) {
 		verbose(VERB_DETAIL, "request has QR bit on, discarded");
 		return -1;
@@ -167,7 +278,7 @@ worker_check_request(ldns_buffer* pkt)
 			LDNS_NSCOUNT(ldns_buffer_begin(pkt)));
 		return LDNS_RCODE_FORMERR;
 	}
-	if(LDNS_ARCOUNT(ldns_buffer_begin(pkt)) != 0) {
+	if(LDNS_ARCOUNT(ldns_buffer_begin(pkt)) > 1) {
 		verbose(VERB_DETAIL, "request wrong nr ar=%d", 
 			LDNS_ARCOUNT(ldns_buffer_begin(pkt)));
 		return LDNS_RCODE_FORMERR;
@@ -175,7 +286,12 @@ worker_check_request(ldns_buffer* pkt)
 	return 0;
 }
 
-/** process control messages from the main thread. */
+/** process control messages from the main thread. 
+ * @param c: comm point to read from.
+ * @param arg: worker.
+ * @param error: error status of comm point.
+ * @param reply_info: not used.
+ */
 static int 
 worker_handle_control_cmd(struct comm_point* c, void* arg, int error, 
 	struct comm_reply* ATTR_UNUSED(reply_info))
@@ -205,6 +321,340 @@ worker_handle_control_cmd(struct comm_point* c, void* arg, int error,
 	return 0;
 }
 
+/** check if a delegation is secure */
+static enum sec_status
+check_delegation_secure(struct reply_info *rep) 
+{
+	/* return smallest security status */
+	size_t i;
+	enum sec_status sec = sec_status_secure;
+	enum sec_status s;
+	size_t num = rep->an_numrrsets + rep->ns_numrrsets;
+	/* check if answer and authority are OK */
+	for(i=0; i<num; i++) {
+		s = ((struct packed_rrset_data*)rep->rrsets[i]->entry.data)
+			->security;
+		if(s < sec)
+			sec = s;
+	}
+	/* in additional, only unchecked triggers revalidation */
+	for(i=num; i<rep->rrset_count; i++) {
+		s = ((struct packed_rrset_data*)rep->rrsets[i]->entry.data)
+			->security;
+		if(s == sec_status_unchecked)
+			return s;
+	}
+	return sec;
+}
+
+/** remove nonsecure from a delegation referral additional section */
+static void
+deleg_remove_nonsecure_additional(struct reply_info* rep)
+{
+	/* we can simply edit it, since we are working in the scratch region */
+	size_t i;
+	enum sec_status s;
+
+	for(i = rep->an_numrrsets+rep->ns_numrrsets; i<rep->rrset_count; i++) {
+		s = ((struct packed_rrset_data*)rep->rrsets[i]->entry.data)
+			->security;
+		if(s != sec_status_secure) {
+			memmove(rep->rrsets+i, rep->rrsets+i+1, 
+				sizeof(struct ub_packed_rrset_key*)* 
+				(rep->rrset_count - i - 1));
+			rep->ar_numrrsets--; 
+			rep->rrset_count--;
+		}
+	}
+}
+
+/** answer nonrecursive query from the cache */
+static int
+answer_norec_from_cache(struct worker* worker, struct query_info* qinfo,
+	uint16_t id, uint16_t flags, struct comm_reply* repinfo, 
+	struct edns_data* edns)
+{
+	/* for a nonrecursive query return either:
+	 * 	o an error (servfail; we try to avoid this)
+	 * 	o a delegation (closest we have; this routine tries that)
+	 * 	o the answer (checked by answer_from_cache) 
+	 *
+	 * So, grab a delegation from the rrset cache. 
+	 * Then check if it needs validation, if so, this routine fails,
+	 * so that iterator can prime and validator can verify rrsets.
+	 */
+	uint16_t udpsize = edns->udp_size;
+	int secure = 0;
+	uint32_t timenow = (uint32_t)time(0);
+	int must_validate = !(flags&BIT_CD) && worker->env.need_to_validate;
+	struct dns_msg *msg = NULL;
+	struct delegpt *dp;
+
+	dp = dns_cache_find_delegation(&worker->env, qinfo->qname, 
+		qinfo->qname_len, qinfo->qtype, qinfo->qclass,
+		worker->scratchpad, &msg, timenow);
+	if(!dp) { /* no delegation, need to reprime */
+		region_free_all(worker->scratchpad);
+		return 0;
+	}
+	if(must_validate) {
+		switch(check_delegation_secure(msg->rep)) {
+		case sec_status_unchecked:
+			/* some rrsets have not been verified yet, go and 
+			 * let validator do that */
+			region_free_all(worker->scratchpad);
+			return 0;
+		case sec_status_bogus:
+			/* some rrsets are bogus, reply servfail */
+			edns->edns_version = EDNS_ADVERTISED_VERSION;
+			edns->udp_size = EDNS_ADVERTISED_SIZE;
+			edns->ext_rcode = 0;
+			edns->bits &= EDNS_DO;
+			error_encode(repinfo->c->buffer, LDNS_RCODE_SERVFAIL, 
+				&msg->qinfo, id, flags, edns);
+			region_free_all(worker->scratchpad);
+			return 1;
+		case sec_status_secure:
+			/* all rrsets are secure */
+			/* remove non-secure rrsets from the add. section*/
+			if(worker->env.cfg->val_clean_additional)
+				deleg_remove_nonsecure_additional(msg->rep);
+			secure = 1;
+			break;
+		case sec_status_indeterminate:
+		case sec_status_insecure:
+		default:
+			/* not secure */
+			secure = 0;
+			break;
+		}
+	}
+	/* return this delegation from the cache */
+	edns->edns_version = EDNS_ADVERTISED_VERSION;
+	edns->udp_size = EDNS_ADVERTISED_SIZE;
+	edns->ext_rcode = 0;
+	edns->bits &= EDNS_DO;
+	msg->rep->flags |= BIT_QR|BIT_RA;
+	if(!reply_info_answer_encode(&msg->qinfo, msg->rep, id, flags, 
+		repinfo->c->buffer, 0, 1, worker->scratchpad,
+		udpsize, edns, (int)(edns->bits & EDNS_DO), secure)) {
+		error_encode(repinfo->c->buffer, LDNS_RCODE_SERVFAIL, 
+			&msg->qinfo, id, flags, edns);
+	}
+	region_free_all(worker->scratchpad);
+	return 1;
+}
+
+/** check cname chain in cache reply */
+static int
+check_cache_chain(struct reply_info* rep) {
+	/* check only answer section rrs for matching cname chain.
+	 * the cache may return changed rdata, but owner names are untouched.*/
+	size_t i;
+	uint8_t* sname = rep->rrsets[0]->rk.dname;
+	size_t snamelen = rep->rrsets[0]->rk.dname_len;
+	for(i=0; i<rep->an_numrrsets; i++) {
+		uint16_t t = ntohs(rep->rrsets[i]->rk.type);
+		if(t == LDNS_RR_TYPE_DNAME)
+			continue; /* skip dnames; note TTL 0 not cached */
+		/* verify that owner matches current sname */
+		if(query_dname_compare(sname, rep->rrsets[i]->rk.dname) != 0){
+			/* cname chain broken */
+			return 0;
+		}
+		/* if this is a cname; move on */
+		if(t == LDNS_RR_TYPE_CNAME) {
+			get_cname_target(rep->rrsets[i], &sname, &snamelen);
+		}
+	}
+	return 1;
+}
+
+/** check security status in cache reply */
+static int
+all_rrsets_secure(struct reply_info* rep) {
+	size_t i;
+	for(i=0; i<rep->rrset_count; i++) {
+		if( ((struct packed_rrset_data*)rep->rrsets[i]->entry.data)
+			->security != sec_status_secure )
+			return 0;
+	}
+	return 1;
+}
+
+/** answer query from the cache */
+static int
+answer_from_cache(struct worker* worker, struct lruhash_entry* e, uint16_t id,
+	uint16_t flags, struct comm_reply* repinfo, struct edns_data* edns)
+{
+	struct msgreply_entry* mrentry = (struct msgreply_entry*)e->key;
+	struct reply_info* rep = (struct reply_info*)e->data;
+	uint32_t timenow = time(0);
+	uint16_t udpsize = edns->udp_size;
+	int secure;
+	int must_validate = !(flags&BIT_CD) && worker->env.need_to_validate;
+	/* see if it is possible */
+	if(rep->ttl <= timenow) {
+		/* the rrsets may have been updated in the meantime.
+		 * we will refetch the message format from the
+		 * authoritative server 
+		 */
+		return 0;
+	}
+	if(!rrset_array_lock(rep->ref, rep->rrset_count, timenow))
+		return 0;
+	/* locked and ids and ttls are OK. */
+	/* check CNAME chain (if any) */
+	if(rep->an_numrrsets > 0 && (rep->rrsets[0]->rk.type == 
+		htons(LDNS_RR_TYPE_CNAME) || rep->rrsets[0]->rk.type == 
+		htons(LDNS_RR_TYPE_DNAME))) {
+		if(!check_cache_chain(rep)) {
+			/* cname chain invalid, redo iterator steps */
+			verbose(VERB_ALGO, "Cache reply: cname chain broken");
+		bail_out:
+			rrset_array_unlock_touch(worker->env.rrset_cache, 
+				worker->scratchpad, rep->ref, rep->rrset_count);
+			region_free_all(worker->scratchpad);
+			return 0;
+		}
+	}
+	/* check security status of the cached answer */
+	if( rep->security == sec_status_bogus && must_validate) {
+		/* BAD cached */
+		edns->edns_version = EDNS_ADVERTISED_VERSION;
+		edns->udp_size = EDNS_ADVERTISED_SIZE;
+		edns->ext_rcode = 0;
+		edns->bits &= EDNS_DO;
+		error_encode(repinfo->c->buffer, LDNS_RCODE_SERVFAIL, 
+			&mrentry->key, id, flags, edns);
+		rrset_array_unlock_touch(worker->env.rrset_cache, 
+			worker->scratchpad, rep->ref, rep->rrset_count);
+		region_free_all(worker->scratchpad);
+		return 1;
+	} else if( rep->security == sec_status_unchecked && must_validate) {
+		verbose(VERB_ALGO, "Cache reply: unchecked entry needs "
+			"validation");
+		goto bail_out; /* need to validate cache entry first */
+	} else if(rep->security == sec_status_secure) {
+		if(all_rrsets_secure(rep))
+			secure = 1;
+		else	{
+			if(must_validate) {
+				verbose(VERB_ALGO, "Cache reply: secure entry"
+					" changed status");
+				goto bail_out; /* rrset changed, re-verify */
+			}
+			secure = 0;
+		}
+	} else	secure = 0;
+
+	edns->edns_version = EDNS_ADVERTISED_VERSION;
+	edns->udp_size = EDNS_ADVERTISED_SIZE;
+	edns->ext_rcode = 0;
+	edns->bits &= EDNS_DO;
+	if(!reply_info_answer_encode(&mrentry->key, rep, id, flags, 
+		repinfo->c->buffer, timenow, 1, worker->scratchpad,
+		udpsize, edns, (int)(edns->bits & EDNS_DO), secure)) {
+		error_encode(repinfo->c->buffer, LDNS_RCODE_SERVFAIL, 
+			&mrentry->key, id, flags, edns);
+	}
+	/* cannot send the reply right now, because blocking network syscall
+	 * is bad while holding locks. */
+	rrset_array_unlock_touch(worker->env.rrset_cache, worker->scratchpad,
+		rep->ref, rep->rrset_count);
+	region_free_all(worker->scratchpad);
+	/* go and return this buffer to the client */
+	return 1;
+}
+
+/**
+ * Fill CH class answer into buffer. Keeps query.
+ * @param pkt: buffer
+ * @param str: string to put into text record (<255).
+ * @param edns: edns reply information.
+ */
+static void
+chaos_replystr(ldns_buffer* pkt, const char* str, struct edns_data* edns)
+{
+	size_t len = strlen(str);
+	unsigned int rd = LDNS_RD_WIRE(ldns_buffer_begin(pkt));
+	unsigned int cd = LDNS_CD_WIRE(ldns_buffer_begin(pkt));
+	if(len>255) len=255; /* cap size of TXT record */
+	ldns_buffer_clear(pkt);
+	ldns_buffer_skip(pkt, (ssize_t)sizeof(uint16_t)); /* skip id */
+	ldns_buffer_write_u16(pkt, (uint16_t)(BIT_QR|BIT_RA));
+	if(rd) LDNS_RD_SET(ldns_buffer_begin(pkt));
+	if(cd) LDNS_CD_SET(ldns_buffer_begin(pkt));
+	ldns_buffer_write_u16(pkt, 1); /* qdcount */
+	ldns_buffer_write_u16(pkt, 1); /* ancount */
+	ldns_buffer_write_u16(pkt, 0); /* nscount */
+	ldns_buffer_write_u16(pkt, 0); /* arcount */
+	(void)query_dname_len(pkt); /* skip qname */
+	ldns_buffer_skip(pkt, (ssize_t)sizeof(uint16_t)); /* skip qtype */
+	ldns_buffer_skip(pkt, (ssize_t)sizeof(uint16_t)); /* skip qclass */
+	ldns_buffer_write_u16(pkt, 0xc00c); /* compr ptr to query */
+	ldns_buffer_write_u16(pkt, LDNS_RR_TYPE_TXT);
+	ldns_buffer_write_u16(pkt, LDNS_RR_CLASS_CH);
+	ldns_buffer_write_u32(pkt, 0); /* TTL */
+	ldns_buffer_write_u16(pkt, sizeof(uint8_t) + len);
+	ldns_buffer_write_u8(pkt, len);
+	ldns_buffer_write(pkt, str, len);
+	ldns_buffer_flip(pkt);
+	edns->edns_version = EDNS_ADVERTISED_VERSION;
+	edns->udp_size = EDNS_ADVERTISED_SIZE;
+	edns->bits &= EDNS_DO;
+	attach_edns_record(pkt, edns);
+}
+
+/**
+ * Answer CH class queries.
+ * @param w: worker
+ * @param qinfo: query info. Pointer into packet buffer.
+ * @param edns: edns info from query.
+ * @param pkt: packet buffer.
+ * @return: true if a reply is to be sent.
+ */
+static int
+answer_chaos(struct worker* w, struct query_info* qinfo, 
+	struct edns_data* edns, ldns_buffer* pkt)
+{
+	struct config_file* cfg = w->env.cfg;
+	if(qinfo->qtype != LDNS_RR_TYPE_ANY && qinfo->qtype != LDNS_RR_TYPE_TXT)
+		return 0;
+	if(query_dname_compare(qinfo->qname, 
+		(uint8_t*)"\002id\006server") == 0 ||
+		query_dname_compare(qinfo->qname, 
+		(uint8_t*)"\010hostname\004bind") == 0)
+	{
+		if(cfg->hide_identity) 
+			return 0;
+		if(cfg->identity==NULL || cfg->identity[0]==0) {
+			char buf[MAXHOSTNAMELEN];
+			if (gethostname(buf, MAXHOSTNAMELEN) == 0)
+				chaos_replystr(pkt, buf, edns);
+			else 	{
+				log_err("gethostname: %s", strerror(errno));
+				chaos_replystr(pkt, "no hostname", edns);
+			}
+		}
+		else 	chaos_replystr(pkt, cfg->identity, edns);
+		return 1;
+	}
+	if(query_dname_compare(qinfo->qname, 
+		(uint8_t*)"\007version\006server") == 0 ||
+		query_dname_compare(qinfo->qname, 
+		(uint8_t*)"\007version\004bind") == 0)
+	{
+		if(cfg->hide_version) 
+			return 0;
+		if(cfg->version==NULL || cfg->version[0]==0)
+			chaos_replystr(pkt, PACKAGE_STRING, edns);
+		else 	chaos_replystr(pkt, cfg->version, edns);
+		return 1;
+	}
+	return 0;
+}
+
 /** handles callbacks from listening event interface */
 static int 
 worker_handle_request(struct comm_point* c, void* arg, int error,
@@ -212,12 +662,17 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 {
 	struct worker* worker = (struct worker*)arg;
 	int ret;
-	verbose(VERB_DETAIL, "worker handle request");
+	hashvalue_t h;
+	struct lruhash_entry* e;
+	struct query_info qinfo;
+	struct edns_data edns;
+
 	if(error != NETEVENT_NOERROR) {
-		log_err("called with err=%d", error);
+		log_err("handle request called with err=%d", error);
 		return 0;
 	}
-	if((ret=worker_check_request(c->buffer)) != 0) {
+	if((ret=worker_check_request(c->buffer, worker)) != 0) {
+		verbose(VERB_ALGO, "worker check request: bad query.");
 		if(ret != -1) {
 			LDNS_QR_SET(ldns_buffer_begin(c->buffer));
 			LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), ret);
@@ -226,22 +681,115 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		comm_point_drop_reply(repinfo);
 		return 0;
 	}
-	if(worker->num_requests > 0) {
+	worker->stats.num_queries++;
+	/* see if query is in the cache */
+	if(!query_info_parse(&qinfo, c->buffer)) {
+		verbose(VERB_ALGO, "worker parse request: formerror.");
+		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
+		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), 
+			LDNS_RCODE_FORMERR);
+		return 1;
+	}
+	if(qinfo.qtype == LDNS_RR_TYPE_AXFR || 
+		qinfo.qtype == LDNS_RR_TYPE_IXFR) {
+		verbose(VERB_ALGO, "worker request: refused zone transfer.");
+		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
+		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), 
+			LDNS_RCODE_REFUSED);
+		return 1;
+	}
+	if((ret=parse_edns_from_pkt(c->buffer, &edns)) != 0) {
+		verbose(VERB_ALGO, "worker parse edns: formerror.");
+		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
+		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), ret);
+		return 1;
+	}
+	if(edns.edns_present && edns.edns_version != 0) {
+		edns.ext_rcode = (uint8_t)(EDNS_RCODE_BADVERS>>4);
+		edns.edns_version = EDNS_ADVERTISED_VERSION;
+		edns.udp_size = EDNS_ADVERTISED_SIZE;
+		edns.bits &= EDNS_DO;
+		verbose(VERB_ALGO, "query with bad edns version.");
+		error_encode(c->buffer, EDNS_RCODE_BADVERS&0xf, &qinfo,
+			*(uint16_t*)ldns_buffer_begin(c->buffer),
+			ldns_buffer_read_u16_at(c->buffer, 2), NULL);
+		attach_edns_record(c->buffer, &edns);
+		return 1;
+	}
+	if(edns.edns_present && edns.udp_size < NORMAL_UDP_SIZE &&
+		worker->daemon->cfg->harden_short_bufsize) {
+		verbose(VERB_DETAIL, "worker request: EDNS bufsize %d ignored",
+			(int)edns.udp_size);
+		edns.udp_size = NORMAL_UDP_SIZE;
+	}
+	if(edns.edns_present && edns.udp_size < LDNS_HEADER_SIZE) {
+		verbose(VERB_ALGO, "worker request: edns is too small.");
+		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
+		LDNS_TC_SET(ldns_buffer_begin(c->buffer));
+		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), 
+			LDNS_RCODE_SERVFAIL);
+		ldns_buffer_set_position(c->buffer, LDNS_HEADER_SIZE);
+		ldns_buffer_flip(c->buffer);
+		return 1;
+	}
+	if(c->type != comm_udp)
+		edns.udp_size = 65535; /* max size for TCP replies */
+	if(qinfo.qclass == LDNS_RR_CLASS_CH && answer_chaos(worker, &qinfo,
+		&edns, c->buffer)) {
+		return 1;
+	}
+	h = query_info_hash(&qinfo);
+	if((e=slabhash_lookup(worker->env.msg_cache, h, &qinfo, 0))) {
+		/* answer from cache - we have acquired a readlock on it */
+		if(answer_from_cache(worker, e, 
+			*(uint16_t*)ldns_buffer_begin(c->buffer), 
+			ldns_buffer_read_u16_at(c->buffer, 2), repinfo, 
+			&edns)) {
+			lock_rw_unlock(&e->lock);
+			return 1;
+		}
+		verbose(VERB_ALGO, "answer from the cache failed");
+		lock_rw_unlock(&e->lock);
+	}
+	if(!LDNS_RD_WIRE(ldns_buffer_begin(c->buffer))) {
+		if(answer_norec_from_cache(worker, &qinfo,
+			*(uint16_t*)ldns_buffer_begin(c->buffer), 
+			ldns_buffer_read_u16_at(c->buffer, 2), repinfo, 
+			&edns)) {
+			return 1;
+		}
+		verbose(VERB_ALGO, "answer norec from cache -- "
+			"need to validate or not primed");
+	}
+	ldns_buffer_rewind(c->buffer);
+	server_stats_querymiss(&worker->stats, worker);
+
+	/* grab a work request structure for this new request */
+	/* @@@ TODO implement overload mode */
+	if(0 && worker->env.mesh->all.count > worker->request_size) {
 		/* we could get this due to a slow tcp incoming query, 
 		   that started before we performed listen_pushback */
 		verbose(VERB_DETAIL, "worker: too many incoming requests "
 			"active. dropping incoming query.");
+		verbose(VERB_ALGO, "currently servicing %d of %d queries", 
+			(int)worker->env.mesh->all.count, 
+			(int)worker->request_size);
+		worker->stats.num_query_list_exceeded++;
 		comm_point_drop_reply(repinfo);
+		query_info_clear(&qinfo);
 		return 0;
 	}
-	/* answer it */
-	worker->num_requests ++;
-	if(worker->num_requests >= 1)  {
+	mesh_new_client(worker->env.mesh, &qinfo, 
+		ldns_buffer_read_u16_at(c->buffer, 2),
+		&edns, repinfo, *(uint16_t*)ldns_buffer_begin(c->buffer));
+
+	if(0) { /* TODO overload mode does not work yet. */
+	if(worker->env.mesh->all.count == worker->request_size)  {
 		/* the max request number has been reached, stop accepting */
 		listen_pushback(worker->front);
 	}
-	memcpy(&worker->query_reply, repinfo, sizeof(struct comm_reply));
-	worker_process_query(worker);
+	}
+	worker_mem_report(worker, NULL);
 	return 0;
 }
 
@@ -259,14 +807,17 @@ worker_sighandler(int sig, void* arg)
 			break;
 		case SIGINT:
 			log_info("caught signal SIGINT");
+			worker->need_to_restart = 0;
 			comm_base_exit(worker->base);
 			break;
 		case SIGQUIT:
 			log_info("caught signal SIGQUIT");
+			worker->need_to_restart = 0;
 			comm_base_exit(worker->base);
 			break;
 		case SIGTERM:
 			log_info("caught signal SIGTERM");
+			worker->need_to_restart = 0;
 			comm_base_exit(worker->base);
 			break;
 		default:
@@ -308,7 +859,7 @@ worker_create(struct daemon* daemon, int id)
 
 int
 worker_init(struct worker* worker, struct config_file *cfg, 
-	struct listen_port* ports, size_t buffer_size, int do_sigs)
+	struct listen_port* ports, int do_sigs)
 {
 	unsigned int seed;
 	int startport;
@@ -339,23 +890,6 @@ worker_init(struct worker* worker, struct config_file *cfg,
 	} else { /* !do_sigs */
 		worker->comsig = 0;
 	}
-	worker->front = listen_create(worker->base, ports,
-		buffer_size, worker_handle_request, worker);
-	if(!worker->front) {
-		log_err("could not create listening sockets");
-		worker_delete(worker);
-		return 0;
-	}
-	startport  = cfg->outgoing_base_port + 
-		cfg->outgoing_num_ports * worker->thread_num;
-	worker->back = outside_network_create(worker->base,
-		buffer_size, (size_t)cfg->outgoing_num_ports, cfg->ifs, 
-		cfg->num_ifs, cfg->do_ip4, cfg->do_ip6, startport);
-	if(!worker->back) {
-		log_err("could not create outgoing sockets");
-		worker_delete(worker);
-		return 0;
-	}
 	/* init random(), large table size. */
 	if(!(worker->rndstate = (struct ub_randstate*)calloc(1,
 		sizeof(struct ub_randstate)))) {
@@ -364,31 +898,74 @@ worker_init(struct worker* worker, struct config_file *cfg,
 		return 0;
 	}
 	seed = (unsigned int)time(NULL) ^ (unsigned int)getpid() ^
-		(unsigned int)worker->thread_num;
+		(((unsigned int)worker->thread_num)<<17);
+		/* shift thread_num so it does not match out pid bits */
 	if(!ub_initstate(seed, worker->rndstate, RND_STATE_SIZE)) {
 		log_err("could not init random numbers.");
 		worker_delete(worker);
 		return 0;
 	}
+	worker->front = listen_create(worker->base, ports,
+		cfg->msg_buffer_size, (int)cfg->incoming_num_tcp, 
+		worker_handle_request, worker);
+	if(!worker->front) {
+		log_err("could not create listening sockets");
+		worker_delete(worker);
+		return 0;
+	}
+	startport  = cfg->outgoing_base_port + 
+		cfg->outgoing_num_ports * worker->thread_num;
+	worker->back = outside_network_create(worker->base,
+		cfg->msg_buffer_size, (size_t)cfg->outgoing_num_ports, 
+		cfg->ifs, cfg->num_ifs, cfg->do_ip4, cfg->do_ip6, startport, 
+		cfg->do_tcp?cfg->outgoing_num_tcp:0, 
+		worker->daemon->env->infra_cache, worker->rndstate);
+	if(!worker->back) {
+		log_err("could not create outgoing sockets");
+		worker_delete(worker);
+		return 0;
+	}
 	if(worker->thread_num != 0) {
 		/* start listening to commands */
 		if(!(worker->cmd_com=comm_point_create_local(worker->base, 
-			worker->cmd_recv_fd, buffer_size, 
+			worker->cmd_recv_fd, cfg->msg_buffer_size, 
 			worker_handle_control_cmd, worker))) {
 			log_err("could not create control compt.");
 			worker_delete(worker);
 			return 0;
 		}
 	}
-
-	/* set forwarder address */
-	if(cfg->fwd_address && cfg->fwd_address[0]) {
-		if(!worker_set_fwd(worker, cfg->fwd_address, cfg->fwd_port)) {
-			worker_delete(worker);
-			fatal_exit("could not set forwarder address");
-		}
+	/* we use the msg_buffer_size as a good estimate for what the 
+	 * user wants for memory usage sizes */
+	worker->scratchpad = region_create_custom(malloc, free, 
+		cfg->msg_buffer_size, cfg->msg_buffer_size/4, 32, 1);
+	if(!worker->scratchpad) {
+		log_err("malloc failure");
+		worker_delete(worker);
+		return 0;
 	}
-	alloc_init(&worker->alloc, &worker->daemon->superalloc);
+	worker->request_size = cfg->num_queries_per_thread;
+
+	server_stats_init(&worker->stats);
+	alloc_init(&worker->alloc, &worker->daemon->superalloc, 
+		worker->thread_num);
+	worker->env = *worker->daemon->env;
+	worker->env.worker = worker;
+	worker->env.alloc = &worker->alloc;
+	worker->env.rnd = worker->rndstate;
+	worker->env.scratch = worker->scratchpad;
+	worker->env.mesh = mesh_create(worker->daemon->num_modules,
+		worker->daemon->modfunc, &worker->env);
+	worker->env.detach_subs = &mesh_detach_subs;
+	worker->env.attach_sub = &mesh_attach_sub;
+	worker->env.kill_sub = &mesh_state_delete;
+	worker->env.detect_cycle = &mesh_detect_cycle;
+	worker->env.scratch_buffer = ldns_buffer_new(cfg->msg_buffer_size);
+	if(!worker->env.mesh || !worker->env.scratch_buffer) {
+		worker_delete(worker);
+		return 0;
+	}
+	worker_mem_report(worker, NULL);
 	return 1;
 }
 
@@ -403,6 +980,11 @@ worker_delete(struct worker* worker)
 {
 	if(!worker) 
 		return;
+	mesh_stats(worker->env.mesh, "mesh has");
+	server_stats_log(&worker->stats, worker->thread_num);
+	worker_mem_report(worker, NULL);
+	mesh_delete(worker->env.mesh);
+	ldns_buffer_free(worker->env.scratch_buffer);
 	listen_delete(worker->front);
 	outside_network_delete(worker->back);
 	comm_signal_delete(worker->comsig);
@@ -419,38 +1001,54 @@ worker_delete(struct worker* worker)
 		close(worker->cmd_recv_fd);
 	worker->cmd_recv_fd = -1;
 	alloc_clear(&worker->alloc);
+	region_destroy(worker->scratchpad);
 	free(worker);
 }
 
 int 
-worker_set_fwd(struct worker* worker, const char* ip, int port)
+worker_send_packet(ldns_buffer* pkt, struct sockaddr_storage* addr,
+        socklen_t addrlen, int timeout, struct module_qstate* q, int use_tcp)
 {
-	uint16_t p;
-	log_assert(worker && ip);
-	p = (uint16_t) port;
-	if(str_is_ip6(ip)) {
-		struct sockaddr_in6* sa = 
-			(struct sockaddr_in6*)&worker->fwd_addr;
-		worker->fwd_addrlen = (socklen_t)sizeof(struct sockaddr_in6);
-		memset(sa, 0, worker->fwd_addrlen);
-		sa->sin6_family = AF_INET6;
-		sa->sin6_port = (in_port_t)htons(p);
-		if(inet_pton((int)sa->sin6_family, ip, &sa->sin6_addr) <= 0) {
-			log_err("Bad ip6 address %s", ip);
-			return 0;
-		}
-	} else { /* ip4 */
-		struct sockaddr_in* sa = 
-			(struct sockaddr_in*)&worker->fwd_addr;
-		worker->fwd_addrlen = (socklen_t)sizeof(struct sockaddr_in);
-		memset(sa, 0, worker->fwd_addrlen);
-		sa->sin_family = AF_INET;
-		sa->sin_port = (in_port_t)htons(p);
-		if(inet_pton((int)sa->sin_family, ip, &sa->sin_addr) <= 0) {
-			log_err("Bad ip4 address %s", ip);
-			return 0;
-		}
+	struct worker* worker = q->env->worker;
+	if(use_tcp) {
+		return pending_tcp_query(worker->back, pkt, addr, addrlen, 
+			timeout, worker_handle_reply, q, 
+			worker->rndstate) != 0;
 	}
-	verbose(VERB_ALGO, "fwd queries to: %s %d", ip, p);
-	return 1;
+	return pending_udp_query(worker->back, pkt, addr, addrlen, 
+		timeout*1000, worker_handle_reply, q, 
+		worker->rndstate) != 0;
+}
+
+/** compare outbound entry qstates */
+static int
+outbound_entry_compare(void* a, void* b)
+{
+	struct outbound_entry* e1 = (struct outbound_entry*)a;
+	struct outbound_entry* e2 = (struct outbound_entry*)b;
+	if(e1->qstate == e2->qstate)
+		return 1;
+	return 0;
+}
+
+struct outbound_entry*
+worker_send_query(uint8_t* qname, size_t qnamelen, uint16_t qtype,
+	uint16_t qclass, uint16_t flags, int dnssec,
+	struct sockaddr_storage* addr, socklen_t addrlen,
+	struct module_qstate* q)
+{
+	struct worker* worker = q->env->worker;
+	struct outbound_entry* e = (struct outbound_entry*)region_alloc(
+		q->region, sizeof(*e));
+	if(!e) 
+		return NULL;
+	e->qstate = q;
+	e->qsent = outnet_serviced_query(worker->back, qname,
+		qnamelen, qtype, qclass, flags, dnssec, addr, addrlen, 
+		worker_handle_service_reply, e, worker->back->udp_buff,
+		&outbound_entry_compare);
+	if(!e->qsent) {
+		return NULL;
+	}
+	return e;
 }

daemon/worker.h

@@ -47,12 +47,17 @@
 #include "util/netevent.h"
 #include "util/locks.h"
 #include "util/alloc.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgparse.h"
+#include "daemon/stats.h"
+#include "util/module.h"
 struct listen_dnsport;
 struct outside_network;
 struct config_file;
 struct daemon;
 struct listen_port;
 struct ub_randstate;
+struct region;
 
 /** size of table used for random numbers. large to be more secure. */
 #define RND_STATE_SIZE 256
@@ -63,7 +68,6 @@ enum worker_commands {
 	worker_cmd_quit
 };
 
-
 /**
  * Structure holding working information for unbound.
  * Holds globally visible information.
@@ -90,17 +94,8 @@ struct worker {
 	/** commpoint to listen to commands. */
 	struct comm_point* cmd_com;
 
-	/** number of requests currently active */
-	int num_requests;
-	/** our one and only query, packet buffer and where to send. */
-	struct comm_reply query_reply;
-	/** id of query */
-	uint16_t query_id;
-
-	/** address to forward to */
-	struct sockaddr_storage fwd_addr;
-	/** length of fwd_addr */
-	socklen_t fwd_addrlen;
+	/** number of requests that can be handled by this worker */
+	size_t request_size;
 
 	/** random() table for this worker. */
 	struct ub_randstate* rndstate;
@@ -108,6 +103,13 @@ struct worker {
 	int need_to_restart;
 	/** allocation cache for this thread */
 	struct alloc_cache alloc;
+	/** per thread statistics */
+	struct server_stats stats;
+	/** thread scratch region */
+	struct region* scratchpad;
+
+	/** module environment passed to modules, changed for this thread */
+	struct module_env env;
 };
 
 /**
@@ -125,12 +127,11 @@ struct worker* worker_create(struct daemon* daemon, int id);
  * @param worker: worker to initialize, created with worker_create.
  * @param cfg: configuration settings.
  * @param ports: list of shared query ports.
- * @param buffer_size: size of datagram buffer.
  * @param do_sigs: if true, worker installs signal handlers.
  * @return: false on error.
  */
 int worker_init(struct worker* worker, struct config_file *cfg, 
-	struct listen_port* ports, size_t buffer_size, int do_sigs);
+	struct listen_port* ports, int do_sigs);
 
 /**
  * Make worker work.
@@ -142,15 +143,6 @@ void worker_work(struct worker* worker);
  */
 void worker_delete(struct worker* worker);
 
-/**
- * Set forwarder
- * @param worker: the worker to modify.
- * @param ip: the server name.
- * @param port: port on server or NULL for default 53.
- * @return: false on error.
- */
-int worker_set_fwd(struct worker* worker, const char* ip, int port);
-
 /**
  * Send a command to a worker. Uses blocking writes.
  * @param worker: worker to send command to.
@@ -167,4 +159,37 @@ void worker_send_cmd(struct worker* worker, ldns_buffer* buffer,
  */
 void worker_sighandler(int sig, void* arg);
 
+/**
+ * Worker service routine to send udp messages for modules.
+ * @param pkt: packet to send.
+ * @param addr: where to.
+ * @param addrlen: length of addr.
+ * @param timeout: seconds to wait until timeout.
+ * @param q: wich query state to reactivate upon return.
+ * @param use_tcp: true to use TCP, false for UDP.
+ * @return: false on failure (memory or socket related). no query was
+ *      sent.
+ */
+int worker_send_packet(ldns_buffer* pkt, struct sockaddr_storage* addr,
+	socklen_t addrlen, int timeout, struct module_qstate* q, int use_tcp);
+
+/**
+ * Worker service routine to send serviced queries to authoritative servers.
+ * @param qname: query name. (host order)
+ * @param qnamelen: length in bytes of qname, including trailing 0.
+ * @param qtype: query type. (host order)
+ * @param qclass: query class. (host order)
+ * @param flags: host order flags word, with opcode and CD bit.
+ * @param dnssec: if set, EDNS record will have DO bit set.
+ * @param addr: where to.
+ * @param addrlen: length of addr.
+ * @param q: wich query state to reactivate upon return.
+ * @return: false on failure (memory or socket related). no query was
+ *      sent.
+ */
+struct outbound_entry* worker_send_query(uint8_t* qname, size_t qnamelen, 
+	uint16_t qtype, uint16_t qclass, uint16_t flags, int dnssec, 
+	struct sockaddr_storage* addr, socklen_t addrlen,
+	struct module_qstate* q);
+
 #endif /* DAEMON_WORKER_H */

doc/CREDITS

@@ -0,0 +1,7 @@
+Unbound was developed at NLnet Labs by Wouter Wijngaards.
+
+Unbound was devised by Bill Manning, David Blacka, and Matt Larson 
+from the University of California and from Verisign. The Java prototype
+was made in further cooperation with Geoff Sisson and Roy Arends from Nominet.
+
+At NLnet Labs, Jelte Jansen and Mark Santcroos reviewed the unbound C sources.

doc/Changelog

@@ -1,3 +1,898 @@
+25 September 2007: Wouter
+	- tests for NSEC3. Fixup bitmap checks for NSEC3.
+	- positive ANY response needs to check if wildcard expansion, and
+	  check that original data did not exist.
+	- tests for NSEC3 that wrong use of OPTOUT is bad. For insecure
+	  delegation, for abuse of child zone apex nsec3.
+
+24 September 2007: Wouter
+	- do not make test programs by default.
+	- But 'make test' will perform all of the tests.
+	- Advertise builtin select libevent alternative when no libevent
+	  is found.
+	- signit can generate NSEC3 hashes, for generating tests.
+	- multiple nsec3 paramaters in message test.
+	- too high nsec3 iterations becomes insecure test.
+
+21 September 2007: Wouter
+	- fixup empty_DS_name allocated in wrong region (port DEC Alpha).
+	- fixup testcode lock safety (port FreeBSD).
+	- removes subscript has type char warnings (port Solaris 9).
+	- fixup of field with format type to int (port MacOS/X intel).
+	- added test for infinite loop case in nonRD answer validation.
+	  It was a more general problem, but hard to reproduce. When an
+	  unsigned rrset is being validated and the key fetched, the DS
+	  sequence is followed, but if the final name has no DS, then no
+	  proof is possible - the signature has been stripped off.
+
+20 September 2007: Wouter
+	- fixup and test for NSEC wildcard with empty nonterminals. 
+	- makedist.sh fixup for svn info.
+	- acl features request in plan.
+	- improved DS empty nonterminal handling.
+	- compat with ANS nxdomain for empty nonterminals. Attempts the nodata
+	  proof anyway, which succeeds in ANS failure case.
+	- striplab protection in case it becomes -1.
+	- plans for static and blacklist config.
+
+19 September 2007: Wouter
+	- comments about non-packed usage.
+	- plan for overload support in 0.6.
+	- added testbound tests for a failed resolution from the logs
+	  and for failed prime when missing glue.
+	- fixup so useless delegation points are not returned from the
+	  cache. Also the safety belt is used if priming fails to complete.
+	- fixup NSEC rdata not to be lowercased, bind compat.
+
+18 September 2007: Wouter
+	- wildcard nsec3 testcases, and fixup to get correct wildcard name.
+	- validator prints subtype classification for debug.
+
+17 September 2007: Wouter
+	- NSEC3 hash cache unit test.
+	- validator nsec3 nameerror test.
+
+14 September 2007: Wouter
+	- nsec3 nodata proof, nods proof, wildcard proof.
+	- nsec3 support for cname chain ending in noerror or nodata.
+	- validator calls nsec3 proof routines if no NSECs prove anything.
+	- fixup iterator bug where it stored the answer to a cname under
+	  the wrong qname into the cache. When prepending the cnames, the
+	  qname has to be reset to the original qname.
+
+13 September 2007: Wouter
+	- nsec3 find matching and covering, ce proof, prove namerror msg.
+
+12 September 2007: Wouter
+	- fixup of manual page warnings, like for NSD bugreport.
+	- nsec3 work, config, max iterations, filter, and hash cache. 
+
+6 September 2007: Wouter
+	- fixup to find libevent on mac port install.
+	- fixup size_t vs unsigned portability in validator/sigcrypt.
+	- please compiler on different platforms, for unreachable code.
+	- val_nsec3 file.
+	- pthread_rwlock type is optional, in case of old pthread libs.
+
+5 September 2007: Wouter
+	- cname, name error validator tests.
+	- logging of qtype ANY works.
+	- ANY type answers get RRSIG in answer section of replies (but not
+	  in other sections, unless DO bit is on).
+	- testbound can replay a TCP query (set MATCH TCP in the QUERY).
+	- DS and noDS referral validation test.
+	- if you configure many trust anchors, parent trust anchors can
+	  securely deny existance of child trust anchors, if validated.
+	- not all *.name NSECs are present because a wildcard was matched,
+	  and *.name NSECs can prove nodata for empty nonterminals.
+	  Also, for wildcard name NSECs, check they are not from the parent
+	  zone (for wildcarded zone cuts), and check absence of CNAME bit,
+	  for a nodata proof.
+	- configure option for memory allocation debugging.
+	- port configure option for memory allocation to solaris10.
+
+4 September 2007: Wouter
+	- fixup of Leakage warning when serviced queries processed multiple
+	  callbacks for the same query from the same server.
+	- testbound removes config file from /tmp on failed exit.
+	- fixup for referral cleanup of the additional section.
+	- tests for cname, referral validation.
+	- neater testbound tpkg output.
+	- DNAMEs no longer match their apex when synthesized from the cache.
+	- find correct signer name for DNAME responses.
+	- wildcarded DNAME test and fixup code to detect.
+	- prepend NSEC and NSEC3 rrsets in the iterator while chasing CNAMEs.
+	  So that wildcarded CNAMEs get their NSEC with them to the answer.
+	- test for a CNAME to a DNAME to a CNAME to an answer, all from
+	  different domains, for key fetching and signature checking of
+	  CNAME'd messages.
+
+3 September 2007: Wouter
+	- Fixed error in iterator that would cause assertion failure in 
+	  validator. CNAME to a NXDOMAIN response was collated into a response
+	  with both a CNAME and the NXDOMAIN rcode. Added a test that the
+	  rcode is changed to NOERROR (because of the CNAME).
+	- timeout on tcp does not lead to spurious leakage detect.
+	- account memory for name of lame zones, so that memory leakages does
+	  not show lame cache growth as a leakage growth.
+	- config setting for lameness cache expressed in bytes, instead of
+	  number of entries.
+	- tool too summarize allocations per code line.
+
+31 August 2007: Wouter
+	- can read bind trusted-keys { ... }; files, in a compatibility mode. 
+	- iterator should not detach target queries that it still could need.
+	  the protection against multiple outstanding queries is moved to a
+	  current_query num check.
+	- validator nodata, positive, referral tests.
+	- dname print can print '*' wildcard.
+
+30 August 2007: Wouter
+	- fixup override date config option.
+	- config options to control memory usage.
+	- caught bad free of un-alloced data in worker_send error case.
+	- memory accounting for key cache (trust anchors and temporary cache).
+	- memory accounting fixup for outside network tcp pending waits.
+	- memory accounting fixup for outside network tcp callbacks.
+	- memory accounting for iterator fixed storage.
+	- key cache size and slabs config options.
+	- lib crypto cleanups at exit. 
+
+29 August 2007: Wouter
+	- test tool to sign rrsets for testing validator with.
+	- added RSA and DSA test keys, public and private pairs, 512 bits.
+	- default configuration is with validation enabled.
+	  Only a trust-anchor needs to be configured for DNSSEC to work.
+	- do not convert to DER for DSA signature verification.
+	- validator replay test file, for a DS to DNSKEY DSA key prime and
+	  positive response.
+
+28 August 2007: Wouter
+	- removed double use for udp buffers, that could fail,
+	  instead performs a malloc to do the backup.
+	- validator validates referral messages, by validating all the rrsets
+	  and stores the rrsets in the cache. Further referral (nonRD queries)
+	  replies are made from the rrset cache directly. Unless unchecked
+	  rrsets are encountered, there are then validated.
+	- enforce that signing is done by a parent domain (or same domain).
+	- adjust TTL downwards if rrset TTL bigger than signature allows.
+	- permissive mode feature, sets AD bit for secure, but bogus does
+	  not give servfail (bogus is changed into indeterminate).
+	- optimization of rrset verification. rr canonical sorting is reused,
+	  for the same rrset. canonical rrset image in buffer is reused for
+	  the same signature.
+	- if the rrset is too big (64k exactly + large owner name) the
+	  canonicalization routine will fail if it does not fit in buffer.
+	- faster verification for large sigsets.
+	- verb_detail mode reports validation failures, but not the entire
+	  algorithm for validation. Key prime failures are reported as 
+	  verb_ops level.
+
+27 August 2007: Wouter
+	- do not garble the edns if a cache answer fails.
+	- answer norecursive from cache if possible.
+	- honor clean_additional setting when returning secure non-recursive
+	  referrals.
+	- do not store referral in msg cache for nonRD queries.
+	- store verification status in the rrset cache to speed up future
+	  verification.
+	- mark rrsets indeterminate and insecure if they are found to be so.
+	  and store this in the cache.
+
+24 August 2007: Wouter
+	- message is bogus if unsecure authority rrsets are present.
+	- val-clean-additional option, so you can turn it off.
+	- move rrset verification out of the specific proof types into one
+	  routine. This makes the proof routines prettier.
+	- fixup cname handling in validator, cname-to-positive and cname-to-
+	  nodata work.
+	- Do not synthesize DNSKEY and DS responses from the rrset cache if
+	  the rrset is from the additional section. Signatures may have
+	  fallen off the packet, and cause validation failure.
+	- more verbose signature date errors (with the date attached).
+	- increased default infrastructure cache size. It is important for
+	  performance, and 1000 entries are only 212k (or a 400 k total cache
+	  size). To 10000 entries (for 2M entries, 4M cache size).
+
+23 August 2007: Wouter
+	- CNAME handling - move needs_validation to before val_new().
+	  val_new() setups the chase-reply to be an edited copy of the msg.
+	  new classification, and find signer can find for it. 
+	  removal of unsigned crap from additional, and query restart for
+	  cname.
+	- refuse to follow wildcarded DNAMEs when validating.
+	  But you can query for qtype ANY, or qtype DNAME and validate that.
+
+22 August 2007: Wouter
+	- bogus TTL.
+	- review - use val_error().
+
+21 August 2007: Wouter
+	- ANY response validation.
+	- store security status in cache.
+	- check cache security status and either send the query to be
+	  validated, return the query to client, or send servfail to client.
+	  Sets AD bit on validated replies.
+	- do not examine security status on an error reply in mesh_done.
+	- construct DS, DNSKEY messages from rrset cache.
+	- manual page entry for override-date.
+
+20 August 2007: Wouter
+	- validate and positive validation, positive wildcard NSEC validation.
+	- nodata validation, nxdomain validation.
+
+18 August 2007: Wouter
+	- process DNSKEY response in FINDKEY state.
+
+17 August 2007: Wouter
+	- work on DS2KE routine.
+	- val_nsec.c for validator NSEC proofs.
+	- unit test for NSEC bitmap reading.
+	- dname iswild and canonical_compare with unit tests.
+
+16 August 2007: Wouter
+	- DS sig unit test.
+	- latest release libevent 1.3c and 1.3d have threading fixed.
+	- key entry fixup data pointer and ttl absolute.
+	- This makes a key-prime succeed in validator, with DS or DNSKEY as
+	  trust-anchor.
+	- fixup canonical compare byfield routine, fix bug and also neater.
+	- fixed iterator response type classification for queries of type
+	  ANY and NS.
+	  dig ANY gives sometimes NS rrset in AN and NS section, and parser
+	  removes the NS section duplicate. dig NS gives sometimes the NS
+	  in the answer section, as referral.
+	- validator FINDKEY state.
+
+15 August 2007: Wouter
+	- crypto calls to verify signatures.
+	- unit test for rrsig verification.
+
+14 August 2007: Wouter
+	- default outgoing ports changed to avoid port 2049 by default.
+	  This port is widely blocked by firewalls.
+	- count infra lameness cache in memory size.
+	- accounting of memory improved
+	- outbound entries are allocated in the query region they are for.
+	- extensive debugging for memory allocations.
+	- --enable-lock-checks can be used to enable lock checking.
+	- protect undefs in config.h from autoheaders ministrations.
+	- print all received udp packets. log hex will print on multiple
+	  lines if needed.
+	- fixed error in parser with backwards rrsig references.
+	- mark cycle targets for iterator did not have CD flag so failed
+	  its task.
+
+13 August 2007: Wouter
+	- fixup makefile, if lexer is missing give nice error and do not
+	  mess up the dependencies.
+	- canonical compare routine updated.
+	- canonical hinfo compare.
+	- printout list of the queries that the mesh is working on.
+
+10 August 2007: Wouter
+	- malloc and free overrides that track total allocation and frees.
+	  for memory debugging.
+	- work on canonical sort.
+
+9 August 2007: Wouter
+	- canonicalization, signature checks
+	- dname signature label count and unit test.
+	- added debug heap size print to memory printout.
+	- typo fixup in worker.c
+	- -R needed on solaris.
+	- validator override option for date check testing.
+
+8 August 2007: Wouter
+	- ldns _raw routines created (in ldns trunk).
+	- sigcrypt DS digest routines
+	- val_utils uses sigcrypt to perform signature cryptography.
+	- sigcrypt keyset processing
+
+7 August 2007: Wouter
+	- security status type.
+	- security status is copied when rdata is equal for rrsets.
+	- rrset id is updated to invalidate all the message cache entries
+	  that refer to NSEC, NSEC3, DNAME rrsets that have changed.
+	- val_util work
+	- val_sigcrypt file for validator signature checks.
+
+6 August 2007: Wouter
+	- key cache for validator.
+	- moved isroot and dellabel to own dname routines, with unit test.
+
+3 August 2007: Wouter
+	- replanning.
+	- scrubber check section of lame NS set.
+	- trust anchors can be in config file or read from zone file,
+	  DS and DNSKEY entries.
+	- unit test trust anchor storage.
+	- trust anchors converted to packed rrsets.
+	- key entry definition.
+
+2 August 2007: Wouter
+	- configure change for latest libevent trunk version (needs -lrt).
+	- query_done and walk_supers are moved out of module interface.
+	- fixup delegation point duplicates.
+	- fixup iterator scrubber; lame NS set is let through the scrubber
+	  so that the classification is lame.
+	- validator module exists, and does nothing but pass through,
+	  with calling of next module and return.
+	- validator work.
+
+1 August 2007: Wouter
+	- set version to 0.5
+	- module work for module to module interconnections.
+	- config of modules.
+	- detect cycle takes flags.
+
+31 July 2007: Wouter
+	- updated plan
+	- release 0.4 tag.
+
+30 July 2007: Wouter
+	- changed random state init, so that sequential process IDs are not
+	  cancelled out by sequential thread-ids in the random number seed.
+	- the fwd_three test, which sends three queries to unbound, and 
+	  unbound is kept waiting by ldns-testns for 3 seconds, failed
+	  because the retry timeout for default by unbound is 3 seconds too,
+	  it would hit that timeout and fail the test. Changed so that unbound
+	  is kept waiting for 2 seconds instead.
+
+27 July 2007: Wouter
+	- removed useless -C debug option. It did not work.
+	- text edit of documentation.
+	- added doc/CREDITS file, referred to by the manpages.
+	- updated planning.
+
+26 July 2007: Wouter
+	- cycle detection, for query state dependencies. Will attempt to
+	  circumvent the cycle, but if no other targets available fails.
+	- unit test for AXFR, IXFR response.
+	- test for cycle detection.
+
+25 July 2007: Wouter
+	- testbound read ADDRESS and check it.
+	- test for version.bind and friends.
+	- test for iterator chaining through several referrals.
+	- test and fixup for refetch for glue. Refetch fails if glue
+	  is still not provided.
+
+24 July 2007: Wouter
+	- Example section in config manual.
+	- Addr stored for range and moment in replay.
+
+20 July 2007: Wouter
+	- Check CNAME chain before returning cache entry with CNAMEs.
+	- Option harden-glue, default is on. It will discard out of zone
+	  data. If disabled, performance is faster, but spoofing attempts
+	  become a possibility. Note that still normalize scrubbing is done,
+	  and that the potentially spoofed data is used for infrastructure
+	  and not returned to the client.
+	- if glue times out, refetch by asking parent of delegation again.
+	  Much like asking for DS at the parent side.
+	- TODO items from forgery-resilience draft.
+	  and on memory handling improvements.
+	- renamed module_event_timeout to module_event_noreply.
+	- memory reporting code; reports on memory usage after handling
+	  a network packet (not on cache replies).
+
+19 July 2007: Wouter
+	- shuffle NS selection when getting nameserver target addresses.
+	- fixup of deadlock warnings, yield cpu in checklock code so that
+	  freebsd scheduler selects correct process to run.
+	- added identity and version config options and replies.
+	- store cname messages complete answers.
+
+18 July 2007: Wouter
+	- do not query addresses, 127.0.0.1, and ::1 by default.
+
+17 July 2007: Wouter
+	- forward zone options in config file.
+	- forward per zone in iterator. takes precendence over stubs.
+	- fixup commithooks.
+	- removed forward-to and forward-to-port features, subsumed by
+	  new forward zones.
+	- fix parser to handle absent server: clause.
+	- change untrusted rrset test to account for scrubber that is now
+	  applied during the test (which removes the poison, by the way).
+	- feature, addresses can be specified with @portnumber, like nsd.conf.
+	- test config files changed over to new forwarder syntax.
+
+27 June 2007: Wouter
+	- delete of mesh does a postorder traverse of the tree.
+	- found and fixed a memory leak. For TTL=0 messages, that would
+	  not be cached, instead the msg-replyinfo structure was leaked.
+	- changed server selection so it will filter out hosts that are
+	  unresponsive. This is defined as a host with the maximum rto value.
+	  This means that unbound tried the host for retries up to 120 secs.
+	  The rto value will time out after host-ttl seconds from the cache.
+	  This keeps such unresolvable queries from taking up resources.
+	- utility for keeping histogram.
+
+26 June 2007: Wouter
+	- mesh is called by worker, and iterator uses it.
+	  This removes the hierarchical code.
+	  QueryTargets state and Finished state are merged for iterator.
+	- forwarder mode no longer sets AA bit on first reply.
+	- rcode in walk_supers is not needed.
+
+25 June 2007: Wouter
+	- more mesh work.
+	- error encode routine for ease.
+
+22 June 2007: Wouter
+	- removed unused _node iterator value from rbtree_t. Takes up space.
+	- iterator can handle querytargets state without a delegation point
+	  set, so that a priming(stub) subquery error can be handled.
+	- iterator stores if it is priming or not.
+	- log_query_info() neater logging.
+	- changed iterator so that it does not alter module_qstate.qinfo
+	  but keeps a chase query info. Also query_flags are not altered,
+	  the iterator uses chase_flags.
+	- fixup crash in case no ports for the family exist.
+
+21 June 2007: Wouter
+	- Fixup secondary buffer in case of error callback.
+	- cleanup slumber list of runnable states.
+	- module_subreq_depth fails to work in slumber list.
+	- fixup query release for cached results to sub targets.
+	- neater error for tcp connection failure, shows addr in verbose.
+	- rbtree_init so that it can be used with preallocated memory.
+
+20 June 2007: Wouter
+	- new -C option to enable coredumps after forking away.
+	- doc update.
+	- fixup CNAME generation by scrubber, and memory allocation of it.
+	- fixup deletion of serviced queries when all callbacks delete too.
+	- set num target queries to 0 when you move them to slumber list.
+	- typo in check caused subquery errors to be ignored, fixed.
+	- make lint happy about rlim_t.
+	- freeup of modules after freeup of module-states.
+	- duplicate replies work, this uses secondary udp buffer in outnet.
+
+19 June 2007: Wouter
+	- nicer layout in stats.c, review 0.3 change.
+	- spelling improvement, review 0.3 change.
+	- uncapped timeout for server selection, so that very fast or slow
+	  servers will stand out from the rest.
+	- target-fetch-policy: "3 2 1 0 0" config setting.
+	- fixup queries answered without RD bit (for root prime results).
+	- refuse AXFR and IXFR requests.
+	- fixup RD flag in error reply from iterator. fixup RA flag from
+	  worker error reply.
+	- fixup encoding of very short edns buffer sizes, now sets TC bit.
+	- config options harden-short-bufsize and harden-large-queries.
+
+18 June 2007: Wouter
+	- same, move subqueries to slumber list when first has resolved.
+	- fixup last fix for duplicate callbacks.
+	- another offbyone in targetcounter. Also in Java prototype by the way.
+
+15 June 2007: Wouter
+	- if a query asks to be notified of the same serviced query result
+	  multiple times, this will succeed. Only one callback will happen;
+	  multiple outbound-list entries result (but the double cleanup of it
+	  will not matter).
+	- when iterator moves on due to CNAME or referral, it will remove
+	  the subqueries (for other targets). These are put on the slumber
+	  list.
+	- state module wait subq is OK with no new subqs, an old one may have
+	  stopped, with an error, and it is still waiting for other ones.
+	- if a query loops, halt entire query (easy way to clean up properly).
+
+14 June 2007: Wouter
+	- num query targets was > 0 , not >= 0 compared, so that fetch
+	  policy of 0 did nothing.
+
+13 June 2007: Wouter
+	- debug option: configure --enable-static-exe for compile where
+	  ldns and libevent are linked statically. Default is off.
+	- make install and make uninstall. Works with static-exe and without.
+	  installation of unbound binary and manual pages.
+	- alignement problem fix on solaris 64.
+	- fixup address in case of TCP error.
+
+12 June 2007: Wouter
+	- num target queries was set to 0 at a bad time. Default it to 0 and
+	  increase as target queries are done.
+	- synthesize CNAME and DNAME responses from the cache.
+	- Updated doxygen config for doxygen 1.5.
+	- aclocal newer version.
+	- doxygen 1.5 fixes for comments (for the strict check on docs).
+
+11 June 2007: Wouter
+	- replies on TCP queries have the address field set in replyinfo,
+	  for serviced queries, because the initiator does not know that
+	  a TCP fallback has occured.
+	- omit DNSSEC types from nonDO replies, except if qtype is ANY or
+	  if qtype directly queries for the type (and then only show that
+	  'unknown type' in the answer section).
+	- fixed message parsing where rrsigs on their own would be put
+	  in the signature list over the rrsig type.
+
+7 June 2007: Wouter
+	- fixup error in double linked list insertion for subqueries and
+	  for outbound list of serviced queries for iterator module.
+	- nicer printout of outgoing port selection. 
+	- fixup cname target readout.
+	- nicer debug output.
+	- fixup rrset counts when prepending CNAMEs to the answer.
+	- fixup rrset TTL for prepended CNAMEs.
+	- process better check for looping modules, and which submodule to
+	  run next.
+	- subreq insertion code fixup for slumber list.
+	- VERB_DETAIL, verbosity: 2 level gives short but readable output.
+	  VERB_ALGO, verbosity: 3 gives extensive output.
+	- fixup RA bit in cached replies.
+	- fixup CNAME responses from the cache no longer partial response.
+	- error in network send handled without leakage.
+	- enable ip6 from config, and try ip6 addresses if available,
+	  if ip6 is not connected, skips to next server.
+
+5 June 2007: Wouter
+	- iterator state finished.
+	- subrequests without parent store in cache and stop.
+	- worker slumber list for ongoing promiscuous queries.
+	- subrequest error handling.
+	- priming failure returns SERVFAIL.
+	- priming gives LAME result, returns SERVFAIL.
+	- debug routine to print dns_msg as handled by iterator.
+	- memleak in config file stubs fixup.
+	- more small bugs, in scrubber, query compare no ID for lookup,
+	  in dname validation for NS targets.
+	- sets entry.key for new special allocs.
+	- lognametypeclass can display unknown types and classes.
+
+4 June 2007: Wouter
+	- random selection of equally preferred nameserver targets.
+	- reply info copy routine. Reuses existing code.
+	- cache lameness in response handling.
+	- do not touch qstate after worker_process_query because it may have
+	  been deleted by that routine.
+	- Prime response state.
+	- Process target response state.
+	- some memcmp changed to dname_compare for case preservation.
+
+1 June 2007: Wouter
+	- normalize incoming messages. Like unbound-java, with CNAME chain
+	  checked, DNAME checked, CNAME's synthesized, glue checked.
+	- sanitize incoming messages.
+	- split msgreply encode functions into own file msgencode.c.
+	- msg_parse to queryinfo/replyinfo conversion more versatile.
+	- process_response, classify response, delegpt_from_message. 
+
+31 May 2007: Wouter
+	- querytargets state.
+	- dname_subdomain_c() routine.
+	- server selection, based on RTT. ip6 is filtered out if not available,
+	  and lameness is checked too.
+	- delegation point copy routine.
+
+30 May 2007: Wouter
+	- removed FLAG_CD from message and rrset caches. This was useful for
+	  an agnostic forwarder, but not for a sophisticated (trust value per
+	  rrset enabled) cache.
+	- iterator reponse typing.
+	- iterator cname handle.
+	- iterator prime start.
+	- subquery work.
+	- processInitRequest and processInitRequest2.
+	- cache synthesizes referral messages, with DS and NSEC.
+	- processInitRequest3.
+	- if a request creates multiple subrequests these are all activated.
+
+29 May 2007: Wouter
+	- routines to lock and unlock array of rrsets moved to cache/rrset.
+	- lookup message from msg cache (and copy to region).
+	- fixed cast error in dns msg lookup.
+	- message with duplicate rrset does not increase its TTLs twice.
+	- 'qnamesize' changed to 'qname_len' for similar naming scheme.
+
+25 May 2007: Wouter
+	- Acknowledge use of unbound-java code in iterator. Nicer readme.
+	- services/cache/dns.c DNS Cache. Hybrid cache uses msgcache and
+	  rrset cache from module environment.
+	- packed rrset key has type and class as easily accessable struct
+	  members. They are still kept in network format for fast msg encode.
+	- dns cache find_delegation routine.
+	- iterator main functions setup.
+	- dns cache lookup setup.
+
+24 May 2007: Wouter
+	- small changes to prepare for subqueries.
+	- iterator forwarder feature separated out.
+	- iterator hints stub code, config file stub code, so that first
+	  testing can proceed locally.
+	- replay tests now have config option to enable forwarding mode.
+
+23 May 2007: Wouter
+	- outside network does precise timers for roundtrip estimates for rtt
+	  and for setting timeout for UDP. Pending_udp takes milliseconds.
+	- cleaner iterator sockaddr conversion of forwarder address.
+	- iterator/iter_utils and iter_delegpt setup.
+	- root hints.
+
+22 May 2007: Wouter
+	- outbound query list for modules and support to callback with the
+	  outbound entry to the module.
+	- testbound support for new serviced queries.
+	- test for retry to TCP cannot use testbound any longer.
+	- testns test for EDNS fallback, test for TCP fallback already exists.
+	- fixes for no-locking compile.
+	- mini_event timer precision and fix for change in timeouts during
+	  timeout callback. Fix for fwd_three tests, performed nonexit query.
+
+21 May 2007: Wouter
+	- small comment on hash table locking.
+	- outside network serviced queries, contain edns and tcp fallback,
+	  and udp retries and rtt timing.
+
+16 May 2007: Wouter
+	- lruhash_touch() would cause locking order problems. Fixup in 
+	  lock-verify in case locking cycle is found.
+	- services/cache/rrset.c for rrset cache code.
+	- special rrset_cache LRU updating function that uses the rrset id.
+	- no dependencies calculation when make clean is called.
+	- config settings for infra cache.
+	- daemon code slightly cleaner, only creates caches once.
+
+15 May 2007: Wouter
+	- host cache code.
+	- unit test for host cache.
+
+14 May 2007: Wouter
+	- Port to OS/X and Dec Alpha. Printf format and alignment fixes.
+	- extensive lock debug report on join timeout.
+	- proper RTT calculation, in utility code.
+	- setup of services/cache/infra, host cache.
+
+11 May 2007: Wouter
+	- iterator/iterator.c module.
+	- fixup to pass reply_info in testcode and in netevent.
+
+10 May 2007: Wouter
+	- created release-0.3 svn tag.
+	- util/module.h
+	- fixed compression - no longer compresses root name.
+
+9 May 2007: Wouter
+	- outside network cleans up waiting tcp queries on exit.
+	- fallback to TCP.
+	- testbound replay with retry in TCP mode.
+	- tpkg test for retry in TCP mode, against ldns-testns server.
+	- daemon checks max number of open files and complains if not enough.
+	- test where data expires in the cache.
+	- compiletests: fixed empty body ifstatements in alloc.c, in case
+	  locks are disabled.
+
+8 May 2007: Wouter
+	- outgoing network keeps list of available tcp buffers for outgoing 
+	  tcp queries.
+	- outgoing-num-tcp config option.
+	- outgoing network keeps waiting list of queries waiting for buffer.
+	- netevent supports outgoing tcp commpoints, nonblocking connects.
+
+7 May 2007: Wouter
+	- EDNS read from query, used to make reply smaller.
+	- advertised edns value constants.
+	- EDNS BADVERS response, if asked for too high edns version.
+	- EDNS extended error reponses once the EDNS record from the query
+	  has successfully been parsed.
+
+4 May 2007: Wouter
+	- msgreply sizefunc is more accurate.
+	- config settings for rrset cache size and slabs.
+	- hashtable insert takes argument so that a thread can use its own
+	  alloc cache to store released keys.
+	- alloc cache special_release() locks if necessary.
+	- rrset trustworthiness type added.
+	- thread keeps a scratchpad region for handling messages.
+	- writev used in netevent to write tcp length and data after another.
+	  This saves a roundtrip on tcp replies.
+	- test for one rrset updated in the cache.
+	- test for one rrset which is not updated, as it is not deemed
+	  trustworthy enough.
+	- test for TTL refreshed in rrset.
+
+3 May 2007: Wouter
+	- fill refs. Use new parse and encode to answer queries.
+	- stores rrsets in cache.
+	- uses new msgreply format in cache.
+
+2 May 2007: Wouter
+	- dname unit tests in own file and spread out neatly in functions.
+	- more dname unit tests.
+	- message encoding creates truncated TC flagged messages if they do 
+	  not fit, and will leave out (whole)rrsets from additional if needed.
+
+1 May 2007: Wouter
+	- decompress query section, extremely lenient acceptance.
+	  But only for answers from other servers, not for plain queries.
+	- compression and decompression test cases.
+	- some stats added.
+	- example.conf interface: line is changed from 127.0.0.1 which leads
+	  to problems if used (restricting communication to the localhost),
+	  to a documentation and test address.
+
+27 April 2007: Wouter
+	- removed iov usage, it is not good for dns message encoding.
+	- owner name compression more optimal.
+	- rrsig owner name compression.
+	- rdata domain name compression.
+
+26 April 2007: Wouter
+	- floating point exception fix in lock-verify.
+	- lint uses make dependency
+	- fixup lint in dname owner domain name compression code.
+	- define for offset range that can be compressed to.
+
+25 April 2007: Wouter
+	- prettier code; parse_rrset->type kept in host byte order.
+	- datatype used for hashvalue of converted rrsig structure.
+	- unit test compares edns section data too.
+
+24 April 2007: Wouter
+	- ttl per RR, for RRSIG rrsets and others.
+	- dname_print debug function.
+	- if type is not known, size calc will skip DNAME decompression.
+	- RRSIG parsing and storing and putting in messages.
+	- dnssec enabled unit tests (from nlnetlabs.nl and se queries).
+	- EDNS extraction routine.
+
+20 April 2007: Wouter
+	- code comes through all of the unit tests now.
+	- disabled warning about spurious extra data.
+	- documented the RRSIG parse plan in msgparse.h.
+	- rrsig reading and outputting.
+
+19 April 2007: Wouter
+	- fix unit test to actually to tests.
+	- fix write iov helper, and fakevent code.
+	- extra builtin testcase (small packet).
+	- ttl converted to network format in packets.
+	- flags converted correctly
+	- rdatalen off by 2 error fixup.
+	- uses less iov space for header.
+
+18 April 2007: Wouter
+	- review of msgparse code.
+	- smaller test cases.
+
+17 April 2007: Wouter
+	- copy and decompress dnames.
+	- store calculated hash value too.
+	- routine to create message out of stored information.
+	- util/data/msgparse.c for message parsing code.
+	- unit test, and first fixes because of test.
+		* forgot rrset_count addition.
+		* did & of ptr on stack for memory position calculation.
+		* dname_pkt_copy forgot to read next label length.
+	- test from file and fixes
+		* double frees fixed in error conditions.
+		* types with less than full rdata allowed by parser.
+		  Some dynamic update packets seem to use it.
+
+16 April 2007: Wouter
+	- following a small change in LDNS, parsing code calculates the
+	  memory size to allocate for rrs.
+	- code to handle ID creation.
+
+13 April 2007: Wouter
+	- parse routines. Code that parses rrsets, rrs.
+
+12 April 2007: Wouter
+	- dname compare routine that preserves case, with unit tests.
+	
+11 April 2007: Wouter
+	- parse work - dname packet parse, msgparse, querysection parse,
+	  start of sectionparse.
+
+10 April 2007: Wouter
+	- Improved alignment of reply_info packet, nice for 32 and 64 bit.
+	- Put RRset counts in reply_info, because the number of RRs can change
+	  due to RRset updates.
+	- import of region-allocator code from nsd.
+	- set alloc special type to ub_packed_rrset_key.
+	  Uses lruhash entry overflow chain next pointer in alloc cache.
+	- doxygen documentation for region-allocator.
+	- setup for parse scratch data.
+
+5 April 2007: Wouter
+	- discussed packed rrset with Jelte.
+
+4 April 2007: Wouter
+	- moved to version 0.3.
+	- added util/data/dname.c
+	- layout of memory for rrsets.
+
+3 April 2007: Wouter
+	- detect sign of msghdr.msg_iovlen so that the cast to that type
+	  in netevent (which is there to please lint) can be correct.
+	  The type on several OSes ranges from int, int32, uint32, size_t.
+	  Detects unsigned or signed using math trick.
+	- constants for DNS flags. 
+	- compilation without locks fixup.
+	- removed include of unportable header from lookup3.c.
+	- more portable use of struct msghdr.
+	- casts for printf warning portability.
+	- tweaks to tests to port them to the testbed.
+	- 0.2 tag created.
+
+2 April 2007: Wouter
+	- check sizes of udp received messages, not too short.
+	- review changes. Some memmoves can be memcpys: 4byte aligned.
+	  set id correctly on cached answers. 
+	- review changes msgreply.c, memleak on error condition. AA flag
+	  clear on cached reply. Lowercase queries on hashing.
+	  unit test on lowercasing. Test AA bit not set on cached reply.
+	  Note that no TTLs are managed.
+
+29 March 2007: Wouter
+	- writev or sendmsg used when answering from cache.
+	  This avoids a copy of the data.
+	- do not do useless byteswap on query id. Store reply flags in uint16
+	  for easier access (and no repeated byteswapping).
+	- reviewed code.
+	- configure detects and config.h includes sys/uio.h for writev decl.
+
+28 March 2007: Wouter
+	- new config option: num-queries-per-thread.
+	- added tpkg test for answering three queries at the same time
+	  using one thread (from the query service list).
+
+27 March 2007: Wouter
+	- added test for cache and not cached answers, in testbound replays.
+	- testbound can give config file and commandline options from the
+	  replay file to unbound.
+	- created test that checks if items drop out of the cache.
+	- added word 'partitioned hash table' to documentation on slab hash.
+	  A slab hash is a partitioned hash table.
+	- worker can handle multiple queries at a time.
+
+26 March 2007: Wouter
+	- config settings for slab hash message cache.
+	- test for cached answer.
+	- Fixup deleting fake answer from testbound list.
+
+23 March 2007: Wouter
+	- review of yesterday's commits.
+	- covered up memory leak of the entry locks.
+	- answers from the cache correctly. Copies flags correctly.
+	- sanity check for incoming query replies.
+	- slabbed hash table. Much nicer contention, need dual cpu to see.
+
+22 March 2007: Wouter
+	- AIX configure check.
+	- lock-verify can handle references to locks that are created
+	  in files it has not yet read in.
+	- threaded hash table test. 
+	- unit test runs lock-verify afterwards and checks result.
+	- need writelock to update data on hash_insert.
+	- message cache code, msgreply code.
+
+21 March 2007: Wouter
+	- unit test of hash table, fixup locking problem in table_grow().
+	- fixup accounting of sizes for removing items from hashtable.
+	- unit test for hash table, single threaded test of integrity.
+	- lock-verify reports errors nicely. More quiet in operation.
+
+16 March 2007: Wouter
+	- lock-verifier, checks consistent order of locking.
+
+14 March 2007: Wouter
+	- hash table insert (and subroutines) and lookup implemented.
+	- hash table remove.
+	- unit tests for hash internal bin, lru functions.
+
+13 March 2007: Wouter
+	- lock_unprotect in checklocks.
+	- util/storage/lruhash.h for LRU hash table structure.
+
+12 March 2007: Wouter
+	- configure.ac moved to 0.2.
+	- query_info and replymsg util/data structure.
+
 9 March 2007: Wouter
 	- added rwlock writelock checking.
 	  So it will keep track of the writelock, and readlocks are enforced
@@ -6,6 +901,7 @@
 	- checklocks zeroes its destroyed lock after checking memory areas.
 	- unit test for alloc.
 	- identifier for union in checklocks to please older compilers.
+	- created 0.1 tag.
 
 8 March 2007: Wouter
 	- Reviewed checklock code.

doc/README

@@ -4,25 +4,56 @@ http://unbound.net
 
 This software is under BSD license, see LICENSE for details.
 
+* Download the latest version of this software from 
+  	http://unbound.net 
+  or get a beta version from the svn repository at 
+  	http://unbound.net/svn/
+
 * Needs the following libraries
   * ldns	http://www.nlnetlabs.nl/ldns/			(BSD license)
   * libevent	http://www.monkey.org/~provos/libevent/		(BSD license)
 
-* Make and install: ./configure; make
+* Create build environment 
+  * run libtoolize -c if config.sub is missing, or run glibtoolize.
+  * autoreconf (autoheader && autoconf), if ./configure is missing.
+
+* Make and install: ./configure; make; make install
   * --with-ldns=/path/to/ldns
   * --with-libevent=/path/to/libevent
   	Can be set to either the system install or the build directory.
-	--with-libevent=no gives an alternative to libevent (based on select).
+	--with-libevent=no gives a builtin alternative implementation.
+  * --without-pthreads 
+	This disables pthreads, and uses Solaris thr library or no threading. 
+	Without this option the pthreads library is detected automatically.
+  * --enable-static-exe
+	This enables a debug option to statically link, against ldns and 
+	libevent libraries.
+  * --enable-lock-checks
+  	This enables a debug option to check lock and unlock calls. It needs
+	a recent pthreads library to work.
+  * --enable-alloc-checks
+	This enables a debug option to check malloc (calloc, realloc, free).
+	The server periodically checks if the amount of memory used fits with
+	the amount of memory it thinks it should be using, and reports 
+	memory usage in detail.
 
-* Create build environment 
-  * run libtoolize -c if config.sub is missing.
-  * autoreconf (autoheader && autoconf), if ./configure is missing.
+* 'make test' attempts to run a series of tests, depending on the support
+  programs that are installed.
 
 Known issues
 ------------
-o If libevent is older (1.3 and before), unbound will exit instead of reload
+o If libevent is older (before 1.3c), unbound will exit instead of reload
   on sighup. On a restart 'did not exit gracefully last time' warning is 
   printed. Perform ./configure --with-libevent=no or update libevent, rerun 
   configure and recompile unbound to make sighup work correctly.
 
+Acknowledgements
+----------------
+o Thanks to David Blacka and Matt Larson (Verisign) for the unbound-java
+  prototype. Design and code from that prototype has been used to create
+  this program. Such as the iterator state machine and the cache design.
+o Other code origins are from the NSD (NLnet Labs) and LDNS (NLnet Labs)
+  projects. Such as buffer, region-allocator and red-black tree code.
+o See Credits file for contributors.
+
 * mailto:wouter@nlnetlabs.nl

doc/TODO

@@ -3,3 +3,53 @@ o use real entropy to make random (ID, port) numbers more random.
 o in production mode, do not free memory on exit. In debug mode, test leaks.
 o profile memory allocation, and if performance issues, use special memory
   allocator. For example, with caches per thread.
+o #define BIT_... different on bigendian and smallendian systems so that
+  the htons on flags is not needed to send a message from the cache.
+o speed up pkt domain name decompression loop detection using counter perhaps.
+o detect OS/400 pthreads implementation that allows upgrading to writelock
+  on pthreads rwlocks and use it to examine-rd before storing-wr rrset cache.
+o understand synthesized DNAMEs, so those TTL=0 packets are cached properly.
+o understand NSEC/NSEC3, aggressive negative caching, so that updates to
+  NSEC/NSEC3 will result in proper negative responses.
+o fallback without EDNS if result is NOTIMPL, now only on FORMERR like in java.
+o scrubber has slow pkt_subdomain and pkt_strict_subdomain functions.
+o get serverselection algorithm out of local optimum.
+  make subtargets to get rtt info for a couple of targets, like fetch-policy.
+  or send out multiple queries to multiple servers.
+o donotqueryaddresses allow specification of subnets, use trie to store.
+  extend default list to contain illegal ip4 and ip6 addresses.
+o memory profile; and if needed put serviced queries under qstate->region
+  and special purpose region code, reuse blocks, shrink if too big.
+o check query, option to enforce presence of qdsection in noerror answers.
+o configuration option where port 53 is used for send and receive, no other
+  ports are used.
+o prime sequence in multiple queries, if glue missing from prime sequence.
+o (option) to not send replies to clients after a timeout of (say 5 secs) has
+  passed, but keep task active for later retries by client.
+o private TTL feature
+o pretend-dnssec-unaware, and pretend-edns-unaware modes for debug/workshops.
+o delegpt use rbtree for ns-list, to avoid slowdown for very large NS sets.
+o be able to have different listen and query-to addresses to bind to,
+  so you can listen to localhost and query-to to the internet.
+o reprime and refresh oft used data before timeout.
+o retain prime results in a overlaid roothints file.
+o store primed key data in a overlaid keyhints file (sort of like drafttimers).
+o windows version, auto update feature, a query to check for the version.
+o autoreport of problems
+o logrotation, syslog
+o command the server with TSIG inband. get-config, clearcache, 
+	get stats, get memstats, get ..., reload, clear one zone from cache
+o watch for spoof nearmisses.
+o improve compression of DNS packets by first puttig uncompressible rrs, then
+  compress to their rdata.
+o if one server is not responsive do not spend 75 secs on that server, but
+  try other servers with lower rtt.
+o NSID rfc 5001 support.
+o timers rfc 5011 support.
+o Treat YXDOMAIN from a DNAME properly, in iterator (not throwaway), validator.
+o grab ports nonconsequtive and change the set after a while (change within
+  a given range). Could be bad for OS if wrong port. unsure if it helps secure.
+o workaround for nxdomain responses for ENT DS queries. Not look at rcode and
+  look at valid empty nonterminal proof that is inside the packet.
+o make timeout backoffs randomized (a couple percent random) to spread traffic.
+o inspect date on executable, then warn user in log if its more than 1 year.

doc/example.conf

@@ -20,7 +20,10 @@ server:
 
 	# specify the interfaces to answer queries from by ip-address.
 	# If you give none the default (all) interface is used.
-	# interface: 127.0.0.1
+	# specify every interface on a new 'interface:' labelled line.
+	# interface: 192.0.2.153
+	# interface: 192.0.2.154
+	# interface: 2001:DB8::5
 
 	# port to answer queries from
 	# port: 53
@@ -36,6 +39,55 @@ server:
 	# But also takes more system resources (for open sockets).
 	# outgoing-range: 16
 
+	# number of outgoing simultaneous tcp buffers to hold per thread.
+	# outgoing-num-tcp: 10
+
+	# number of incoming simultaneous tcp buffers to hold per thread.
+	# incoming-num-tcp: 10
+	
+	# buffer size for handling DNS data. No messages larger than this
+	# size can be sent or received, by UDP or TCP. In bytes.
+	# msg-buffer-size: 65552
+
+	# the amount of memory to use for the message cache.
+	# in bytes. default is 4 Mb
+	# msg-cache-size: 4194304
+
+	# the number of slabs to use for the message cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# msg-cache-slabs: 4
+
+	# the number of queries that a thread gets to service.
+	# num-queries-per-thread: 1024
+
+	# the amount of memory to use for the RRset cache.
+	# in bytes. default is 4 Mb
+	# rrset-cache-size: 4194304
+
+	# the number of slabs to use for the RRset cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# rrset-cache-slabs: 4
+
+	# the time to live (TTL) value for cached roundtrip times and
+	# EDNS version information for hosts. In seconds.
+	# infra-host-ttl: 900
+
+	# the time to live (TTL) value for cached lame delegations. In sec.
+	# infra-lame-ttl: 900
+
+	# the number of slabs to use for the Infrastructure cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# infra-cache-slabs: 4
+
+	# the maximum number of hosts that are cached (roundtrip times, EDNS).
+	# infra-cache-numhosts: 10000
+
+	# the maximum size of the lame zones cached per host. in bytes.
+	# infra-cache-lame-size: 10240
+
 	# Enable IPv4, "yes" or "no".
 	# do-ip4: yes
 
@@ -48,14 +100,6 @@ server:
 	# Enable TCP, "yes" or "no".
 	# do-tcp: yes
 
-	# Set this to configure unbound to act as a forwarder. All queries are
-	# sent to the remote nameserver that will resolve them.
-	# Set to "" to disable forwarding, or give ip-address to enable.
-	# forward-to: ""
-
-	# The port number to send forwarded queries to.
-	# forward-to-port: 53
-
 	# if given, a chroot(2) is done to the given directory.
 	# i.e. you can chroot to the working directory, for example,
 	# for extra security, but make sure all files are in that directory.
@@ -73,3 +117,123 @@ server:
 
 	# the pid file.
 	# pidfile: "unbound.pid"
+	
+	# enable to not answer id.server and hostname.bind queries.
+	# hide-identity: no
+	
+	# enable to not answer version.server and version.bind queries.
+	# hide-version: no
+	
+	# the identity to report. Leave "" or default to return hostname.
+	# identity: ""
+	
+	# the version to report. Leave "" or default to return package version.
+	# version: ""
+	
+	# the target fetch policy.
+	# series of integers describing the policy per dependency depth. 
+	# The number of values in the list determines the maximum dependency 
+	# depth the recursor will pursue before giving up. Each integer means:
+	# 	-1 : fetch all targets opportunistically,
+	# 	0: fetch on demand,
+	#	positive value: fetch that many targets opportunistically.
+	# Enclose the list of numbers between quotes ("").
+	# target-fetch-policy: "3 2 1 0 0"
+	
+	# Harden against very small EDNS buffer sizes. 
+	# harden-short-bufsize: no
+	
+	# Harden against unseemly large queries.
+	# harden-large-queries: no
+	
+	# Harden against out of zone rrsets, to avoid spoofing attempts. 
+	# harden-glue: yes
+	
+	# Do not query the following addresses. No DNS queries are sent there.
+	# List one address per entry. To block other ports than the default
+	# DNS port, use "1.2.3.4@123" to block port 123 for 1.2.3.4.
+	# do-not-query-address: 127.0.0.1
+	# do-not-query-address: ::1
+	
+	# module configuration of the server. A string with identifiers
+	# separated by spaces. "iterator" or "validator iterator"
+	# module-config: "validator iterator"
+	
+	# File with trusted keys for validation. Specify more than one file
+	# with several entries, one file per entry.
+	# Zone file format, with DS and DNSKEY entries.
+	# trust-anchor-file: ""
+	
+	# Trusted key for validation. DS or DNSKEY. specify the RR on a
+	# single line, surrounded by "". TTL is ignored. class is IN default.
+	# (These examples are from August 2007 and may not be valid anymore).
+	# trust-anchor: "nlnetlabs.nl. DNSKEY 257 3 5 AQPzzTWMz8qSWIQlfRnPckx2BiVmkVN6LPupO3mbz7FhLSnm26n6iG9N Lby97Ji453aWZY3M5/xJBSOS2vWtco2t8C0+xeO1bc/d6ZTy32DHchpW 6rDH1vp86Ll+ha0tmwyy9QP7y2bVw5zSbFCrefk8qCUBgfHm9bHzMG1U BYtEIQ=="
+	# trust-anchor: "jelte.nlnetlabs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"
+
+	# File with trusted keys for validation. Specify more than one file
+	# with several entries, one file per entry. Like trust-anchor-file
+	# but has a different file format. Format is BIND-9 style format, 
+	# the trusted-keys { name flag proto algo "key"; }; clauses are read.
+	# trusted-keys-file: ""
+
+	# Override the date for validation with a specific fixed date.
+	# Do not set this unless you are debugging signature inception
+	# and expiration. "" or "0" turns the feature off. 
+	# val-override-date: ""
+	
+	# The time to live for bogus data, rrsets and messages. This avoids
+	# some of the revalidation, until the time interval expires. in secs.
+	# val-bogus-ttl: 900
+	
+	# Should additional section of secure message also be kept clean of
+	# unsecure data. Useful to shield the users of this validator from
+	# potential bogus data in the additional section. All unsigned data 
+	# in the additional section is removed from secure messages.
+	# val-clean-additional: yes
+	
+	# Turn permissive mode on to permit bogus messages. Thus, messages
+	# for which security checks failed will be returned to clients,
+	# instead of SERVFAIL. It still performs the security checks, which
+	# result in interesting log files and possibly the AD bit in
+	# replies if the message is found secure. The default is off.
+	# val-permissive-mode: no
+	
+	# It is possible to configure NSEC3 maximum iteration counts per
+	# keysize. Keep this table very short, as linear search is done.
+	# A message with an NSEC3 with larger count is marked insecure.
+	# List in ascending order the keysize and count values.
+	# val-nsec3-keysize-iterations: "1024 150 2048 500 4096 2500"
+
+	# the amount of memory to use for the key cache.
+	# in bytes. default is 4 Mb
+	# key-cache-size: 4194304
+
+	# the number of slabs to use for the key cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# key-cache-slabs: 4
+
+
+# Stub zones.
+# Create entries like below, to make all queries for 'example.com' and 
+# 'example.org' go to the given list of nameservers. list zero or more 
+# nameservers by hostname or by ipaddress.
+# stub-zone:
+#	name: "example.com"
+#	stub-addr: 192.0.2.68
+# stub-zone:
+#	name: "example.org"
+#	stub-host: ns.example.com.
+
+# Forward zones
+# Create entries like below, to make all queries for 'example.com' and
+# 'example.org' go to the given list of servers. These servers have to handle
+# recursion to other nameservers. List zero or more nameservers by hostname
+# or by ipaddress. Use an entry with name "." to forward all queries.
+# forward-zone:
+# 	name: "example.com"
+# 	forward-addr: 192.0.2.68
+# 	forward-addr: 192.0.2.73@5355  # forward to port 5355.
+# forward-zone:
+# 	name: "example.org"
+# 	forward-host: fwd.example.com

doc/plan

@@ -29,14 +29,26 @@ Roughly the boxes are as follows:
     This stage takes longer, due to complexity in the iterator module.
     Twice as long; one box for module layout, one box for iterator module.
 0.5 Validator - validator module.
-0.6 Bigger and better - good syslog, compressed packets, port ranges,
-    config file format with config checker. 
-0.7 Local zones feature - localzones stubzones fwdzones, no leak rfc1918.
-0.8 Library use - resolver validator lib (and test apps)
-0.9 Corner cases - be able to resolve in the wild. Run fuzzers.
+0.6 Bigger and better - Operational useful features (config, log, memory)
+0.7 Put to a limited audience.
+    gamma/alpha core functionality test release, to a small audience.
+    partial functionality. For more extensive use and testing.
+0.8 Local zones feature - localzones stubzones fwdzones, no leak rfc1918.
+    views support; for selective recursive service.
+0.9 Library use - resolver validator lib (and test apps)
+0.10 Corner cases - be able to resolve in the wild. Run fuzzers.
     Run as many tests as we can think of.
-0.10 Beta release. Run shadow for a resolver in production for several
+    Go through logs and check for long, unresolved cases
+    Use profiler.
+0.11 Beta release. Run shadow for a resolver in production for several
     weeks.
+0.12 Features features
+    aggressive negative caching for NSEC, NSEC3.
+    multiple queries per question, server exploration, server selection.
+    option to use real entropy for randomness (mix it in once in a while).
+    check query, option to enforce qdsection checking (forgery-resilience).
+    NSID support.
+    Be able to prime roots using several queries (only NS on first).
 
 For boxes 0.5-1.0 the planning is to be revised, at the 0.5 stage external
 coders are welcome.  Since the project is bigger, there is room for them.
@@ -115,7 +127,7 @@ Styleguide:
 * Speed test of caching forwarder.
 
 *** Basic Resolver
-* Create module interfacce and module caller algorithm.
+* Create module interface and module caller algorithm.
 * Daemon config to use modules. Test the module caller.
 * Create basic iterator and scrubber modules.
 	* Test every state of the iterator by passing test data into
@@ -133,20 +145,59 @@ Styleguide:
 	* Test resolver.
 	* Speed test.
 
+*** Put to a limited audience
+* The alpha/gamma core functionality, svn access to limited audience.
+* Support features and requests as they arise.
+* Provide real-world experiences.
+
 *** Bigger and Better
-* DNS packet compression of output.
-	* test packet compression (that it compresses.)
-	* config option to compress yes/no
-	* speed test with and without compression.
 * Config file syntax checker program. Tests on checker.
-* Use a huge number of random outgoing ports.
 * Logging first class feature with config options.
+	* with logfile turnover to avoid Gbs of logs.
+* donotqueryaddresses with trie for blocking entire netblocks.
+* Memory overhaul, special allocators for hashtable caches, and mesh qstates.
+	* keep a preallocated list of region-chunks per worker thread.
+	* allocate region struct and cleanup list in region itself; use
+	  linked list cleanup list. unit test on this. do not call region
+	  to avoid name-collision with nsd regions, 'regional'.
+* read root hints from file.
+* failover to next server in 1 second, instead of 100 seconds on one server.
+* failure to return answer, w. reason (donotq, noanswer servers, cannot
+  find servers, validationfail w.classification, error), 
+  with threadno, starttime and endtime and qname/type/class, prime/qflags, 
+  from-clients, from-internal, has-subrequests, a nice error report,
+  so that an excerpt from those times can be made from the logs.
+  logfileparsing tool that makes these excerpts and emails them.
+* clear cache as a callback from the new-rrset-id routine.
+* make overload mode work; phase 0 all ok, phase 1 some threads close ports,
+  to let other threads pick up work. phase 2, all threads closed, so all open
+  the ports again and drop all non-cache-reply queries.
+  Keep mutexed num-overloaded-threads counter. thread incs it when it hits
+  max number of user queries serviced in mesh. threads decs it when it
+  falls below 90% of the max. if incs, and not all threads closed, phase 1,
+  else, phase 2 start is broadcast over command pipes. if decs, open ports
+  if phase 1, start servicing, phase is 0 again. Make robust against delays.
+  readme: max about 1 second worth of incoming queries, 10k perhaps,
+	or 1/number of seconds it takes start up of 10k.
+* should the source include a copy of the ldns lib for ease of building by 
+  new users.
+* no greedy TTL algo (and test). 
+* maximum TTL, cap incoming values, and config option.
 
 *** Local zones feature.
 * Build in local zone features. First the total stop for1912.
 * Then 'local content' for minimal serving of localhost.localdomain,
   and so on. 
-* Remember jakob's diagram.
+* Remember jakob's diagram. views support, selective recursive service:
+	* acl for allowed recursion (RD=1), then drop or refused query.
+	  like 10.0.0.0/8 allow, 0.0.0.0/0 refuse, ... in-order.
+	  perhaps also, same list to disallow RD=0 access, like;
+	  allow_recursion, drop_recursion, refuse_recursion, drop_all
+	* static answers for queries, fixed RRs from cfg, option
+	  query for that RR returns answer with that RR.
+	* blacklist (return fixed nxdomain for domain and below), option
+	  can be used to block AS112 traffic, option to unblock a zone.
+	* after checking acl, do iter: static, blacklist, forwards, recurse.
 * Forward-local-zone to NSD.
 	- include in package, autoforkexec on localhost to do so.
 * forward local zone to remote server.
@@ -169,6 +220,32 @@ Styleguide:
 * Try to setup corner cases of (mis)configured DNS service/websites.
 * Resolve msoft, google, yahoo, etc weird websites.
 * Try to resolve many many different queries, perhaps compared with bind.
+* create module testers, specific for the modules
+	* read a file with cache contents and settings, provide fake
+	  environment for module-handle-state-X functions, then check
+	  resulting module state structure to correct answer.
+* speed test cache responses.
+* using two servers, compare answer differences between bind and unbound.
+  this gives false differences due to changes in the rest of internet.
+
+*** Beta release. 
+* Run shadow for a resolver in production for several weeks.
+* Check logs for errors, long queries.
+* Run in valgrind, speed profiling (as production shadow).
+
+*** Features features
+* aggressive negative caching for NSEC, NSEC3.
+* multiple queries per question, server exploration, server selection.
+* option to use real entropy for randomness (mix it in once in a while).
+* check query, enforce qdsection, checking for forgery-resilience.
+* NSID support.
+* support TSIG on queries, for validating resolver deployment.
+* Be able to prime roots using several queries (like, get only NS first).
+* Nicer statistics
+* private TTL, dTLS features.
+* retry-mode, where a bogus result triggers a retry-mode query, where a list
+  of responses over a time interval is collected, and each is validated.
+* draft-timers, DLV features.
 
 treeshrew/
 	validator/ *.c *.h

doc/requirements.txt

@@ -79,3 +79,45 @@ o An authoritative name server.
 o Too many Features.
 
 
+5. Choices
+----------
+o rfc2181 decourages duplicates RRs in RRsets. unbound does not create
+  duplicates, but when presented with duplicates on the wire from the
+  authoritative servers, does not perform duplicate removal.
+  It does do some rrsig duplicate removal, in the msgparser, for dnssec qtype
+  rrsig and any, because of special rrsig processing in the msgparser.
+o The harden-glue feature, when yes all out of zone glue is deleted, when
+  no out of zone glue is used for further resolving, is more complicated 
+  than that, see below.
+  Main points:
+  	* rfc2182 trust handling is used. 
+	* data is let through only in very specific cases
+	* spoofability remains possible.
+  Not all glue is let through (despite the name of the option). Only glue 
+  which is present in a delegation, of type A and AAAA, where the name is
+  present in the NS record in the authority section is let through.
+  The glue that is let through is stored in the cache (marked as 'from the
+  additional section'). And will then be used for sending queries to. It
+  will not be present in the reply to the client (if RD is off).
+  A direct query for that name will attempt to get a msg into the message
+  cache. Since A and AAAA queries are not synthesized by the unbound cache,
+  this query will be (eventually) sent to the authoritative server and its
+  answer will be put in the cache, marked as 'from the answer section' and
+  thus remove the 'from the additional section' data, and this record is 
+  returned to the client.
+  The message has a TTL smaller or equal to the TTL of the answer RR.
+  If the cache memory is low; the answer RR may be dropped, and a glue
+  RR may be inserted, within the message TTL time, and thus return the
+  spoofed glue to a client. When the message expires, it is refetched and
+  the cached RR is updated with the correct content.
+  The server can be spoofed by getting it to visit a especially prepared 
+  domain. This domain then inserts an address for another authoritative 
+  server into the cache, when visiting that other domain, this address may
+  then be used to send queries to. And fake answers may be returned.
+  If the other domain is signed by DNSSEC, the fakes will be detected.
+
+  In summary, the harden glue feature presents a security risk if
+  disabled. Disabling the feature leads to possible better performance
+  as more glue is present for the recursive service to use. The feature
+  is implemented so as to minimise the security risk, while trying to 
+  keep this performance gain.

doc/unbound.8

@@ -1,38 +1,11 @@
-.ig
-unbound.8 -- unbound manual
-
-Copyright (c) 2007, NLnet Labs. All rights reserved.
-
-This software is open source.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-Redistributions of source code must retain the above copyright notice,
-this list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-Neither the name of the NLNET LABS nor the names of its contributors may
-be used to endorse or promote products derived from this software without
-specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-..
+.\"
+.\" unbound.8 -- unbound manual
+.\"
+.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
+.\"
+.\" See LICENSE for the license.
+.\"
+.\"
 .Dd @date@
 .Dt unbound 8
 .Sh NAME
@@ -44,7 +17,6 @@ unbound
 .Op Fl d
 .Op Fl v
 .Op Fl c Ar cfgfile
-
 .Sh DESCRIPTION
 .Ic Unbound 
 is an implementation of a DNS resolver, that does caching and 
@@ -52,27 +24,21 @@ DNSSEC validation.
 .Pp
 The available options are:
 .Bl -tag -width indent
-
 .It Fl h
 Show the version and commandline option help.
-
 .It Fl c Ar cfgfile
 Set the config file to read with settings for unbound. The syntax is
 described in 
 .Xr unbound.conf 5 .
-
 .It Fl d
 Debug flag, do not fork into the background, but stay attached to the
 console.
-
 .It Fl v
 Increase verbosity. If given multiple times, more information is logged.
 This is in addition to the verbosity (if any) from the config file.
-
 .El
 .Sh SEE ALSO
 .Xr unbound.conf 5 .
-
 .Sh AUTHORS
 .Ic Unbound
 developers are mentioned in the CREDITS file in the distribution.

doc/unbound.conf.5

@@ -20,22 +20,40 @@ is used to configure
 .Xr unbound 8 .
 The file format has attributes and values. Some attributes have attributes inside them.
 The notation is: attribute: value.
-
+.Pp
 Comments start with # and last to the end of line. Empty lines are
 ignored as is whitespace at the beginning of a line.
-
-.El
+.Sh EXAMPLE
+An example config file is shown below. Copy this to /etc/unbound/unbound.conf
+and start the server with:
+.nf
+	$ unbound -c /etc/unbound/unbound.conf 
+.fi
+Most settings are the defaults. Stop the server with:
+.nf
+	$ kill `cat /etc/unbound/unbound.pid`
+.fi
+Below is a minimal config file. The source distribution contains an extensive
+example.conf file with all the options.
+.nf
+# unbound.conf(5) config file for unbound(8).
+server:
+	directory: "/etc/unbound"
+	username: unbound 	# make sure it can write to pidfile, logfile.
+	chroot: "/etc/unbound"
+	logfile: "/etc/unbound/unbound.log"
+	pidfile: "/etc/unbound/unbound.pid"
+	# verbosity: 1		# uncomment and increase to get more logging.
+.fi
 .Sh FILE FORMAT
 There must be whitespace between keywords. Attribute keywords end with a colon ':'. An attribute
 is followed by its containing attributes, or a value.
-
 .Pp
 Files can be included using the
 .Ic include:
 directive. It can appear anywhere, and takes a single filename as an argument.
 Processing continues as if the text from the included file was copied into
 the config file at that point.
-
 .Ss Server Options
 There may only be one 
 .Ic server:
@@ -63,6 +81,45 @@ Number of ports to open. This number is opened per thread for every outgoing
 query interface. Must be at least 1. Default is 16.
 Larger numbers give more protection against spoofing attempts, but need
 extra resources from the operating system.
+.It \fBoutgoing-num-tcp:\fR <number>
+Number of outgoing TCP buffers to allocate per thread. Default is 10. If set
+to 0, or if do_tcp is "no", no TCP queries to authoritative servers are done.
+.It \fBincoming-num-tcp:\fR <number>
+Number of incoming TCP buffers to allocate per thread. Default is 10. If set
+to 0, or if do_tcp is "no", no TCP queries from clients are accepted.
+.It \fBmsg-buffer-size:\fR <number>
+Number of bytes size of the message buffers. Default is 65552 bytes, enough
+for 64 Kb packets, the maximum DNS message size. No message larger than this
+can be sent or received. Can be reduced to use less memory, but some requests
+for DNS data, such as for huge resource records, will result in a SERVFAIL 
+reply to the client.
+.It \fBmsg-cache-size:\fR <number>
+Number of bytes size of the message cache. Default is 4 megabytes.
+.It \fBmsg-cache-slabs:\fR <number>
+Number of slabs in the message cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. Setting (close) to the number of cpus is a 
+reasonable guess.
+.It \fBnum-queries-per-thread:\fR <number>
+The number of queries that every thread will service simultaneously.
+.It \fBrrset-cache-size:\fR <number>
+Number of bytes size of the RRset cache. Default is 4 megabytes.
+.It \fBrrset-cache-slabs:\fR <number>
+Number of slabs in the RRset cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. 
+.It \fBinfra-host-ttl:\fR <seconds>
+Time to live for entries in the host cache. The host cache contains 
+roundtrip timing and EDNS support information. Default is 900.
+.It \fBinfra-lame-ttl:\fR <seconds>
+The time to live when a delegation is discovered to be lame. Default is 900.
+.It \fBinfra-cache-slabs:\fR <number>
+Number of slabs in the infrastructure cache. Slabs reduce lock contention 
+by threads. Must be set to a power of 2. 
+.It \fBinfra-cache-numhosts:\fR <number>
+Number of hosts for which information is cached. Default is 10000.
+.It \fBinfra-cache-lame-size:\fR <number>
+Number of bytes that the lameness cache per host is allowed to use. Default
+is 10 kb, which gives maximum storage for a couple score zones, depending on 
+the lame zone name lengths.
 .It \fBdo-ip4:\fR <yes or no>
 Enable or disable whether ip4 queries are answered. Default is yes.
 .It \fBdo-ip6:\fR <yes or no>
@@ -71,20 +128,16 @@ Enable or disable whether ip6 queries are answered. Default is yes.
 Enable or disable whether UDP queries are answered. Default is yes.
 .It \fBdo-tcp:\fR <yes or no>
 Enable or disable whether TCP queries are answered. Default is yes.
-.It \fBforward-to:\fR <ip address>
-If set (not "") then forwarder mode is enabled. Default is "" (disabled).
-The ip address is used to forward all DNS queries to.
-.It \fBforward-to-port:\fR <port number>
-The port on which the remote server is running that answers forwarded queries.
-Default is 53.
 .It \fBchroot:\fR <directory>
 If given a chroot is done to the given directory. The default is none ("").
 .It \fBusername:\fR <name>
 If given, after binding the port the user privileges are dropped. Default is
-not to change user, username: "". If this user is not capable of binding the
+not to change user, username: "". 
+.Pp
+If this user is not capable of binding the
 port, reloads (by signal HUP) will still retain the opened ports.
 If you change the port number in the config file, and that new port number 
-requires privileges, then a restart is needed.
+requires privileges, then a reload will fail; a restart is needed.
 .It \fBdirectory:\fR <directory>
 Sets the working directory for the program.
 .It \fBlogfile:\fR <filename>
@@ -95,11 +148,176 @@ The logfile is appended to, in the following format:
 The process id is written to the file. Default is "unbound.pid". So,
 kill -HUP `cat /etc/unbound/unbound.pid` will trigger a reload,
 kill -QUIT `cat /etc/unbound/unbound.pid` will gracefully terminate.
-
+.It \fBhide-identity:\fR <yes or no>
+If enabled id.server and hostname.bind queries are refused.
+.It \fBidentity:\fR <string>
+Set the identity to report. If set to "", the default, then the hostname
+of the server is returned.
+.It \fBhide-version:\fR <yes or no>
+If enabled version.server and version.bind queries are refused.
+.It \fBversion:\fR <string>
+Set the version to report. If set to "", the default, then the package
+version is returned.
+.It \fBtarget-fetch-policy:\fR <"list of numbers">
+Set the target fetch policy used by unbound to determine if it should fetch
+nameserver target addresses opportunistically. The policy is described per
+dependency depth. 
+.Pp
+The number of values determines the maximum dependency depth
+that unbound will pursue in answering a query.  
+A value of -1 means to fetch all targets opportunistically for that dependency
+depth. A value of 0 means to fetch on demand only. A positive value fetches
+that many targets opportunistically. 
+.Pp
+Enclose the list between quotes ("") and put spaces between numbers.
+The default is "3 2 1 0 0". Setting all zeroes, "0 0 0 0 0" gives behaviour
+closer to that of BIND 9, while setting "-1 -1 -1 -1 -1" gives behaviour 
+rumoured to be closer to that of BIND 8.
+.It \fBharden-short-bufsize:\fR <yes or no>
+Very small EDNS buffer sizes from queries are ignored. Default is off, since
+it is legal protocol wise to send these, and unbound tries to give very 
+small answers to these queries, where possible.
+.It \fBharden-large-queries:\fR <yes or no>
+Very large queries are ignored. Default is off, since it is legal protocol
+wise to send these, and could be necessary for operation if TSIG or EDNS
+payload is very large.
+.It \fBharden-glue:\fR <yes or no>
+Will trust glue only if it is within the servers authority. Default is on.
+.It \fBdo-not-query-address:\fR <IP address>
+Do not query the given IP address. Can be IP4 or IP6. By default the
+DNS port is blocked for that address. Appending the character '@' and then
+the portnumber will block other port numbers. 
+.It \fBmodule-config:\fR <"module names">
+Module configuration, a list of module names separated by spaces, surround
+the string with quotes (""). The modules can be validator, iterator.
+Setting this to "iterator" will result in a non-validating server.
+Setting this to "validator iterator" will turn on DNSSEC validation.
+You must also set trust-anchors for validation to be useful.
+.It \fBtrust-anchor-file:\fR <filename>
+File with trusted keys for validation. Both DS and DNSKEY entries can appear
+in the file. The format of the file is the standard DNS Zone file format.
+Default is "", or no trust anchor file.
+.It \fBtrust-anchor:\fR <"Resource Record">
+A DS or DNSKEY RR for a key to use for validation. Multiple entries can be
+given to specify multiple trusted keys, in addition to the trust-anchor-files.
+The resource record is entered in the same format as 'dig' or 'drill' prints
+them, the same format as in the zone file. Has to be on a single line, with
+"" around it. A TTL can be specified for ease of cut and paste, but is ignored. 
+A class can be specified, but class IN is default.
+.It \fBtrusted-keys-file:\fR <filename>
+File with trusted keys for validation. Specify more than one file
+with several entries, one file per entry. Like \fBtrust-anchor-file\fR
+but has a different file format. Format is BIND-9 style format, 
+the trusted-keys { name flag proto algo "key"; }; clauses are read.
+.It \fBval-override-date:\fR <rrsig-style date spec>
+Default is "" or "0", which disables this debugging feature. If enabled by
+giving a RRSIG style date, that date is used for verifying RRSIG inception
+and expiration dates, instead of the current date. Do not set this unless 
+you are debugging signature inception and expiration.
+.It \fBval-bogus-ttl:\fR <number>
+The time to live for bogus data. This is data that has failed validation;
+due to invalid signatures or other checks. The TTL from that data cannot be
+trusted, and this value is used instead. The value is in seconds, default 900.
+The time interval prevents repeated revalidation of bogus data.
+.It \fBval-clean-additional:\fR <yes or no>
+Instruct the validator to remove data from the additional section of secure
+messages that are not signed properly. Messages that are insecure, bogus,
+indeterminate or unchecked are not affected. Default is yes. Use this setting
+to protect the users that rely on this validator for authentication from 
+protentially bad data in the additional section.
+.It \fBval-permissive-mode:\fR <yes or no>
+Instruct the validator to mark bogus messages as indeterminate. The security
+checks are performed, but if the result is bogus (failed security), the
+reply is not withheld from the client with SERVFAIL as usual. The client 
+receives the bogus data. For messages that are found to be secure the AD bit 
+is set in replies. Also logging is performed as for full validation.
+The default value is "no". 
+.It \fBval-nsec3-keysize-iterations:\fR <"list of values">
+List of keysize and iteration count values, separated by spaces, surrounded
+by quotes. Default is "1024 150 2048 500 4096 2500". This determines the
+maximum allowed NSEC3 iteration count before a message is simply marked
+insecure instead of performing the many hashing iterations. The list must
+be in ascending order and have at least one entry. If you set it to 
+"1024 65535" there is no restriction to NSEC3 iteration values.
+This table must be kept short; a very long list could cause slower operation.
+.It \fBkey-cache-size:\fR <number>
+Number of bytes size of the key cache. Default is 4 megabytes.
+.It \fBkey-cache-slabs:\fR <number>
+Number of slabs in the key cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. Setting (close) to the number of cpus is a 
+reasonable guess.
+.El
+.Ss Stub Zone Options
+There may be multiple
+.Ic stub-zone:
+clauses. Each with a name: and zero or more hostnames or IP addresses.
+For the stub zone this list of nameservers is used. Class IN is assumed.
+.Bl -tag -width indent
+.It \fBname:\fR <domain name>
+Name of the stub zone.
+.It \fBstub-host:\fR <domain name>
+Name of stub zone nameserver. Is itself resolved before it is used.
+.It \fBstub-addr:\fR <IP address>
+IP address of stub zone nameserver. Can be IP 4 or IP 6.
+To use a nondefault port for DNS communication append '@' with the port number.
+.El
+.Ss Forward Zone Options
+There may be multiple
+.Ic forward-zone:
+clauses. Each with a name: and zero or more hostnames or IP addresses.
+For the forward zone this list of nameservers is used to forward the queries
+to. The servers have to handle further recursion for the query. Class IN is 
+assumed. A forward-zone entry with name "." and a forward-addr target will 
+forward all queries to that other server (unless it can answer from the cache).
+.Bl -tag -width indent
+.It \fBname:\fR <domain name>
+Name of the forward zone.
+.It \fBforward-host:\fR <domain name>
+Name of server to forward to. Is itself resolved before it is used.
+.It \fBforward-addr:\fR <IP address>
+IP address of server to forward to. Can be IP 4 or IP 6.
+To use a nondefault port for DNS communication append '@' with the port number.
+.El
+.Sh MEMORY CONTROL EXAMPLE
+In the example config settings below memory usage is reduced. Some service
+levels are lower, notable very large data and a high TCP load are no longer
+supported. Very large data and high TCP loads are exceptional for the DNS.
+DNSSEC validation is enabled, just add trust anchors.
+If you do not have to worry about programs using more than 1 meg of memory,
+the below example is not for you. Use the defaults to receive full service.
+.Pp
+.nf
+# example settings that reduce memory usage
+server:
+	num-threads: 1
+	outgoing-num-tcp: 1	# this limits TCP service, uses less buffers.
+	incoming-num-tcp: 1
+	outgoing-range: 1	# uses less memory, but less port randomness.
+	msg-buffer-size: 8192   # note this limits service, 'no huge stuff'.
+	msg-cache-size: 102400  # 100 Kb.
+	msg-cache-slabs: 1
+	rrset-cache-size: 102400  # 100 Kb.
+	rrset-cache-slabs: 1
+	infra-cache-numhosts: 200
+	infra-cache-numlame: 10
+	key-cache-size: 102400  # 100 Kb.
+	key-cache-slabs: 1
+	num-queries-per-thread: 30
+	target-fetch-policy: "2 1 0 0 0 0"
+	harden-large-queries: "yes"
+	harden-short-bufsize: "yes"
+	do-ip6: no	# save a bit of memory if not used.
+.fi
 .Sh FILES
 .Bl -tag -width indent
+.It Pa /etc/unbound
+default unbound working directory
 .It Pa unbound.conf
 unbound configuration file.
+.It Pa unbound.pid
+default unbound pidfile with process ID of the running daemon.
+.It Pa unbound.log
+unbound log file.
 .El
 .Sh SEE ALSO
 .Xr unbound 8

doc/unbound.doxygen

@@ -1,4 +1,4 @@
-# Doxyfile 1.4.6
+# Doxyfile 1.5.2
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project
@@ -14,6 +14,14 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 
+# This tag specifies the encoding used for all characters in the config file that 
+# follow. The default is UTF-8 which is also the encoding used for all text before 
+# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into 
+# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of 
+# possible encodings.
+
+# DOXYFILE_ENCODING      = UTF-8
+
 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded 
 # by quotes) that should identify the project.
 
@@ -45,24 +53,14 @@ CREATE_SUBDIRS         = NO
 # documentation generated by doxygen is written. Doxygen will use this 
 # information to generate all constant output in the proper language. 
 # The default language is English, other supported languages are: 
-# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, 
-# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, 
-# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, 
-# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, 
-# Swedish, and Ukrainian.
+# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, 
+# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, 
+# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, 
+# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, 
+# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian.
 
 OUTPUT_LANGUAGE        = English
 
-# This tag can be used to specify the encoding used in the generated output. 
-# The encoding is not always determined by the language that is chosen, 
-# but also whether or not the output is meant for Windows or non-Windows users. 
-# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES 
-# forces the Windows encoding (this is the default for the Windows binary), 
-# whereas setting the tag to NO uses a Unix-style encoding (the default for 
-# all platforms other than Windows).
-
-USE_WINDOWS_ENCODING   = NO
-
 # If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will 
 # include brief member descriptions after the members that are listed in 
 # the file and class documentation (similar to JavaDoc). 
@@ -204,6 +202,11 @@ OPTIMIZE_OUTPUT_JAVA   = NO
 
 BUILTIN_STL_SUPPORT    = NO
 
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+
+# CPP_CLI_SUPPORT        = NO
+
 # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC 
 # tag is set to YES, then doxygen will reuse the documentation of the first 
 # member in the group (if any) for the other members of the group. By default 
@@ -459,7 +462,14 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories 
 # with spaces.
 
-INPUT                  =  .
+INPUT                  = .
+
+# This tag can be used to specify the character encoding of the source files that 
+# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default 
+# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. 
+# See http://www.gnu.org/software/libiconv for the list of possible encodings.
+
+# INPUT_ENCODING         = UTF-8
 
 # If the value of the INPUT tag contains directories, you can use the 
 # FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
@@ -468,7 +478,7 @@ INPUT                  =  .
 # *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx 
 # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py
 
-#FILE_PATTERNS          = *.h *.c *.dox
+FILE_PATTERNS          = 
 
 # The RECURSIVE tag can be used to turn specify whether or not subdirectories 
 # should be searched for input files as well. Possible values are YES and NO. 
@@ -480,7 +490,12 @@ RECURSIVE              = YES
 # excluded from the INPUT source files. This way you can easily exclude a 
 # subdirectory from a directory tree whose root is specified with the INPUT tag.
 
-EXCLUDE                =  ./build ./compat util/configparser.c util/configparser.h util/configlexer.c util/locks.h
+EXCLUDE                = ./build \
+                         ./compat \
+                         util/configparser.c \
+                         util/configparser.h \
+                         util/configlexer.c \
+                         util/locks.h
 
 # The EXCLUDE_SYMLINKS tag can be used select whether or not files or 
 # directories that are symbolic links (a Unix filesystem feature) are excluded 
@@ -496,6 +511,13 @@ EXCLUDE_SYMLINKS       = NO
 
 EXCLUDE_PATTERNS       = 
 
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names 
+# (namespaces, classes, functions, etc.) that should be excluded from the output. 
+# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, 
+# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test
+
+# EXCLUDE_SYMBOLS        = 
+
 # The EXAMPLE_PATH tag can be used to specify one or more files or 
 # directories that contain example code fragments that are included (see 
 # the \include command).
@@ -581,6 +603,13 @@ REFERENCED_BY_RELATION = YES
 
 REFERENCES_RELATION    = YES
 
+# If the REFERENCES_LINK_SOURCE tag is set to YES (the default)
+# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from
+# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will
+# link to the source code.  Otherwise they will link to the documentstion.
+
+# REFERENCES_LINK_SOURCE = YES
+
 # If the USE_HTAGS tag is set to YES then the references to source code 
 # will point to the HTML generated by the htags(1) tool instead of doxygen 
 # built-in source browser. The htags tool is part of GNU's global source 
@@ -1078,6 +1107,14 @@ PERL_PATH              = /usr/bin/perl
 
 CLASS_DIAGRAMS         = YES
 
+# You can define message sequence charts within doxygen comments using the \msc 
+# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to 
+# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to 
+# specify the directory where the mscgen tool resides. If left empty the tool is assumed to 
+# be found in the default search path.
+
+# MSCGEN_PATH            = 
+
 # If set to YES, the inheritance and collaboration graphs will hide 
 # inheritance and usage relations if the target is undocumented 
 # or is not a class.
@@ -1143,6 +1180,14 @@ INCLUDED_BY_GRAPH      = YES
 
 CALL_GRAPH             = NO
 
+# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will 
+# generate a caller dependency graph for every global function or class method. 
+# Note that enabling this option will significantly increase the time of a run. 
+# So in most cases it will be better to enable caller graphs for selected 
+# functions only using the \callergraph command.
+
+# CALLER_GRAPH           = NO
+
 # If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen 
 # will graphical hierarchy of all classes instead of a textual one.
 
@@ -1172,33 +1217,13 @@ DOT_PATH               =
 
 DOTFILE_DIRS           = 
 
-# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width 
-# (in pixels) of the graphs generated by dot. If a graph becomes larger than 
-# this value, doxygen will try to truncate the graph, so that it fits within 
-# the specified constraint. Beware that most browsers cannot cope with very 
-# large images.
+# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of 
+# nodes that will be shown in the graph. If the number of nodes in a graph 
+# becomes larger than this value, doxygen will truncate the graph, which is 
+# visualized by representing a node as a red box. Note that doxygen will always 
+# show the root nodes and its direct children regardless of this setting.
 
-MAX_DOT_GRAPH_WIDTH    = 1024
-
-# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height 
-# (in pixels) of the graphs generated by dot. If a graph becomes larger than 
-# this value, doxygen will try to truncate the graph, so that it fits within 
-# the specified constraint. Beware that most browsers cannot cope with very 
-# large images.
-
-MAX_DOT_GRAPH_HEIGHT   = 1024
-
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the 
-# graphs generated by dot. A depth value of 3 means that only nodes reachable 
-# from the root by following a path via at most 3 edges will be shown. Nodes 
-# that lay further from the root node will be omitted. Note that setting this 
-# option to 1 or 2 may greatly reduce the computation time needed for large 
-# code bases. Also note that a graph may be further truncated if the graph's 
-# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH 
-# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), 
-# the graph is not depth-constrained.
-
-MAX_DOT_GRAPH_DEPTH    = 0
+# DOT_GRAPH_MAX_NODES    = 50
 
 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent 
 # background. This is disabled by default, which results in a white background. 

iterator/iter_delegpt.c

@@ -0,0 +1,383 @@
+/*
+ * iterator/iter_delegpt.c - delegation point with NS and address information.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file implements the Delegation Point. It contains a list of name servers
+ * and their addresses if known.
+ */
+#include "config.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "util/region-allocator.h"
+#include "util/data/dname.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/msgreply.h"
+#include "util/net_help.h"
+
+struct delegpt* 
+delegpt_create(struct region* region)
+{
+	struct delegpt* dp=(struct delegpt*)region_alloc(region, sizeof(*dp));
+	if(!dp)
+		return NULL;
+	memset(dp, 0, sizeof(*dp));
+	return dp;
+}
+
+struct delegpt* delegpt_copy(struct delegpt* dp, struct region* region)
+{
+	struct delegpt* copy = delegpt_create(region);
+	struct delegpt_ns* ns;
+	struct delegpt_addr* a;
+	if(!copy) 
+		return NULL;
+	if(!delegpt_set_name(copy, region, dp->name))
+		return NULL;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(!delegpt_add_ns(copy, region, ns->name))
+			return NULL;
+		copy->nslist->resolved = ns->resolved;
+	}
+	for(a = dp->target_list; a; a = a->next_target) {
+		if(!delegpt_add_addr(copy, region, &a->addr, a->addrlen))
+			return NULL;
+	}
+	return copy;
+}
+
+int 
+delegpt_set_name(struct delegpt* dp, struct region* region, uint8_t* name)
+{
+	dp->namelabs = dname_count_size_labels(name, &dp->namelen);
+	dp->name = region_alloc_init(region, name, dp->namelen);
+	return dp->name != 0;
+}
+
+int 
+delegpt_add_ns(struct delegpt* dp, struct region* region, uint8_t* name)
+{
+	struct delegpt_ns* ns;
+	size_t len;
+	(void)dname_count_size_labels(name, &len);
+	/* slow check for duplicates to avoid counting failures when
+	 * adding the same server as a dependency twice */
+	if(delegpt_find_ns(dp, name, len))
+		return 1;
+	ns = (struct delegpt_ns*)region_alloc(region,
+		sizeof(struct delegpt_ns));
+	if(!ns)
+		return 0;
+	ns->next = dp->nslist;
+	ns->namelen = len;
+	dp->nslist = ns;
+	ns->name = region_alloc_init(region, name, ns->namelen);
+	ns->resolved = 0;
+	return 1;
+}
+
+struct delegpt_ns*
+delegpt_find_ns(struct delegpt* dp, uint8_t* name, size_t namelen)
+{
+	struct delegpt_ns* p = dp->nslist;
+	while(p) {
+		if(namelen == p->namelen && 
+			query_dname_compare(name, p->name) == 0) {
+			return p;
+		}
+		p = p->next;
+	}
+	return NULL;
+}
+
+int 
+delegpt_add_target(struct delegpt* dp, struct region* region, 
+	uint8_t* name, size_t namelen, struct sockaddr_storage* addr, 
+	socklen_t addrlen)
+{
+	struct delegpt_ns* ns = delegpt_find_ns(dp, name, namelen);
+	if(!ns) {
+		/* ignore it */
+		return 1;
+	}
+	ns->resolved = 1;
+	return delegpt_add_addr(dp, region, addr, addrlen);
+}
+
+int 
+delegpt_add_addr(struct delegpt* dp, struct region* region, 
+	struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	struct delegpt_addr* a = (struct delegpt_addr*)region_alloc(region,
+		sizeof(struct delegpt_addr));
+	if(!a)
+		return 0;
+	a->next_target = dp->target_list;
+	dp->target_list = a;
+	a->next_result = 0;
+	a->next_usable = dp->usable_list;
+	dp->usable_list = a;
+	memcpy(&a->addr, addr, addrlen);
+	a->addrlen = addrlen;
+	return 1;
+}
+
+/** count NS and number missing */
+static void
+delegpt_count_ns(struct delegpt* dp, size_t* numns, size_t* missing)
+{
+	struct delegpt_ns* ns;
+	*numns = 0;
+	*missing = 0;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		(*numns)++;
+		if(!ns->resolved)
+			(*missing)++;
+	}
+}
+
+/** count addresses, and number in result and available lists */
+static void
+delegpt_count_addr(struct delegpt* dp, size_t* numaddr, size_t* numres, 
+	size_t* numavail)
+{
+	struct delegpt_addr* a;
+	*numaddr = 0;
+	*numres = 0;
+	*numavail = 0;
+	for(a = dp->target_list; a; a = a->next_target) {
+		(*numaddr)++;
+	}
+	for(a = dp->result_list; a; a = a->next_result) {
+		(*numres)++;
+	}
+	for(a = dp->usable_list; a; a = a->next_usable) {
+		(*numavail)++;
+	}
+}
+
+void delegpt_log(struct delegpt* dp)
+{
+	char buf[LDNS_MAX_DOMAINLEN+1];
+	struct delegpt_ns* ns;
+	struct delegpt_addr* a;
+	size_t missing=0, numns=0, numaddr=0, numres=0, numavail=0;
+	dname_str(dp->name, buf);
+	if(dp->nslist == NULL && dp->target_list == NULL) {
+		log_info("DelegationPoint<%s>: empty", buf);
+		return;
+	}
+	delegpt_count_ns(dp, &numns, &missing);
+	delegpt_count_addr(dp, &numaddr, &numres, &numavail);
+	log_info("DelegationPoint<%s>: %u names (%u missing), "
+		"%u addrs (%u result, %u avail)", 
+		buf, (unsigned)numns, (unsigned)missing, 
+		(unsigned)numaddr, (unsigned)numres, (unsigned)numavail);
+	if(verbosity >= VERB_ALGO) {
+		for(ns = dp->nslist; ns; ns = ns->next) {
+			dname_str(ns->name, buf);
+			log_info("  %s%s", buf, (ns->resolved?"*":""));
+		}
+		for(a = dp->target_list; a; a = a->next_target) {
+			log_addr("  ", &a->addr, a->addrlen);
+		}
+	}
+}
+
+void 
+delegpt_add_unused_targets(struct delegpt* dp)
+{
+	struct delegpt_addr* usa = dp->usable_list;
+	dp->usable_list = NULL;
+	while(usa) {
+		usa->next_result = dp->result_list;
+		dp->result_list = usa;
+		usa = usa->next_usable;
+	}
+}
+
+size_t 
+delegpt_count_missing_targets(struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	size_t n = 0;
+	for(ns = dp->nslist; ns; ns = ns->next)
+		if(!ns->resolved)
+			n++;
+	return n;
+}
+
+/** find NS rrset in given list */
+static struct ub_packed_rrset_key*
+find_NS(struct reply_info* rep, size_t from, size_t to)
+{
+	size_t i;
+	for(i=from; i<to; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == LDNS_RR_TYPE_NS)
+			return rep->rrsets[i];
+	}
+	return NULL;
+}
+
+struct delegpt* 
+delegpt_from_message(struct dns_msg* msg, struct region* region)
+{
+	struct ub_packed_rrset_key* ns_rrset = NULL;
+	struct delegpt* dp;
+	size_t i;
+	/* look for NS records in the authority section... */
+	ns_rrset = find_NS(msg->rep, msg->rep->an_numrrsets, 
+		msg->rep->an_numrrsets+msg->rep->ns_numrrsets);
+
+	/* In some cases (even legitimate, perfectly legal cases), the 
+	 * NS set for the "referral" might be in the answer section. */
+	if(!ns_rrset)
+		ns_rrset = find_NS(msg->rep, 0, msg->rep->an_numrrsets);
+	
+	/* If there was no NS rrset in the authority section, then this 
+	 * wasn't a referral message. (It might not actually be a 
+	 * referral message anyway) */
+	if(!ns_rrset)
+		return NULL;
+	
+	/* If we found any, then Yay! we have a delegation point. */
+	dp = delegpt_create(region);
+	if(!dp)
+		return NULL;
+	if(!delegpt_set_name(dp, region, ns_rrset->rk.dname))
+		return NULL;
+	if(!delegpt_rrset_add_ns(dp, region, ns_rrset))
+		return NULL;
+
+	/* add glue, A and AAAA in answer and additional section */
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+		/* skip auth section. FIXME really needed?*/
+		if(msg->rep->an_numrrsets <= i && 
+			i < (msg->rep->an_numrrsets+msg->rep->ns_numrrsets))
+			continue;
+
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_A) {
+			if(!delegpt_add_rrset_A(dp, region, s))
+				return NULL;
+		} else if(ntohs(s->rk.type) == LDNS_RR_TYPE_AAAA) {
+			if(!delegpt_add_rrset_AAAA(dp, region, s))
+				return NULL;
+		}
+	}
+	return dp;
+}
+
+int 
+delegpt_rrset_add_ns(struct delegpt* dp, struct region* region,
+        struct ub_packed_rrset_key* ns_rrset)
+{
+	struct packed_rrset_data* nsdata = (struct packed_rrset_data*)
+		ns_rrset->entry.data;
+	size_t i;
+	for(i=0; i<nsdata->count; i++) {
+		if(nsdata->rr_len[i] < 2+1) continue; /* len + root label */
+		if(dname_valid(nsdata->rr_data[i]+2, nsdata->rr_len[i]-2) !=
+			(size_t)ldns_read_uint16(nsdata->rr_data[i]))
+			continue; /* bad format */
+		/* add rdata of NS (= wirefmt dname), skip rdatalen bytes */
+		if(!delegpt_add_ns(dp, region, nsdata->rr_data[i]+2))
+			return 0;
+	}
+	return 1;
+}
+
+int 
+delegpt_add_rrset_A(struct delegpt* dp, struct region* region,
+	struct ub_packed_rrset_key* ak)
+{
+        struct packed_rrset_data* d=(struct packed_rrset_data*)ak->entry.data;
+        size_t i;
+        struct sockaddr_in sa;
+        socklen_t len = (socklen_t)sizeof(sa);
+        memset(&sa, 0, len);
+        sa.sin_family = AF_INET;
+        sa.sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+        for(i=0; i<d->count; i++) {
+                if(d->rr_len[i] != 2 + INET_SIZE)
+                        continue;
+                memmove(&sa.sin_addr, d->rr_data[i]+2, INET_SIZE);
+                if(!delegpt_add_target(dp, region, ak->rk.dname,
+                        ak->rk.dname_len, (struct sockaddr_storage*)&sa,
+                        len))
+                        return 0;
+        }
+        return 1;
+}
+
+int 
+delegpt_add_rrset_AAAA(struct delegpt* dp, struct region* region,
+	struct ub_packed_rrset_key* ak)
+{
+        struct packed_rrset_data* d=(struct packed_rrset_data*)ak->entry.data;
+        size_t i;
+        struct sockaddr_in6 sa;
+        socklen_t len = (socklen_t)sizeof(sa);
+        memset(&sa, 0, len);
+        sa.sin6_family = AF_INET6;
+        sa.sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+        for(i=0; i<d->count; i++) {
+                if(d->rr_len[i] != 2 + INET6_SIZE) /* rdatalen + len of IP6 */
+                        continue;
+                memmove(&sa.sin6_addr, d->rr_data[i]+2, INET6_SIZE);
+                if(!delegpt_add_target(dp, region, ak->rk.dname,
+                        ak->rk.dname_len, (struct sockaddr_storage*)&sa,
+                        len))
+                        return 0;
+        }
+        return 1;
+}
+
+int 
+delegpt_add_rrset(struct delegpt* dp, struct region* region,
+        struct ub_packed_rrset_key* rrset)
+{
+	if(!rrset)
+		return 1;
+	if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_NS)
+		return delegpt_rrset_add_ns(dp, region, rrset);
+	else if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_A)
+		return delegpt_add_rrset_A(dp, region, rrset);
+	else if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_AAAA)
+		return delegpt_add_rrset_AAAA(dp, region, rrset);
+	log_warn("Unknown rrset type added to delegpt");
+	return 1;
+}

iterator/iter_delegpt.h

@@ -0,0 +1,257 @@
+/*
+ * iterator/iter_delegpt.h - delegation point with NS and address information.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file implements the Delegation Point. It contains a list of name servers
+ * and their addresses if known.
+ */
+
+#ifndef ITERATOR_ITER_DELEGPT_H
+#define ITERATOR_ITER_DELEGPT_H
+struct region;
+struct delegpt_ns;
+struct delegpt_addr;
+struct dns_msg;
+struct ub_packed_rrset_key;
+
+/**
+ * Delegation Point.
+ * For a domain name, the NS rrset, and the A and AAAA records for those.
+ */
+struct delegpt {
+	/** the domain name of the delegation point. */
+	uint8_t* name;
+	/** length of the delegation point name */
+	size_t namelen;
+	/** number of labels in delegation point */
+	int namelabs;
+
+	/** the nameservers, names from the NS RRset rdata. */
+	struct delegpt_ns* nslist;
+	/** the target addresses for delegation */
+	struct delegpt_addr* target_list;
+	/** the list of usable targets; subset of target_list 
+	 * the items in this list are not part of the result list.  */
+	struct delegpt_addr* usable_list;
+	/** the list of returned targets; subset of target_list */
+	struct delegpt_addr* result_list;
+};
+
+/**
+ * Nameservers for a delegation point.
+ */
+struct delegpt_ns {
+	/** next in list */
+	struct delegpt_ns* next;
+	/** name of nameserver */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** 
+	 * If the name has been resolved. false if not queried for yet.
+	 * true if the address is known, or marked true if failed.
+	 */
+	int resolved;
+};
+
+/**
+ * Address of target nameserver in delegation point.
+ */
+struct delegpt_addr {
+	/** next delegation point in results */
+	struct delegpt_addr* next_result;
+	/** next delegation point in usable list */
+	struct delegpt_addr* next_usable;
+	/** next delegation point in all targets list */
+	struct delegpt_addr* next_target;
+
+	/** delegation point address */
+	struct sockaddr_storage addr;
+	/** length of addr */
+	socklen_t addrlen;
+};
+
+/**
+ * Create new delegation point.
+ * @param region: where to allocate it.
+ * @return new delegation point or NULL on error.
+ */
+struct delegpt* delegpt_create(struct region* region);
+
+/**
+ * Create a copy of a delegation point.
+ * @param dp: delegation point to copy.
+ * @param region: where to allocate it.
+ * @return new delegation point or NULL on error.
+ */
+struct delegpt* delegpt_copy(struct delegpt* dp, struct region* region);
+
+/**
+ * Set name of delegation point.
+ * @param dp: delegation point.
+ * @param region: where to allocate the name copy.
+ * @param name: name to use.
+ * @return false on error.
+ */
+int delegpt_set_name(struct delegpt* dp, struct region* region, uint8_t* name);
+
+/**
+ * Add a name to the delegation point.
+ * @param dp: delegation point.
+ * @param region: where to allocate the info.
+ * @param name: domain name in wire format.
+ * @return false on error.
+ */
+int delegpt_add_ns(struct delegpt* dp, struct region* region, uint8_t* name);
+
+/**
+ * Add NS rrset; calls add_ns repeatedly.
+ * @param dp: delegation point.
+ * @param region: where to allocate the info.
+ * @param ns_rrset: NS rrset.
+ * return 0 on alloc error.
+ */
+int delegpt_rrset_add_ns(struct delegpt* dp, struct region* region,
+	struct ub_packed_rrset_key* ns_rrset);
+
+/**
+ * Add target address to the delegation point.
+ * @param dp: delegation point.
+ * @param region: where to allocate the info.
+ * @param name: name for which target was found (must be in nslist).
+ *	This name is marked resolved.
+ * @param namelen: length of name.
+ * @param addr: the address.
+ * @param addrlen: the length of addr.
+ * @return false on error.
+ */
+int delegpt_add_target(struct delegpt* dp, struct region* region, 
+	uint8_t* name, size_t namelen, struct sockaddr_storage* addr, 
+	socklen_t addrlen);
+
+/**
+ * Add A RRset to delegpt.
+ * @param dp: delegation point.
+ * @param region: where to allocate the info.
+ * @param rrset: RRset A to add.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset_A(struct delegpt* dp, struct region* region, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add AAAA RRset to delegpt.
+ * @param dp: delegation point.
+ * @param region: where to allocate the info.
+ * @param rrset: RRset AAAA to add.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset_AAAA(struct delegpt* dp, struct region* region, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add any RRset to delegpt.
+ * @param dp: delegation point.
+ * @param region: where to allocate the info.
+ * @param rrset: RRset to add, NS, A, AAAA.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset(struct delegpt* dp, struct region* region, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add address to the delegation point. No servername is associated or checked.
+ * @param dp: delegation point.
+ * @param region: where to allocate the info.
+ * @param addr: the address.
+ * @param addrlen: the length of addr.
+ * @return false on error.
+ */
+int delegpt_add_addr(struct delegpt* dp, struct region* region, 
+	struct sockaddr_storage* addr, socklen_t addrlen);
+
+/** 
+ * Find NS record in name list of delegation point.
+ * @param dp: delegation point.
+ * @param name: name of nameserver to look for, uncompressed wireformat.
+ * @param namelen: length of name.
+ * @return the ns structure or NULL if not found.
+ */
+struct delegpt_ns* delegpt_find_ns(struct delegpt* dp, uint8_t* name, 
+	size_t namelen);
+
+/**
+ * Print the delegation point to the log. For debugging.
+ * @param dp: delegation point.
+ */
+void delegpt_log(struct delegpt* dp);
+
+/**
+ * Add all usable targets to the result list.
+ * @param dp: delegation point.
+ */
+void delegpt_add_unused_targets(struct delegpt* dp);
+
+/**
+ * Count number of missing targets. These are ns names with no resolved flag.
+ * @param dp: delegation point.
+ * @return number of missing targets (or 0).
+ */
+size_t delegpt_count_missing_targets(struct delegpt* dp);
+
+/**
+ * Create new delegation point from a dns message
+ *
+ * Note that this method does not actually test to see if the message is an
+ * actual referral. It really is just checking to see if it can construct a
+ * delegation point, so the message could be of some other type (some ANSWER
+ * messages, some CNAME messages, generally.) Note that the resulting
+ * DelegationPoint will contain targets for all "relevant" glue (i.e.,
+ * address records whose ownernames match the target of one of the NS
+ * records), so if policy dictates that some glue should be discarded beyond
+ * that, discard it before calling this method. Note that this method will
+ * find "glue" in either the ADDITIONAL section or the ANSWER section.
+ *
+ * @param msg: the dns message, referral.
+ * @param region: where to allocate delegation point.
+ * @return new delegation point or NULL on alloc error, or if the
+ *         message was not appropriate.
+ */
+struct delegpt* delegpt_from_message(struct dns_msg* msg, 
+	struct region* region);
+
+#endif /* ITERATOR_ITER_DELEGPT_H */

iterator/iter_donotq.c

@@ -0,0 +1,156 @@
+/*
+ * iterator/iter_donotq.c - iterative resolver donotqueryaddresses storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * The donotqueryaddresses are stored and looked up. These addresses
+ * (like 127.0.0.1) must not be used to send queries to, and can be
+ * discarded immediately from the server selection.
+ */
+#include "config.h"
+#include "iterator/iter_donotq.h"
+#include "util/region-allocator.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+
+/** compare two donotq entries */
+static int
+donotq_cmp(const void* k1, const void* k2)
+{
+	struct iter_donotq_addr* n1 = (struct iter_donotq_addr*)k1;
+	struct iter_donotq_addr* n2 = (struct iter_donotq_addr*)k2;
+	return sockaddr_cmp(&n1->addr, n1->addrlen, &n2->addr, n2->addrlen);
+}
+
+struct iter_donotq* 
+donotq_create()
+{
+	struct iter_donotq* dq = (struct iter_donotq*)calloc(1,
+		sizeof(struct iter_donotq));
+	if(!dq)
+		return NULL;
+	dq->region = region_create(malloc, free);
+	if(!dq->region) {
+		donotq_delete(dq);
+		return NULL;
+	}
+	return dq;
+}
+
+void 
+donotq_delete(struct iter_donotq* dq)
+{
+	if(!dq) 
+		return;
+	region_destroy(dq->region);
+	free(dq->tree);
+	free(dq);
+}
+
+/** insert new address into donotq structure */
+static int
+donotq_insert(struct iter_donotq* dq, struct sockaddr_storage* addr, 
+	socklen_t addrlen)
+{
+	struct iter_donotq_addr* node = region_alloc(dq->region,
+		sizeof(struct iter_donotq_addr));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	memcpy(&node->addr, addr, addrlen);
+	node->addrlen = addrlen;
+	if(!rbtree_insert(dq->tree, &node->node)) {
+		log_warn("duplicate donotquery address ignored.");
+	}
+	return 1;
+}
+
+/** read donotq config */
+static int 
+read_donotq(struct iter_donotq* dq, struct config_file* cfg)
+{
+	struct config_strlist* p;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	for(p = cfg->donotqueryaddrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!extstrtoaddr(p->str, &addr, &addrlen)) {
+			log_err("cannot parse donotquery ip address: '%s'", 
+				p->str);
+			return 0;
+		}
+		if(!donotq_insert(dq, &addr, addrlen)) {
+			log_err("out of memory");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+int 
+donotq_apply_cfg(struct iter_donotq* dq, struct config_file* cfg)
+{
+	free(dq->tree);
+	dq->tree = rbtree_create(donotq_cmp);
+	if(!dq->tree)
+		return 0;
+	if(!read_donotq(dq, cfg))
+		return 0;
+	return 1;
+}
+
+int 
+donotq_lookup(struct iter_donotq* donotq, struct sockaddr_storage* addr,
+        socklen_t addrlen)
+{
+	/* lookup in the tree */
+	struct iter_donotq_addr key;
+	key.node.key = &key;
+	memcpy(&key.addr, addr, addrlen);
+	key.addrlen = addrlen;
+	if(rbtree_search(donotq->tree, &key))
+		return 1;
+	return 0;
+}
+
+size_t 
+donotq_get_mem(struct iter_donotq* donotq)
+{
+	if(!donotq) return 0;
+	return sizeof(*donotq) + region_get_mem(donotq->region);
+}

iterator/iter_donotq.h

@@ -0,0 +1,114 @@
+/*
+ * iterator/iter_donotq.h - iterative resolver donotqueryaddresses storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of the donotquery addresses and lookup fast.
+ */
+
+#ifndef ITERATOR_ITER_DONOTQ_H
+#define ITERATOR_ITER_DONOTQ_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct region;
+
+/**
+ * Iterator donotqueryaddresses structure
+ */
+struct iter_donotq {
+	/** region for allocation */
+	struct region* region;
+	/** 
+	 * Tree of the single addresses that are blocked.
+	 * contents of type iter_donotq_addr.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator donotquery address.
+ * A single address that must not be used to send queries to.
+ */
+struct iter_donotq_addr {
+	/** redblacktree node, key is this structure: addr and addrlen */
+	rbnode_t node;
+	/** address */
+	struct sockaddr_storage addr;
+	/** length of addr */
+	socklen_t addrlen;
+};
+
+/**
+ * Create donotqueryaddresses structure 
+ * @return new structure or NULL on error.
+ */
+struct iter_donotq* donotq_create();
+
+/**
+ * Delete donotqueryaddresses structure.
+ * @param donotq: to delete.
+ */
+void donotq_delete(struct iter_donotq* donotq);
+
+/**
+ * Process donotqueryaddresses config.
+ * @param donotq: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int donotq_apply_cfg(struct iter_donotq* donotq, struct config_file* cfg);
+
+/**
+ * See if an address is blocked.
+ * @param donotq: structure for address storage.
+ * @param addr: address to check
+ * @param addrlen: length of addr.
+ * @return: true if the address must not be queried. false if unlisted.
+ */
+int donotq_lookup(struct iter_donotq* donotq, struct sockaddr_storage* addr,
+	socklen_t addrlen);
+
+/**
+ * Get memory used by donotqueryaddresses structure.
+ * @param donotq: structure for address storage.
+ * @return bytes in use.
+ */
+size_t donotq_get_mem(struct iter_donotq* donotq);
+
+
+#endif /* ITERATOR_ITER_DONOTQ_H */

iterator/iter_fwd.c

@@ -0,0 +1,293 @@
+/*
+ * iterator/iter_fwd.c - iterative resolver module forward zones.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of forward zones and config settings.
+ */
+#include "config.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_delegpt.h"
+#include "util/region-allocator.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+/** compare two fwd entries */
+static int
+fwd_cmp(const void* k1, const void* k2)
+{
+	int m;
+	struct iter_forward_zone* n1 = (struct iter_forward_zone*)k1;
+	struct iter_forward_zone* n2 = (struct iter_forward_zone*)k2;
+	if(n1->dclass != n2->dclass) {
+		if(n1->dclass < n2->dclass)
+			return -1;
+		return 1;
+	}
+	return dname_lab_cmp(n1->name, n1->namelabs, n2->name, n2->namelabs, 
+		&m);
+}
+
+struct iter_forwards* 
+forwards_create()
+{
+	struct iter_forwards* fwd = (struct iter_forwards*)calloc(1,
+		sizeof(struct iter_forwards));
+	if(!fwd)
+		return NULL;
+	fwd->region = region_create(malloc, free);
+	if(!fwd->region) {
+		forwards_delete(fwd);
+		return NULL;
+	}
+	return fwd;
+}
+
+void 
+forwards_delete(struct iter_forwards* fwd)
+{
+	if(!fwd) 
+		return;
+	region_destroy(fwd->region);
+	free(fwd->tree);
+	free(fwd);
+}
+
+/** insert new info into forward structure */
+static int
+forwards_insert(struct iter_forwards* fwd, uint16_t c, struct delegpt* dp)
+{
+	struct iter_forward_zone* node = region_alloc(fwd->region,
+		sizeof(struct iter_forward_zone));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	node->dclass = c;
+	node->name = region_alloc_init(fwd->region, dp->name, dp->namelen);
+	if(!node->name)
+		return 0;
+	node->namelen = dp->namelen;
+	node->namelabs = dp->namelabs;
+	node->dp = dp;
+	if(!rbtree_insert(fwd->tree, &node->node)) {
+		log_err("duplicate forward zone ignored.");
+	}
+	return 1;
+}
+
+/** initialise parent pointers in the tree */
+static void
+fwd_init_parents(struct iter_forwards* fwd)
+{
+	struct iter_forward_zone* node, *prev = NULL, *p;
+	int m;
+	RBTREE_FOR(node, struct iter_forward_zone*, fwd->tree) {
+		node->parent = NULL;
+		if(!prev || prev->dclass != node->dclass) {
+			prev = node;
+			continue;
+		}
+		(void)dname_lab_cmp(prev->name, prev->namelabs, node->name,
+			node->namelabs, &m); /* we know prev is smaller */
+		/* sort order like: . com. bla.com. zwb.com. net. */
+		/* find the previous, or parent-parent-parent */
+		for(p = prev; p; p = p->parent)
+			/* looking for name with few labels, a parent */
+			if(p->namelabs <= m) {
+				/* ==: since prev matched m, this is closest*/
+				/* <: prev matches more, but is not a parent,
+				 * this one is a (grand)parent */
+				node->parent = p;
+				break;
+			}
+		prev = node;
+	}
+}
+
+/** set zone name */
+static int 
+read_fwds_name(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	ldns_rdf* rdf;
+	if(!s->name) {
+		log_err("forward zone without a name (use name \".\" to forward everything)");
+		return 0;
+	}
+	rdf = ldns_dname_new_frm_str(s->name);
+	if(!rdf) {
+		log_err("cannot parse forward zone name %s", s->name);
+		return 0;
+	}
+	if(!delegpt_set_name(dp, fwd->region, ldns_rdf_data(rdf))) {
+		ldns_rdf_deep_free(rdf);
+		log_err("out of memory");
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** set fwd host names */
+static int 
+read_fwds_host(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	ldns_rdf* rdf;
+	for(p = s->hosts; p; p = p->next) {
+		log_assert(p->str);
+		rdf = ldns_dname_new_frm_str(p->str);
+		if(!rdf) {
+			log_err("cannot parse forward %s server name: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_ns(dp, fwd->region, ldns_rdf_data(rdf))) {
+			ldns_rdf_deep_free(rdf);
+			log_err("out of memory");
+			return 0;
+		}
+		ldns_rdf_deep_free(rdf);
+	}
+	return 1;
+}
+
+/** set fwd server addresses */
+static int 
+read_fwds_addr(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	for(p = s->addrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!extstrtoaddr(p->str, &addr, &addrlen)) {
+			log_err("cannot parse forward %s ip address: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_addr(dp, fwd->region, &addr, addrlen)) {
+			log_err("out of memory");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/** read forwards config */
+static int 
+read_forwards(struct iter_forwards* fwd, struct config_file* cfg)
+{
+	struct config_stub* s;
+	for(s = cfg->forwards; s; s = s->next) {
+		struct delegpt* dp = delegpt_create(fwd->region);
+		if(!dp) {
+			log_err("out of memory");
+			return 0;
+		}
+		if(!read_fwds_name(fwd, s, dp) ||
+			!read_fwds_host(fwd, s, dp) ||
+			!read_fwds_addr(fwd, s, dp))
+			return 0;
+		if(!forwards_insert(fwd, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		log_info("Forward zone server list:");
+		delegpt_log(dp);
+	}
+	return 1;
+}
+
+int 
+forwards_apply_cfg(struct iter_forwards* fwd, struct config_file* cfg)
+{
+	free(fwd->tree);
+	fwd->tree = rbtree_create(fwd_cmp);
+	if(!fwd->tree)
+		return 0;
+
+	/* read forward zones */
+	if(!read_forwards(fwd, cfg))
+		return 0;
+	fwd_init_parents(fwd);
+	return 1;
+}
+
+struct delegpt* 
+forwards_lookup(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass)
+{
+	/* lookup the forward zone in the tree */
+	rbnode_t* res = NULL;
+	struct iter_forward_zone *result;
+	struct iter_forward_zone key;
+	key.node.key = &key;
+	key.dclass = qclass;
+	key.name = qname;
+	key.namelabs = dname_count_size_labels(qname, &key.namelen);
+	if(rbtree_find_less_equal(fwd->tree, &key, &res)) {
+		/* exact */
+		result = (struct iter_forward_zone*)res;
+	} else {
+		/* smaller element (or no element) */
+		int m;
+		result = (struct iter_forward_zone*)res;
+		if(!result || result->dclass != qclass)
+			return NULL;
+		/* count number of labels matched */
+		(void)dname_lab_cmp(result->name, result->namelabs, key.name,
+			key.namelabs, &m);
+		while(result) { /* go up until qname is subdomain of stub */
+			if(result->namelabs <= m)
+				break;
+			result = result->parent;
+		}
+	}
+	if(result)
+		return result->dp;
+	return NULL;
+}
+
+size_t 
+forwards_get_mem(struct iter_forwards* fwd)
+{
+	if(!fwd)
+		return 0;
+	return sizeof(*fwd) + region_get_mem(fwd->region);
+}

iterator/iter_fwd.h

@@ -0,0 +1,128 @@
+/*
+ * iterator/iter_fwd.h - iterative resolver module forward zones.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of forward zones, and read those from config.
+ */
+
+#ifndef ITERATOR_ITER_FWD_H
+#define ITERATOR_ITER_FWD_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct delegpt;
+struct region;
+
+/**
+ * Iterator forward zones structure
+ */
+struct iter_forwards {
+	/** region where forward zone server addresses are allocated */
+	struct region* region;
+	/** 
+	 * Zones are stored in this tree. Sort order is specially chosen.
+	 * first sorted on qtype. Then on dname in nsec-like order, so that
+	 * a lookup on class, name will return an exact match or the closest
+	 * match which gives the ancestor needed.
+	 * contents of type iter_forward_zone.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator forward servers for a particular zone.
+ */
+struct iter_forward_zone {
+	/** redblacktree node, key is this structure: class and name */
+	rbnode_t node;
+	/** name */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** number of labels in name */
+	int namelabs;
+	/** delegation point with forward server information for this zone. */
+	struct delegpt* dp;
+	/** pointer to parent in tree (or NULL if none) */
+	struct iter_forward_zone* parent;
+	/** class. host order. */
+	uint16_t dclass;
+};
+
+/**
+ * Create forwards 
+ * @return new forwards or NULL on error.
+ */
+struct iter_forwards* forwards_create();
+
+/**
+ * Delete forwards.
+ * @param fwd: to delete.
+ */
+void forwards_delete(struct iter_forwards* fwd);
+
+/**
+ * Process forwards config.
+ * @param fwd: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int forwards_apply_cfg(struct iter_forwards* fwd, struct config_file* cfg);
+
+/**
+ * Find forward zone information
+ * For this qname/qclass find forward zone information, returns delegation
+ * point with server names and addresses, or NULL if no forwarding is needed.
+ *
+ * @param fwd: forward storage.
+ * @param qname: The qname of the query.
+ * @param qclass: The qclass of the query.
+ * @return: A delegation point if the query has to be forwarded to that list,
+ *         otherwise null.
+ */
+struct delegpt* forwards_lookup(struct iter_forwards* fwd, 
+	uint8_t* qname, uint16_t qclass);
+
+/**
+ * Get memory in use by forward storage
+ * @param fwd: forward storage.
+ * @return bytes in use
+ */
+size_t forwards_get_mem(struct iter_forwards* fwd);
+
+#endif /* ITERATOR_ITER_FWD_H */

iterator/iter_hints.c

@@ -0,0 +1,384 @@
+/*
+ * iterator/iter_hints.c - iterative resolver module stub and root hints.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of stub and root hints, and read those from config.
+ */
+#include "config.h"
+#include "iterator/iter_hints.h"
+#include "iterator/iter_delegpt.h"
+#include "util/region-allocator.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+/** compare two hint entries */
+static int
+stub_cmp(const void* k1, const void* k2)
+{
+	int m;
+	struct iter_hints_stub* n1 = (struct iter_hints_stub*)k1;
+	struct iter_hints_stub* n2 = (struct iter_hints_stub*)k2;
+	if(n1->hint_class != n2->hint_class) {
+		if(n1->hint_class < n2->hint_class)
+			return -1;
+		return 1;
+	}
+	return dname_lab_cmp(n1->name, n1->namelabs, n2->name, n2->namelabs, 
+		&m);
+}
+
+struct iter_hints* 
+hints_create()
+{
+	struct iter_hints* hints = (struct iter_hints*)calloc(1,
+		sizeof(struct iter_hints));
+	if(!hints)
+		return NULL;
+	hints->region = region_create(malloc, free);
+	if(!hints->region) {
+		hints_delete(hints);
+		return NULL;
+	}
+	return hints;
+}
+
+void 
+hints_delete(struct iter_hints* hints)
+{
+	if(!hints) 
+		return;
+	region_destroy(hints->region);
+	free(hints->tree);
+	free(hints);
+}
+
+/** add hint to delegation hints */
+static int
+ah(struct delegpt* dp, struct region* r, const char* sv, const char* ip)
+{
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	ldns_rdf* rdf = ldns_dname_new_frm_str(sv);
+	if(!rdf) {
+		log_err("could not parse %s", sv);
+		return 0;
+	}
+	if(!delegpt_add_ns(dp, r, ldns_rdf_data(rdf)) ||
+	   !extstrtoaddr(ip, &addr, &addrlen) ||
+	   !delegpt_add_target(dp, r, ldns_rdf_data(rdf), ldns_rdf_size(rdf),
+		&addr, addrlen)) {
+		ldns_rdf_deep_free(rdf);
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** obtain compiletime provided root hints */
+static struct delegpt* 
+compile_time_root_prime(struct region* r)
+{
+	/* from:
+	 ;       This file is made available by InterNIC
+	 ;       under anonymous FTP as
+	 ;           file                /domain/named.cache
+	 ;           on server           FTP.INTERNIC.NET
+	 ;       -OR-                    RS.INTERNIC.NET
+	 ;
+	 ;       last update:    Jan 29, 2004
+	 ;       related version of root zone:   2004012900
+	 */
+	struct delegpt* dp = delegpt_create(r);
+	if(!dp)
+		return NULL;
+	if(!delegpt_set_name(dp, r, (uint8_t*)"\000"))
+		return NULL;
+	if(!ah(dp, r, "A.ROOT-SERVERS.NET.", "198.41.0.4"))	return 0;
+	if(!ah(dp, r, "B.ROOT-SERVERS.NET.", "192.228.79.201")) return 0;
+	if(!ah(dp, r, "C.ROOT-SERVERS.NET.", "192.33.4.12"))	return 0;
+	if(!ah(dp, r, "D.ROOT-SERVERS.NET.", "128.8.10.90"))	return 0;
+	if(!ah(dp, r, "E.ROOT-SERVERS.NET.", "192.203.230.10")) return 0;
+	if(!ah(dp, r, "F.ROOT-SERVERS.NET.", "192.5.5.241"))	return 0;
+	if(!ah(dp, r, "G.ROOT-SERVERS.NET.", "192.112.36.4"))	return 0;
+	if(!ah(dp, r, "H.ROOT-SERVERS.NET.", "128.63.2.53"))	return 0;
+	if(!ah(dp, r, "I.ROOT-SERVERS.NET.", "192.36.148.17"))	return 0;
+	if(!ah(dp, r, "J.ROOT-SERVERS.NET.", "192.58.128.30"))	return 0;
+	if(!ah(dp, r, "K.ROOT-SERVERS.NET.", "193.0.14.129"))	return 0;
+	if(!ah(dp, r, "L.ROOT-SERVERS.NET.", "198.32.64.12"))	return 0;
+	if(!ah(dp, r, "M.ROOT-SERVERS.NET.", "202.12.27.33"))	return 0;
+	return dp;
+}
+
+/** insert new hint info into hint structure */
+static int
+hints_insert(struct iter_hints* hints, uint16_t c, struct delegpt* dp)
+{
+	struct iter_hints_stub* node = region_alloc(hints->region,
+		sizeof(struct iter_hints_stub));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	node->hint_class = c;
+	node->name = region_alloc_init(hints->region, dp->name, dp->namelen);
+	if(!node->name)
+		return 0;
+	node->namelen = dp->namelen;
+	node->namelabs = dp->namelabs;
+	node->dp = dp;
+	if(!rbtree_insert(hints->tree, &node->node)) {
+		log_err("second hints ignored.");
+	}
+	return 1;
+}
+
+/** initialise parent pointers in the tree */
+static void
+init_parents(struct iter_hints* hints)
+{
+	struct iter_hints_stub* node, *prev = NULL, *p;
+	int m;
+	RBTREE_FOR(node, struct iter_hints_stub*, hints->tree) {
+		node->parent = NULL;
+		if(!prev || prev->hint_class != node->hint_class) {
+			prev = node;
+			continue;
+		}
+		(void)dname_lab_cmp(prev->name, prev->namelabs, node->name,
+			node->namelabs, &m); /* we know prev is smaller */
+		/* sort order like: . com. bla.com. zwb.com. net. */
+		/* find the previous, or parent-parent-parent */
+		for(p = prev; p; p = p->parent)
+			/* looking for name with few labels, a parent */
+			if(p->namelabs <= m) {
+				/* ==: since prev matched m, this is closest*/
+				/* <: prev matches more, but is not a parent,
+				 * this one is a (grand)parent */
+				node->parent = p;
+				break;
+			}
+		prev = node;
+	}
+}
+
+/** set stub name */
+static int 
+read_stubs_name(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	ldns_rdf* rdf;
+	if(!s->name) {
+		log_err("stub zone without a name");
+		return 0;
+	}
+	rdf = ldns_dname_new_frm_str(s->name);
+	if(!rdf) {
+		log_err("cannot parse stub zone name %s", s->name);
+		return 0;
+	}
+	if(!delegpt_set_name(dp, hints->region, ldns_rdf_data(rdf))) {
+		ldns_rdf_deep_free(rdf);
+		log_err("out of memory");
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** set stub host names */
+static int 
+read_stubs_host(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	ldns_rdf* rdf;
+	for(p = s->hosts; p; p = p->next) {
+		log_assert(p->str);
+		rdf = ldns_dname_new_frm_str(p->str);
+		if(!rdf) {
+			log_err("cannot parse stub %s nameserver name: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_ns(dp, hints->region, ldns_rdf_data(rdf))) {
+			ldns_rdf_deep_free(rdf);
+			log_err("out of memory");
+			return 0;
+		}
+		ldns_rdf_deep_free(rdf);
+	}
+	return 1;
+}
+
+/** set stub server addresses */
+static int 
+read_stubs_addr(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	for(p = s->addrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!extstrtoaddr(p->str, &addr, &addrlen)) {
+			log_err("cannot parse stub %s ip address: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_addr(dp, hints->region, &addr, addrlen)) {
+			log_err("out of memory");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/** read stubs config */
+static int 
+read_stubs(struct iter_hints* hints, struct config_file* cfg)
+{
+	struct config_stub* s;
+	for(s = cfg->stubs; s; s = s->next) {
+		struct delegpt* dp = delegpt_create(hints->region);
+		if(!dp) {
+			log_err("out of memory");
+			return 0;
+		}
+		if(!read_stubs_name(hints, s, dp) ||
+			!read_stubs_host(hints, s, dp) ||
+			!read_stubs_addr(hints, s, dp))
+			return 0;
+		if(!hints_insert(hints, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		delegpt_log(dp);
+	}
+	return 1;
+}
+
+int 
+hints_apply_cfg(struct iter_hints* hints, struct config_file* cfg)
+{
+	free(hints->tree);
+	hints->tree = rbtree_create(stub_cmp);
+	if(!hints->tree)
+		return 0;
+	/* TODO: read root hints from file named in cfg */
+
+	/* read stub hints */
+	if(!read_stubs(hints, cfg))
+		return 0;
+
+	/* use fallback compiletime root hints */
+	if(!hints_lookup_root(hints, LDNS_RR_CLASS_IN)) {
+		struct delegpt* dp = compile_time_root_prime(hints->region);
+		verbose(VERB_ALGO, "no config, using builtin root hints.");
+		if(!dp) 
+			return 0;
+		if(!hints_insert(hints, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		delegpt_log(dp);
+	}
+
+	init_parents(hints);
+	return 1;
+}
+
+struct delegpt* 
+hints_lookup_root(struct iter_hints* hints, uint16_t qclass)
+{
+	uint8_t rootlab = 0;
+	struct iter_hints_stub key, *stub;
+	key.node.key = &key;
+	key.hint_class = qclass;
+	key.name = &rootlab;
+	key.namelen = 1;
+	key.namelabs = 1;
+	stub = (struct iter_hints_stub*)rbtree_search(hints->tree, &key);
+	if(!stub)
+		return NULL;
+	return stub->dp;
+}
+
+struct delegpt* 
+hints_lookup_stub(struct iter_hints* hints, uint8_t* qname, 
+	uint16_t qclass, struct delegpt* cache_dp)
+{
+	/* first lookup the stub */
+	rbnode_t* res = NULL;
+	struct iter_hints_stub *result;
+	struct iter_hints_stub key;
+	key.node.key = &key;
+	key.hint_class = qclass;
+	key.name = qname;
+	key.namelabs = dname_count_size_labels(qname, &key.namelen);
+	if(rbtree_find_less_equal(hints->tree, &key, &res)) {
+		/* exact */
+		result = (struct iter_hints_stub*)res;
+	} else {
+		/* smaller element (or no element) */
+		int m;
+		result = (struct iter_hints_stub*)res;
+		if(!result || result->hint_class != qclass)
+			return NULL;
+		/* count number of labels matched */
+		(void)dname_lab_cmp(result->name, result->namelabs, key.name,
+			key.namelabs, &m);
+		while(result) { /* go up until qname is subdomain of stub */
+			if(result->namelabs <= m)
+				break;
+			result = result->parent;
+		}
+		if(!result)
+			return NULL;
+	}
+	/* 
+	 * If our cached delegation point is above the hint, we need to prime.
+	 */
+	if(dname_strict_subdomain(result->dp->name, result->dp->namelabs,
+		cache_dp->name, cache_dp->namelabs))
+		return result->dp; /* need to prime this stub */
+	return NULL;
+}
+
+size_t 
+hints_get_mem(struct iter_hints* hints)
+{
+	if(!hints) return 0;
+	return sizeof(*hints) + region_get_mem(hints->region);
+}

iterator/iter_hints.h

@@ -0,0 +1,137 @@
+/*
+ * iterator/iter_hints.h - iterative resolver module stub and root hints.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of stub and root hints, and read those from config.
+ */
+
+#ifndef ITERATOR_ITER_HINTS_H
+#define ITERATOR_ITER_HINTS_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct delegpt;
+struct region;
+
+/**
+ * Iterator hints structure
+ */
+struct iter_hints {
+	/** region where hints are allocated */
+	struct region* region;
+	/** 
+	 * Hints are stored in this tree. Sort order is specially chosen.
+	 * first sorted on qtype. Then on dname in nsec-like order, so that
+	 * a lookup on class, name will return an exact match or the closest
+	 * match which gives the ancestor needed.
+	 * contents of type iter_hints_stub. The class IN root is in here.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator hints for a particular stub.
+ */
+struct iter_hints_stub {
+	/** redblacktree node, key is this structure: class and name */
+	rbnode_t node;
+	/** name */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** number of labels in name */
+	int namelabs;
+	/** delegation point with hint information for this stub. */
+	struct delegpt* dp;
+	/** pointer to parent in stub hint tree (or NULL if none) */
+	struct iter_hints_stub* parent;
+	/** class of hints. host order. */
+	uint16_t hint_class;
+};
+
+/**
+ * Create hints 
+ * @return new hints or NULL on error.
+ */
+struct iter_hints* hints_create();
+
+/**
+ * Delete hints.
+ * @param hints: to delete.
+ */
+void hints_delete(struct iter_hints* hints);
+
+/**
+ * Process hints config. Sets default values for root hints if no config.
+ * @param hints: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int hints_apply_cfg(struct iter_hints* hints, struct config_file* cfg);
+
+/**
+ * Find root hints for the given class.
+ * @param hints: hint storage.
+ * @param qclass: class for which root hints are requested. host order.
+ * @return: NULL if no hints, or a ptr to stored hints.
+ */
+struct delegpt* hints_lookup_root(struct iter_hints* hints, uint16_t qclass);
+
+/**
+ * Given a qname/qclass combination, and the delegation point from the cache
+ * for this qname/qclass, determine if this combination indicates that a
+ * stub hint exists and must be primed.
+ *
+ * @param hints: hint storage.
+ * @param qname: The qname that generated the delegation point.
+ * @param qclass: The qclass that generated the delegation point.
+ * @param dp: The cache generated delegation point.
+ * @return: A priming delegation point if there is a stub hint that must
+ *         be primed, otherwise null.
+ */
+struct delegpt* hints_lookup_stub(struct iter_hints* hints, 
+	uint8_t* qname, uint16_t qclass, struct delegpt* dp);
+
+/**
+ * Get memory in use by hints
+ * @param hints: hint storage.
+ * @return bytes in use
+ */
+size_t hints_get_mem(struct iter_hints* hints);
+
+#endif /* ITERATOR_ITER_HINTS_H */

iterator/iter_resptype.c

@@ -0,0 +1,235 @@
+/*
+ * iterator/iter_resptype.c - response type information and classification.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file defines the response type. DNS Responses can be classified as
+ * one of the response types.
+ */
+#include "config.h"
+#include "iterator/iter_resptype.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+enum response_type 
+response_type_from_cache(struct dns_msg* msg, 
+	struct query_info* request)
+{
+	/* If the message is NXDOMAIN, then it is an ANSWER. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_NXDOMAIN)
+		return RESPONSE_TYPE_ANSWER;
+	
+	/* First we look at the answer section. This can tell us if this is
+	 * CNAME or positive ANSWER. */
+	if(msg->rep->an_numrrsets > 0) {
+		/* Now look at the answer section first. 3 states: 
+		 *	o our answer is there directly,
+		 *	o our answer is there after a cname,
+		 *	o or there is just a cname. */
+		uint8_t* mname = request->qname;
+		size_t mname_len = request->qname_len;
+		size_t i;
+		for(i=0; i<msg->rep->an_numrrsets; i++) {
+			struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+
+			/* If we have encountered an answer (before or 
+			 * after a CNAME), then we are done! Note that 
+			 * if qtype == CNAME then this will be noted as 
+			 * an ANSWER before it gets treated as a CNAME, 
+			 * as it should */
+			if(ntohs(s->rk.type) == request->qtype &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				return RESPONSE_TYPE_ANSWER;
+			}
+
+			/* If we have encountered a CNAME, make sure that 
+			 * it is relevant. */
+			if(ntohs(s->rk.type) == LDNS_RR_TYPE_CNAME &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				get_cname_target(s, &mname, &mname_len);
+			}
+		}
+
+		/* if we encountered a CNAME (or a bunch of CNAMEs), and 
+		 * still got to here, then it is a CNAME response. (i.e., 
+		 * the CNAME chain didn't terminate in an answer rrset.) */
+		if(mname != request->qname) {
+			return RESPONSE_TYPE_CNAME;
+		}
+	}
+
+	/* At this point, since we don't need to detect REFERRAL or LAME 
+	 * messages, it can only be an ANSWER. */
+	return RESPONSE_TYPE_ANSWER;
+}
+
+enum response_type 
+response_type_from_server(struct dns_msg* msg, struct query_info* request,
+	struct delegpt* dp)
+{
+	uint8_t* origzone = (uint8_t*)"\000"; /* the default */
+	size_t origzonelen = 1;
+	size_t i;
+
+	if(!msg || !request)
+		return RESPONSE_TYPE_THROWAWAY;
+	
+	/* If the message is NXDOMAIN, then it answers the question. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_NXDOMAIN)
+		return RESPONSE_TYPE_ANSWER;
+	
+	/* Other response codes mean (so far) to throw the response away as
+	 * meaningless and move on to the next nameserver. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) != LDNS_RCODE_NOERROR)
+		return RESPONSE_TYPE_THROWAWAY;
+
+	/* Note: TC bit has already been handled */
+
+	if(dp) {
+		origzone = dp->name;
+		origzonelen = dp->namelen;
+	}
+
+	/* First we look at the answer section. This can tell us if this is a
+	 * CNAME or ANSWER or (provisional) ANSWER. */
+	if(msg->rep->an_numrrsets > 0) {
+		uint8_t* mname = request->qname;
+		size_t mname_len = request->qname_len;
+
+		/* Now look at the answer section first. 3 states: our 
+		 * answer is there directly, our answer is there after 
+		 * a cname, or there is just a cname. */
+		for(i=0; i<msg->rep->an_numrrsets; i++) {
+			struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+			
+			/* if the answer section has NS rrset, and qtype ANY 
+		 	 * and the delegation is lower, and no CNAMEs followed,
+		 	 * this is a referral where the NS went to AN section */
+			if((request->qtype == LDNS_RR_TYPE_ANY ||
+				request->qtype == LDNS_RR_TYPE_NS) &&
+				ntohs(s->rk.type) == LDNS_RR_TYPE_NS &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				dname_strict_subdomain_c(s->rk.dname, 
+				origzone)) {
+				return RESPONSE_TYPE_REFERRAL;
+			}
+
+			/* If we have encountered an answer (before or 
+			 * after a CNAME), then we are done! Note that 
+			 * if qtype == CNAME then this will be noted as an
+			 * ANSWER before it gets treated as a CNAME, as 
+			 * it should. */
+			if(ntohs(s->rk.type) == request->qtype &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				if((msg->rep->flags&BIT_AA))
+					return RESPONSE_TYPE_ANSWER;
+				/* If the AA bit isn't on, and we've seen 
+				 * the answer, we only provisionally say 
+				 * 'ANSWER' -- it very well could be a 
+				 * REFERRAL. */
+				break;
+			}
+
+			/* If we have encountered a CNAME, make sure that 
+			 * it is relevant. */
+			if(ntohs(s->rk.type) == LDNS_RR_TYPE_CNAME &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				get_cname_target(s, &mname, &mname_len);
+			}
+		}
+		/* if we encountered a CNAME (or a bunch of CNAMEs), and 
+		 * still got to here, then it is a CNAME response. 
+		 * (This is regardless of the AA bit at this point) */
+		if(mname != request->qname) {
+			return RESPONSE_TYPE_CNAME;
+		}
+	}
+
+	/* Looking at the authority section, we just look and see if 
+	 * there is a delegation NS set, turning it into a delegation. 
+	 * Otherwise, we will have to conclude ANSWER (either it is 
+	 * NOERROR/NODATA, or an non-authoritative answer). */
+	for(i = msg->rep->an_numrrsets; i < (msg->rep->an_numrrsets +
+		msg->rep->ns_numrrsets); i++) {
+		struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+
+		/* The normal way of detecting NOERROR/NODATA. */
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_SOA &&
+			dname_subdomain_c(request->qname, s->rk.dname)) {
+			return RESPONSE_TYPE_ANSWER;
+		}
+
+		/* Detect REFERRAL/LAME/ANSWER based on the relationship 
+		 * of the NS set to the originating zone name. */
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_NS) {
+			/* If we are getting an NS set for the zone we 
+			 * thought we were contacting, then it is an answer.*/
+			/* FIXME: is this correct? */
+			if(query_dname_compare(s->rk.dname, origzone) == 0) {
+				return RESPONSE_TYPE_ANSWER;
+			}
+			/* If we are getting a referral upwards (or to 
+			 * the same zone), then the server is 'lame'. */
+			if(dname_subdomain_c(origzone, s->rk.dname)) {
+				return RESPONSE_TYPE_LAME;
+			}
+			/* If the NS set is below the delegation point we 
+			 * are on, and it is non-authoritative, then it is 
+			 * a referral, otherwise it is an answer. */
+			if(dname_subdomain_c(s->rk.dname, origzone)) {
+				/* NOTE: I no longer remember in what case 
+				 * we would like this to be an answer. 
+				 * NODATA should have a SOA or nothing, 
+				 * not an NS rrset. 
+				 * True, referrals should not have the AA 
+				 * bit set, but... */
+				 
+				/* if((msg->rep->flags&BIT_AA))
+					return RESPONSE_TYPE_ANSWER; */
+				return RESPONSE_TYPE_REFERRAL;
+			}
+			/* Otherwise, the NS set is irrelevant. */
+		}
+	}
+
+	/* If we've gotten this far, this is NOERROR/NODATA (which could 
+	 * be an entirely empty message) */
+	return RESPONSE_TYPE_ANSWER;
+}

iterator/iter_resptype.h

@@ -0,0 +1,119 @@
+/*
+ * iterator/iter_resptype.h - response type information and classification.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file defines the response type. DNS Responses can be classified as
+ * one of the response types.
+ */
+
+#ifndef ITERATOR_ITER_RESPTYPE_H
+#define ITERATOR_ITER_RESPTYPE_H
+struct dns_msg;
+struct query_info;
+struct delegpt;
+
+/**
+ * The response type is used to interpret the response.
+ */
+enum response_type {
+	/** 
+	 * 'untyped' means that the type of this response hasn't been 
+	 * assigned. 
+	 */
+	RESPONSE_TYPE_UNTYPED   = 0,
+
+	/** 
+	 * 'answer' means that the response terminates the resolution 
+	 * process. 
+	 */
+	RESPONSE_TYPE_ANSWER,
+
+	/** 'delegation' means that the response is a delegation. */
+	RESPONSE_TYPE_REFERRAL,
+
+	/**
+	 * 'cname' means that the response is a cname without the final 
+	 * answer, and thus must be restarted.
+	 */
+	RESPONSE_TYPE_CNAME,
+
+	/**
+	 * 'throwaway' means that this particular response should be 
+	 * discarded and the next nameserver should be contacted
+	 */
+	RESPONSE_TYPE_THROWAWAY,
+
+	/**
+	 * 'lame' means that this particular response indicates that 
+	 * the nameserver knew nothing about the question.
+	 */
+	RESPONSE_TYPE_LAME
+};
+
+/**
+ * Classifies a response message from cache based on the current request.
+ * Note that this routine assumes that THROWAWAY or LAME responses will not
+ * occur. Also, it will not detect REFERRAL type messages, since those are
+ * (currently) automatically classified based on how they came from the
+ * cache (findDelegation() instead of lookup()).
+ *
+ * @param msg: the message from the cache.
+ * @param request: the request that generated the response.
+ * @return the response type (CNAME or ANSWER).
+ */
+enum response_type response_type_from_cache(struct dns_msg* msg, 
+	struct query_info* request);
+
+/**
+ * Classifies a response message (from the wire) based on the current
+ * request.
+ *
+ * NOTE: currently this routine uses the AA bit in the response to help
+ * distinguish between some non-standard referrals and answers. It also
+ * relies somewhat on the originating zone to be accurate (for lameness
+ * detection, mostly).
+ *
+ * @param msg: the message from the cache.
+ * @param request: the request that generated the response.
+ * @param dp: The delegation point that was being queried
+ *          when the response was returned.
+ * @return the response type (CNAME or ANSWER).
+ */
+enum response_type response_type_from_server(struct dns_msg* msg, 
+	struct query_info* request, struct delegpt* dp);
+
+#endif /* ITERATOR_ITER_RESPTYPE_H */

iterator/iter_scrub.c

@@ -0,0 +1,575 @@
+/*
+ * iterator/iter_scrub.c - scrubbing, normalization, sanitization of DNS msgs.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file has routine(s) for cleaning up incoming DNS messages from 
+ * possible useless or malicious junk in it.
+ */
+#include "config.h"
+#include "iterator/iter_scrub.h"
+#include "services/cache/rrset.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/region-allocator.h"
+#include "util/config_file.h"
+#include "util/module.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/data/msgreply.h"
+#include "util/alloc.h"
+
+/** RRset flag used during scrubbing. The RRset is OK. */
+#define RRSET_SCRUB_OK	0x80
+
+/** remove rrset, update loop variables */
+static void
+remove_rrset(const char* str, ldns_buffer* pkt, struct msg_parse* msg, 
+	struct rrset_parse* prev, struct rrset_parse** rrset)
+{
+	if(verbosity >= VERB_DETAIL 
+		&& (*rrset)->dname_len <= LDNS_MAX_DOMAINLEN) {
+		uint8_t buf[LDNS_MAX_DOMAINLEN+1];
+		dname_pkt_copy(pkt, buf, (*rrset)->dname);
+		log_nametypeclass(VERB_DETAIL, str, buf, 
+			(*rrset)->type, ntohs((*rrset)->rrset_class));
+	}
+	if(prev)
+		prev->rrset_all_next = (*rrset)->rrset_all_next;
+	else	msg->rrset_first = (*rrset)->rrset_all_next;
+	if(msg->rrset_last == *rrset)
+		msg->rrset_last = prev;
+	msg->rrset_count --;
+	switch((*rrset)->section) {
+		case LDNS_SECTION_ANSWER: msg->an_rrsets--; break;
+		case LDNS_SECTION_AUTHORITY: msg->ns_rrsets--; break;
+		case LDNS_SECTION_ADDITIONAL: msg->ar_rrsets--; break;
+		default: log_assert(0);
+	}
+	msgparse_bucket_remove(msg, *rrset);
+	*rrset = (*rrset)->rrset_all_next;
+}
+
+/** return true if rr type has additional names in it */
+static int
+has_additional(uint16_t t)
+{
+	switch(t) {
+		case LDNS_RR_TYPE_MB:
+		case LDNS_RR_TYPE_MD:
+		case LDNS_RR_TYPE_MF:
+		case LDNS_RR_TYPE_NS:
+		case LDNS_RR_TYPE_MX:
+		case LDNS_RR_TYPE_KX:
+		case LDNS_RR_TYPE_SRV:
+			return 1;
+		case LDNS_RR_TYPE_NAPTR:
+			/* TODO: NAPTR not supported, glue stripped off */
+			return 0;
+		default:
+			return 0;
+	}
+	return 0;
+}
+
+/** get additional name from rrset RR, return false if no name present */
+static int
+get_additional_name(struct rrset_parse* rrset, struct rr_parse* rr, 
+	uint8_t** nm, size_t* nmlen, ldns_buffer* pkt) 
+{
+	size_t offset = 0;
+	size_t len, oldpos;
+	switch(rrset->type) {
+		case LDNS_RR_TYPE_MB:
+		case LDNS_RR_TYPE_MD:
+		case LDNS_RR_TYPE_MF:
+		case LDNS_RR_TYPE_NS:
+			offset = 0;
+			break;
+		case LDNS_RR_TYPE_MX:
+		case LDNS_RR_TYPE_KX:
+			offset = 2;
+			break;
+		case LDNS_RR_TYPE_SRV:
+			offset = 6;
+			break;
+		case LDNS_RR_TYPE_NAPTR:
+			/* TODO: NAPTR not supported, glue stripped off */
+			return 0;
+		default:
+			return 0;
+	}
+	len = ldns_read_uint16(rr->ttl_data+sizeof(uint32_t));
+	if(len < offset+1)
+		return 0; /* rdata field too small */
+	*nm = rr->ttl_data+sizeof(uint32_t)+sizeof(uint16_t)+offset;
+	oldpos = ldns_buffer_position(pkt);
+	ldns_buffer_set_position(pkt, (size_t)(*nm - ldns_buffer_begin(pkt)));
+	*nmlen = pkt_dname_len(pkt);
+	ldns_buffer_set_position(pkt, oldpos);
+	if(*nmlen == 0)
+		return 0;
+	return 1;
+}
+
+/** Place mark on rrsets in additional section they are OK */
+static void
+mark_additional_rrset(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct rrset_parse* rrset)
+{
+	/* Mark A and AAAA for NS as appropriate additional section info. */
+	uint8_t* nm = NULL;
+	size_t nmlen = 0;
+	struct rr_parse* rr;
+
+	if(!has_additional(rrset->type))
+		return;
+	for(rr = rrset->rr_first; rr; rr = rr->next) {
+		if(get_additional_name(rrset, rr, &nm, &nmlen, pkt)) {
+			/* mark A */
+			hashvalue_t h = pkt_hash_rrset(pkt, nm, LDNS_RR_TYPE_A, 
+				rrset->rrset_class, 0);
+			struct rrset_parse* r = msgparse_hashtable_lookup(
+				msg, pkt, h, 0, nm, nmlen, 
+				LDNS_RR_TYPE_A, rrset->rrset_class);
+			if(r && r->section == LDNS_SECTION_ADDITIONAL) {
+				r->flags |= RRSET_SCRUB_OK;
+			}
+			
+			/* mark AAAA */
+			h = pkt_hash_rrset(pkt, nm, LDNS_RR_TYPE_AAAA, 
+				rrset->rrset_class, 0);
+			r = msgparse_hashtable_lookup(msg, pkt, h, 0, nm, 
+				nmlen, LDNS_RR_TYPE_AAAA, rrset->rrset_class);
+			if(r && r->section == LDNS_SECTION_ADDITIONAL) {
+				r->flags |= RRSET_SCRUB_OK;
+			}
+		}
+	}
+}
+
+/** Get target name of a CNAME */
+static int
+parse_get_cname_target(struct rrset_parse* rrset, uint8_t** sname, 
+	size_t* snamelen)
+{
+	if(rrset->rr_count != 1) {
+		verbose(VERB_ALGO, "Found CNAME rrset with "
+			"size > 1: %u", (unsigned)rrset->rr_count);
+		return 0;
+	}
+	if(rrset->rr_first->size < sizeof(uint16_t)+1)
+		return 0; /* CNAME rdata too small */
+	*sname = rrset->rr_first->ttl_data + sizeof(uint32_t)
+		+ sizeof(uint16_t); /* skip ttl, rdatalen */
+	*snamelen = rrset->rr_first->size - sizeof(uint16_t);
+	return 1;
+}
+
+/** Synthesize CNAME from DNAME, false if too long */
+static int 
+synth_cname(uint8_t* qname, size_t qnamelen, struct rrset_parse* dname_rrset, 
+	uint8_t* alias, size_t* aliaslen, ldns_buffer* pkt)
+{
+	/* we already know that sname is a strict subdomain of DNAME owner */
+	uint8_t* dtarg = NULL;
+	size_t dtarglen;
+	if(!parse_get_cname_target(dname_rrset, &dtarg, &dtarglen))
+		return 0; 
+	log_assert(qnamelen > dname_rrset->dname_len);
+	/* DNAME from com. to net. with qname example.com. -> example.net. */
+	/* so: \3com\0 to \3net\0 and qname \7example\3com\0 */
+	*aliaslen = qnamelen + dtarglen - dname_rrset->dname_len;
+	if(*aliaslen > LDNS_MAX_DOMAINLEN)
+		return 0; /* should have been RCODE YXDOMAIN */
+	/* decompress dnames into buffer, we know it fits */
+	dname_pkt_copy(pkt, alias, qname);
+	dname_pkt_copy(pkt, alias+(qnamelen-dname_rrset->dname_len), dtarg);
+	return 1;
+}
+
+/** synthesize a CNAME rrset */
+static struct rrset_parse*
+synth_cname_rrset(uint8_t** sname, size_t* snamelen, uint8_t* alias, 
+	size_t aliaslen, struct region* region, struct msg_parse* msg, 
+	struct rrset_parse* rrset, struct rrset_parse* prev,
+	struct rrset_parse* nx, ldns_buffer* pkt)
+{
+	struct rrset_parse* cn = (struct rrset_parse*)region_alloc(region,
+		sizeof(struct rrset_parse));
+	if(!cn)
+		return NULL;
+	memset(cn, 0, sizeof(*cn));
+	cn->rr_first = (struct rr_parse*)region_alloc(region, 
+		sizeof(struct rr_parse));
+	if(!cn->rr_first)
+		return NULL;
+	cn->rr_last = cn->rr_first;
+	/* CNAME from sname to alias */
+	cn->dname = (uint8_t*)region_alloc(region, *snamelen);
+	if(!cn->dname)
+		return NULL;
+	dname_pkt_copy(pkt, cn->dname, *sname);
+	cn->dname_len = *snamelen;
+	cn->type = LDNS_RR_TYPE_CNAME;
+	cn->section = rrset->section;
+	cn->rrset_class = rrset->rrset_class;
+	cn->rr_count = 1;
+	cn->size = sizeof(uint16_t) + aliaslen;
+	cn->hash=pkt_hash_rrset(pkt, cn->dname, cn->type, cn->rrset_class, 0);
+	/* allocate TTL + rdatalen + uncompressed dname */
+	memset(cn->rr_first, 0, sizeof(struct rr_parse));
+	cn->rr_first->outside_packet = 1;
+	cn->rr_first->ttl_data = (uint8_t*)region_alloc(region, 
+		sizeof(uint32_t)+sizeof(uint16_t)+aliaslen);
+	if(!cn->rr_first->ttl_data)
+		return NULL;
+	ldns_write_uint32(cn->rr_first->ttl_data, 0); /* TTL = 0 */
+	ldns_write_uint16(cn->rr_first->ttl_data+4, aliaslen);
+	memmove(cn->rr_first->ttl_data+6, alias, aliaslen);
+	cn->rr_first->size = sizeof(uint16_t)+aliaslen;
+
+	/* link it in */
+	cn->rrset_all_next = nx;
+	if(prev)
+		prev->rrset_all_next = cn;
+	else	msg->rrset_first = cn;
+	if(nx == NULL)
+		msg->rrset_last = cn;
+	msg->rrset_count ++;
+	msg->an_rrsets++;
+	/* it is not inserted in the msg hashtable. */
+
+	*sname = cn->rr_first->ttl_data + sizeof(uint32_t)+sizeof(uint16_t);
+	*snamelen = aliaslen;
+	return cn;
+}
+
+/** check if DNAME applies to a name */
+static int
+pkt_strict_sub(ldns_buffer* pkt, uint8_t* sname, uint8_t* dr)
+{
+	uint8_t buf1[LDNS_MAX_DOMAINLEN+1];
+	uint8_t buf2[LDNS_MAX_DOMAINLEN+1];
+	/* decompress names */
+	dname_pkt_copy(pkt, buf1, sname);
+	dname_pkt_copy(pkt, buf2, dr);
+	return dname_strict_subdomain_c(buf1, buf2);
+}
+
+/** check subdomain with decompression */
+static int
+pkt_sub(ldns_buffer* pkt, uint8_t* comprname, uint8_t* zone)
+{
+	uint8_t buf[LDNS_MAX_DOMAINLEN+1];
+	dname_pkt_copy(pkt, buf, comprname);
+	return dname_subdomain_c(buf, zone);
+}
+
+/**
+ * This routine normalizes a response. This includes removing "irrelevant"
+ * records from the answer and additional sections and (re)synthesizing
+ * CNAMEs from DNAMEs, if present.
+ *
+ * @param pkt: packet.
+ * @param msg: msg to normalize.
+ * @param qinfo: original query.
+ * @param region: where to allocate synthesized CNAMEs.
+ * @return 0 on error.
+ */
+static int
+scrub_normalize(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, struct region* region)
+{
+	uint8_t* sname = qinfo->qname;
+	size_t snamelen = qinfo->qname_len;
+	struct rrset_parse* rrset, *prev;
+
+	if(FLAGS_GET_RCODE(msg->flags) != LDNS_RCODE_NOERROR &&
+		FLAGS_GET_RCODE(msg->flags) != LDNS_RCODE_NXDOMAIN)
+		return 1;
+
+	/* For the ANSWER section, remove all "irrelevant" records and add
+	 * synthesized CNAMEs from DNAMEs
+	 * This will strip out-of-order CNAMEs as well. */
+
+	/* walk through the parse packet rrset list, keep track of previous
+	 * for insert and delete ease, and examine every RRset */
+	prev = NULL;
+	rrset = msg->rrset_first;
+	while(rrset && rrset->section == LDNS_SECTION_ANSWER) {
+		if(rrset->type == LDNS_RR_TYPE_DNAME && 
+			pkt_strict_sub(pkt, sname, rrset->dname)) {
+			/* check if next rrset is correct CNAME. else,
+			 * synthesize a CNAME */
+			struct rrset_parse* nx = rrset->rrset_all_next;
+			uint8_t alias[LDNS_MAX_DOMAINLEN+1];
+			size_t aliaslen = 0;
+			if(rrset->rr_count != 1) {
+				verbose(VERB_ALGO, "Found DNAME rrset with "
+					"size > 1: %u", 
+					(unsigned)rrset->rr_count);
+				return 0;
+			}
+			if(!synth_cname(sname, snamelen, rrset, alias, 
+				&aliaslen, pkt)) {
+				verbose(VERB_ALGO, "synthesized CNAME "
+					"too long");
+				return 0;
+			}
+			if(nx && nx->type == LDNS_RR_TYPE_CNAME && 
+			   dname_pkt_compare(pkt, sname, nx->dname) == 0) {
+				/* check next cname */
+				uint8_t* t = NULL;
+				size_t tlen = 0;
+				if(!parse_get_cname_target(rrset, &t, &tlen))
+					return 0;
+				if(dname_pkt_compare(pkt, alias, t) == 0) {
+					/* it's OK and better capitalized */
+					prev = rrset;
+					rrset = nx;
+					continue;
+				}
+				/* synth ourselves */
+			}
+			/* synth a CNAME rrset */
+			prev = synth_cname_rrset(&sname, &snamelen, alias, 
+				aliaslen, region, msg, rrset, rrset, nx, pkt);
+			if(!prev) {
+				log_err("out of memory synthesizing CNAME");
+				return 0;
+			}
+			/* FIXME: resolve the conflict between synthesized 
+			 * CNAME ttls and the cache. */
+			rrset = nx;
+			continue;
+
+		}
+
+		/* The only records in the ANSWER section not allowed to */
+		if(dname_pkt_compare(pkt, sname, rrset->dname) != 0) {
+			remove_rrset("normalize: removing irrelevant RRset:", 
+				pkt, msg, prev, &rrset);
+			continue;
+		}
+
+		/* Follow the CNAME chain. */
+		if(rrset->type == LDNS_RR_TYPE_CNAME) {
+			if(!parse_get_cname_target(rrset, &sname, &snamelen))
+				return 0;
+			prev = rrset;
+			rrset = rrset->rrset_all_next;
+			continue;
+		}
+
+		/* Otherwise, make sure that the RRset matches the qtype. */
+		if(qinfo->qtype != LDNS_RR_TYPE_ANY && 
+			qinfo->qtype != rrset->type) {
+			remove_rrset("normalize: removing irrelevant RRset:", 
+				pkt, msg, prev, &rrset);
+			continue;
+		}
+
+		/* Mark the additional names from relevant rrset as OK. */
+		mark_additional_rrset(pkt, msg, rrset);
+		
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+
+	/* Mark additional names from AUTHORITY */
+	while(rrset && rrset->section == LDNS_SECTION_AUTHORITY) {
+		mark_additional_rrset(pkt, msg, rrset);
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+
+	/* For each record in the additional section, remove it if it is an
+	 * address record and not in the collection of additional names 
+	 * found in ANSWER and AUTHORITY. */
+	/* These records have not been marked OK previously */
+	while(rrset && rrset->section == LDNS_SECTION_ADDITIONAL) {
+		/* FIXME: what about other types? */
+		if(rrset->type==LDNS_RR_TYPE_A || 
+			rrset->type==LDNS_RR_TYPE_AAAA) 
+		{
+			if((rrset->flags & RRSET_SCRUB_OK)) {
+				/* remove flag to clean up flags variable */
+				rrset->flags &= ~RRSET_SCRUB_OK;
+			} else {
+				remove_rrset("normalize: removing irrelevant "
+					"RRset:", pkt, msg, prev, &rrset);
+				continue;
+			}
+		}
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+	
+	return 1;
+}
+
+/**
+ * Store potential poison in the cache (only if hardening disabled).
+ * The rrset is stored in the cache but removed from the message.
+ * So that it will be used for infrastructure purposes, but not be 
+ * returned to the client.
+ * @param pkt: packet
+ * @param msg: message parsed
+ * @param env: environment with cache
+ * @param rrset: to store.
+ */
+static void
+store_rrset(ldns_buffer* pkt, struct msg_parse* msg, struct module_env* env,
+	struct rrset_parse* rrset)
+{
+	struct ub_packed_rrset_key* k;
+	struct packed_rrset_data* d;
+	struct rrset_ref ref;
+	uint32_t now = time(NULL);
+
+	k = alloc_special_obtain(env->alloc);
+	if(!k)
+		return;
+	k->entry.data = NULL;
+	if(!parse_copy_decompress_rrset(pkt, msg, rrset, NULL, k)) {
+		alloc_special_release(env->alloc, k);
+		return;
+	}
+	d = (struct packed_rrset_data*)k->entry.data;
+	packed_rrset_ttl_add(d, now);
+	ref.key = k;
+	ref.id = k->id;
+	/*ignore ret: it was in the cache, ref updated */
+	(void)rrset_cache_update(env->rrset_cache, &ref,
+		env->alloc, now);
+}
+
+/**
+ * Given a response event, remove suspect RRsets from the response.
+ * "Suspect" rrsets are potentially poison. Note that this routine expects
+ * the response to be in a "normalized" state -- that is, all "irrelevant"
+ * RRsets have already been removed, CNAMEs are in order, etc.
+ *
+ * @param pkt: packet.
+ * @param msg: msg to normalize.
+ * @param zonename: name of server zone.
+ * @param env: module environment with config and cache.
+ * @return 0 on error.
+ */
+static int
+scrub_sanitize(ldns_buffer* pkt, struct msg_parse* msg, uint8_t* zonename,
+	struct module_env* env)
+{
+	struct rrset_parse* rrset, *prev;
+	prev = NULL;
+	rrset = msg->rrset_first;
+
+	/* At this point, we brutally remove ALL rrsets that aren't 
+	 * children of the originating zone. The idea here is that, 
+	 * as far as we know, the server that we contacted is ONLY 
+	 * authoritative for the originating zone. It, of course, MAY 
+	 * be authoriative for any other zones, and of course, MAY 
+	 * NOT be authoritative for some subdomains of the originating 
+	 * zone. */
+	while(rrset) {
+		
+		/* skip DNAME records -- they will always be followed by a 
+		 * synthesized CNAME, which will be relevant.
+		 * FIXME: should this do something differently with DNAME 
+		 * rrsets NOT in Section.ANSWER? */
+		/* But since DNAME records are also subdomains of the zone,
+		 * same check can be used */
+
+		if(!pkt_sub(pkt, rrset->dname, zonename)) {
+			if(msg->an_rrsets == 0 && 
+				rrset->type == LDNS_RR_TYPE_NS && 
+				rrset->section == LDNS_SECTION_AUTHORITY &&
+				FLAGS_GET_RCODE(msg->flags) == 
+				LDNS_RCODE_NOERROR) {
+				/* noerror, nodata and this NS rrset is above
+				 * the zone. This is LAME! 
+				 * Leave in the NS for lame classification. */
+			} else if(!env->cfg->harden_glue) {
+				/* store in cache! Since it is relevant
+				 * (from normalize) it will be picked up 
+				 * from the cache to be used later */
+				store_rrset(pkt, msg, env, rrset);
+				remove_rrset("sanitize: storing potential "
+				"poison RRset:", pkt, msg, prev, &rrset);
+				continue;
+			} else {
+				remove_rrset("sanitize: removing potential "
+				"poison RRset:", pkt, msg, prev, &rrset);
+				continue;
+			}
+		}
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+	return 1;
+}
+
+int 
+scrub_message(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, uint8_t* zonename, struct region* region,
+	struct module_env* env)
+{
+	/* basic sanity checks */
+	log_nametypeclass(VERB_ALGO, "scrub for", zonename, LDNS_RR_TYPE_NS, 
+		qinfo->qclass);
+	if(msg->qdcount > 1)
+		return 0;
+	if( !(msg->flags&BIT_QR) )
+		return 0;
+	
+	/* if a query is echoed back, make sure it is correct. Otherwise,
+	 * this may be not a reply to our query. */
+	if(msg->qdcount == 1) {
+		if(dname_pkt_compare(pkt, msg->qname, qinfo->qname) != 0)
+			return 0;
+		if(msg->qtype != qinfo->qtype || msg->qclass != qinfo->qclass)
+			return 0;
+	}
+
+	/* normalize the response, this cleans up the additional.  */
+	if(!scrub_normalize(pkt, msg, qinfo, region))
+		return 0;
+	/* delete all out-of-zone information */
+	if(!scrub_sanitize(pkt, msg, zonename, env))
+		return 0;
+	return 1;
+}

iterator/iter_scrub.h

@@ -0,0 +1,66 @@
+/*
+ * iterator/iter_scrub.h - scrubbing, normalization, sanitization of DNS msgs.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file has routine(s) for cleaning up incoming DNS messages from 
+ * possible useless or malicious junk in it.
+ */
+
+#ifndef ITERATOR_ITER_SCRUB_H
+#define ITERATOR_ITER_SCRUB_H
+struct msg_parse;
+struct query_info;
+struct region;
+struct module_env;
+
+/**
+ * Cleanup the passed dns message.
+ * @param pkt: the packet itself, for resolving name compression pointers.
+ *	the packet buffer is unaltered.
+ * @param msg: the parsed packet, this structure is cleaned up.
+ * @param qinfo: the query info that was sent to the server. Checked.
+ * @param zonename: the name of the last delegation point.
+ *	Used to determine out of bailiwick information.
+ * @param region: where to allocate (new) parts of the message.
+ * @param env: module environment with config settings and cache. 
+ * @return: false if the message is total waste. true if scrubbed with success.
+ */
+int scrub_message(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, uint8_t* zonename, struct region* region,
+	struct module_env* env);
+
+#endif /* ITERATOR_ITER_SCRUB_H */

iterator/iter_utils.c

@@ -0,0 +1,347 @@
+/*
+ * iterator/iter_utils.c - iterative resolver module utility functions.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Configuration options. Forward zones. 
+ */
+#include "config.h"
+#include "iterator/iter_utils.h"
+#include "iterator/iterator.h"
+#include "iterator/iter_hints.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_donotq.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/infra.h"
+#include "services/cache/dns.h"
+#include "services/cache/rrset.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/region-allocator.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/random.h"
+
+/** fillup fetch policy array */
+static void
+fetch_fill(struct iter_env* ie, const char* str)
+{
+	char* s = (char*)str, *e;
+	int i;
+	for(i=0; i<ie->max_dependency_depth+1; i++) {
+		ie->target_fetch_policy[i] = strtol(s, &e, 10);
+		if(s == e)
+			fatal_exit("cannot parse fetch policy number %s", s);
+		s = e;
+	}
+}
+
+/** Read config string that represents the target fetch policy */
+static int
+read_fetch_policy(struct iter_env* ie, const char* str)
+{
+	int count = cfg_count_numbers(str);
+	if(count < 1) {
+		log_err("Cannot parse target fetch policy: \"%s\"", str);
+		return 0;
+	}
+	ie->max_dependency_depth = count - 1;
+	ie->target_fetch_policy = (int*)calloc(
+		(size_t)ie->max_dependency_depth+1, sizeof(int));
+	if(!ie->target_fetch_policy) {
+		log_err("alloc fetch policy: out of memory");
+		return 0;
+	}
+	fetch_fill(ie, str);
+	return 1;
+}
+
+int 
+iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
+{
+	int i;
+	/* target fetch policy */
+	if(!read_fetch_policy(iter_env, cfg->target_fetch_policy))
+		return 0;
+	for(i=0; i<iter_env->max_dependency_depth+1; i++)
+		verbose(VERB_DETAIL, "target fetch policy for level %d is %d",
+			i, iter_env->target_fetch_policy[i]);
+	
+	if(!iter_env->hints)
+		iter_env->hints = hints_create();
+	if(!iter_env->hints || !hints_apply_cfg(iter_env->hints, cfg)) {
+		log_err("Could not set root or stub hints");
+		return 0;
+	}
+	if(!iter_env->fwds)
+		iter_env->fwds = forwards_create();
+	if(!iter_env->fwds || !forwards_apply_cfg(iter_env->fwds, cfg)) {
+		log_err("Could not set forward zones");
+		return 0;
+	}
+	if(!iter_env->donotq)
+		iter_env->donotq = donotq_create();
+	if(!iter_env->donotq || !donotq_apply_cfg(iter_env->donotq, cfg)) {
+		log_err("Could not set donotqueryaddresses");
+		return 0;
+	}
+	iter_env->supports_ipv6 = cfg->do_ip6;
+	return 1;
+}
+
+/** filter out unsuitable targets, return rtt or -1 */
+static int
+iter_filter_unsuitable(struct iter_env* iter_env, struct module_env* env,
+	uint8_t* name, size_t namelen, time_t now, struct delegpt_addr* a)
+{
+	int rtt;
+	int lame;
+	if(donotq_lookup(iter_env->donotq, &a->addr, a->addrlen)) {
+		return -1; /* server is on the donotquery list */
+	}
+	if(!iter_env->supports_ipv6 && addr_is_ip6(&a->addr)) {
+		return -1; /* there is no ip6 available */
+	}
+	/* check lameness - need zone , class info */
+	if(infra_get_lame_rtt(env->infra_cache, &a->addr, a->addrlen, 
+		name, namelen, &lame, &rtt, now)) {
+		if(lame)
+			return -1; /* server is lame */
+		else if(rtt >= USEFUL_SERVER_TOP_TIMEOUT)
+			return -1; /* server is unresponsive */
+		else	return rtt;
+	}
+	/* no server information present */
+	return UNKNOWN_SERVER_NICENESS;
+}
+
+/** filter the addres list, putting best targets at front,
+ * returns number of best targets (or 0, no suitable targets) */
+static int
+iter_filter_order(struct iter_env* iter_env, struct module_env* env,
+	uint8_t* name, size_t namelen, time_t now, struct delegpt* dp)
+{
+	int got_num = 0, got_rtt = 0, thisrtt, swap_to_front;
+	struct delegpt_addr* a, *n, *prev=NULL;
+
+	a = dp->result_list;
+	while(a) {
+		/* filter out unsuitable targets */
+		thisrtt = iter_filter_unsuitable(iter_env, env, name, namelen, 
+			now, a);
+		if(thisrtt == -1) {
+			prev = a;
+			a = a->next_result;
+			continue;
+		}
+		/* classify the server address and determine what to do */
+		swap_to_front = 0;
+		if(got_num == 0) {
+			got_rtt = thisrtt;
+			got_num = 1;
+			swap_to_front = 1;
+		} else if(thisrtt == got_rtt) {
+			got_num++;
+			swap_to_front = 1;
+		} else if(thisrtt < got_rtt) {
+			got_rtt = thisrtt;
+			got_num = 1; /* start back at count of 1 */
+			swap_to_front = 1;
+		}
+		/* swap to front if necessary, or move to next result */
+		if(swap_to_front && prev) {
+			n = a->next_result;
+			prev->next_result = n;
+			a->next_result = dp->result_list;
+			dp->result_list = a;
+			a = n;
+		} else {
+			prev = a;
+			a = a->next_result;
+		}
+	}
+	return got_num;
+}
+
+struct delegpt_addr* 
+iter_server_selection(struct iter_env* iter_env, 
+	struct module_env* env, struct delegpt* dp, 
+	uint8_t* name, size_t namelen)
+{
+	time_t now = time(NULL);
+	int sel;
+	struct delegpt_addr* a, *prev;
+	int num = iter_filter_order(iter_env, env, name, namelen, now, dp);
+
+	if(num == 0)
+		return NULL;
+	if(num == 1) {
+		a = dp->result_list;
+		dp->result_list = a->next_result;
+		return a;
+	}
+	/* randomly select a target from the list */
+	log_assert(num > 1);
+	/* we do not need secure random numbers here, but
+	 * we do need it to be threadsafe, so we use this */
+	sel = ub_random(env->rnd) % num; 
+	a = dp->result_list;
+	prev = NULL;
+	while(sel > 0 && a) {
+		prev = a;
+		a = a->next_result;
+		sel--;
+	}
+	if(!a)  /* robustness */
+		return NULL;
+	/* remove it from the delegation point result list */
+	if(prev)
+		prev->next_result = a->next_result;
+	else	dp->result_list = a->next_result;
+	return a;
+}
+
+struct dns_msg* 
+dns_alloc_msg(ldns_buffer* pkt, struct msg_parse* msg, struct region* region)
+{
+	struct dns_msg* m = (struct dns_msg*)region_alloc(region,
+		sizeof(struct dns_msg));
+	if(!m)
+		return NULL;
+	memset(m, 0, sizeof(*m));
+	if(!parse_create_msg(pkt, msg, NULL, &m->qinfo, &m->rep, region)) {
+		log_err("malloc failure: allocating incoming dns_msg");
+		return NULL;
+	}
+	return m;
+}
+
+struct dns_msg* 
+dns_copy_msg(struct dns_msg* from, struct region* region)
+{
+	struct dns_msg* m = (struct dns_msg*)region_alloc(region,
+		sizeof(struct dns_msg));
+	if(!m)
+		return NULL;
+	m->qinfo = from->qinfo;
+	if(!(m->qinfo.qname = region_alloc_init(region, from->qinfo.qname,
+		from->qinfo.qname_len)))
+		return NULL;
+	if(!(m->rep = reply_info_copy(from->rep, NULL, region)))
+		return NULL;
+	return m;
+}
+
+int 
+iter_dns_store(struct module_env* env, struct query_info* msgqinf,
+	struct reply_info* msgrep, int is_referral)
+{
+	return dns_cache_store(env, msgqinf, msgrep, is_referral);
+}
+
+int 
+iter_ns_probability(struct ub_randstate* rnd, int n, int m)
+{
+	int sel;
+	if(n == m) /* 100% chance */
+		return 1;
+	/* we do not need secure random numbers here, but
+	 * we do need it to be threadsafe, so we use this */
+	sel = ub_random(rnd) % m; 
+	return (sel < n);
+}
+
+/** detect dependency cycle for query and target */
+static int
+causes_cycle(struct module_qstate* qstate, uint8_t* name, size_t namelen,
+	uint16_t t, uint16_t c)
+{
+	struct query_info qinf;
+	qinf.qname = name;
+	qinf.qname_len = namelen;
+	qinf.qtype = t;
+	qinf.qclass = c;
+	return (*qstate->env->detect_cycle)(qstate, &qinf, 
+		(uint16_t)(BIT_RD|BIT_CD), qstate->is_priming);
+}
+
+void 
+iter_mark_cycle_targets(struct module_qstate* qstate, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved)
+			continue;
+		/* see if this ns as target causes dependency cycle */
+		if(causes_cycle(qstate, ns->name, ns->namelen, 
+			LDNS_RR_TYPE_AAAA, qstate->qinfo.qclass) ||
+		   causes_cycle(qstate, ns->name, ns->namelen, 
+			LDNS_RR_TYPE_A, qstate->qinfo.qclass)) {
+			log_nametypeclass(VERB_DETAIL, "skipping target due "
+			 	"to dependency cycle (harden-glue: no may "
+				"fix this)", ns->name, LDNS_RR_TYPE_A, 
+				qstate->qinfo.qclass);
+			ns->resolved = 1;
+		}
+	}
+}
+
+int 
+iter_dp_is_useless(uint16_t flags, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	/* check:
+	 *      o all NS items are required glue.
+	 *      o no addresses are provided.
+	 *      o RD qflag is on.
+	 */
+	if(!(flags&BIT_RD))
+		return 0;
+	/* either available or unused targets */
+	if(dp->usable_list || dp->result_list) 
+		return 0;
+	
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved) /* skip failed targets */
+			continue;
+		if(!dname_subdomain_c(ns->name, dp->name))
+			return 0; /* one address is not required glue */
+	}
+	return 1;
+}

iterator/iter_utils.h

@@ -0,0 +1,139 @@
+/*
+ * iterator/iter_utils.h - iterative resolver module utility functions.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Configuration options. Forward zones. 
+ */
+
+#ifndef ITERATOR_ITER_UTILS_H
+#define ITERATOR_ITER_UTILS_H
+struct iter_env;
+struct config_file;
+struct module_env;
+struct delegpt_addr;
+struct delegpt;
+struct region;
+struct msg_parse;
+struct ub_randstate;
+struct query_info;
+struct reply_info;
+struct module_qstate;
+
+/**
+ * Process config options and set iterator module state.
+ * Sets default values if no config is found.
+ * @param iter_env: iterator module state.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg);
+
+/**
+ * Select a valid, nice target to send query to.
+ * Sorting and removing unsuitable targets is combined.
+ *
+ * @param iter_env: iterator module global state, with ip6 enabled and 
+ *	do-not-query-addresses.
+ * @param env: environment with infra cache (lameness, rtt info).
+ * @param dp: delegation point with result list.
+ * @param name: zone name (for lameness check).
+ * @param namelen: length of name.
+ * @return best target or NULL if no target.
+ *	if not null, that target is removed from the result list in the dp.
+ */
+struct delegpt_addr* iter_server_selection(struct iter_env* iter_env, 
+	struct module_env* env, struct delegpt* dp, uint8_t* name, 
+	size_t namelen);
+
+/**
+ * Allocate dns_msg from parsed msg, in region.
+ * @param pkt: packet.
+ * @param msg: parsed message (cleaned and ready for region allocation).
+ * @param region: region to use for allocation.
+ * @return newly allocated dns_msg, or NULL on memory error.
+ */
+struct dns_msg* dns_alloc_msg(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct region* region);
+
+/**
+ * Copy a dns_msg to this region.
+ * @param from: dns message, also in region.
+ * @param region: region to use for allocation.
+ * @return newly allocated dns_msg, or NULL on memory error.
+ */
+struct dns_msg* dns_copy_msg(struct dns_msg* from, struct region* region);
+
+/**
+ * Allocate a dns_msg with malloc/alloc structure and store in dns cache.
+ * @param env: environment, with alloc structure and dns cache.
+ * @param qinf: query info, the query for which answer is stored.
+ * @param rep: reply in dns_msg from dns_alloc_msg for example.
+ * @param is_referral: If true, then the given message to be stored is a
+ *	referral. The cache implementation may use this as a hint.
+ * @return 0 on alloc error (out of memory).
+ */
+int iter_dns_store(struct module_env* env, struct query_info* qinf,
+	struct reply_info* rep, int is_referral);
+
+/**
+ * Select randomly with n/m probability.
+ * For shuffle NS records for address fetching.
+ * @param rnd: random table
+ * @param n: probability.
+ * @param m: divisor for probability.
+ * @return true with n/m probability.
+ */
+int iter_ns_probability(struct ub_randstate* rnd, int n, int m);
+
+/**
+ * Mark targets that result in a dependency cycle as done, so they
+ * will not get selected as targets.
+ * @param qstate: query state.
+ * @param dp: delegpt to mark ns in.
+ */
+void iter_mark_cycle_targets(struct module_qstate* qstate, struct delegpt* dp);
+
+/**
+ * See if delegation is useful or offers immediately no targets for 
+ * further recursion.
+ * @param flags: query flags.
+ * @param dp: delegpt to check.
+ */
+int iter_dp_is_useless(uint16_t flags, struct delegpt* dp);
+
+#endif /* ITERATOR_ITER_UTILS_H */

iterator/iterator.h

@@ -0,0 +1,276 @@
+/*
+ * iterator/iterator.h - iterative resolver DNS query response module
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains a module that performs recusive iterative DNS query
+ * processing.
+ */
+
+#ifndef ITERATOR_ITERATOR_H
+#define ITERATOR_ITERATOR_H
+#include "services/outbound_list.h"
+#include "util/data/msgreply.h"
+struct module_func_block;
+struct delegpt;
+struct iter_hints;
+struct iter_forwards;
+struct iter_donotq;
+struct iter_prep_list;
+
+/** max number of query restarts. Determines max number of CNAME chain. */
+#define MAX_RESTART_COUNT       8
+/** max number of referrals. Makes sure resolver does not run away */
+#define MAX_REFERRAL_COUNT	30
+/** how nice is a server without further information, in msec 
+ * Equals rtt initial timeout value.
+ */
+#define UNKNOWN_SERVER_NICENESS 3000
+/** maximum timeout before a host is deemed unsuitable, in msec. 
+ * After host_ttl this will be timed out and the host will be tried again. 
+ * Equals RTT_MAX_TIMEOUT
+ */
+#define USEFUL_SERVER_TOP_TIMEOUT	120000
+
+/**
+ * Global state for the iterator. 
+ */
+struct iter_env {
+	/** 
+	 * The hints -- these aren't stored in the cache because they don't 
+	 * expire. The hints are always used to "prime" the cache. Note 
+	 * that both root hints and stub zone "hints" are stored in this 
+	 * data structure.
+	 */
+	struct iter_hints* hints;
+
+	/** A flag to indicate whether or not we have an IPv6 route */
+	int supports_ipv6;
+
+	/** Mapping of forwarding zones to targets. */
+	struct iter_forwards* fwds;
+
+	/** A set of inetaddrs that should never be queried. */
+	struct iter_donotq* donotq;
+
+	/** The maximum dependency depth that this resolver will pursue. */
+	int max_dependency_depth;
+
+	/**
+	 * The target fetch policy for each dependency level. This is 
+	 * described as a simple number (per dependency level): 
+	 *	negative numbers (usually just -1) mean fetch-all, 
+	 *	0 means only fetch on demand, and 
+	 *	positive numbers mean to fetch at most that many targets.
+	 * array of max_dependency_depth+1 size.
+	 */
+	int* target_fetch_policy;
+};
+
+/**
+ * State of the iterator for a query.
+ */
+enum iter_state {
+	/**
+	 * Externally generated queries start at this state. Query restarts are
+	 * reset to this state.
+	 */
+	INIT_REQUEST_STATE = 0,
+
+	/**
+	 * Root priming events reactivate here, most other events pass 
+	 * through this naturally as the 2nd part of the INIT_REQUEST_STATE.
+	 */
+	INIT_REQUEST_2_STATE,
+
+	/**
+	 * Stub priming events reactivate here, most other events pass 
+	 * through this naturally as the 3rd part of the INIT_REQUEST_STATE.
+	 */
+	INIT_REQUEST_3_STATE,
+
+	/**
+	 * Each time a delegation point changes for a given query or a 
+	 * query times out and/or wakes up, this state is (re)visited. 
+	 * This state is reponsible for iterating through a list of 
+	 * nameserver targets.
+	 */
+	QUERYTARGETS_STATE,
+
+	/**
+	 * Responses to queries start at this state. This state handles 
+	 * the decision tree associated with handling responses.
+	 */
+	QUERY_RESP_STATE,
+
+	/** Responses to priming queries finish at this state. */
+	PRIME_RESP_STATE,
+
+	/** Responses that are to be returned upstream end at this state. 
+	 * As well as responses to target queries. */
+	FINISHED_STATE
+};
+
+/**
+ * Per query state for the iterator module.
+ */
+struct iter_qstate {
+	/** 
+	 * State of the iterator module.
+	 * This is the state that event is in or should sent to -- all 
+	 * requests should start with the INIT_REQUEST_STATE. All 
+	 * responses should start with QUERY_RESP_STATE. Subsequent 
+	 * processing of the event will change this state.
+	 */
+	enum iter_state state;
+
+	/** 
+	 * Final state for the iterator module.
+	 * This is the state that responses should be routed to once the 
+	 * response is final. For externally initiated queries, this 
+	 * will be FINISHED_STATE, locally initiated queries will have 
+	 * different final states.
+	 */
+	enum iter_state final_state;
+
+	/** 
+	 * The depth of this query, this means the depth of recursion.
+	 * This address is needed for another query, which is an address
+	 * needed for another query, etc. Original client query has depth 0.
+	 */
+	int depth;
+
+	/**
+	 * The response
+	 */
+	struct dns_msg* response;
+
+	/** 
+	 * This is a list of RRsets that must be prepended to the 
+	 * ANSWER section of a response before being sent upstream.
+	 */
+	struct iter_prep_list* an_prepend_list;
+	/** Last element of the prepend list */
+	struct iter_prep_list* an_prepend_last;
+
+	/**
+	 * This is the list of RRsets that must be prepended to the
+	 * AUTHORITY section of the response before being sent upstream.
+	 */
+	struct iter_prep_list* ns_prepend_list;
+	/** Last element of the authority prepend list */
+	struct iter_prep_list* ns_prepend_last;
+
+	/** query name used for chasing the results. Initially the same as
+	 * the state qinfo, but after CNAMEs this will be different. 
+	 * The query info used to elicit the results needed. */
+	struct query_info qchase;
+	/** query flags to use when chasing the answer (i.e. RD flag) */
+	uint16_t chase_flags;
+
+	/** 
+	 * This is the current delegation point for an in-progress query. This
+	 * object retains state as to which delegation targets need to be
+	 * (sub)queried for vs which ones have already been visited.
+	 */
+	struct delegpt* dp;
+
+	/** Current delegation message - returned for non-RD queries */
+	struct dns_msg* deleg_msg;
+
+	/** number of outstanding target sub queries */
+	int num_target_queries;
+
+	/** outstanding direct queries */
+	int num_current_queries;
+
+	/** the number of times this query has been restarted. */
+	int query_restart_count;
+
+	/** the number of times this query as followed a referral. */
+	int referral_count;
+
+	/**
+	 * This is flag that, if true, means that this event is 
+	 * waiting for a stub priming query. 
+	 */
+	int wait_priming_stub;
+
+	/**
+	 * This is a flag that, if true, means that this query is
+	 * for (re)fetching glue from a zone. Since the address should
+	 * have been glue, query again to the servers that should have
+	 * been returning it as glue.
+	 * The delegation point must be set to the one that should *not*
+	 * be used when creating the state. A higher one will be attempted.
+	 */
+	int refetch_glue;
+
+	/** list of pending queries to authoritative servers. */
+	struct outbound_list outlist;
+};
+
+/**
+ * List of prepend items
+ */
+struct iter_prep_list {
+	/** next in list */
+	struct iter_prep_list* next;
+	/** rrset */
+	struct ub_packed_rrset_key* rrset;
+};
+
+/**
+ * Get the iterator function block.
+ * @return: function block with function pointers to iterator methods.
+ */
+struct module_func_block* iter_get_funcblock();
+
+/**
+ * Get iterator state as a string
+ * @param state: to convert
+ * @return constant string that is printable.
+ */
+const char* iter_state_to_string(enum iter_state state);
+
+/**
+ * See if iterator state is a response state
+ * @param s: to inspect
+ * @return true if response state.
+ */
+int iter_state_is_responsestate(enum iter_state s);
+
+#endif /* ITERATOR_ITERATOR_H */

makedist.sh

@@ -134,7 +134,7 @@ done
 # Check if SVNROOT is specified.
 if [ -z "$SVNROOT" ]; then
     if test -f .svn/entries; then
-	  eval `grep 'url=' .svn/entries | head -1`
+	  eval `svn info | grep 'URL:' | sed -e 's/URL: /url=/' | head -1`
 	  SVNROOT="$url"
     fi
     if test -z "$SVNROOT"; then

services/cache/dns.c

@@ -0,0 +1,666 @@
+/*
+ * services/cache/dns.c - Cache services for DNS using msg and rrset caches.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the DNS cache.
+ */
+#include "config.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "services/cache/rrset.h"
+#include "util/data/msgreply.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/dname.h"
+#include "util/module.h"
+#include "util/net_help.h"
+#include "util/region-allocator.h"
+
+/** store rrsets in the rrset cache. 
+ * @param env: module environment with caches.
+ * @param rep: contains list of rrsets to store.
+ * @param now: current time(NULL).
+ */
+static void
+store_rrsets(struct module_env* env, struct reply_info* rep, uint32_t now)
+{
+        size_t i;
+        /* see if rrset already exists in cache, if not insert it. */
+        for(i=0; i<rep->rrset_count; i++) {
+                rep->ref[i].key = rep->rrsets[i];
+                rep->ref[i].id = rep->rrsets[i]->id;
+                if(rrset_cache_update(env->rrset_cache, &rep->ref[i],
+                        env->alloc, now)) /* it was in the cache */
+                        rep->rrsets[i] = rep->ref[i].key;
+        }
+}
+
+void 
+dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
+	hashvalue_t hash, struct reply_info* rep)
+{
+	struct msgreply_entry* e;
+	uint32_t now = time(NULL), ttl = rep->ttl;
+	size_t i;
+
+	/* store RRsets */
+        for(i=0; i<rep->rrset_count; i++) {
+		rep->ref[i].key = rep->rrsets[i];
+		rep->ref[i].id = rep->rrsets[i]->id;
+	}
+	reply_info_sortref(rep);
+	reply_info_set_ttls(rep, now);
+	store_rrsets(env, rep, now);
+	if(ttl == 0) {
+		/* we do not store the message, but we did store the RRs,
+		 * which could be useful for delegation information */
+		verbose(VERB_ALGO, "TTL 0: dropped msg from cache");
+		free(rep);
+		return;
+	}
+
+	/* store msg in the cache */
+	reply_info_sortref(rep);
+	if(!(e = query_info_entrysetup(qinfo, rep, hash))) {
+		log_err("store_msg: malloc failed");
+		return;
+	}
+	slabhash_insert(env->msg_cache, hash, &e->entry, rep, env->alloc);
+}
+
+/** allocate rrset in region - no more locks needed */
+static struct ub_packed_rrset_key*
+copy_rrset(struct ub_packed_rrset_key* key, struct region* region, 
+	uint32_t now)
+{
+	struct ub_packed_rrset_key* ck = region_alloc(region, 
+		sizeof(struct ub_packed_rrset_key));
+	struct packed_rrset_data* d;
+	struct packed_rrset_data* data = (struct packed_rrset_data*)
+		key->entry.data;
+	size_t dsize, i;
+	if(!ck)
+		return NULL;
+	ck->id = key->id;
+	memset(&ck->entry, 0, sizeof(ck->entry));
+	ck->entry.hash = key->entry.hash;
+	ck->entry.key = ck;
+	ck->rk = key->rk;
+	ck->rk.dname = region_alloc_init(region, key->rk.dname, 
+		key->rk.dname_len);
+	if(!ck->rk.dname)
+		return NULL;
+	dsize = packed_rrset_sizeof(data);
+	d = (struct packed_rrset_data*)region_alloc_init(region, data, dsize);
+	if(!d)
+		return NULL;
+	ck->entry.data = d;
+	packed_rrset_ptr_fixup(d);
+	/* make TTLs relative - once per rrset */
+	for(i=0; i<d->count + d->rrsig_count; i++)
+		d->rr_ttl[i] -= now;
+	d->ttl -= now;
+	return ck;
+}
+
+/** find closest NS or DNAME and returns the rrset (locked) */
+static struct ub_packed_rrset_key*
+find_closest_of_type(struct module_env* env, uint8_t* qname, size_t qnamelen, 
+	uint16_t qclass, uint32_t now, uint16_t searchtype, int stripfront)
+{
+	struct ub_packed_rrset_key *rrset;
+	uint8_t lablen;
+
+	if(stripfront) {
+		/* strip off so that DNAMEs have strict subdomain match */
+		lablen = *qname;
+		qname += lablen + 1;
+		qnamelen -= lablen + 1;
+	}
+
+	/* snip off front part of qname until the type is found */
+	while(qnamelen > 0) {
+		if((rrset = rrset_cache_lookup(env->rrset_cache, qname, 
+			qnamelen, searchtype, qclass, 0, now, 0)))
+			return rrset;
+
+		/* snip off front label */
+		lablen = *qname;
+		qname += lablen + 1;
+		qnamelen -= lablen + 1;
+	}
+	return NULL;
+}
+
+/** add addr to additional section */
+static void
+addr_to_additional(struct ub_packed_rrset_key* rrset, struct region* region,
+	struct dns_msg* msg, uint32_t now)
+{
+	if((msg->rep->rrsets[msg->rep->rrset_count] = 
+		copy_rrset(rrset, region, now))) {
+		msg->rep->ar_numrrsets++;
+		msg->rep->rrset_count++;
+	}
+}
+
+/** find and add A and AAAA records for nameservers in delegpt */
+static int
+find_add_addrs(struct module_env* env, uint16_t qclass, struct region* region,
+	struct delegpt* dp, uint32_t now, struct dns_msg** msg)
+{
+	struct delegpt_ns* ns;
+	struct ub_packed_rrset_key* akey;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_A, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_A(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			if(msg)
+				addr_to_additional(akey, region, *msg, now);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_AAAA(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			if(msg)
+				addr_to_additional(akey, region, *msg, now);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+	}
+	return 1;
+}
+
+/** find and add A and AAAA records for missing nameservers in delegpt */
+int
+cache_fill_missing(struct module_env* env, uint16_t qclass, 
+	struct region* region, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	struct ub_packed_rrset_key* akey;
+	uint32_t now = time(NULL);
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved)
+			continue;
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_A, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_A(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			log_nametypeclass(VERB_ALGO, "found in cache",
+				ns->name, LDNS_RR_TYPE_A, qclass);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_AAAA(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			log_nametypeclass(VERB_ALGO, "found in cache",
+				ns->name, LDNS_RR_TYPE_AAAA, qclass);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+	}
+	return 1;
+}
+
+/** find and add DS or NSEC to delegation msg */
+static void
+find_add_ds(struct module_env* env, struct region* region, 
+	struct dns_msg* msg, struct delegpt* dp, uint32_t now)
+{
+	/* Lookup the DS or NSEC at the delegation point. */
+	struct ub_packed_rrset_key* rrset = rrset_cache_lookup(
+		env->rrset_cache, dp->name, dp->namelen, LDNS_RR_TYPE_DS, 
+		msg->qinfo.qclass, 0, now, 0);
+	if(!rrset) {
+		/* NOTE: this won't work for alternate NSEC schemes 
+		 *	(opt-in, NSEC3) */
+		rrset = rrset_cache_lookup(env->rrset_cache, dp->name, 
+			dp->namelen, LDNS_RR_TYPE_NSEC, msg->qinfo.qclass, 
+			0, now, 0);
+		/* Note: the PACKED_RRSET_NSEC_AT_APEX flag is not used.
+		 * since this is a referral, we need the NSEC at the parent
+		 * side of the zone cut, not the NSEC at apex side. */
+	}
+	if(rrset) {
+		/* add it to auth section. This is the second rrset. */
+		if((msg->rep->rrsets[msg->rep->rrset_count] = 
+			copy_rrset(rrset, region, now))) {
+			msg->rep->ns_numrrsets++;
+			msg->rep->rrset_count++;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+}
+
+/** create referral message with NS and query */
+static struct dns_msg*
+create_msg(uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct region* region, struct ub_packed_rrset_key* nskey, 
+	struct packed_rrset_data* nsdata, uint32_t now)
+{
+	struct dns_msg* msg = (struct dns_msg*)region_alloc(region,
+		sizeof(struct dns_msg));
+	if(!msg)
+		return NULL;
+	msg->qinfo.qname = region_alloc_init(region, qname, qnamelen);
+	if(!msg->qinfo.qname)
+		return NULL;
+	msg->qinfo.qname_len = qnamelen;
+	msg->qinfo.qtype = qtype;
+	msg->qinfo.qclass = qclass;
+	/* non-packed reply_info, because it needs to grow the array */
+	msg->rep = (struct reply_info*)region_alloc(region, 
+		sizeof(struct reply_info)-sizeof(struct rrset_ref));
+	if(!msg->rep)
+		return NULL;
+	memset(msg->rep, 0, 
+		sizeof(struct reply_info)-sizeof(struct rrset_ref));
+	msg->rep->flags = BIT_QR; /* with QR, no AA */
+	msg->rep->qdcount = 1;
+	/* allocate the array to as much as we could need:
+	 *	NS rrset + DS/NSEC rrset +
+	 *	A rrset for every NS RR
+	 *	AAAA rrset for every NS RR
+	 */
+	msg->rep->rrsets = (struct ub_packed_rrset_key**)region_alloc(region,
+		(2 + nsdata->count*2)*sizeof(struct ub_packed_rrset_key*));
+	if(!msg->rep->rrsets)
+		return NULL;
+	msg->rep->rrsets[0] = copy_rrset(nskey, region, now);
+	if(!msg->rep->rrsets[0])
+		return NULL;
+	msg->rep->ns_numrrsets++;
+	msg->rep->rrset_count++;
+	return msg;
+}
+
+struct delegpt* 
+dns_cache_find_delegation(struct module_env* env, uint8_t* qname, 
+	size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct region* region, struct dns_msg** msg, uint32_t now)
+{
+	/* try to find closest NS rrset */
+	struct ub_packed_rrset_key* nskey;
+	struct packed_rrset_data* nsdata;
+	struct delegpt* dp;
+
+	nskey = find_closest_of_type(env, qname, qnamelen, qclass, now,
+		LDNS_RR_TYPE_NS, 0);
+	if(!nskey) /* hope the caller has hints to prime or something */
+		return NULL;
+	nsdata = (struct packed_rrset_data*)nskey->entry.data;
+	/* got the NS key, create delegation point */
+	dp = delegpt_create(region);
+	if(!dp || !delegpt_set_name(dp, region, nskey->rk.dname)) {
+		lock_rw_unlock(&nskey->entry.lock);
+		log_err("find_delegation: out of memory");
+		return NULL;
+	}
+	/* create referral message */
+	if(msg) {
+		*msg = create_msg(qname, qnamelen, qtype, qclass, region, 
+			nskey, nsdata, now);
+		if(!*msg) {
+			lock_rw_unlock(&nskey->entry.lock);
+			log_err("find_delegation: out of memory");
+			return NULL;
+		}
+	}
+	if(!delegpt_rrset_add_ns(dp, region, nskey))
+		log_err("find_delegation: addns out of memory");
+	lock_rw_unlock(&nskey->entry.lock); /* first unlock before next lookup*/
+	/* find and add DS/NSEC (if any) */
+	if(msg)
+		find_add_ds(env, region, *msg, dp, now);
+	/* find and add A entries */
+	if(!find_add_addrs(env, qclass, region, dp, now, msg))
+		log_err("find_delegation: addrs out of memory");
+	return dp;
+}
+
+/** allocate dns_msg from query_info and reply_info */
+static struct dns_msg*
+gen_dns_msg(struct region* region, struct query_info* q, size_t num)
+{
+	struct dns_msg* msg = (struct dns_msg*)region_alloc(region, 
+		sizeof(struct dns_msg));
+	if(!msg)
+		return NULL;
+	memcpy(&msg->qinfo, q, sizeof(struct query_info));
+	msg->qinfo.qname = region_alloc_init(region, q->qname, q->qname_len);
+	if(!msg->qinfo.qname)
+		return NULL;
+	/* allocate replyinfo struct and rrset key array separately */
+	msg->rep = (struct reply_info*)region_alloc(region,
+		sizeof(struct reply_info) - sizeof(struct rrset_ref));
+	if(!msg->rep)
+		return NULL;
+	msg->rep->rrsets = (struct ub_packed_rrset_key**)region_alloc(region,
+		num * sizeof(struct ub_packed_rrset_key*));
+	if(!msg->rep->rrsets)
+		return NULL;
+	return msg;
+}
+
+/** generate dns_msg from cached message */
+static struct dns_msg*
+tomsg(struct module_env* env, struct msgreply_entry* e, struct reply_info* r, 
+	struct region* region, uint32_t now, struct region* scratch)
+{
+	struct dns_msg* msg;
+	size_t i;
+	if(now > r->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, &e->key, r->rrset_count);
+	if(!msg)
+		return NULL;
+	msg->rep->flags = r->flags;
+	msg->rep->qdcount = r->qdcount;
+	msg->rep->ttl = r->ttl;
+	msg->rep->security = r->security;
+	msg->rep->an_numrrsets = r->an_numrrsets;
+	msg->rep->ns_numrrsets = r->ns_numrrsets;
+	msg->rep->ar_numrrsets = r->ar_numrrsets;
+	msg->rep->rrset_count = r->rrset_count;
+	if(!rrset_array_lock(r->ref, r->rrset_count, now))
+		return NULL;
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		msg->rep->rrsets[i] = copy_rrset(r->rrsets[i], region, now);
+		if(!msg->rep->rrsets[i]) {
+			rrset_array_unlock(r->ref, r->rrset_count);
+			return NULL;
+		}
+	}
+	rrset_array_unlock_touch(env->rrset_cache, scratch, r->ref, 
+		r->rrset_count);
+	return msg;
+}
+
+/** synthesize RRset-only response from cached RRset item */
+static struct dns_msg*
+rrset_msg(struct ub_packed_rrset_key* rrset, struct region* region, 
+	uint32_t now, struct query_info* q)
+{
+	struct dns_msg* msg;
+	struct packed_rrset_data* d = (struct packed_rrset_data*)
+		rrset->entry.data;
+	if(now > d->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, q, 1); /* only the CNAME (or other) RRset */
+	if(!msg)
+		return NULL;
+	msg->rep->flags = BIT_QR; /* reply, no AA, no error */
+	msg->rep->qdcount = 1;
+	msg->rep->ttl = d->ttl - now;
+	msg->rep->security = sec_status_unchecked;
+	msg->rep->an_numrrsets = 1;
+	msg->rep->ns_numrrsets = 0;
+	msg->rep->ar_numrrsets = 0;
+	msg->rep->rrset_count = 1;
+	msg->rep->rrsets[0] = copy_rrset(rrset, region, now);
+	if(!msg->rep->rrsets[0]) /* copy CNAME */
+		return NULL;
+	return msg;
+}
+
+/** synthesize DNAME+CNAME response from cached DNAME item */
+static struct dns_msg*
+synth_dname_msg(struct ub_packed_rrset_key* rrset, struct region* region, 
+	uint32_t now, struct query_info* q)
+{
+	struct dns_msg* msg;
+	struct ub_packed_rrset_key* ck;
+	struct packed_rrset_data* newd, *d = (struct packed_rrset_data*)
+		rrset->entry.data;
+	uint8_t* newname, *dtarg = NULL;
+	size_t newlen, dtarglen;
+	if(now > d->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, q, 2); /* DNAME + CNAME RRset */
+	if(!msg)
+		return NULL;
+	msg->rep->flags = BIT_QR; /* reply, no AA, no error */
+	msg->rep->qdcount = 1;
+	msg->rep->ttl = d->ttl - now;
+	msg->rep->security = sec_status_unchecked;
+	msg->rep->an_numrrsets = 1;
+	msg->rep->ns_numrrsets = 0;
+	msg->rep->ar_numrrsets = 0;
+	msg->rep->rrset_count = 1;
+	msg->rep->rrsets[0] = copy_rrset(rrset, region, now);
+	if(!msg->rep->rrsets[0]) /* copy DNAME */
+		return NULL;
+	/* synth CNAME rrset */
+	get_cname_target(rrset, &dtarg, &dtarglen);
+	if(!dtarg)
+		return NULL;
+	newlen = q->qname_len + dtarglen - rrset->rk.dname_len;
+	if(newlen > LDNS_MAX_DOMAINLEN) {
+		msg->rep->flags |= LDNS_RCODE_YXDOMAIN;
+		return msg;
+	}
+	newname = (uint8_t*)region_alloc(region, newlen);
+	if(!newname)
+		return NULL;
+	/* new name is concatenation of qname front (without DNAME owner)
+	 * and DNAME target name */
+	memcpy(newname, q->qname, q->qname_len-rrset->rk.dname_len);
+	memmove(newname+(q->qname_len-rrset->rk.dname_len), dtarg, dtarglen);
+	/* create rest of CNAME rrset */
+	ck = (struct ub_packed_rrset_key*)region_alloc(region, 
+		sizeof(struct ub_packed_rrset_key));
+	if(!ck)
+		return NULL;
+	memset(&ck->entry, 0, sizeof(ck->entry));
+	msg->rep->rrsets[1] = ck;
+	ck->entry.key = ck;
+	ck->rk.type = htons(LDNS_RR_TYPE_CNAME);
+	ck->rk.rrset_class = rrset->rk.rrset_class;
+	ck->rk.flags = 0;
+	ck->rk.dname = region_alloc_init(region, q->qname, q->qname_len);
+	if(!ck->rk.dname)
+		return NULL;
+	ck->rk.dname_len = q->qname_len;
+	ck->entry.hash = rrset_key_hash(&ck->rk);
+	newd = (struct packed_rrset_data*)region_alloc(region,
+		sizeof(struct packed_rrset_data) + sizeof(size_t) + 
+		sizeof(uint8_t*) + sizeof(uint32_t) + sizeof(uint16_t) 
+		+ newlen);
+	if(!newd)
+		return NULL;
+	ck->entry.data = newd;
+	newd->ttl = 0; /* 0 for synthesized CNAME TTL */
+	newd->count = 1;
+	newd->rrsig_count = 0;
+	newd->trust = rrset_trust_ans_noAA;
+	newd->rr_len = (size_t*)((uint8_t*)newd + 
+		sizeof(struct packed_rrset_data));
+	newd->rr_len[0] = newlen + sizeof(uint16_t);
+	packed_rrset_ptr_fixup(newd);
+	newd->rr_ttl[0] = newd->ttl;
+	msg->rep->ttl = newd->ttl;
+	ldns_write_uint16(newd->rr_data[0], newlen);
+	memmove(newd->rr_data[0] + sizeof(uint16_t), newname, newlen);
+	msg->rep->an_numrrsets ++;
+	msg->rep->rrset_count ++;
+	return msg;
+}
+
+struct dns_msg* 
+dns_cache_lookup(struct module_env* env,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	struct region* region, struct region* scratch)
+{
+	struct lruhash_entry* e;
+	struct query_info k;
+	hashvalue_t h;
+	uint32_t now = (uint32_t)time(NULL);
+	struct ub_packed_rrset_key* rrset;
+
+	/* lookup first, this has both NXdomains and ANSWER responses */
+	k.qname = qname;
+	k.qname_len = qnamelen;
+	k.qtype = qtype;
+	k.qclass = qclass;
+	h = query_info_hash(&k);
+	e = slabhash_lookup(env->msg_cache, h, &k, 0);
+	if(e) {
+		struct msgreply_entry* key = (struct msgreply_entry*)e->key;
+		struct reply_info* data = (struct reply_info*)e->data;
+		struct dns_msg* msg = tomsg(env, key, data, region, now, 
+			scratch);
+		if(msg) {
+			lock_rw_unlock(&e->lock);
+			return msg;
+		}
+		/* could be msg==NULL; due to TTL or not all rrsets available */
+		lock_rw_unlock(&e->lock);
+	}
+
+	/* see if a DNAME exists. Checked for first, to enforce that DNAMEs
+	 * are more important, the CNAME is resynthesized and thus 
+	 * consistent with the DNAME */
+	if( (rrset=find_closest_of_type(env, qname, qnamelen, qclass, now,
+		LDNS_RR_TYPE_DNAME, 1))) {
+		/* synthesize a DNAME+CNAME message based on this */
+		struct dns_msg* msg = synth_dname_msg(rrset, region, now, &k);
+		if(msg) {
+			lock_rw_unlock(&rrset->entry.lock);
+			return msg;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+
+	/* see if we have CNAME for this domain */
+	if( (rrset=rrset_cache_lookup(env->rrset_cache, qname, qnamelen, 
+		LDNS_RR_TYPE_CNAME, qclass, 0, now, 0))) {
+		struct dns_msg* msg = rrset_msg(rrset, region, now, &k);
+		if(msg) {
+			lock_rw_unlock(&rrset->entry.lock);
+			return msg;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+
+	/* construct DS, DNSKEY messages from rrset cache. */
+	if((qtype == LDNS_RR_TYPE_DS || qtype == LDNS_RR_TYPE_DNSKEY) &&
+		(rrset=rrset_cache_lookup(env->rrset_cache, qname, qnamelen, 
+		qtype, qclass, 0, now, 0))) {
+		/* if the rrset is from the additional section, and the
+		 * signatures have fallen off, then do not synthesize a msg
+		 * instead, allow a full query for signed results to happen.
+		 * Forego all rrset data from additional section, because
+		 * some signatures may not be present and cause validation
+		 * failure.
+		 */
+		struct packed_rrset_data *d = (struct packed_rrset_data*)
+			rrset->entry.data;
+		if(d->trust != rrset_trust_add_noAA && 
+			d->trust != rrset_trust_add_AA) {
+			struct dns_msg* msg = rrset_msg(rrset, region, now, &k);
+			if(msg) {
+				lock_rw_unlock(&rrset->entry.lock);
+				return msg;
+			}
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+	return NULL;
+}
+
+int 
+dns_cache_store(struct module_env* env, struct query_info* msgqinf,
+        struct reply_info* msgrep, int is_referral)
+{
+	struct reply_info* rep = NULL;
+	/* alloc, malloc properly (not in region, like msg is) */
+	rep = reply_info_copy(msgrep, env->alloc, NULL);
+	if(!rep)
+		return 0;
+
+	if(is_referral) {
+		/* store rrsets */
+		struct rrset_ref ref;
+		uint32_t now = time(NULL);
+		size_t i;
+		for(i=0; i<rep->rrset_count; i++) {
+			packed_rrset_ttl_add((struct packed_rrset_data*)
+				rep->rrsets[i]->entry.data, now);
+			ref.key = rep->rrsets[i];
+			ref.id = rep->rrsets[i]->id;
+			/*ignore ret: it was in the cache, ref updated */
+			(void)rrset_cache_update(env->rrset_cache, &ref, 
+				env->alloc, now);
+		}
+		free(rep);
+		return 1;
+	} else {
+		/* store msg, and rrsets */
+		struct query_info qinf;
+		hashvalue_t h;
+
+		qinf = *msgqinf;
+		qinf.qname = memdup(msgqinf->qname, msgqinf->qname_len);
+		if(!qinf.qname) {
+			reply_info_parsedelete(rep, env->alloc);
+			return 0;
+		}
+		/* fixup flags to be sensible for a reply based on the cache */
+		/* this module means that RA is available. It is an answer QR. 
+		 * Not AA from cache. Not CD in cache (depends on client bit). */
+		rep->flags |= (BIT_RA | BIT_QR);
+		rep->flags &= ~(BIT_AA | BIT_CD);
+		h = query_info_hash(&qinf);
+		dns_cache_store_msg(env, &qinf, h, rep);
+		/* qname is used inside query_info_entrysetup, and set to 
+		 * NULL. If it has not been used, free it. free(0) is safe. */
+		free(qinf.qname);
+	}
+	return 1;
+}

services/cache/dns.h

@@ -0,0 +1,142 @@
+/*
+ * services/cache/dns.h - Cache services for DNS using msg and rrset caches.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the DNS cache.
+ */
+
+#ifndef SERVICES_CACHE_DNS_H
+#define SERVICES_CACHE_DNS_H
+#include "util/storage/lruhash.h"
+#include "util/data/msgreply.h"
+struct module_env;
+struct query_info;
+struct reply_info;
+struct region;
+struct delegpt;
+
+/**
+ * Region allocated message reply
+ */
+struct dns_msg {
+	/** query info */
+	struct query_info qinfo;
+	/** reply info - ptr to packed repinfo structure */
+	struct reply_info *rep;
+};
+
+/**
+ * Allocate a dns_msg with malloc/alloc structure and store in dns cache.
+ *
+ * @param env: environment, with alloc structure and dns cache.
+ * @param qinf: query info, the query for which answer is stored.
+ * 	this is allocated in a region, and will be copied to malloc area
+ * 	before insertion.
+ * @param rep: reply in dns_msg from dns_alloc_msg for example.
+ * 	this is allocated in a region, and will be copied to malloc area
+ * 	before insertion.
+ * @param is_referral: If true, then the given message to be stored is a
+ *      referral. The cache implementation may use this as a hint.
+ *      It will store only the RRsets, not the message.
+ * @return 0 on alloc error (out of memory).
+ */
+int dns_cache_store(struct module_env* env, struct query_info* qinf,
+        struct reply_info* rep, int is_referral); 
+
+/**
+ * Store message in the cache. Stores in message cache and rrset cache.
+ * Both qinfo and rep should be malloced and are put in the cache.
+ * They should not be used after this call, as they are then in shared cache.
+ * Does not return errors, they are logged and only lead to less cache.
+ *
+ * @param env: module environment with the DNS cache.
+ * @param qinfo: query info
+ * @param hash: hash over qinfo.
+ * @param rep: reply info, together with qinfo makes up the message.
+ *	Adjusts the reply info TTLs to absolute time.
+ */
+void dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
+	hashvalue_t hash, struct reply_info* rep);
+
+/**
+ * Find a delegation from the cache.
+ * @param env: module environment with the DNS cache.
+ * @param qname: query name.
+ * @param qnamelen: length of qname.
+ * @param qtype: query type.
+ * @param qclass: query class.
+ * @param region: where to allocate result delegation.
+ * @param msg: if not NULL, delegation message is returned here, synthesized
+ *	from the cache.
+ * @param timenow: the time now, for checking if TTL on cache entries is OK.
+ * @return new delegation or NULL on error or if not found in cache.
+ */
+struct delegpt* dns_cache_find_delegation(struct module_env* env, 
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct region* region, struct dns_msg** msg, uint32_t timenow);
+
+/** 
+ * Find cached message 
+ * @param env: module environment with the DNS cache.
+ * @param qname: query name.
+ * @param qnamelen: length of qname.
+ * @param qtype: query type.
+ * @param qclass: query class.
+ * @param region: where to allocate result.
+ * @param scratch: where to allocate temporary data.
+ * @return new response message (alloced in region, rrsets do not have IDs).
+ * 	or NULL on error or if not found in cache.
+ *	TTLs are made relative to the current time.
+ */
+struct dns_msg* dns_cache_lookup(struct module_env* env,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	struct region* region, struct region* scratch);
+
+/** 
+ * find and add A and AAAA records for missing nameservers in delegpt 
+ * @param env: module environment with rrset cache
+ * @param qclass: which class to look in.
+ * @param region: where to store new dp info.
+ * @param dp: delegation point to fill missing entries.
+ * @return false on alloc failure.
+ */
+int cache_fill_missing(struct module_env* env, uint16_t qclass, 
+	struct region* region, struct delegpt* dp);
+
+/** Find covering DNAME */
+
+#endif /* SERVICES_CACHE_DNS_H */

services/cache/infra.c

@@ -0,0 +1,528 @@
+/*
+ * services/cache/infra.c - infrastructure cache, server rtt and capabilities
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the infrastructure cache.
+ */
+#include "config.h"
+#include "services/cache/infra.h"
+#include "util/storage/slabhash.h"
+#include "util/storage/lookup3.h"
+#include "util/data/dname.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/config_file.h"
+
+/** calculate size for the hashtable, does not count size of lameness,
+ * so the hashtable is a fixed number of items */
+static size_t 
+infra_host_sizefunc(void* k, void* ATTR_UNUSED(d))
+{
+	struct infra_host_key* key = (struct infra_host_key*)k;
+	return sizeof(*key) + sizeof(struct infra_host_data) 
+		+ lock_get_mem(&key->entry.lock);
+}
+
+/** compare two addresses, returns -1, 0, or +1 */
+static int 
+infra_host_compfunc(void* key1, void* key2)
+{
+	struct infra_host_key* k1 = (struct infra_host_key*)key1;
+	struct infra_host_key* k2 = (struct infra_host_key*)key2;
+	return sockaddr_cmp(&k1->addr, k1->addrlen, &k2->addr, k2->addrlen);
+}
+
+/** delete key, and destroy the lock */
+static void 
+infra_host_delkeyfunc(void* k, void* ATTR_UNUSED(arg), int il)
+{
+	struct infra_host_key* key = (struct infra_host_key*)k;
+	if(!key)
+		return;
+	if(il) { lock_rw_unlock(&key->entry.lock); }
+	lock_rw_destroy(&key->entry.lock);
+	free(key);
+}
+
+/** delete data and destroy the lameness hashtable */
+static void 
+infra_host_deldatafunc(void* d, void* ATTR_UNUSED(arg))
+{
+	struct infra_host_data* data = (struct infra_host_data*)d;
+	lruhash_delete(data->lameness);
+	free(data);
+}
+
+struct infra_cache* 
+infra_create(struct config_file* cfg)
+{
+	struct infra_cache* infra = (struct infra_cache*)calloc(1, 
+		sizeof(struct infra_cache));
+	/* the size of the lameness tables are not counted */
+	size_t maxmem = cfg->infra_cache_numhosts * 
+		(sizeof(struct infra_host_key)+sizeof(struct infra_host_data));
+	infra->hosts = slabhash_create(cfg->infra_cache_slabs,
+		INFRA_HOST_STARTSIZE, maxmem, &infra_host_sizefunc,
+		&infra_host_compfunc, &infra_host_delkeyfunc,
+		&infra_host_deldatafunc, NULL);
+	if(!infra->hosts) {
+		free(infra);
+		return NULL;
+	}
+	infra->host_ttl = cfg->host_ttl;
+	infra->lame_ttl = cfg->lame_ttl;
+	infra->max_lame_size = cfg->infra_cache_lame_size;
+	return infra;
+}
+
+void 
+infra_delete(struct infra_cache* infra)
+{
+	if(!infra)
+		return;
+	slabhash_delete(infra->hosts);
+	free(infra);
+}
+
+struct infra_cache* 
+infra_adjust(struct infra_cache* infra, struct config_file* cfg)
+{
+	size_t maxmem;
+	if(!infra)
+		return infra_create(cfg);
+	infra->host_ttl = cfg->host_ttl;
+	infra->lame_ttl = cfg->lame_ttl;
+	infra->max_lame_size = cfg->infra_cache_lame_size;
+	maxmem = cfg->infra_cache_numhosts * 
+		(sizeof(struct infra_host_key)+sizeof(struct infra_host_data));
+	if(maxmem != slabhash_get_size(infra->hosts) ||
+		cfg->infra_cache_slabs != infra->hosts->size) {
+		infra_delete(infra);
+		infra = infra_create(cfg);
+	}
+	return infra;
+}
+
+/** calculate the hash value for a host key */
+static hashvalue_t
+hash_addr(struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	hashvalue_t h = 0xab;
+	h = hashlittle(&addrlen, sizeof(addrlen), h);
+	h = hashlittle(addr, addrlen, h);
+	return h;
+}
+
+/** lookup version that does not check host ttl (you check it) */
+static struct lruhash_entry* 
+infra_lookup_host_nottl(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, int wr)
+{
+	struct infra_host_key k;
+	k.addrlen = addrlen;
+	memcpy(&k.addr, addr, addrlen);
+	k.entry.hash = hash_addr(addr, addrlen);
+	k.entry.key = (void*)&k;
+	k.entry.data = NULL;
+	return slabhash_lookup(infra->hosts, k.entry.hash, &k, wr);
+}
+
+struct infra_host_data* 
+infra_lookup_host(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, int wr,
+        time_t timenow, struct infra_host_key** key)
+{
+	struct infra_host_data* data;
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, wr);
+	*key = NULL;
+	if(!e)
+		return NULL;
+	/* check TTL */
+	data = (struct infra_host_data*)e->data;
+	if(data->ttl < timenow) {
+		lock_rw_unlock(&e->lock);
+		return NULL;
+	}
+	*key = (struct infra_host_key*)e->key;
+	return data;
+}
+
+/** 
+ * Create and init a new entry for a host 
+ * @param infra: infra structure with config parameters.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param tm: time now.
+ * @return: the new entry or NULL on malloc failure.
+ */
+static struct lruhash_entry*
+new_host_entry(struct infra_cache* infra, struct sockaddr_storage* addr, 
+	socklen_t addrlen, time_t tm)
+{
+	struct infra_host_data* data;
+	struct infra_host_key* key = (struct infra_host_key*)malloc(
+		sizeof(struct infra_host_key));
+	if(!key)
+		return NULL;
+	data = (struct infra_host_data*)malloc(
+		sizeof(struct infra_host_data));
+	if(!data) {
+		free(key);
+		return NULL;
+	}
+	lock_rw_init(&key->entry.lock);
+	key->entry.hash = hash_addr(addr, addrlen);
+	key->entry.key = (void*)key;
+	key->entry.data = (void*)data;
+	key->addrlen = addrlen;
+	memcpy(&key->addr, addr, addrlen);
+	data->ttl = tm + infra->host_ttl;
+	data->lameness = NULL;
+	data->edns_version = 0;
+	rtt_init(&data->rtt);
+	return &key->entry;
+}
+
+int 
+infra_host(struct infra_cache* infra, struct sockaddr_storage* addr,
+        socklen_t addrlen, time_t timenow, int* edns_vs, int* to)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 0);
+	struct infra_host_data* data;
+	if(e && ((struct infra_host_data*)e->data)->ttl < timenow) {
+		/* it expired, try to reuse existing entry */
+		lock_rw_unlock(&e->lock);
+		e = infra_lookup_host_nottl(infra, addr, addrlen, 1);
+		if(e) {
+			/* if its still there we have a writelock, init */
+			/* re-initialise */
+			data = (struct infra_host_data*)e->data;
+			data->ttl = timenow + infra->host_ttl;
+			rtt_init(&data->rtt);
+			/* do not touch lameness, it may be valid still */
+			data->edns_version = 0;
+		}
+	}
+	if(!e) {
+		/* insert new entry */
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		data = (struct infra_host_data*)e->data;
+		*to = rtt_timeout(&data->rtt);
+		*edns_vs = data->edns_version;
+		slabhash_insert(infra->hosts, e->hash, e, data, NULL);
+		return 1;
+	}
+	/* use existing entry */
+	data = (struct infra_host_data*)e->data;
+	*to = rtt_timeout(&data->rtt);
+	*edns_vs = data->edns_version;
+	lock_rw_unlock(&e->lock);
+	return 1;
+}
+
+/** hash lameness key */
+static hashvalue_t
+hash_lameness(uint8_t* name, size_t namelen)
+{
+	return hashlittle(name, namelen, 0xab);
+}
+
+int 
+infra_lookup_lame(struct infra_host_data* host,
+        uint8_t* name, size_t namelen, time_t timenow)
+{
+	struct lruhash_entry* e;
+	struct infra_lame_key k;
+	struct infra_lame_data *d;
+	if(!host->lameness)
+		return 0;
+	k.entry.hash = hash_lameness(name, namelen);
+	k.zonename = name;
+	k.namelen = namelen;
+	k.entry.key = (void*)&k;
+	k.entry.data = NULL;
+	e = lruhash_lookup(host->lameness, k.entry.hash, &k, 0);
+	if(!e)
+		return 0;
+	d = (struct infra_lame_data*)e->data;
+	if(d->ttl < timenow) {
+		lock_rw_unlock(&e->lock);
+		return 0;
+	}
+	lock_rw_unlock(&e->lock);
+	return 1;
+}
+
+/** calculate size, which is fixed, zonename does not count so that
+ * a fixed number of items is stored */
+static size_t 
+infra_lame_sizefunc(void* k, void* ATTR_UNUSED(d))
+{
+	struct infra_lame_key* key = (struct infra_lame_key*)k;
+	return sizeof(*key) + sizeof(struct infra_lame_data)
+		+ key->namelen + lock_get_mem(&key->entry.lock);
+}
+
+/** compare zone names, returns -1, 0, +1 */
+static int 
+infra_lame_compfunc(void* key1, void* key2) 
+{
+	struct infra_lame_key* k1 = (struct infra_lame_key*)key1;
+	struct infra_lame_key* k2 = (struct infra_lame_key*)key2;
+	if(k1->namelen != k2->namelen) {
+		if(k1->namelen < k2->namelen)
+			return -1;
+		return 1;
+	}
+	return query_dname_compare(k1->zonename, k2->zonename);
+}
+
+/** free key, lock and zonename */
+static void 
+infra_lame_delkeyfunc(void* k, void* ATTR_UNUSED(arg), int il)
+{
+	struct infra_lame_key* key = (struct infra_lame_key*)k;
+	if(!key) 
+		return;
+	if(il) { lock_rw_unlock(&key->entry.lock); }
+	lock_rw_destroy(&key->entry.lock);
+	free(key->zonename);
+	free(key);
+}
+
+/** free the lameness data */
+static void 
+infra_lame_deldatafunc(void* d, void* ATTR_UNUSED(arg))
+{
+	if(!d) 
+		return;
+	free(d);
+}
+
+int 
+infra_set_lame(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        uint8_t* name, size_t namelen, time_t timenow)
+{
+	struct infra_host_data* data;
+	struct lruhash_entry* e;
+	int needtoinsert = 0;
+	struct infra_lame_key* k;
+	struct infra_lame_data* d;
+	/* allocate at start, easier cleanup (no locks held) */
+	k = (struct infra_lame_key*)malloc(sizeof(*k));
+	if(!k) {
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	d = (struct infra_lame_data*)malloc(sizeof(*d));
+	if(!d) {
+		free(k);
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	k->zonename = memdup(name, namelen);
+	if(!k->zonename) {
+		free(d);
+		free(k);
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	lock_rw_init(&k->entry.lock);
+	k->entry.hash = hash_lameness(name, namelen);
+	k->entry.key = (void*)k;
+	k->entry.data = (void*)d;
+	d->ttl = timenow + infra->lame_ttl;
+	k->namelen = namelen;
+	e = infra_lookup_host_nottl(infra, addr, addrlen, 1);
+	if(!e) {
+		/* insert it */
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow))) {
+			free(k->zonename);
+			free(k);
+			free(d);
+			log_err("set_lame: malloc failure");
+			return 0;
+		}
+		needtoinsert = 1;
+	}
+	/* got an entry, now set the zone lame */
+	data = (struct infra_host_data*)e->data;
+	if(!data->lameness) {
+		/* create hash table if not there already */
+		data->lameness = lruhash_create(INFRA_LAME_STARTSIZE,
+			infra->max_lame_size, infra_lame_sizefunc, 
+			infra_lame_compfunc, infra_lame_delkeyfunc,
+			infra_lame_deldatafunc, NULL);
+		if(!data->lameness) {
+			log_err("set_lame: malloc failure");
+			if(needtoinsert) slabhash_insert(infra->hosts, 
+				e->hash, e, e->data, NULL);
+			else 	{ lock_rw_unlock(&e->lock); }
+			free(k->zonename);
+			free(k);
+			free(d);
+			return 0;
+		}
+	}
+	/* inserts new entry, or updates TTL of older entry */
+	lruhash_insert(data->lameness, k->entry.hash, &k->entry, d, NULL);
+	
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return 1;
+}
+
+int 
+infra_rtt_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        int roundtrip, time_t timenow)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 1);
+	struct infra_host_data* data;
+	int needtoinsert = 0;
+	if(!e) {
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		needtoinsert = 1;
+	}
+	/* have an entry, update the rtt, and the ttl */
+	data = (struct infra_host_data*)e->data;
+	data->ttl = timenow + infra->host_ttl;
+	if(roundtrip == -1)
+		rtt_lost(&data->rtt);
+	else	rtt_update(&data->rtt, roundtrip);
+
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return 1;
+}
+
+int 
+infra_edns_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        int edns_version, time_t timenow)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 1);
+	struct infra_host_data* data;
+	int needtoinsert = 0;
+	if(!e) {
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		needtoinsert = 1;
+	}
+	/* have an entry, update the rtt, and the ttl */
+	data = (struct infra_host_data*)e->data;
+	data->ttl = timenow + infra->host_ttl;
+	data->edns_version = edns_version;
+
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return 1;
+}
+
+int 
+infra_get_lame_rtt(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        uint8_t* name, size_t namelen, int* lame, int* rtt, time_t timenow)
+{
+	struct infra_host_data* host;
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 0);
+	if(!e) 
+		return 0;
+	host = (struct infra_host_data*)e->data;
+	*rtt = rtt_unclamped(&host->rtt);
+	/* check lameness first, if so, ttl on host does not matter anymore */
+	if(infra_lookup_lame(host, name, namelen, timenow)) {
+		lock_rw_unlock(&e->lock);
+		*lame = 1;
+		return 1;
+	}
+	*lame = 0;
+	if(timenow > host->ttl) {
+		lock_rw_unlock(&e->lock);
+		return 0;
+	}
+	lock_rw_unlock(&e->lock);
+	return 1;
+}
+
+/** helper memory count for a host lame cache */
+static size_t
+count_host_lame(struct lruhash_entry* e)
+{
+	struct infra_host_data* host_data = (struct infra_host_data*)e->data;
+	if(!host_data->lameness)
+		return 0;
+	return lruhash_get_mem(host_data->lameness);
+}
+
+size_t 
+infra_get_mem(struct infra_cache* infra)
+{
+	size_t i, bin;
+	size_t s = sizeof(*infra) +
+		slabhash_get_mem(infra->hosts);
+	struct lruhash_entry* e;
+	for(i=0; i<infra->hosts->size; i++) {
+		lock_quick_lock(&infra->hosts->array[i]->lock);
+		for(bin=0; bin<infra->hosts->array[i]->size; bin++) {
+			lock_quick_lock(&infra->hosts->array[i]->
+				array[bin].lock);
+			/* count data size in bin items. */
+			for(e = infra->hosts->array[i]->array[bin].
+				overflow_list; e; e = e->overflow_next) {
+				lock_rw_rdlock(&e->lock);
+				s += count_host_lame(e);
+				lock_rw_unlock(&e->lock);
+			}
+			lock_quick_unlock(&infra->hosts->array[i]->
+				array[bin].lock);
+		}
+		lock_quick_unlock(&infra->hosts->array[i]->lock);
+	}
+	return s;
+}

services/cache/infra.h

@@ -0,0 +1,245 @@
+/*
+ * services/cache/infra.h - infrastructure cache, server rtt and capabilities
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the infrastructure cache.
+ */
+
+#ifndef SERVICES_CACHE_INFRA_H
+#define SERVICES_CACHE_INFRA_H
+#include "util/storage/lruhash.h"
+#include "util/rtt.h"
+struct slabhash;
+struct config_file;
+
+/**
+ * Host information kept for every server.
+ */
+struct infra_host_key {
+	/** the host address. */
+	struct sockaddr_storage addr;
+	/** length of addr. */
+	socklen_t addrlen;
+	/** hash table entry, data of type infra_host_data. */
+	struct lruhash_entry entry;
+};
+
+/**
+ * Host information encompasses host capabilities and retransmission timeouts.
+ */
+struct infra_host_data {
+	/** TTL value for this entry. absolute time. */
+	time_t ttl;
+	/** round trip times for timeout calculation */
+	struct rtt_info rtt;
+	/** Names of the zones that are lame. NULL=no lame zones. */
+	struct lruhash* lameness;
+	/** edns version that the host supports, -1 means no EDNS */
+	int edns_version;
+};
+
+/**
+ * Lameness information, per host, per zone. 
+ */
+struct infra_lame_key {
+	/** key is zone name in wireformat */
+	uint8_t* zonename;
+	/** length of zonename */
+	size_t namelen;
+	/** lruhash entry */
+	struct lruhash_entry entry;
+};
+
+/**
+ * Lameness information. Expires.
+ * This host is lame because it is in the cache.
+ */
+struct infra_lame_data {
+	/** TTL of this entry. absolute time. */
+	time_t ttl;
+};
+
+/**
+ * Infra cache 
+ */
+struct infra_cache {
+	/** The hash table with hosts */
+	struct slabhash* hosts;
+	/** TTL value for host information, in seconds */
+	int host_ttl;
+	/** TTL for Lameness information, in seconds */
+	int lame_ttl;
+	/** infra lame cache max memory per host, in bytes */
+	size_t max_lame_size;
+};
+
+/** infra host cache default hash lookup size */
+#define INFRA_HOST_STARTSIZE 32
+/** infra lame cache default hash lookup size */
+#define INFRA_LAME_STARTSIZE 2
+
+/**
+ * Create infra cache.
+ * @param cfg: config parameters or NULL for defaults.
+ * @return: new infra cache, or NULL.
+ */
+struct infra_cache* infra_create(struct config_file* cfg);
+
+/**
+ * Delete infra cache.
+ * @param infra: infrastructure cache to delete.
+ */
+void infra_delete(struct infra_cache* infra);
+
+/**
+ * Adjust infra cache to use updated configuration settings.
+ * This may clean the cache. Operates a bit like realloc.
+ * There may be no threading or use by other threads.
+ * @param infra: existing cache. If NULL a new infra cache is returned.
+ * @param cfg: config options.
+ * @return the new infra cache pointer or NULL on error.
+ */
+struct infra_cache* infra_adjust(struct infra_cache* infra, 
+	struct config_file* cfg);
+
+/**
+ * Lookup host data
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param wr: set to true to get a writelock on the entry.
+ * @param timenow: what time it is now.
+ * @param key: the key for the host, returned so caller can unlock when done.
+ * @return: host data or NULL if not found or expired.
+ */
+struct infra_host_data* infra_lookup_host(struct infra_cache* infra, 
+	struct sockaddr_storage* addr, socklen_t addrlen, int wr, 
+	time_t timenow, struct infra_host_key** key);
+
+/**
+ * Find host information to send a packet. Creates new entry if not found.
+ * Lameness is empty. EDNS is 0 (try with first), and rtt is returned for 
+ * the first message to it.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param timenow: what time it is now.
+ * @param edns_vs: edns version it supports, is returned.
+ * @param to: timeout to use, is returned.
+ * @return: 0 on error.
+ */
+int infra_host(struct infra_cache* infra, struct sockaddr_storage* addr, 
+	socklen_t addrlen, time_t timenow, int* edns_vs, int* to);
+
+/**
+ * Check for lameness of this server for a particular zone.
+ * You must have a lock on the host structure.
+ * @param host: infrastructure cache data for the host. Caller holds lock.
+ * @param name: domain name of zone apex.
+ * @param namelen: length of domain name.
+ * @param timenow: what time it is now.
+ * @return: 0 if not lame or unknown or timed out, true if lame.
+ */
+int infra_lookup_lame(struct infra_host_data* host,
+	uint8_t* name, size_t namelen, time_t timenow);
+
+/**
+ * Set a host to be lame for the given zone.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param name: domain name of zone apex.
+ * @param namelen: length of domain name.
+ * @param timenow: what time it is now.
+ * @return: 0 on error.
+ */
+int infra_set_lame(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	uint8_t* name, size_t namelen, time_t timenow);
+
+/**
+ * Update rtt information for the host.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param roundtrip: estimate of roundtrip time in milliseconds or -1 for 
+ * 	timeout.
+ * @param timenow: what time it is now.
+ * @return: 0 on error.
+ */
+int infra_rtt_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	int roundtrip, time_t timenow);
+
+/**
+ * Update edns information for the host.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param edns_version: the version that it publishes.
+ * @param timenow: what time it is now.
+ * @return: 0 on error.
+ */
+int infra_edns_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	int edns_version, time_t timenow);
+
+/**
+ * Get Lameness information and average RTT if host is in the cache.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param name: zone name.
+ * @param namelen: zone name length.
+ * @param lame: if function returns true, this returns lameness of the zone.
+ * @param rtt: if function returns true, this returns avg rtt of the server.
+ * 	The rtt value is unclamped and reflects recent timeouts.
+ * @param timenow: what time it is now.
+ * @return if found in cache, or false if not (or TTL bad).
+ */
+int infra_get_lame_rtt(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, 
+	uint8_t* name, size_t namelen, int* lame, int* rtt, time_t timenow);
+
+/**
+ * Get memory used by the infra cache.
+ * @param infra: infrastructure cache.
+ * @return memory in use in bytes.
+ */
+size_t infra_get_mem(struct infra_cache* infra);
+
+#endif /* SERVICES_CACHE_INFRA_H */

services/cache/rrset.c

@@ -0,0 +1,372 @@
+/*
+ * services/cache/rrset.c - Resource record set cache.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the rrset cache.
+ */
+#include "config.h"
+#include "services/cache/rrset.h"
+#include "util/storage/slabhash.h"
+#include "util/config_file.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/msgreply.h"
+#include "util/region-allocator.h"
+#include "util/alloc.h"
+
+struct rrset_cache* rrset_cache_create(struct config_file* cfg, 
+	struct alloc_cache* alloc)
+{
+	size_t slabs = (cfg?cfg->rrset_cache_slabs:HASH_DEFAULT_SLABS);
+	size_t startarray = HASH_DEFAULT_STARTARRAY;
+	size_t maxmem = (cfg?cfg->rrset_cache_size:HASH_DEFAULT_MAXMEM);
+
+	struct rrset_cache *r = (struct rrset_cache*)slabhash_create(slabs,
+		startarray, maxmem, ub_rrset_sizefunc, ub_rrset_compare,
+		ub_rrset_key_delete, rrset_data_delete, alloc);
+	return r;
+}
+
+void rrset_cache_delete(struct rrset_cache* r)
+{
+	if(!r) 
+		return;
+	slabhash_delete(&r->table);
+}
+
+struct rrset_cache* rrset_cache_adjust(struct rrset_cache *r, 
+	struct config_file* cfg, struct alloc_cache* alloc)
+{
+	if(!r || !cfg || cfg->rrset_cache_slabs != r->table.size ||
+		cfg->rrset_cache_size != slabhash_get_size(&r->table))
+	{
+		rrset_cache_delete(r);
+		r = rrset_cache_create(cfg, alloc);
+	}
+	return r;
+}
+
+void 
+rrset_cache_touch(struct rrset_cache* r, struct ub_packed_rrset_key* key,
+        hashvalue_t hash, rrset_id_t id)
+{
+	struct lruhash* table = slabhash_gettable(&r->table, hash);
+	/* 
+	 * This leads to locking problems, deadlocks, if the caller is 
+	 * holding any other rrset lock.
+	 * Because a lookup through the hashtable does:
+	 *	tablelock -> entrylock  (for that entry caller holds)
+	 * And this would do
+	 *	entrylock(already held) -> tablelock
+	 * And if two threads do this, it results in deadlock.
+	 * So, the caller must not hold entrylock.
+	 */
+	lock_quick_lock(&table->lock);
+	/* we have locked the hash table, the item can still be deleted.
+	 * because it could already have been reclaimed, but not yet set id=0.
+	 * This is because some lruhash routines have lazy deletion.
+	 * so, we must acquire a lock on the item to verify the id != 0.
+	 * also, with hash not changed, we are using the right slab.
+	 */
+	lock_rw_rdlock(&key->entry.lock);
+	if(key->id == id && key->entry.hash == hash) {
+		lru_touch(table, &key->entry);
+	}
+	lock_rw_unlock(&key->entry.lock);
+	lock_quick_unlock(&table->lock);
+}
+
+/** see if rrset needs to be updated in the cache */
+static int
+need_to_update_rrset(void* nd, void* cd, uint32_t timenow, int equal)
+{
+	struct packed_rrset_data* newd = (struct packed_rrset_data*)nd;
+	struct packed_rrset_data* cached = (struct packed_rrset_data*)cd;
+	/* 	o store if rrset has been validated */
+	if( newd->security > cached->security) {
+		return 1;
+	}
+        /*      o if current RRset is more trustworthy - insert it */
+        if( newd->trust > cached->trust ) {
+		/* if the cached rrset is bogus, and this one equal,
+		 * do not update the TTL - let it expire. */
+		if(equal && cached->ttl >= timenow && 
+			cached->security == sec_status_bogus)
+			return 0;
+                return 1;
+	}
+	/*	o item in cache has expired */
+	if( cached->ttl < timenow )
+		return 1;
+        /*      o same trust, but different in data - insert it */
+        if( newd->trust == cached->trust && !equal )
+                return 1;
+        /*      o see if TTL is better than TTL in cache. */
+        /*        if so, see if rrset+rdata is the same */
+        /*        if so, update TTL in cache, even if trust is worse. */
+        if( newd->ttl > cached->ttl && equal ) {
+		/* if the cached rrset is bogus, and this one equal,
+		 * do not update the TTL - let it expire. */
+		if(cached->security == sec_status_bogus)
+			return 0;
+		/* since all else is the same, use the best trust value */
+		if(newd->trust < cached->trust) {
+			newd->trust = cached->trust;
+			newd->security = cached->security;
+		}
+                return 1;
+	}
+        return 0;
+}
+
+/** Update RRSet special key ID */
+static void
+rrset_update_id(struct rrset_ref* ref, struct alloc_cache* alloc)
+{
+	/* this may clear the cache and invalidate lock below */
+	uint64_t newid = alloc_get_id(alloc);
+	/* obtain writelock */
+	lock_rw_wrlock(&ref->key->entry.lock);
+	/* check if it was deleted in the meantime, if so, skip update */
+	if(ref->key->id == ref->id) {
+		ref->key->id = newid;
+		ref->id = newid;
+	}
+	lock_rw_unlock(&ref->key->entry.lock);
+}
+
+int 
+rrset_cache_update(struct rrset_cache* r, struct rrset_ref* ref,
+	struct alloc_cache* alloc, uint32_t timenow)
+{
+	struct lruhash_entry* e;
+	struct ub_packed_rrset_key* k = ref->key;
+	hashvalue_t h = k->entry.hash;
+	uint16_t rrset_type = ntohs(k->rk.type);
+	int equal = 0;
+	/* looks up item with a readlock - no editing! */
+	if((e=slabhash_lookup(&r->table, h, k, 0)) != 0) {
+		/* return id and key as they will be used in the cache
+		 * since the lruhash_insert, if item already exists, deallocs
+		 * the passed key in favor of the already stored key.
+		 * because of the small gap (see below) this key ptr and id
+		 * may prove later to be already deleted, which is no problem
+		 * as it only makes a cache miss. 
+		 */
+		ref->key = (struct ub_packed_rrset_key*)e->key;
+		ref->id = ref->key->id;
+		equal = rrsetdata_equal((struct packed_rrset_data*)k->entry.
+			data, (struct packed_rrset_data*)e->data);
+		if(!need_to_update_rrset(k->entry.data, e->data, timenow,
+			equal)) {
+			/* cache is superior, return that value */
+			lock_rw_unlock(&e->lock);
+			ub_packed_rrset_parsedelete(k, alloc);
+			return 1;
+		}
+		lock_rw_unlock(&e->lock);
+		/* Go on and insert the passed item.
+		 * small gap here, where entry is not locked.
+		 * possibly entry is updated with something else.
+		 * we then overwrite that with our data.
+		 * this is just too bad, its cache anyway. */
+		/* use insert to update entry to manage lruhash
+		 * cache size values nicely. */
+	}
+	slabhash_insert(&r->table, h, &k->entry, k->entry.data, alloc);
+	if(e) {
+		/* For NSEC, NSEC3, DNAME, when rdata is updated, update 
+		 * the ID number so that proofs in message cache are 
+		 * invalidated */
+		if((rrset_type == LDNS_RR_TYPE_NSEC 
+			|| rrset_type == LDNS_RR_TYPE_NSEC3
+			|| rrset_type == LDNS_RR_TYPE_DNAME) && !equal) {
+			rrset_update_id(ref, alloc);
+		}
+		return 1;
+	}
+	return 0;
+}
+
+struct ub_packed_rrset_key* 
+rrset_cache_lookup(struct rrset_cache* r, uint8_t* qname, size_t qnamelen, 
+	uint16_t qtype, uint16_t qclass, uint32_t flags, uint32_t timenow,
+	int wr)
+{
+	struct lruhash_entry* e;
+	struct ub_packed_rrset_key key;
+	
+	key.entry.key = &key;
+	key.entry.data = NULL;
+	key.rk.dname = qname;
+	key.rk.dname_len = qnamelen;
+	key.rk.type = htons(qtype);
+	key.rk.rrset_class = htons(qclass);
+	key.rk.flags = flags;
+
+	key.entry.hash = rrset_key_hash(&key.rk);
+
+	if((e = slabhash_lookup(&r->table, key.entry.hash, &key, wr))) {
+		/* check TTL */
+		struct packed_rrset_data* data = 
+			(struct packed_rrset_data*)e->data;
+		if(timenow > data->ttl) {
+			lock_rw_unlock(&e->lock);
+			return NULL;
+		}
+		/* we're done */
+		return (struct ub_packed_rrset_key*)e->key;
+	}
+	return NULL;
+}
+
+int 
+rrset_array_lock(struct rrset_ref* ref, size_t count, uint32_t timenow)
+{
+	size_t i;
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only lock items once */
+		lock_rw_rdlock(&ref[i].key->entry.lock);
+		if(ref[i].id != ref[i].key->id || timenow >
+			((struct packed_rrset_data*)(ref[i].key->entry.data))
+			->ttl) {
+			/* failure! rollback our readlocks */
+			rrset_array_unlock(ref, i+1);
+			return 0;
+		}
+	}
+	return 1;
+}
+
+void 
+rrset_array_unlock(struct rrset_ref* ref, size_t count)
+{
+	size_t i;
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only unlock items once */
+		lock_rw_unlock(&ref[i].key->entry.lock);
+	}
+}
+
+void 
+rrset_array_unlock_touch(struct rrset_cache* r, struct region* scratch,         
+	struct rrset_ref* ref, size_t count)
+{
+	hashvalue_t* h;
+	size_t i;
+	if(!(h = (hashvalue_t*)region_alloc(scratch, 
+		sizeof(hashvalue_t)*count)))
+		log_warn("rrset LRU: memory allocation failed");
+	else 	/* store hash values */
+		for(i=0; i<count; i++)
+			h[i] = ref[i].key->entry.hash;
+	/* unlock */
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only unlock items once */
+		lock_rw_unlock(&ref[i].key->entry.lock);
+	}
+	if(h) {
+		/* LRU touch, with no rrset locks held */
+		for(i=0; i<count; i++) {
+			if(i>0 && ref[i].key == ref[i-1].key)
+				continue; /* only touch items once */
+			rrset_cache_touch(r, ref[i].key, h[i], ref[i].id);
+		}
+	}
+}
+
+void 
+rrset_update_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset)
+{
+	uint32_t now = (uint32_t)time(0);
+	struct packed_rrset_data* updata = 
+		(struct packed_rrset_data*)rrset->entry.data;
+	struct lruhash_entry* e;
+	struct packed_rrset_data* cachedata;
+
+	/* hash it again to make sure it has a hash */
+	rrset->entry.hash = rrset_key_hash(&rrset->rk);
+
+	e = slabhash_lookup(&r->table, rrset->entry.hash, rrset, 1);
+	if(!e)
+		return; /* not in the cache anymore */
+	cachedata = (struct packed_rrset_data*)e->data;
+	if(!rrsetdata_equal(updata, cachedata)) {
+		lock_rw_unlock(&e->lock);
+		return; /* rrset has changed in the meantime */
+	}
+	/* update the cached rrset */
+	if(updata->security > cachedata->security) {
+		cachedata->trust = updata->trust;
+		cachedata->security = updata->security;
+		cachedata->ttl = updata->ttl + now;
+	}
+	lock_rw_unlock(&e->lock);
+}
+
+void 
+rrset_check_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset)
+{
+	uint32_t now = (uint32_t)time(0);
+	struct packed_rrset_data* updata = 
+		(struct packed_rrset_data*)rrset->entry.data;
+	struct lruhash_entry* e;
+	struct packed_rrset_data* cachedata;
+
+	/* hash it again to make sure it has a hash */
+	rrset->entry.hash = rrset_key_hash(&rrset->rk);
+
+	e = slabhash_lookup(&r->table, rrset->entry.hash, rrset, 0);
+	if(!e)
+		return; /* not in the cache anymore */
+	cachedata = (struct packed_rrset_data*)e->data;
+	if(now > cachedata->ttl || !rrsetdata_equal(updata, cachedata)) {
+		lock_rw_unlock(&e->lock);
+		return; /* expired, or rrset has changed in the meantime */
+	}
+	if(cachedata->security > updata->security) {
+		updata->security = cachedata->security;
+		if(cachedata->security == sec_status_bogus)
+			updata->ttl = cachedata->ttl - now;
+		updata->trust = cachedata->trust;
+	}
+	lock_rw_unlock(&e->lock);
+}

services/cache/rrset.h

@@ -0,0 +1,210 @@
+/*
+ * services/cache/rrset.h - Resource record set cache.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the rrset cache.
+ */
+
+#ifndef SERVICES_CACHE_RRSET_H
+#define SERVICES_CACHE_RRSET_H
+#include "util/storage/lruhash.h"
+#include "util/storage/slabhash.h"
+#include "util/data/packed_rrset.h"
+struct config_file;
+struct alloc_cache;
+struct rrset_ref;
+struct region;
+
+/**
+ * The rrset cache
+ * Thin wrapper around hashtable, like a typedef.
+ */
+struct rrset_cache {
+	/** uses partitioned hash table */
+	struct slabhash table;
+};
+
+/**
+ * Create rrset cache
+ * @param cfg: config settings or NULL for defaults.
+ * @param alloc: initial default rrset key allocation.
+ * @return: NULL on error.
+ */
+struct rrset_cache* rrset_cache_create(struct config_file* cfg, 
+	struct alloc_cache* alloc);
+
+/**
+ * Delete rrset cache
+ * @param r: rrset cache to delete.
+ */
+void rrset_cache_delete(struct rrset_cache* r);
+
+/**
+ * Adjust settings of the cache to settings from the config file.
+ * May purge the cache. May recreate the cache.
+ * There may be no threading or use by other threads.
+ * @param r: rrset cache to adjust (like realloc).
+ * @param cfg: config settings or NULL for defaults.
+ * @param alloc: initial default rrset key allocation.
+ * @return 0 on error, or new rrset cache pointer on success.
+ */
+struct rrset_cache* rrset_cache_adjust(struct rrset_cache* r, 
+	struct config_file* cfg, struct alloc_cache* alloc);
+
+/**
+ * Touch rrset, with given pointer and id.
+ * Caller may not hold a lock on ANY rrset, this could give deadlock.
+ *
+ * This routine is faster than a hashtable lookup:
+ *	o no bin_lock is acquired.
+ *	o no walk through the bin-overflow-list. 
+ *	o no comparison of the entry key to find it.
+ *
+ * @param r: rrset cache.
+ * @param key: rrset key. Marked recently used (if it was not deleted
+ *	before the lock is acquired, in that case nothing happens).
+ * @param hash: hash value of the item. Please read it from the key when
+ *	you have it locked. Used to find slab from slabhash.
+ * @param id: used to check that the item is unchanged and not deleted.
+ */
+void rrset_cache_touch(struct rrset_cache* r, struct ub_packed_rrset_key* key,
+	hashvalue_t hash, rrset_id_t id);
+
+/**
+ * Update an rrset in the rrset cache. Stores the information for later use.
+ * Will lookup if the rrset is in the cache and perform an update if necessary.
+ * If the item was present, and superior, references are returned to that.
+ * The passed item is then deallocated with rrset_parsedelete.
+ *
+ * A superior rrset is:
+ *	o rrset with better trust value.
+ *	o same trust value, different rdata, newly passed rrset is inserted.
+ * If rdata is the same, TTL in the cache is updated.
+ *
+ * @param r: the rrset cache.
+ * @param ref: reference (ptr and id) to the rrset. Pass reference setup for
+ *	the new rrset. The reference may be changed if the cached rrset is
+ *	superior.
+ *	Before calling the rrset is presumed newly allocated and changeable.
+ *	Afer calling you do not hold a lock, and the rrset is inserted in
+ *	the hashtable so you need a lock to change it.
+ * @param alloc: how to allocate (and deallocate) the special rrset key.
+ * @param timenow: current time (to see if ttl in cache is expired).
+ * @return: true if the passed reference is updated, false if it is unchanged.
+ */
+int rrset_cache_update(struct rrset_cache* r, struct rrset_ref* ref, 
+	struct alloc_cache* alloc, uint32_t timenow);
+
+/**
+ * Lookup rrset. You obtain read/write lock. You must unlock before lookup
+ * anything of else.
+ * @param r: the rrset cache.
+ * @param qname: name of rrset to lookup.
+ * @param qnamelen: length of name of rrset to lookup.
+ * @param qtype: type of rrset to lookup (host order).
+ * @param qclass: class of rrset to lookup (host order).
+ * @param flags: rrset flags, or 0.
+ * @param timenow: used to compare with TTL.
+ * @param wr: set true to get writelock.
+ * @return packed rrset key pointer. Remember to unlock the key.entry.lock.
+ * 	or NULL if could not be found or it was timed out.
+ */
+struct ub_packed_rrset_key* rrset_cache_lookup(struct rrset_cache* r,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint32_t flags, uint32_t timenow, int wr);
+
+/**
+ * Obtain readlock on a (sorted) list of rrset references.
+ * Checks TTLs and IDs of the rrsets and rollbacks locking if not Ok.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ * @param timenow: used to compare with TTL.
+ * @return true on success, false on a failure, which can be that some
+ * 	RRsets have timed out, or that they do not exist any more, the
+ *	RRsets have been purged from the cache.
+ *	If true, you hold readlocks on all the ref items. 
+ */
+int rrset_array_lock(struct rrset_ref* ref, size_t count, uint32_t timenow);
+
+/**
+ * Unlock array (sorted) of rrset references.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ */
+void rrset_array_unlock(struct rrset_ref* ref, size_t count);
+
+/**
+ * Unlock array (sorted) of rrset references and at the same time
+ * touch LRU on the rrsets. It needs the scratch region for temporary
+ * storage as it uses the initial locks to obtain hash values.
+ * @param r: the rrset cache. In this cache LRU is updated.
+ * @param scratch: region for temporary storage of hash values.
+ *	if memory allocation fails, the lru touch fails silently,
+ *	but locks are released. memory errors are logged.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ */
+void rrset_array_unlock_touch(struct rrset_cache* r, struct region* scratch,
+	struct rrset_ref* ref, size_t count);
+
+/**
+ * Update security status of an rrset. Looks up the rrset.
+ * If found, checks if rdata is equal.
+ * If so, it will update the security, trust and rrset-ttl values.
+ * The values are only updated if security is increased (towards secure).
+ * @param r: the rrset cache. 
+ * @param rrset: which rrset to attempt to update. This rrset is left 
+ * 	untouched. The rrset in the cache is updated in-place.
+ */
+void rrset_update_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Looks up security status of an rrset. Looks up the rrset.
+ * If found, checks if rdata is equal, and entry did not expire.
+ * If so, it will update the security, trust and rrset-ttl values.
+ * @param r: the rrset cache. 
+ * @param rrset: This rrset may change security status due to the cache.
+ * 	But its status will only improve, towards secure.
+ */
+void rrset_check_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset);
+
+#endif /* SERVICES_CACHE_RRSET_H */

services/listen_dnsport.c

@@ -54,8 +54,6 @@
 
 /** number of queued TCP connections for listen() */
 #define TCP_BACKLOG 5 
-/** number of simultaneous open TCP connections for queries */
-#define TCP_COUNT 10 
 
 /**
  * Debug print of the getaddrinfo returned address.
@@ -290,7 +288,8 @@ listen_cp_insert(struct comm_point* c, struct listen_dnsport* front)
 
 struct listen_dnsport* 
 listen_create(struct comm_base* base, struct listen_port* ports,
-	size_t bufsize, comm_point_callback_t* cb, void *cb_arg)
+	size_t bufsize, int tcp_accept_count,
+	comm_point_callback_t* cb, void *cb_arg)
 {
 	struct listen_dnsport* front = (struct listen_dnsport*)
 		malloc(sizeof(struct listen_dnsport));
@@ -310,7 +309,7 @@ listen_create(struct comm_base* base, struct listen_port* ports,
 			cp = comm_point_create_udp(base, ports->fd, 
 				front->udp_buff, cb, cb_arg);
 		else 	cp = comm_point_create_tcp(base, ports->fd, 
-				TCP_COUNT, bufsize, cb, cb_arg);
+				tcp_accept_count, bufsize, cb, cb_arg);
 		if(!cp) {
 			log_err("can't create commpoint");	
 			listen_delete(front);
@@ -383,10 +382,14 @@ listening_ports_open(struct config_file* cfg)
 	struct listen_port* list = NULL;
 	struct addrinfo hints;
 	int i, do_ip4, do_ip6;
+	int do_tcp;
 	char portbuf[32];
 	snprintf(portbuf, sizeof(portbuf), "%d", cfg->port);
 	do_ip4 = cfg->do_ip4;
 	do_ip6 = cfg->do_ip6;
+	do_tcp = cfg->do_tcp;
+	if(cfg->incoming_num_tcp == 0)
+		do_tcp = 0;
 
 	/* getaddrinfo */
 	memset(&hints, 0, sizeof(hints));
@@ -405,7 +408,7 @@ listening_ports_open(struct config_file* cfg)
 	if(cfg->num_ifs == 0) {
 		if(do_ip6) {
 			hints.ai_family = AF_INET6;
-			if(!ports_create_if(NULL, cfg->do_udp, cfg->do_tcp, 
+			if(!ports_create_if(NULL, cfg->do_udp, do_tcp, 
 				&hints, portbuf, &list)) {
 				listening_ports_free(list);
 				return NULL;
@@ -413,7 +416,7 @@ listening_ports_open(struct config_file* cfg)
 		}
 		if(do_ip4) {
 			hints.ai_family = AF_INET;
-			if(!ports_create_if(NULL, cfg->do_udp, cfg->do_tcp, 
+			if(!ports_create_if(NULL, cfg->do_udp, do_tcp, 
 				&hints, portbuf, &list)) {
 				listening_ports_free(list);
 				return NULL;
@@ -425,7 +428,7 @@ listening_ports_open(struct config_file* cfg)
 				continue;
 			hints.ai_family = AF_INET6;
 			if(!ports_create_if(cfg->ifs[i], cfg->do_udp, 
-				cfg->do_tcp, &hints, portbuf, &list)) {
+				do_tcp, &hints, portbuf, &list)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -434,7 +437,7 @@ listening_ports_open(struct config_file* cfg)
 				continue;
 			hints.ai_family = AF_INET;
 			if(!ports_create_if(cfg->ifs[i], cfg->do_udp, 
-				cfg->do_tcp, &hints, portbuf, &list)) {
+				do_tcp, &hints, portbuf, &list)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -454,3 +457,16 @@ void listening_ports_free(struct listen_port* list)
 		list = nx;
 	}
 }
+
+size_t listen_get_mem(struct listen_dnsport* listen)
+{
+	size_t s = sizeof(*listen) + sizeof(*listen->base) + 
+		sizeof(*listen->udp_buff) + 
+		ldns_buffer_capacity(listen->udp_buff);
+	struct listen_list* p;
+	for(p = listen->cps; p; p = p->next) {
+		s += sizeof(*p);
+		s += comm_point_get_mem(p->com);
+	}
+	return s;
+}

services/listen_dnsport.h

@@ -108,13 +108,15 @@ void listening_ports_free(struct listen_port* list);
  *	for default all ifs.
  * @param ports: the list of shared ports.
  * @param bufsize: size of datagram buffer.
+ * @param tcp_accept_count: max number of simultaneous TCP connections 
+ * 	from clients.
  * @param cb: callback function when a request arrives. It is passed
  *	  the packet and user argument. Return true to send a reply.
  * @param cb_arg: user data argument for callback function.
  * @return: the malloced listening structure, ready for use. NULL on error.
  */
 struct listen_dnsport* listen_create(struct comm_base* base,
-	struct listen_port* ports, size_t bufsize, 
+	struct listen_port* ports, size_t bufsize, int tcp_accept_count,
 	comm_point_callback_t* cb, void* cb_arg);
 
 /**
@@ -138,9 +140,17 @@ void listen_resume(struct listen_dnsport* listen);
 
 /**
  * delete the listening structure
+ * @param listen: listening structure.
  */
 void listen_delete(struct listen_dnsport* listen);
 
+/**
+ * get memory size used by the listening structs
+ * @param listen: listening structure.
+ * @return: size in bytes.
+ */
+size_t listen_get_mem(struct listen_dnsport* listen);
+
 /**
  * Create and bind nonblocking UDP socket
  * @param addr: address info ready to make socket.

services/mesh.c

@@ -0,0 +1,695 @@
+/*
+ * services/mesh.c - deal with mesh of query states and handle events for that.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist in dealing with a mesh of
+ * query states. This mesh is supposed to be thread-specific.
+ * It consists of query states (per qname, qtype, qclass) and connections
+ * between query states and the super and subquery states, and replies to
+ * send back to clients.
+ */
+#include "config.h"
+#include "services/mesh.h"
+#include "services/outbound_list.h"
+#include "services/cache/dns.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/region-allocator.h"
+#include "util/data/msgencode.h"
+#include "util/timehist.h"
+
+/** compare two mesh_states */
+static int
+mesh_state_compare(const void* ap, const void* bp)
+{
+	struct mesh_state* a = (struct mesh_state*)ap;
+	struct mesh_state* b = (struct mesh_state*)bp;
+
+	if(a->s.is_priming && !b->s.is_priming)
+		return -1;
+	if(!a->s.is_priming && b->s.is_priming)
+		return 1;
+
+	if((a->s.query_flags&BIT_RD) && !(b->s.query_flags&BIT_RD))
+		return -1;
+	if(!(a->s.query_flags&BIT_RD) && (b->s.query_flags&BIT_RD))
+		return 1;
+
+	if((a->s.query_flags&BIT_CD) && !(b->s.query_flags&BIT_CD))
+		return -1;
+	if(!(a->s.query_flags&BIT_CD) && (b->s.query_flags&BIT_CD))
+		return 1;
+
+	return query_info_compare(&a->s.qinfo, &b->s.qinfo);
+}
+
+/** compare two mesh references */
+static int
+mesh_state_ref_compare(const void* ap, const void* bp)
+{
+	struct mesh_state_ref* a = (struct mesh_state_ref*)ap;
+	struct mesh_state_ref* b = (struct mesh_state_ref*)bp;
+	return mesh_state_compare(a->s, b->s);
+}
+
+struct mesh_area* 
+mesh_create(int num_modules, struct module_func_block** modfunc,
+	struct module_env* env)
+{
+	struct mesh_area* mesh = calloc(1, sizeof(struct mesh_area));
+	if(!mesh) {
+		log_err("mesh area alloc: out of memory");
+		return NULL;
+	}
+	mesh->histogram = timehist_setup();
+	if(!mesh->histogram) {
+		free(mesh);
+		log_err("mesh area alloc: out of memory");
+		return NULL;
+	}
+	mesh->num_modules = num_modules;
+	mesh->modfunc = modfunc;
+	mesh->env = env;
+	rbtree_init(&mesh->run, &mesh_state_compare);
+	rbtree_init(&mesh->all, &mesh_state_compare);
+	mesh->num_reply_addrs = 0;
+	mesh->num_reply_states = 0;
+	mesh->num_detached_states = 0;
+	return mesh;
+}
+
+/** help mesh delete delete mesh states */
+static void
+mesh_delete_helper(rbnode_t* n, void* ATTR_UNUSED(arg))
+{
+	struct mesh_state* mstate = (struct mesh_state*)n->key;
+	mesh_state_cleanup(mstate);
+}
+
+void 
+mesh_delete(struct mesh_area* mesh)
+{
+	if(!mesh)
+		return;
+	/* free all query states */
+	traverse_postorder(&mesh->all, &mesh_delete_helper, NULL);
+	timehist_delete(mesh->histogram);
+	free(mesh);
+}
+
+void mesh_new_client(struct mesh_area* mesh, struct query_info* qinfo,
+        uint16_t qflags, struct edns_data* edns, struct comm_reply* rep,
+        uint16_t qid)
+{
+	struct mesh_state* s = mesh_area_find(mesh, qinfo, qflags, 0);
+	int was_detached = 0;
+	int was_noreply = 0;
+	int added = 0;
+	/* see if it already exists, if not, create one */
+	if(!s) {
+		struct rbnode_t* n;
+		s = mesh_state_create(mesh->env,qinfo, qflags, 0);
+		if(!s) {
+			log_err("mesh_state_create: out of memory; SERVFAIL");
+			error_encode(rep->c->buffer, LDNS_RCODE_SERVFAIL,
+				qinfo, qid, qflags, edns);
+			comm_point_send_reply(rep);
+			return;
+		}
+		n = rbtree_insert(&mesh->all, &s->node);
+		log_assert(n != NULL);
+		/* set detached (it is now) */
+		mesh->num_detached_states++;
+		added = 1;
+	}
+	if(!s->reply_list && s->super_set.count == 0)
+		was_detached = 1;
+	if(!s->reply_list)
+		was_noreply = 1;
+	/* add reply to s */
+	if(!mesh_state_add_reply(s, edns, rep, qid, qflags)) {
+			log_err("mesh_new_client: out of memory; SERVFAIL");
+			error_encode(rep->c->buffer, LDNS_RCODE_SERVFAIL,
+				qinfo, qid, qflags, edns);
+			comm_point_send_reply(rep);
+			if(added)
+				mesh_state_delete(&s->s);
+			return;
+	}
+	/* update statistics */
+	if(was_detached) {
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	if(was_noreply) {
+		mesh->num_reply_states ++;
+	}
+	mesh->num_reply_addrs++;
+	if(added)
+		mesh_run(mesh, s, module_event_new, NULL);
+}
+
+void mesh_report_reply(struct mesh_area* mesh, struct outbound_entry* e,
+        int is_ok, struct comm_reply* reply)
+{
+	e->qstate->reply = reply;
+	mesh_run(mesh, e->qstate->mesh_info,
+		is_ok?module_event_reply:module_event_noreply, e);
+}
+
+struct mesh_state* 
+mesh_state_create(struct module_env* env, struct query_info* qinfo, 
+	uint16_t qflags, int prime)
+{
+	region_type* region = region_create(malloc, free);
+	struct mesh_state* mstate;
+	int i;
+	if(!region)
+		return NULL;
+	mstate = (struct mesh_state*)region_alloc(region, 
+		sizeof(struct mesh_state));
+	if(!mstate) {
+		region_destroy(region);
+		return NULL;
+	}
+	memset(mstate, 0, sizeof(*mstate));
+	mstate->node = *RBTREE_NULL;
+	mstate->run_node = *RBTREE_NULL;
+	mstate->node.key = mstate;
+	mstate->run_node.key = mstate;
+	mstate->reply_list = NULL;
+	rbtree_init(&mstate->super_set, &mesh_state_ref_compare);
+	rbtree_init(&mstate->sub_set, &mesh_state_ref_compare);
+	/* init module qstate */
+	mstate->s.qinfo.qtype = qinfo->qtype;
+	mstate->s.qinfo.qclass = qinfo->qclass;
+	mstate->s.qinfo.qname_len = qinfo->qname_len;
+	mstate->s.qinfo.qname = region_alloc_init(region, qinfo->qname,
+		qinfo->qname_len);
+	if(!mstate->s.qinfo.qname) {
+		region_destroy(region);
+		return NULL;
+	}
+	/* remove all weird bits from qflags */
+	mstate->s.query_flags = (qflags & (BIT_RD|BIT_CD));
+	mstate->s.is_priming = prime;
+	mstate->s.reply = NULL;
+	mstate->s.region = region;
+	mstate->s.curmod = 0;
+	mstate->s.return_msg = 0;
+	mstate->s.return_rcode = LDNS_RCODE_NOERROR;
+	mstate->s.env = env;
+	mstate->s.mesh_info = mstate;
+	/* init modules */
+	for(i=0; i<env->mesh->num_modules; i++) {
+		mstate->s.minfo[i] = NULL;
+		mstate->s.ext_state[i] = module_state_initial;
+	}
+	return mstate;
+}
+
+void 
+mesh_state_cleanup(struct mesh_state* mstate)
+{
+	struct mesh_area* mesh;
+	int i;
+	if(!mstate)
+		return;
+	/* de-init modules */
+	mesh = mstate->s.env->mesh;
+	for(i=0; i<mesh->num_modules; i++) {
+		(*mesh->modfunc[i]->clear)(&mstate->s, i);
+		mstate->s.minfo[i] = NULL;
+		mstate->s.ext_state[i] = module_finished;
+	}
+	region_destroy(mstate->s.region);
+}
+
+void 
+mesh_state_delete(struct module_qstate* qstate)
+{
+	struct mesh_area* mesh;
+	struct mesh_state_ref* super, ref;
+	struct mesh_state* mstate;
+	if(!qstate)
+		return;
+	mstate = qstate->mesh_info;
+	mesh = mstate->s.env->mesh;
+	mesh_detach_subs(&mstate->s);
+	if(!mstate->reply_list && mstate->super_set.count == 0) {
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	if(mstate->reply_list) {
+		log_assert(mesh->num_reply_states > 0);
+		mesh->num_reply_states--;
+	}
+	ref.node.key = &ref;
+	ref.s = mstate;
+	RBTREE_FOR(super, struct mesh_state_ref*, &mstate->super_set) {
+		(void)rbtree_delete(&super->s->sub_set, &ref);
+	}
+	(void)rbtree_delete(&mesh->run, mstate);
+	(void)rbtree_delete(&mesh->all, mstate);
+	mesh_state_cleanup(mstate);
+}
+
+void mesh_detach_subs(struct module_qstate* qstate)
+{
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state_ref* ref, lookup;
+	struct rbnode_t* n;
+	lookup.node.key = &lookup;
+	lookup.s = qstate->mesh_info;
+	RBTREE_FOR(ref, struct mesh_state_ref*, &qstate->mesh_info->sub_set) {
+		n = rbtree_delete(&ref->s->super_set, &lookup);
+		log_assert(n != NULL); /* must have been present */
+		if(!ref->s->reply_list && ref->s->super_set.count == 0) {
+			mesh->num_detached_states++;
+			log_assert(mesh->num_detached_states + 
+				mesh->num_reply_states <= mesh->all.count);
+		}
+	}
+	rbtree_init(&qstate->mesh_info->sub_set, &mesh_state_ref_compare);
+}
+
+int mesh_attach_sub(struct module_qstate* qstate, struct query_info* qinfo,
+        uint16_t qflags, int prime, struct module_qstate** newq)
+{
+	/* find it, if not, create it */
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state* sub = mesh_area_find(mesh, qinfo, qflags, prime);
+	if(!sub) {
+		struct rbnode_t* n;
+		/* create a new one */
+		sub = mesh_state_create(qstate->env, qinfo, qflags, prime);
+		if(!sub) {
+			log_err("mesh_attach_sub: out of memory");
+			return 0;
+		}
+		n = rbtree_insert(&mesh->all, &sub->node);
+		log_assert(n != NULL);
+		/* set detached (it is now) */
+		mesh->num_detached_states++;
+		/* set new query state to run */
+		n = rbtree_insert(&mesh->run, &sub->run_node);
+		log_assert(n != NULL);
+		*newq = &sub->s;
+	} else
+		*newq = NULL;
+	if(!mesh_state_attachment(qstate->mesh_info, sub))
+		return 0;
+	if(!sub->reply_list && sub->super_set.count == 1) {
+		/* it used to be detached, before this one got added */
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	/* *newq will be run when inited after the current module stops */
+	return 1;
+}
+
+int mesh_state_attachment(struct mesh_state* super, struct mesh_state* sub)
+{
+	struct rbnode_t* n;
+	struct mesh_state_ref* subref; /* points to sub, inserted in super */
+	struct mesh_state_ref* superref; /* points to super, inserted in sub */
+	if( !(subref = region_alloc(super->s.region,
+		sizeof(struct mesh_state_ref))) ||
+		!(superref = region_alloc(sub->s.region,
+		sizeof(struct mesh_state_ref))) ) {
+		log_err("mesh_state_attachment: out of memory");
+		return 0;
+	}
+	superref->node.key = superref;
+	superref->s = super;
+	subref->node.key = subref;
+	subref->s = sub;
+	n = rbtree_insert(&sub->super_set, &superref->node);
+	log_assert(n != NULL);
+	n = rbtree_insert(&super->sub_set, &subref->node);
+	log_assert(n != NULL);
+	return 1;
+}
+
+/** subtract timers and the values do not overflow or become negative */
+static void
+timeval_subtract(struct timeval* d, struct timeval* end, struct timeval* start)
+{
+#ifndef S_SPLINT_S
+	d->tv_sec = end->tv_sec - start->tv_sec;
+	while(end->tv_usec < start->tv_usec) {
+		end->tv_usec += 1000000;
+		d->tv_sec--;
+	}
+	d->tv_usec = end->tv_usec - start->tv_usec;
+#endif
+}
+
+/** add timers and the values do not overflow or become negative */
+static void
+timeval_add(struct timeval* d, struct timeval* add)
+{
+#ifndef S_SPLINT_S
+	d->tv_sec += add->tv_sec;
+	d->tv_usec += add->tv_usec;
+	while(d->tv_usec > 1000000 ) {
+		d->tv_usec -= 1000000;
+		d->tv_sec++;
+	}
+#endif
+}
+
+/** divide sum of timers to get average */
+static void
+timeval_divide(struct timeval* avg, struct timeval* sum, size_t d)
+{
+#ifndef S_SPLINT_S
+	size_t leftover;
+	if(d == 0) {
+		avg->tv_sec = 0;
+		avg->tv_usec = 0;
+		return;
+	}
+	avg->tv_sec = sum->tv_sec / d;
+	avg->tv_usec = sum->tv_usec / d;
+	/* handle fraction from seconds divide */
+	leftover = sum->tv_sec - avg->tv_sec*d;
+	avg->tv_usec += (leftover*1000000)/d;
+#endif
+}
+
+/**
+ * Send reply to mesh reply entry
+ * @param m: mesh state to send it for.
+ * @param rcode: if not 0, error code.
+ * @param rep: reply to send (or NULL if rcode is set).
+ * @param r: reply entry
+ */
+static void
+mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
+	struct mesh_reply* r)
+{
+	struct timeval end_time;
+	int secure;
+	/* examine security status */
+	if(m->s.env->need_to_validate && !(r->qflags&BIT_CD) && rep && 
+		rep->security <= sec_status_bogus) {
+		rcode = LDNS_RCODE_SERVFAIL;
+	}
+	if(rep && rep->security == sec_status_secure)
+		secure = 1;
+	else	secure = 0;
+	/* send the reply */
+	if(rcode) {
+		error_encode(r->query_reply.c->buffer, rcode, &m->s.qinfo,
+			r->qid, r->qflags, &r->edns);
+		comm_point_send_reply(&r->query_reply);
+	} else {
+		size_t udp_size = r->edns.udp_size;
+		r->edns.edns_version = EDNS_ADVERTISED_VERSION;
+		r->edns.udp_size = EDNS_ADVERTISED_SIZE;
+		r->edns.ext_rcode = 0;
+		r->edns.bits &= EDNS_DO;
+		if(!reply_info_answer_encode(&m->s.qinfo, rep, r->qid, 
+			r->qflags, r->query_reply.c->buffer, 0, 1, 
+			m->s.env->scratch, udp_size, &r->edns, 
+			(int)(r->edns.bits & EDNS_DO), secure)) 
+		{
+			error_encode(r->query_reply.c->buffer, 
+				LDNS_RCODE_SERVFAIL, &m->s.qinfo, r->qid, 
+				r->qflags, &r->edns);
+		}
+		comm_point_send_reply(&r->query_reply);
+	}
+	/* account */
+	m->s.env->mesh->num_reply_addrs--;
+	if(gettimeofday(&end_time, NULL) < 0) {
+		log_err("gettimeofday: %s", strerror(errno));
+		return;
+	} else {
+		struct timeval duration;
+		timeval_subtract(&duration, &end_time, &r->start_time);
+		verbose(VERB_ALGO, "query took %d.%6.6d sec",
+			(int)duration.tv_sec, (int)duration.tv_usec);
+		m->s.env->mesh->replies_sent++;
+		timeval_add(&m->s.env->mesh->replies_sum_wait, &duration);
+		timehist_insert(m->s.env->mesh->histogram, &duration);
+	}
+}
+
+void mesh_query_done(struct mesh_state* mstate)
+{
+	struct mesh_reply* r;
+	struct reply_info* rep = (mstate->s.return_msg?
+		mstate->s.return_msg->rep:NULL);
+	for(r = mstate->reply_list; r; r = r->next) {
+		mesh_send_reply(mstate, mstate->s.return_rcode, rep, r);
+	}
+}
+
+void mesh_walk_supers(struct mesh_area* mesh, struct mesh_state* mstate)
+{
+	struct mesh_state_ref* ref;
+	RBTREE_FOR(ref, struct mesh_state_ref*, &mstate->super_set)
+	{
+		/* make super runnable */
+		(void)rbtree_insert(&mesh->run, &ref->s->run_node);
+		/* callback the function to inform super of result */
+		(*mesh->modfunc[ref->s->s.curmod]->inform_super)(&mstate->s, 
+			ref->s->s.curmod, &ref->s->s);
+	}
+}
+
+struct mesh_state* mesh_area_find(struct mesh_area* mesh,
+	struct query_info* qinfo, uint16_t qflags, int prime)
+{
+	struct mesh_state key;
+	struct mesh_state* result;
+
+	key.node.key = &key;
+	key.s.is_priming = prime;
+	key.s.qinfo = *qinfo;
+	key.s.query_flags = qflags;
+	
+	result = (struct mesh_state*)rbtree_search(&mesh->all, &key);
+	return result;
+}
+
+int mesh_state_add_reply(struct mesh_state* s, struct edns_data* edns,
+        struct comm_reply* rep, uint16_t qid, uint16_t qflags)
+{
+	struct mesh_reply* r = region_alloc(s->s.region, 
+		sizeof(struct mesh_reply));
+	if(!r)
+		return 0;
+	r->query_reply = *rep;
+	r->edns = *edns;
+	r->qid = qid;
+	r->qflags = qflags;
+	if(gettimeofday(&r->start_time, NULL) < 0) {
+		log_err("addrep: gettimeofday: %s", strerror(errno));
+		memset(&r->start_time, 0, sizeof(r->start_time));
+	}
+	r->next = s->reply_list;
+	s->reply_list = r;
+	return 1;
+
+}
+
+/**
+ * Continue processing the mesh state at another module.
+ * Handles module to modules tranfer of control.
+ * Handles module finished.
+ * @param mesh: the mesh area.
+ * @param mstate: currently active mesh state.
+ * 	Deleted if finished, calls _done and _supers to 
+ * 	send replies to clients and inform other mesh states.
+ * 	This in turn may create additional runnable mesh states.
+ * @param s: state at which the current module exited.
+ * @param ev: the event sent to the module.
+ * 	returned is the event to send to the next module.
+ * @return true if continue processing at the new module.
+ * 	false if not continued processing is needed.
+ */
+static int
+mesh_continue(struct mesh_area* mesh, struct mesh_state* mstate,
+	enum module_ext_state s, enum module_ev* ev)
+{
+	if(s == module_wait_module) {
+		/* start next module */
+		mstate->s.curmod++;
+		if(mesh->num_modules == mstate->s.curmod) {
+			log_err("Cannot pass to next module; at last module");
+			log_query_info(VERB_DETAIL, "pass error for qstate",
+				&mstate->s.qinfo);
+			log_assert(0); /* catch this for now */
+			mstate->s.curmod--;
+			return mesh_continue(mesh, mstate, module_error, ev);
+		}
+		*ev = module_event_pass;
+		return 1;
+	}
+	if(s == module_error && mstate->s.return_rcode == LDNS_RCODE_NOERROR) {
+		/* error is bad, handle pass back up below */
+		mstate->s.return_rcode = LDNS_RCODE_SERVFAIL;
+	}
+	if(s == module_error || s == module_finished) {
+		if(mstate->s.curmod == 0) {
+			mesh_query_done(mstate);
+			mesh_walk_supers(mesh, mstate);
+			mesh_state_delete(&mstate->s);
+			return 0;
+		}
+		/* pass along the locus of control */
+		mstate->s.curmod --;
+		*ev = module_event_moddone;
+		return 1;
+	}
+	return 0;
+}
+
+void mesh_run(struct mesh_area* mesh, struct mesh_state* mstate,
+	enum module_ev ev, struct outbound_entry* e)
+{
+	enum module_ext_state s;
+	verbose(VERB_ALGO, "mesh_run: start");
+	while(mstate) {
+		/* run the module */
+		(*mesh->modfunc[mstate->s.curmod]->operate)
+			(&mstate->s, ev, mstate->s.curmod, e);
+
+		/* examine results */
+		mstate->s.reply = NULL;
+		region_free_all(mstate->s.env->scratch);
+		s = mstate->s.ext_state[mstate->s.curmod];
+		verbose(VERB_ALGO, "mesh_run: %s module exit state is %s", 
+			mesh->modfunc[mstate->s.curmod]->name, strextstate(s));
+		e = NULL;
+		if(mesh_continue(mesh, mstate, s, &ev))
+			continue;
+
+		/* run more modules */
+		ev = module_event_pass;
+		if(mesh->run.count > 0) {
+			/* pop random element off the runnable tree */
+			mstate = (struct mesh_state*)mesh->run.root->key;
+			(void)rbtree_delete(&mesh->run, mstate);
+		} else mstate = NULL;
+	}
+	if(verbosity >= VERB_ALGO) {
+		mesh_stats(mesh, "mesh_run: end");
+		mesh_log_list(mesh);
+	}
+}
+
+void 
+mesh_log_list(struct mesh_area* mesh)
+{
+	char buf[30];
+	struct mesh_state* m;
+	int num = 0;
+	RBTREE_FOR(m, struct mesh_state*, &mesh->all) {
+		snprintf(buf, sizeof(buf), "%d%s%s%s%s%s mod%d %s", 
+			num++, (m->s.is_priming)?"p":"",  /* prime */
+			(m->s.query_flags&BIT_RD)?"RD":"",
+			(m->s.query_flags&BIT_CD)?"CD":"",
+			(m->super_set.count==0)?"d":"", /* detached */
+			(m->sub_set.count!=0)?"c":"",  /* children */
+			m->s.curmod, (m->reply_list)?"hr":"nr"); /*hasreply*/
+		log_query_info(VERB_ALGO, buf, &m->s.qinfo);
+	}
+}
+
+void 
+mesh_stats(struct mesh_area* mesh, const char* str)
+{
+	log_info("%s %u states (%u with reply, %u detached), "
+		"%u waiting replies", str, (unsigned)mesh->all.count, 
+		(unsigned)mesh->num_reply_states,
+		(unsigned)mesh->num_detached_states,
+		(unsigned)mesh->num_reply_addrs);
+	if(mesh->replies_sent > 0) {
+		struct timeval avg;
+		timeval_divide(&avg, &mesh->replies_sum_wait, 
+			mesh->replies_sent);
+		log_info("sent %u replies, with average wait "
+			"of %d.%6.6d sec", (unsigned)mesh->replies_sent,
+			(int)avg.tv_sec, (int)avg.tv_usec);
+		log_info("histogram of reply wait times");
+		timehist_log(mesh->histogram);
+	}
+}
+
+size_t 
+mesh_get_mem(struct mesh_area* mesh)
+{
+	struct mesh_state* m;
+	size_t s = sizeof(*mesh) + sizeof(struct timehist) +
+		sizeof(struct th_buck)*mesh->histogram->num;
+	RBTREE_FOR(m, struct mesh_state*, &mesh->all) {
+		/* all, including m itself allocated in qstate region */
+		s += region_get_mem(m->s.region);
+	}
+	return s;
+}
+
+/** helper recursive rbtree find routine */
+static int
+find_in_subsub(struct mesh_state* m, struct mesh_state* tofind)
+{
+	struct mesh_state_ref* r;
+	RBTREE_FOR(r, struct mesh_state_ref*, &m->sub_set) {
+		if(r->s == tofind || find_in_subsub(r->s, tofind))
+			return 1;
+	}
+	return 0;
+}
+
+int 
+mesh_detect_cycle(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t flags, int prime)
+{
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state* cyc_m = qstate->mesh_info;
+	struct mesh_state* dep_m = mesh_area_find(mesh, qinfo, flags, prime);
+	if(!dep_m)
+		return 0;
+	if(dep_m == cyc_m || find_in_subsub(dep_m, cyc_m))
+		return 1;
+	return 0;
+}

services/mesh.h

@@ -0,0 +1,373 @@
+/*
+ * services/mesh.h - deal with mesh of query states and handle events for that.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist in dealing with a mesh of
+ * query states. This mesh is supposed to be thread-specific.
+ * It consists of query states (per qname, qtype, qclass) and connections
+ * between query states and the super and subquery states, and replies to
+ * send back to clients.
+ */
+
+#ifndef SERVICES_MESH_H
+#define SERVICES_MESH_H
+
+#include "util/rbtree.h"
+#include "util/netevent.h"
+#include "util/data/msgparse.h"
+#include "util/module.h"
+struct mesh_state;
+struct mesh_reply;
+struct query_info;
+struct reply_info;
+struct outbound_entry;
+struct timehist;
+
+/** 
+ * Mesh of query states
+ */
+struct mesh_area {
+	/** the number of modules */
+	int num_modules;
+	/** the module callbacks, array of num_modules length (ref only) */
+	struct module_func_block** modfunc;
+	/** environment for new states */
+	struct module_env* env;
+
+	/** set of runnable queries (mesh_state.run_node) */
+	rbtree_t run;
+	/** rbtree of all current queries (mesh_state.node)*/
+	rbtree_t all;
+
+	/** count of the total number of mesh_reply entries */
+	size_t num_reply_addrs;
+	/** count of the number of mesh_states that have mesh_replies 
+	 * Because a state can send results to multiple reply addresses,
+	 * this number must be equal or lower than num_reply_addrs. */
+	size_t num_reply_states;
+	/** number of mesh_states that have no mesh_replies, and also
+	 * an empty set of super-states, thus are 'toplevel' or detached
+	 * internal opportunistic queries */
+	size_t num_detached_states;
+
+	/** number of replies sent */
+	size_t replies_sent;
+	/** sum of waiting times for the replies */
+	struct timeval replies_sum_wait;
+	/** histogram of time values */
+	struct timehist* histogram;
+};
+
+/**
+ * A mesh query state
+ * Unique per qname, qtype, qclass (from the qstate).
+ * And RD / CD flag; in case a client turns it off.
+ * And priming queries are different from ordinary queries (because of hints).
+ *
+ * The entire structure is allocated in a region, this region is the qstate
+ * region. All parts (rbtree nodes etc) are also allocated in the region.
+ */
+struct mesh_state {
+	/** node in mesh_area all tree, key is this struct. Must be first. */
+	rbnode_t node;
+	/** node in mesh_area runnable tree, key is this struct */
+	rbnode_t run_node;
+	/** the query state. Note that the qinfo and query_flags 
+	 * may not change. */
+	struct module_qstate s;
+	/** the list of replies to clients for the results */
+	struct mesh_reply* reply_list;
+	/** set of superstates (that want this state's result) 
+	 * contains struct mesh_state_ref* */
+	rbtree_t super_set;
+	/** set of substates (that this state needs to continue)
+	 * contains struct mesh_state_ref* */
+	rbtree_t sub_set;
+};
+
+/**
+ * Rbtree reference to a mesh_state.
+ * Used in super_set and sub_set. 
+ */
+struct mesh_state_ref {
+	/** node in rbtree for set, key is this structure */
+	rbnode_t node;
+	/** the mesh state */
+	struct mesh_state* s;
+};
+
+/**
+ * Reply to a client
+ */
+struct mesh_reply {
+	/** next in reply list */
+	struct mesh_reply* next;
+	/** the query reply destination, packet buffer and where to send. */
+	struct comm_reply query_reply;
+	/** edns data from query */
+	struct edns_data edns;
+	/** the time when request was entered */
+	struct timeval start_time;
+	/** id of query, in network byteorder. */
+	uint16_t qid;
+	/** flags of query, for reply flags */
+	uint16_t qflags;
+};
+
+/* ------------------- Functions for worker -------------------- */
+
+/**
+ * Allocate mesh, to empty.
+ * @param num_modules: number of modules that are present.
+ * @param modfunc: array passed (alloced and deleted by caller), that has
+ * 	num_modules function callbacks for the modules.
+ * @param env: environment for new queries.
+ * @return mesh: the new mesh or NULL on error.
+ */
+struct mesh_area* mesh_create(int num_modules, 
+	struct module_func_block** modfunc, struct module_env* env);
+
+/**
+ * Delete mesh, and all query states and replies in it.
+ * @param mesh: the mesh to delete.
+ */
+void mesh_delete(struct mesh_area* mesh);
+
+/**
+ * New query incoming from clients. Create new query state if needed, and
+ * add mesh_reply to it. Returns error to client on malloc failures.
+ * Will run the mesh area queries to process if a new query state is created.
+ *
+ * @param mesh: the mesh.
+ * @param qinfo: query from client.
+ * @param qflags: flags from client query.
+ * @param edns: edns data from client query.
+ * @param rep: where to reply to.
+ * @param qid: query id to reply with.
+ */
+void mesh_new_client(struct mesh_area* mesh, struct query_info* qinfo,
+	uint16_t qflags, struct edns_data* edns, struct comm_reply* rep, 
+	uint16_t qid);
+
+/**
+ * Handle new event from the wire. A serviced query has returned.
+ * The query state will be made runnable, and the mesh_area will process
+ * query states until processing is complete.
+ *
+ * @param mesh: the query mesh.
+ * @param e: outbound entry, with query state to run and reply pointer.
+ * @param is_ok: if true, reply is OK, otherwise a timeout happened.
+ * @param reply: the comm point reply info.
+ */
+void mesh_report_reply(struct mesh_area* mesh, struct outbound_entry* e,
+	int is_ok, struct comm_reply* reply);
+
+/* ------------------- Functions for module environment --------------- */
+
+/**
+ * Detach-subqueries.
+ * Remove all sub-query references from this query state.
+ * Keeps super-references of those sub-queries correct.
+ * Updates stat items in mesh_area structure.
+ * @param qstate: used to find mesh state.
+ */
+void mesh_detach_subs(struct module_qstate* qstate);
+
+/**
+ * Attach subquery.
+ * Creates it if it does not exist already.
+ * Keeps sub and super references correct.
+ * Updates stat items in mesh_area structure.
+ * Pass if it is priming query or not.
+ * return:
+ * 	o if error (malloc) happened.
+ * 	o need to initialise the new state (module init; it is a new state).
+ * 	  so that the next run of the query with this module is successful.
+ * 	o no init needed, attachment successful.
+ *
+ * @param qstate: the state to find mesh state, and that wants to receive
+ * 	the results from the new subquery.
+ * @param qinfo: what to query for (copied).
+ * @param qflags: what flags to use (RD / CD flag or not).
+ * @param prime: if it is a (stub) priming query.
+ * @param newq: If the new subquery needs initialisation, it is returned,
+ * 	otherwise NULL is returned.
+ * @return: false on error, true if success (and init may be needed).
+ */
+int mesh_attach_sub(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t qflags, int prime, struct module_qstate** newq);
+
+/**
+ * Query state is done, send messages to reply entries.
+ * Encode messages using reply entry values and the querystate (with original
+ * qinfo), using given reply_info.
+ * Pass errcode != 0 if an error reply is needed.
+ * If no reply entries, nothing is done.
+ * Must be called before a module can module_finished or return module_error.
+ * The module must handle the super query states itself as well.
+ *
+ * @param mstate: mesh state that is done. return_rcode and return_msg
+ * 	are used for replies.
+ * 	return_rcode: if not 0 (NOERROR) an error is sent back (and 
+ * 		return_msg is ignored).
+ * 	return_msg: reply to encode and send back to clients.
+ */
+void mesh_query_done(struct mesh_state* mstate);
+
+/**
+ * Call inform_super for the super query states that are interested in the 
+ * results from this query state. These can then be changed for error 
+ * or results.
+ * Called when a module is module_finished or returns module_error.
+ * The super query states become runnable with event module_event_pass.
+ *
+ * @param mesh: mesh area to add newly runnable modules to.
+ * @param mstate: the state that has results, used to find mesh state.
+ */
+void mesh_walk_supers(struct mesh_area* mesh, struct mesh_state* mstate);
+
+/**
+ * Delete mesh state, cleanup and also rbtrees and so on.
+ * Will detach from all super/subnodes.
+ * @param qstate: to remove.
+ */
+void mesh_state_delete(struct module_qstate* qstate);
+
+/* ------------------- Functions for mesh -------------------- */
+
+/**
+ * Create and initialize a new mesh state and its query state
+ * Does not put the mesh state into rbtrees and so on.
+ * @param env: module environment to set.
+ * @param qinfo: query info that the mesh is for.
+ * @param qflags: flags for query (RD / CD flag).
+ * @param prime: if true, it is a priming query, set is_priming on mesh state.
+ * @return: new mesh state or NULL on allocation error.
+ */
+struct mesh_state* mesh_state_create(struct module_env* env, 
+	struct query_info* qinfo, uint16_t qflags, int prime);
+
+/**
+ * Cleanup a mesh state and its query state. Does not do rbtree or 
+ * reference cleanup.
+ * @param mstate: mesh state to cleanup. Its pointer may no longer be used
+ * 	afterwards. Cleanup rbtrees before calling this function.
+ */
+void mesh_state_cleanup(struct mesh_state* mstate);
+
+/**
+ * Find a mesh state in the mesh area. Pass relevant flags.
+ *
+ * @param mesh: the mesh area to look in.
+ * @param qinfo: what query
+ * @param qflags: if RD / CD bit is set or not.
+ * @param prime: if it is a priming query.
+ * @return: mesh state or NULL if not found.
+ */
+struct mesh_state* mesh_area_find(struct mesh_area* mesh, 
+	struct query_info* qinfo, uint16_t qflags, int prime);
+
+/**
+ * Setup attachment super/sub relation between super and sub mesh state.
+ * The relation must not be present when calling the function.
+ * Does not update stat items in mesh_area.
+ * @param super: super state.
+ * @param sub: sub state.
+ * @return: 0 on alloc error.
+ */
+int mesh_state_attachment(struct mesh_state* super, struct mesh_state* sub);
+
+/**
+ * Create new reply structure and attach it to a mesh state.
+ * Does not update stat items in mesh area.
+ * @param s: the mesh state.
+ * @param edns: edns data for reply (bufsize).
+ * @param rep: comm point reply info.
+ * @param qid: ID of reply.
+ * @param qflags: original query flags.
+ * @return: 0 on alloc error.
+ */
+int mesh_state_add_reply(struct mesh_state* s, struct edns_data* edns, 
+	struct comm_reply* rep, uint16_t qid, uint16_t qflags);
+
+/**
+ * Run the mesh. Run all runnable mesh states. Which can create new
+ * runnable mesh states. Until completion. Automatically called by
+ * mesh_report_reply and mesh_new_client as needed.
+ * @param mesh: mesh area.
+ * @param mstate: first mesh state to run.
+ * @param ev: event the mstate. Others get event_pass.
+ * @param e: if a reply, its outbound entry.
+ */
+void mesh_run(struct mesh_area* mesh, struct mesh_state* mstate, 
+	enum module_ev ev, struct outbound_entry* e);
+
+/**
+ * Print some stats about the mesh to the log.
+ * @param mesh: the mesh to print it for.
+ * @param str: descriptive string to go with it.
+ */
+void mesh_stats(struct mesh_area* mesh, const char* str);
+
+/**
+ * Print all the states in the mesh to the log.
+ * @param mesh: the mesh to print all states of.
+ */
+void mesh_log_list(struct mesh_area* mesh);
+
+/**
+ * Calculate memory size in use by mesh and all queries inside it.
+ * @param mesh: the mesh to examine.
+ * @return size in bytes.
+ */
+size_t mesh_get_mem(struct mesh_area* mesh);
+
+/**
+ * Find cycle; see if the given mesh is in the targets sub, or sub-sub, ...
+ * trees.
+ * @param qstate: given mesh querystate.
+ * @param qinfo: query info for dependency.
+ * @param flags: query flags of dependency.
+ * @param prime: if dependency is a priming query or not.
+ * @return true if the name,type,class exists and the given qstate mesh exists
+ * 	as a dependency of that name. Thus if qstate becomes dependent on
+ * 	name,type,class then a cycle is created.
+ */
+int mesh_detect_cycle(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t flags, int prime);
+
+#endif /* SERVICES_MESH_H */

services/outbound_list.c

@@ -0,0 +1,88 @@
+/*
+ * services/outbound_list.c - keep list of outbound serviced queries.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to help a module keep track of the
+ * queries it has outstanding to authoritative servers.
+ */
+#include "config.h"
+#include "services/outbound_list.h"
+#include "services/outside_network.h"
+
+void 
+outbound_list_init(struct outbound_list* list)
+{
+	list->first = NULL;
+}
+
+void 
+outbound_list_clear(struct outbound_list* list)
+{
+	struct outbound_entry *p, *np;
+	p = list->first;
+	while(p) {
+		np = p->next;
+		outnet_serviced_query_stop(p->qsent, p);
+		/* in region, no free needed */
+		p = np;
+	}
+	outbound_list_init(list);
+}
+
+void 
+outbound_list_insert(struct outbound_list* list, struct outbound_entry* e)
+{
+	if(list->first)
+		list->first->prev = e;
+	e->next = list->first;
+	e->prev = NULL;
+	list->first = e;
+}
+
+void 
+outbound_list_remove(struct outbound_list* list, struct outbound_entry* e)
+{
+	if(!e)
+		return;
+	outnet_serviced_query_stop(e->qsent, e);
+	if(e->next)
+		e->next->prev = e->prev;
+	if(e->prev)
+		e->prev->next = e->next;
+	else	list->first = e->next;
+	/* in region, no free needed */
+}

services/outbound_list.h

@@ -0,0 +1,105 @@
+/*
+ * services/outbound_list.h - keep list of outbound serviced queries.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to help a module keep track of the
+ * queries it has outstanding to authoritative servers.
+ */
+#ifndef SERVICES_OUTBOUND_LIST_H
+#define SERVICES_OUTBOUND_LIST_H
+struct outbound_entry;
+struct serviced_query;
+struct module_qstate;
+
+/**
+ * The outbound list. This structure is part of the module specific query
+ * state.
+ */
+struct outbound_list {
+	/** The linked list of outbound query entries. */
+	struct outbound_entry* first;
+};
+
+/**
+ * Outbound list entry. A serviced query sent by a module processing the
+ * query from the qstate. Double linked list to aid removal.
+ */
+struct outbound_entry {
+	/** next in list */
+	struct outbound_entry* next;
+	/** prev in list */
+	struct outbound_entry* prev;
+	/** The query that was sent out */
+	struct serviced_query* qsent;
+	/** the module query state that sent it */
+	struct module_qstate* qstate;
+};
+
+/**
+ * Init the user allocated outbound list structure
+ * @param list: the list structure.
+ */
+void outbound_list_init(struct outbound_list* list);
+
+/**
+ * Clear the user owner outbound list structure.
+ * Deletes serviced queries.
+ * @param list: the list structure. It is cleared, but the list struct itself
+ * 	is callers responsability to delete.
+ */
+void outbound_list_clear(struct outbound_list* list);
+
+/**
+ * Insert new entry into the list. Caller must allocate the entry with malloc.
+ * qstate and qsent are set by caller.
+ * @param list: the list to add to.
+ * @param e: entry to add, it is only half initialised at call start, fully
+ *	initialised at call end.
+ */
+void outbound_list_insert(struct outbound_list* list, 
+	struct outbound_entry* e);
+
+/**
+ * Remove an entry from the list, and deletes it. 
+ * Deletes serviced query in the entry.
+ * @param list: the list to remove from.
+ * @param e: the entry to remove.
+ */
+void outbound_list_remove(struct outbound_list* list, 
+	struct outbound_entry* e);
+
+#endif /* SERVICES_OUTBOUND_LIST_H */

services/outside_network.h

@@ -49,6 +49,9 @@
 struct pending;
 struct pending_timeout;
 struct ub_randstate;
+struct pending_tcp;
+struct waiting_tcp;
+struct infra_cache;
 
 /**
  * Send queries to outside servers and wait for answers from servers.
@@ -61,24 +64,49 @@ struct outside_network {
 	/** buffer shared by UDP connections, since there is only one
 	    datagram at any time. */
 	ldns_buffer* udp_buff;
+	/** serviced_callbacks malloc overhead when processing multiple
+	 * identical serviced queries to the same server. */
+	size_t svcd_overhead;
 
 	/** 
 	 * Array of udp comm point* that are used to listen to pending events.
 	 * Each is on a different port. This is for ip4 ports.
 	 */
-	struct comm_point **udp4_ports;
+	struct comm_point** udp4_ports;
 	/** number of udp4 ports */
 	size_t num_udp4;
 
 	/**
 	 * The opened ip6 ports.
 	 */
-	struct comm_point **udp6_ports;
+	struct comm_point** udp6_ports;
 	/** number of udp6 ports */
 	size_t num_udp6;
 
-	/** pending answers. sorted by id, addr */
-	rbtree_t *pending;
+	/** pending udp answers. sorted by id, addr */
+	rbtree_t* pending;
+	/** serviced queries, sorted by qbuf, addr, dnssec */
+	rbtree_t* serviced;
+	/** host cache, pointer but not owned by outnet. */
+	struct infra_cache* infra;
+	/** where to get random numbers */
+	struct ub_randstate* rnd;
+
+	/**
+	 * Array of tcp pending used for outgoing TCP connections.
+	 * Each can be used to establish a TCP connection with a server.
+	 * The file descriptors are -1 if they are free, and need to be 
+	 * opened for the tcp connection. Can be used for ip4 and ip6.
+	 */
+	struct pending_tcp **tcp_conns;
+	/** number of tcp communication points. */
+	size_t num_tcp;
+	/** list of tcp comm points that are free for use */
+	struct pending_tcp* tcp_free;
+	/** list of tcp queries waiting for a buffer */
+	struct waiting_tcp* tcp_wait_first;
+	/** last of waiting query list */
+	struct waiting_tcp* tcp_wait_last;
 };
 
 /**
@@ -105,6 +133,113 @@ struct pending {
 	struct outside_network* outnet;
 };
 
+/**
+ * Pending TCP query to server.
+ */
+struct pending_tcp {
+	/** next in list of free tcp comm points, or NULL. */
+	struct pending_tcp* next_free;
+	/** the ID for the query; checked in reply */
+	uint16_t id;
+	/** tcp comm point it was sent on (and reply must come back on). */
+	struct comm_point* c;
+	/** the query being serviced, NULL if the pending_tcp is unused. */
+	struct waiting_tcp* query;
+};
+
+/**
+ * Query waiting for TCP buffer.
+ */
+struct waiting_tcp {
+	/** 
+	 * next in waiting list.
+	 * if pkt==0, this points to the pending_tcp structure.
+	 */
+	struct waiting_tcp* next_waiting;
+	/** timeout event; timer keeps running whether the query is
+	 * waiting for a buffer or the tcp reply is pending */
+	struct comm_timer* timer;
+	/** the outside network it is part of */
+	struct outside_network* outnet;
+	/** remote address. */
+	struct sockaddr_storage addr;
+	/** length of addr field in use. */
+	socklen_t addrlen;
+	/** 
+	 * The query itself, the query packet to send.
+	 * allocated after the waiting_tcp structure.
+	 * set to NULL when the query is serviced and it part of pending_tcp.
+	 * if this is NULL, the next_waiting points to the pending_tcp.
+	 */
+	uint8_t* pkt;
+	/** length of query packet. */
+	size_t pkt_len;
+	/** callback for the timeout, error or reply to the message */
+	comm_point_callback_t* cb;
+	/** callback user argument */
+	void* cb_arg;
+};
+
+/**
+ * Callback to party interested in serviced query results.
+ */
+struct service_callback {
+	/** next in callback list */
+	struct service_callback* next;
+	/** callback function */
+	comm_point_callback_t* cb;
+	/** user argument for callback function */
+	void* cb_arg;
+};
+
+/**
+ * Query service record.
+ * Contains query and destination. UDP, TCP, EDNS are all tried.
+ * complete with retries and timeouts. A number of interested parties can
+ * receive a callback.
+ */
+struct serviced_query {
+	/** The rbtree node, key is this record */
+	rbnode_t node;
+	/** The query that needs to be answered. Starts with flags u16,
+	 * then qdcount, ..., including qname, qtype, qclass. Does not include
+	 * EDNS record. */
+	uint8_t* qbuf;
+	/** length of qbuf. */
+	size_t qbuflen;
+	/** If an EDNS section is included, the DO/CD bit will be turned on. */
+	int dnssec;
+	/** where to send it */
+	struct sockaddr_storage addr;
+	/** length of addr field in use. */
+	socklen_t addrlen;
+	/** current status */
+	enum serviced_query_status {
+		/** initial status */
+		serviced_initial,
+		/** UDP with EDNS sent */
+		serviced_query_UDP_EDNS,
+		/** UDP without EDNS sent */
+		serviced_query_UDP,
+		/** TCP with EDNS sent */
+		serviced_query_TCP_EDNS,
+		/** TCP without EDNS sent */
+		serviced_query_TCP
+	} status;
+	/** true if serviced_query is scheduled for deletion already */
+	int to_be_deleted;
+	/** number of UDP retries */
+	int retry;
+	/** time last UDP was sent */
+	struct timeval last_sent_time;
+	/** outside network this is part of */
+	struct outside_network* outnet;
+	/** list of interested parties that need callback on results. */
+	struct service_callback* cblist;
+	/** the UDP or TCP query that is pending, see status which */
+	void* pending;
+};
+
 /**
  * Create outside_network structure with N udp ports.
  * @param base: the communication base to use for event handling.
@@ -117,11 +252,15 @@ struct pending {
  * @param do_ip6: service IP6.
  * @param port_base: if -1 system assigns ports, otherwise try to get
  *    the ports numbered from this starting number.
+ * @param num_tcp: number of outgoing tcp buffers to preallocate.
+ * @param infra: pointer to infra cached used for serviced queries.
+ * @param rnd: stored to create random numbers for serviced queries.
  * @return: the new structure (with no pending answers) or NULL on error.
  */
 struct outside_network* outside_network_create(struct comm_base* base,
 	size_t bufsize, size_t num_ports, char** ifs, int num_ifs,
-	int do_ip4, int do_ip6, int port_base);
+	int do_ip4, int do_ip6, int port_base, size_t num_tcp, 
+	struct infra_cache* infra, struct ub_randstate* rnd);
 
 /**
  * Delete outside_network structure.
@@ -136,17 +275,36 @@ void outside_network_delete(struct outside_network* outnet);
  * @param packet: wireformat query to send to destination.
  * @param addr: address to send to.
  * @param addrlen: length of addr.
- * @param timeout: in seconds from now.
+ * @param timeout: in milliseconds from now.
  * @param callback: function to call on error, timeout or reply.
- *    The routine does not return an error, instead it calls the callback,
- *    with an error code if an error happens.
  * @param callback_arg: user argument for callback function.
  * @param rnd: random state for generating ID and port.
+ * @return: NULL on error for malloc or socket. Else the pending query object.
  */
-void pending_udp_query(struct outside_network* outnet, ldns_buffer* packet, 
-	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* callback, void* callback_arg,
-	struct ub_randstate* rnd);
+struct pending* pending_udp_query(struct outside_network* outnet, 
+	ldns_buffer* packet, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+	void* callback_arg, struct ub_randstate* rnd);
+
+/**
+ * Send TCP query. May wait for TCP buffer. Selects ID to be random, and 
+ * checks id.
+ * @param outnet: provides the event handling.
+ * @param packet: wireformat query to send to destination. copied from.
+ * @param addr: address to send to.
+ * @param addrlen: length of addr.
+ * @param timeout: in seconds from now.
+ *    Timer starts running now. Timer may expire if all buffers are used,
+ *    without any query been sent to the server yet.
+ * @param callback: function to call on error, timeout or reply.
+ * @param callback_arg: user argument for callback function.
+ * @param rnd: random state for generating ID.
+ * @return: false on error for malloc or socket. Else the pending TCP object.
+ */
+struct waiting_tcp* pending_tcp_query(struct outside_network* outnet, 
+	ldns_buffer* packet, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+	void* callback_arg, struct ub_randstate* rnd);
 
 /**
  * Delete pending answer.
@@ -156,4 +314,59 @@ void pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
  */
 void pending_delete(struct outside_network* outnet, struct pending* p);
 
+/**
+ * Perform a serviced query to the authoritative servers.
+ * Duplicate efforts are detected, and EDNS, TCP and UDP retry is performed.
+ * @param outnet: outside network, with rbtree of serviced queries.
+ * @param qname: what qname to query.
+ * @param qnamelen: length of qname in octets including 0 root label.
+ * @param qtype: rrset type to query (host format)
+ * @param qclass: query class. (host format)
+ * @param flags: flags u16 (host format), includes opcode, CD bit.
+ * @param dnssec: if set, DO bit is set in EDNS queries.
+ *	If the value includes BIT_CD, CD bit is set when in EDNS queries.
+ *	If the value includes BIT_DO, DO bit is set when in EDNS queries.
+ * @param callback: callback function.
+ * @param callback_arg: user argument to callback function.
+ * @param addr: to which server to send the query.
+ * @param addrlen: length of addr.
+ * @param buff: scratch buffer to create query contents in. Empty on exit.
+ * @param arg_compare: function to compare callback args, return true if 
+ * 	identical. It is given the callback_arg and args that are listed.
+ * @return 0 on error, or pointer to serviced query that is used to answer
+ *	this serviced query may be shared with other callbacks as well.
+ */
+struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint16_t flags, int dnssec, struct sockaddr_storage* addr, 
+	socklen_t addrlen, comm_point_callback_t* callback, 
+	void* callback_arg, ldns_buffer* buff, 
+	int (*arg_compare)(void*,void*));
+
+/**
+ * Remove service query callback.
+ * If that leads to zero callbacks, the query is completely cancelled.
+ * @param sq: serviced query to adjust.
+ * @param cb_arg: callback argument of callback that needs removal.
+ *	same as the callback_arg to outnet_serviced_query().
+ */
+void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg);
+
+/**
+ * Get memory size in use by outside network.
+ * Counts buffers and outstanding query (serviced queries) malloced data.
+ * @param outnet: outside network structure.
+ * @return size in bytes.
+ */
+size_t outnet_get_mem(struct outside_network* outnet);
+
+/**
+ * Get memory size in use by serviced query while it is servicing callbacks.
+ * This takes into account the pre-deleted status of it; it will be deleted
+ * when the callbacks are done.
+ * @param sq: serviced query. 
+ * @return size in bytes.
+ */
+size_t serviced_get_mem(struct serviced_query* sq);
+
 #endif /* OUTSIDE_NETWORK_H */

testcode/checklocks.c

@@ -46,18 +46,29 @@
  * are passed to thread_create to make the thread numbers here the same as 
  * those used for logging which is nice.
  *
- * Todo: - check global ordering of instances of locks.
- *	 - refcount statistics.
+ * Todo: 
  *	 - debug status print, of thread lock stacks, and current waiting.
  */
 #ifdef USE_THREAD_DEBUG
 
+/** How long to wait before lock attempt is a failure. */
+#define CHECK_LOCK_TIMEOUT 30 /* seconds */
+/** How long to wait before join attempt is a failure. */
+#define CHECK_JOIN_TIMEOUT 120 /* seconds */
+
 /** if key has been created */
 static int key_created = 0;
 /** we hide the thread debug info with this key. */
 static ub_thread_key_t thr_debug_key;
 /** the list of threads, so all threads can be examined. NULL if unused. */
 static struct thr_check* thread_infos[THRDEBUG_MAX_THREADS];
+/** do we check locking order */
+int check_locking_order = 0;
+/** the pid of this runset, reasonably unique. */
+static pid_t check_lock_pid;
+
+/** print all possible debug info on the state of the system */
+static void total_debug_info();
 
 /** print pretty lock error and exit */
 static void lock_error(struct checked_lock* lock, 
@@ -66,13 +77,16 @@ static void lock_error(struct checked_lock* lock,
 	log_err("lock error (description follows)");
 	log_err("Created at %s %s:%d", lock->create_func, 
 		lock->create_file, lock->create_line);
-	log_err("Previously %s %s:%d", lock->holder_func, 
-		lock->holder_file, lock->holder_line);
+	if(lock->holder_func && lock->holder_file)
+		log_err("Previously %s %s:%d", lock->holder_func, 
+			lock->holder_file, lock->holder_line);
 	log_err("At %s %s:%d", func, file, line);
 	log_err("Error for %s lock: %s",
 		(lock->type==check_lock_mutex)?"mutex": (
 		(lock->type==check_lock_spinlock)?"spinlock": (
 		(lock->type==check_lock_rwlock)?"rwlock": "badtype")), err);
+	log_err("complete status display:");
+	total_debug_info();
 	fatal_exit("bailing out");
 }
 
@@ -132,6 +146,31 @@ lock_protect(void *p, void* area, size_t size)
 	LOCKRET(pthread_mutex_unlock(&lock->lock));
 }
 
+/** remove protected region */
+void
+lock_unprotect(void* mangled, void* area)
+{
+	struct checked_lock* lock = *(struct checked_lock**)mangled;
+	struct protected_area* p, **prevp;
+	if(!lock) 
+		return;
+	acquire_locklock(lock, __func__, __FILE__, __LINE__);
+	p = lock->prot;
+	prevp = &lock->prot;
+	while(p) {
+		if(p->region == area) {
+			*prevp = p->next;
+			free(p->hold);
+			free(p);
+			LOCKRET(pthread_mutex_unlock(&lock->lock));
+			return;
+		}
+		prevp = &p->next;
+		p = p->next;
+	}
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+}
+
 /** 
  * Check protected memory region. Memory compare. Exit on error. 
  * @param lock: which lock to check.
@@ -155,7 +194,7 @@ prot_check(struct checked_lock* lock,
 	}
 }
 
-/** Copy protected memory region. */
+/** Copy protected memory region */
 static void 
 prot_store(struct checked_lock* lock)
 {
@@ -166,6 +205,62 @@ prot_store(struct checked_lock* lock)
 	}
 }
 
+/** get memory held by lock */
+size_t 
+lock_get_mem(void* pp)
+{
+	size_t s;
+	struct checked_lock* lock = *(struct checked_lock**)pp;
+	struct protected_area* p;
+	s = sizeof(struct checked_lock);
+	acquire_locklock(lock, __func__, __FILE__, __LINE__);
+	for(p = lock->prot; p; p = p->next) {
+		s += sizeof(struct protected_area);
+		s += p->size;
+	}
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+	return s;
+}
+
+/** write lock trace info to file, while you hold those locks */
+static void
+ordercheck_locklock(struct thr_check* thr, struct checked_lock* lock)
+{
+	int info[4];
+	if(!check_locking_order) return;
+	if(!thr->holding_first) return; /* no older lock, no info */
+	/* write: <lock id held> <lock id new> <file> <line> */
+	info[0] = thr->holding_first->create_thread;
+	info[1] = thr->holding_first->create_instance;
+	info[2] = lock->create_thread;
+	info[3] = lock->create_instance;
+	if(fwrite(info, 4*sizeof(int), 1, thr->order_info) != 1 ||
+		fwrite(lock->holder_file, strlen(lock->holder_file)+1, 1, 
+		thr->order_info) != 1 ||
+		fwrite(&lock->holder_line, sizeof(int), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
+
+/** write ordercheck lock creation details to file */
+static void 
+ordercheck_lockcreate(struct thr_check* thr, struct checked_lock* lock)
+{
+	/* write: <ffff = create> <lock id> <file> <line> */
+	int cmd = -1;
+	if(!check_locking_order) return;
+
+	if( fwrite(&cmd, sizeof(int), 1, thr->order_info) != 1 ||
+		fwrite(&lock->create_thread, sizeof(int), 1, 
+			thr->order_info) != 1 ||
+		fwrite(&lock->create_instance, sizeof(int), 1, 
+			thr->order_info) != 1 ||
+		fwrite(lock->create_file, strlen(lock->create_file)+1, 1, 
+			thr->order_info) != 1 ||
+		fwrite(&lock->create_line, sizeof(int), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
 
 /** alloc struct, init lock empty */
 void 
@@ -174,13 +269,21 @@ checklock_init(enum check_lock_type type, struct checked_lock** lock,
 {
 	struct checked_lock* e = (struct checked_lock*)calloc(1, 
 		sizeof(struct checked_lock));
+	struct thr_check *thr = (struct thr_check*)pthread_getspecific(
+		thr_debug_key);
 	if(!e)
 		fatal_exit("%s %s %d: out of memory", func, file, line);
+	if(!thr)
+		fatal_exit("%s %s %d: lock_init no thread info", func, file,
+			line);
 	*lock = e;
 	e->type = type;
 	e->create_func = func;
 	e->create_file = file;
 	e->create_line = line;
+	e->create_thread = thr->num;
+	e->create_instance = thr->locks_created++;
+	ordercheck_lockcreate(thr, e);
 	LOCKRET(pthread_mutex_init(&e->lock, NULL));
 	switch(e->type) {
 		case check_lock_mutex:
@@ -228,7 +331,7 @@ void
 checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
         const char* func, const char* file, int line)
 {
-	const size_t contention_interest = 10;
+	const size_t contention_interest = 1; /* promille contented locks */
 	struct checked_lock* e;
 	if(!lock) 
 		return;
@@ -247,11 +350,14 @@ checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
 	*lock = NULL; /* use after free will fail */
 	LOCKRET(pthread_mutex_unlock(&e->lock));
 
-	/* contention */
-	if(e->contention_count > contention_interest) {
-		log_info("lock created %s %s %d has contention %u",
+	/* contention, look at fraction in trouble. */
+	if(e->history_count > 1 &&
+	   1000*e->contention_count/e->history_count > contention_interest) {
+		log_info("lock created %s %s %d has contention %u of %u (%d%%)",
 			e->create_func, e->create_file, e->create_line,
-			(unsigned int)e->contention_count);
+			(unsigned int)e->contention_count, 
+			(unsigned int)e->history_count,
+			(int)(100*e->contention_count/e->history_count));
 	}
 
 	/* delete it */
@@ -276,7 +382,7 @@ checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
 	free(e);
 }
 
-/** finish acquiring lock, shared between _(rd|wr||)lock() routines. */
+/** finish acquiring lock, shared between _(rd|wr||)lock() routines */
 static void 
 finish_acquire_lock(struct thr_check* thr, struct checked_lock* lock,
         const char* func, const char* file, int line)
@@ -288,6 +394,7 @@ finish_acquire_lock(struct thr_check* thr, struct checked_lock* lock,
 	lock->holder_func = func;
 	lock->holder_file = file;
 	lock->holder_line = line;
+	ordercheck_locklock(thr, lock);
 	
 	/* insert in thread lock list, as first */
 	lock->prev_held_lock[thr->num] = NULL;
@@ -355,6 +462,7 @@ checklock_lockit(enum check_lock_type type, struct checked_lock* lock,
 
 	acquire_locklock(lock, func, file, line);
 	lock->contention_count += contend;
+	lock->history_count++;
 	if(exclusive && lock->hold_count > 0)
 		lock_error(lock, func, file, line, "got nonexclusive lock");
 	if(type==check_lock_rwlock && getwr && lock->writeholder)
@@ -425,6 +533,7 @@ static int timed_spinlock(void* arg, struct timespec* to)
 #ifndef S_SPLINT_S
 		if(time(NULL) >= to->tv_sec)
 			return ETIMEDOUT;
+		usleep(1000); /* in 1/1000000s of a second */
 #endif
 	}
 	return err;
@@ -520,7 +629,27 @@ checklock_unlock(enum check_lock_type type, struct checked_lock* lock,
 	}
 }
 
-/** checklock thread main, Inits thread structure. */
+/** open order info debug file, thr->num must be valid */
+static void 
+open_lockorder(struct thr_check* thr)
+{
+	char buf[24];
+	time_t t;
+	snprintf(buf, sizeof(buf), "ublocktrace.%d", thr->num);
+	thr->order_info = fopen(buf, "w");
+	if(!thr->order_info)
+		fatal_exit("could not open %s: %s", buf, strerror(errno));
+	thr->locks_created = 0;
+	t = time(NULL);
+	/* write: <time_stamp> <runpid> <thread_num> */
+	if(fwrite(&t, sizeof(t), 1, thr->order_info) != 1 ||
+		fwrite(&thr->num, sizeof(thr->num), 1, thr->order_info) != 1 || 
+		fwrite(&check_lock_pid, sizeof(check_lock_pid), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
+
+/** checklock thread main, Inits thread structure */
 static void* checklock_main(void* arg)
 {
 	struct thr_check* thr = (struct thr_check*)arg; 
@@ -532,8 +661,12 @@ static void* checklock_main(void* arg)
 	log_assert(thread_infos[thr->num] == NULL);
 	thread_infos[thr->num] = thr;
 	LOCKRET(pthread_setspecific(thr_debug_key, thr));
+	if(check_locking_order)
+		open_lockorder(thr);
 	ret = thr->func(thr->arg);
 	thread_infos[thr->num] = NULL;
+	if(check_locking_order)
+		fclose(thr->order_info);
 	free(thr);
 	return ret;
 }
@@ -547,9 +680,12 @@ void checklock_start()
 		if(!thisthr)
 			fatal_exit("thrcreate: out of memory");
 		key_created = 1;
+		check_lock_pid = getpid();
 		LOCKRET(pthread_key_create(&thr_debug_key, NULL));
 		LOCKRET(pthread_setspecific(thr_debug_key, thisthr));
 		thread_infos[0] = thisthr;
+		if(check_locking_order)
+			open_lockorder(thisthr);
 	}
 }
 
@@ -558,6 +694,8 @@ void checklock_stop()
 {
 	if(key_created) {
 		int i;
+		if(check_locking_order)
+			fclose(thread_infos[0]->order_info);
 		free(thread_infos[0]);
 		thread_infos[0] = NULL;
 		for(i = 0; i < THRDEBUG_MAX_THREADS; i++)
@@ -584,13 +722,102 @@ checklock_thrcreate(pthread_t* id, void* (*func)(void*), void* arg)
 	LOCKRET(pthread_create(id, NULL, checklock_main, thr));
 }
 
-/** signal handler for join timeout, Exits. */
+/** count number of thread infos */
+static int
+count_thread_infos()
+{
+	int cnt = 0;
+	int i;
+	for(i=0; i<THRDEBUG_MAX_THREADS; i++)
+		if(thread_infos[i])
+			cnt++;
+	return cnt;
+}
+
+/** print lots of info on a lock */
+static void
+lock_debug_info(struct checked_lock* lock)
+{
+	if(!lock) return;
+	log_info("+++ Lock %x, %d %d create %s %s %d", (int)lock, 
+		lock->create_thread, lock->create_instance, 
+		lock->create_func, lock->create_file, lock->create_line);
+	log_info("lock type: %s",
+		(lock->type==check_lock_mutex)?"mutex": (
+		(lock->type==check_lock_spinlock)?"spinlock": (
+		(lock->type==check_lock_rwlock)?"rwlock": "badtype")));
+	log_info("lock contention %u, history:%u, hold:%d, wait:%d", 
+		(unsigned)lock->contention_count, (unsigned)lock->history_count,
+		lock->hold_count, lock->wait_count);
+	log_info("last touch %s %s %d", lock->holder_func, lock->holder_file,
+		lock->holder_line);
+	log_info("holder thread %d, writeholder thread %d",
+		lock->holder?lock->holder->num:-1,
+		lock->writeholder?lock->writeholder->num:-1);
+}
+
+/** print debug locks held by a thread */
+static void
+held_debug_info(struct thr_check* thr, struct checked_lock* lock)
+{
+	if(!lock) return;
+	lock_debug_info(lock);
+	held_debug_info(thr, lock->next_held_lock[thr->num]);
+}
+
+/** print debug info for a thread */
+static void
+thread_debug_info(struct thr_check* thr)
+{
+	struct checked_lock* w = NULL;
+	struct checked_lock* f = NULL;
+	struct checked_lock* l = NULL;
+	if(!thr) return;
+	log_info("pthread id is %x", (int)thr->id);
+	log_info("thread func is %x", (int)thr->func);
+	log_info("thread arg is %x (%d)", (int)thr->arg, 
+		(thr->arg?*(int*)thr->arg:0));
+	log_info("thread num is %d", thr->num);
+	log_info("locks created %d", thr->locks_created);
+	log_info("open file for lockinfo: %s", 
+		thr->order_info?"yes, flushing":"no");
+	fflush(thr->order_info);
+	w = thr->waiting;
+	f = thr->holding_first;
+	l = thr->holding_last;
+	log_info("thread waiting for a lock: %s %x", w?"yes":"no", (int)w);
+	lock_debug_info(w);
+	log_info("thread holding first: %s, last: %s", f?"yes":"no", 
+		l?"yes":"no");
+	held_debug_info(thr, f);
+}
+
+static void
+total_debug_info()
+{
+	int i;
+	log_info("checklocks: supervising %d threads.",
+		count_thread_infos());
+	if(!key_created) {
+		log_info("No thread debug key created yet");
+	}
+	for(i=0; i<THRDEBUG_MAX_THREADS; i++) {
+		if(thread_infos[i]) {
+			log_info("*** Thread %d information: ***", i);
+			thread_debug_info(thread_infos[i]);
+		}
+	}
+}
+
+/** signal handler for join timeout, Exits */
 static RETSIGTYPE joinalarm(int ATTR_UNUSED(sig))
 {
+	log_err("join thread timeout. hangup or deadlock. Info follows.");
+	total_debug_info();
 	fatal_exit("join thread timeout. hangup or deadlock.");
 }
 
-/** wait for thread with a timeout. */
+/** wait for thread with a timeout */
 void 
 checklock_thrjoin(pthread_t thread)
 {

testcode/checklocks.h

@@ -71,12 +71,10 @@
 /******************* THREAD DEBUG ************************/
 #include <pthread.h>
 
-/** How long to wait before lock attempt is a failure. */
-#define CHECK_LOCK_TIMEOUT 5 /* seconds */
-/** How long to wait before join attempt is a failure. */
-#define CHECK_JOIN_TIMEOUT 120 /* seconds */
 /** How many threads to allocate for */
 #define THRDEBUG_MAX_THREADS 32 /* threads */
+/** do we check locking order */
+extern int check_locking_order;
 
 /**
  * Protection memory area.
@@ -106,6 +104,10 @@ struct thr_check {
 	void* arg;
 	/** number of thread in list structure */
 	int num;
+	/** instance number - how many locks have been created by thread */
+	int locks_created;
+	/** file to write locking order information to */
+	FILE* order_info;
 	/** 
 	 * List of locks that this thread is holding, double
 	 * linked list. The first element is the most recent lock acquired.
@@ -128,8 +130,12 @@ struct checked_lock {
 	const char* create_func, *create_file;
 	/** where was this lock created */
 	int create_line;
+	/** unique instance identifier */
+	int create_thread, create_instance;
 	/** contention count */
 	size_t contention_count;
+	/** number of times locked, ever */
+	size_t history_count;
 	/** hold count (how many threads are holding this lock) */
 	int hold_count;
 	/** how many threads are waiting for this lock */
@@ -182,6 +188,21 @@ struct checked_lock {
  */
 void lock_protect(void* lock, void* area, size_t size);
 
+/**
+ * Remove protected area from lock.
+ * No need to call this when deleting the lock.
+ * @param lock: the lock, any type, (struct checked_lock**).
+ * @param area: pointer to memory.
+ */
+void lock_unprotect(void* lock, void* area);
+
+/**
+ * Get memory associated with a checked lock
+ * @param lock: the checked lock, any type. (struct checked_lock**).
+ * @return: in bytes, including protected areas.
+ */
+size_t lock_get_mem(void* lock);
+
 /**
  * Initialise checklock. Sets up internal debug structures.
  */

testcode/do-tests.sh

@@ -2,8 +2,10 @@
 
 NEED_SPLINT='00-lint.tpkg'
 NEED_DOXYGEN='01-doc.tpkg'
+NEED_LDNS_TESTNS='fwd_no_edns.tpkg fwd_tcp_tc.tpkg fwd_tcp.tpkg fwd_three_service.tpkg fwd_three.tpkg fwd_ttlexpire.tpkg fwd_udp.tpkg'
 
 cd testdata;
+sh ../testcode/mini_tpkg.sh clean
 for test in `ls *.tpkg`; do
 	SKIP=0
 	if echo $NEED_SPLINT | grep $test >/dev/null; then
@@ -16,10 +18,16 @@ for test in `ls *.tpkg`; do
 			SKIP=1;
 		fi
 	fi
+	if echo $NEED_LDNS_TESTNS | grep $test >/dev/null; then
+		if test ! -x "`which ldns-testns`"; then
+			SKIP=1;
+		fi
+	fi
 	if test $SKIP -eq 0; then
 		echo $test
-		tpkg -a ../.. exe $test
+		sh ../testcode/mini_tpkg.sh -a ../.. exe $test
 	else
 		echo "skip $test"
 	fi
 done
+sh ../testcode/mini_tpkg.sh report

testcode/fake_event.c

@@ -47,6 +47,10 @@
 #include "config.h"
 #include "testcode/fake_event.h"
 #include "util/netevent.h"
+#include "util/net_help.h"
+#include "util/data/msgparse.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgencode.h"
 #include "services/listen_dnsport.h"
 #include "services/outside_network.h"
 #include "testcode/replay.h"
@@ -69,7 +73,7 @@ fake_event_cleanup()
 	saved_scenario = NULL;
 }
 
-/** helper function that logs a ldns_pkt packet to logfile. */
+/** helper function that logs a ldns_pkt packet to logfile */
 static void
 log_pkt(const char* desc, ldns_pkt* pkt)
 {
@@ -139,6 +143,10 @@ pending_matches_current(struct replay_runtime* runtime,
 		return 0;
 	/* see if any of the pending queries matches */
 	for(p = runtime->pending_list; p; p = p->next) {
+		if(runtime->now->addrlen != 0 &&
+			sockaddr_cmp(&p->addr, p->addrlen, &runtime->now->addr,
+			runtime->now->addrlen) != 0)
+			continue;
 		if((e=find_match(runtime->now->match, p->pkt, p->transport))) {
 			*entry = e;
 			*pend = p;
@@ -163,10 +171,15 @@ pending_find_match(struct replay_runtime* runtime, struct entry** entry,
 	struct replay_range* p = runtime->scenario->range_list;
 	while(p) {
 		if(p->start_step <= timenow && timenow <= p->end_step &&
+		  (p->addrlen == 0 || sockaddr_cmp(&p->addr, p->addrlen,
+		  	&pend->addr, pend->addrlen) == 0) &&
 		  (*entry = find_match(p->match, pend->pkt, pend->transport))) {
 			log_info("matched query time %d in range [%d, %d] "
 				"with entry line %d", timenow, 
 				p->start_step, p->end_step, (*entry)->lineno);
+			if(p->addrlen != 0)
+				log_addr("matched ip", &p->addr, p->addrlen);
+			log_pkt("matched pkt: ", (*entry)->reply_list->reply);
 			return 1;
 		}
 		p = p->next_range;
@@ -188,6 +201,7 @@ pending_matches_range(struct replay_runtime* runtime,
 	struct fake_pending* p = runtime->pending_list;
 	/* slow, O(N*N), but it works as advertised with weird matching */
 	while(p) {
+		log_info("check of pending");
 		if(pending_find_match(runtime, entry, p)) {
 			*pend = p;
 			return 1;
@@ -261,6 +275,10 @@ answer_callback_from_entry(struct replay_runtime* runtime,
         struct entry* entry, struct fake_pending* pend)
 {
 	struct comm_point c;
+	struct comm_reply repinfo;
+	void* cb_arg = pend->cb_arg;
+	comm_point_callback_t* cb = pend->callback;
+
 	memset(&c, 0, sizeof(c));
 	c.fd = -1;
 	c.buffer = ldns_buffer_new(runtime->bufsize);
@@ -268,11 +286,15 @@ answer_callback_from_entry(struct replay_runtime* runtime,
 	if(pend->transport == transport_tcp)
 		c.type = comm_tcp;
 	fill_buffer_with_reply(c.buffer, entry, pend->pkt);
-	if((*pend->callback)(&c, pend->cb_arg, NETEVENT_NOERROR, NULL)) {
+	repinfo.c = &c;
+	repinfo.addrlen = pend->addrlen;
+	memcpy(&repinfo.addr, &pend->addr, pend->addrlen);
+	if(!pend->serviced)
+		pending_list_delete(runtime, pend);
+	if((*cb)(&c, cb_arg, NETEVENT_NOERROR, &repinfo)) {
 		fatal_exit("testbound: unexpected: callback returned 1");
 	}
 	ldns_buffer_free(c.buffer);
-	pending_list_delete(runtime, pend);
 }
 
 /** Check the now moment answer check event */
@@ -280,26 +302,33 @@ static void
 answer_check_it(struct replay_runtime* runtime)
 {
 	struct replay_answer* ans = runtime->answer_list, 
-		**prev = &runtime->answer_list;
+		*prev = NULL;
 	log_assert(runtime && runtime->now && 
 		runtime->now->evt_type == repevt_front_reply);
 	while(ans) {
 		enum transport_type tr = transport_tcp;
 		if(ans->repinfo.c->type == comm_udp)
 			tr = transport_udp;
-		if(find_match(runtime->now->match, ans->pkt, tr)) {
+		if((runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&ans->repinfo.addr, ans->repinfo.addrlen) == 0) &&
+			find_match(runtime->now->match, ans->pkt, tr)) {
 			struct replay_answer *n = ans->next;
 			log_info("testbound matched event entry from line %d",
 				runtime->now->match->lineno);
 			log_info("testbound: do STEP %d %s", 
 				runtime->now->time_step,
 				repevt_string(runtime->now->evt_type));
-			*prev = ans->next;
+			if(prev)
+				prev->next = ans->next;
+			else 	runtime->answer_list = ans->next;
+			if(!ans->next)
+				runtime->answer_last = prev;
 			delete_replay_answer(ans);
 			ans = n;
 			return;
 		} else {
-			prev = &ans->next;
+			prev = ans;
 			ans = ans->next;
 		}
 	}
@@ -318,10 +347,16 @@ fake_front_query(struct replay_runtime* runtime, struct replay_moment *todo)
 	memset(&repinfo, 0, sizeof(repinfo));
 	repinfo.c = (struct comm_point*)calloc(1, sizeof(struct comm_point));
 	repinfo.addrlen = (socklen_t)sizeof(struct sockaddr_in);
+	if(todo->addrlen != 0) {
+		repinfo.addrlen = todo->addrlen;
+		memcpy(&repinfo.addr, &todo->addr, todo->addrlen);
+	}
 	repinfo.c->fd = -1;
 	repinfo.c->ev = (struct internal_event*)runtime;
 	repinfo.c->buffer = ldns_buffer_new(runtime->bufsize);
-	repinfo.c->type = comm_udp;
+	if(todo->match->match_transport == transport_tcp)
+		repinfo.c->type = comm_tcp;
+	else	repinfo.c->type = comm_udp;
 	fill_buffer_with_reply(repinfo.c->buffer, todo->match, NULL);
 	log_info("testbound: incoming QUERY");
 	/* call the callback for incoming queries */
@@ -342,6 +377,7 @@ fake_pending_callback(struct replay_runtime* runtime,
 	struct replay_moment* todo, int error)
 {
 	struct fake_pending* p = runtime->pending_list;
+	struct comm_reply repinfo;
 	struct comm_point c;
 	memset(&c, 0, sizeof(c));
 	if(!p) fatal_exit("No pending queries.");
@@ -352,8 +388,11 @@ fake_pending_callback(struct replay_runtime* runtime,
 		c.type = comm_tcp;
 	if(todo->evt_type == repevt_back_reply && todo->match) {
 		fill_buffer_with_reply(c.buffer, todo->match, p->pkt);
-	}	
-	if((*p->callback)(&c, p->cb_arg, error, NULL)) {
+	}
+	repinfo.c = &c;
+	repinfo.addrlen = p->addrlen;
+	memcpy(&repinfo.addr, &p->addr, p->addrlen);
+	if((*p->callback)(&c, p->cb_arg, error, &repinfo)) {
 		fatal_exit("unexpected: pending callback returned 1");
 	}
 	/* delete the pending item. */
@@ -468,6 +507,10 @@ run_scenario(struct replay_runtime* runtime)
 	} while(runtime->now);
 
 	if(runtime->pending_list) {
+		struct fake_pending* p;
+		log_err("testbound: there are still messages pending.");
+		for(p = runtime->pending_list; p; p=p->next)
+			log_pkt("pending msg", p->pkt);
 		fatal_exit("testbound: there are still messages pending.");
 	}
 	if(runtime->answer_list) {
@@ -480,7 +523,8 @@ run_scenario(struct replay_runtime* runtime)
 
 struct listen_dnsport* 
 listen_create(struct comm_base* base, struct listen_port* ATTR_UNUSED(ports),
-	size_t bufsize, comm_point_callback_t* cb, void* cb_arg)
+	size_t bufsize, int ATTR_UNUSED(tcp_accept_count),
+	comm_point_callback_t* cb, void* cb_arg)
 {
 	struct replay_runtime* runtime = (struct replay_runtime*)base;
 	struct listen_dnsport* l= calloc(1, sizeof(struct listen_dnsport));
@@ -605,6 +649,15 @@ comm_point_send_reply(struct comm_reply* repinfo)
 	log_pkt("reply pkt: ", ans->pkt);
 }
 
+void 
+comm_point_send_reply_iov(struct comm_reply* repinfo, struct iovec* iov,
+        size_t iovlen)
+{
+	/* skip tcp len at [0]. */
+	write_iov_buffer(repinfo->c->buffer, iov+1, iovlen-1);
+	comm_point_send_reply(repinfo);
+}
+
 void 
 comm_point_drop_reply(struct comm_reply* repinfo)
 {
@@ -619,7 +672,9 @@ struct outside_network*
 outside_network_create(struct comm_base* base, size_t bufsize, 
 	size_t ATTR_UNUSED(num_ports), char** ATTR_UNUSED(ifs), 
 	int ATTR_UNUSED(num_ifs), int ATTR_UNUSED(do_ip4), 
-	int ATTR_UNUSED(do_ip6), int ATTR_UNUSED(port_base))
+	int ATTR_UNUSED(do_ip6), int ATTR_UNUSED(port_base),
+	size_t ATTR_UNUSED(num_tcp), struct infra_cache* ATTR_UNUSED(infra),
+	struct ub_randstate* ATTR_UNUSED(rnd))
 {
 	struct outside_network* outnet =  calloc(1, 
 		sizeof(struct outside_network));
@@ -641,7 +696,7 @@ outside_network_delete(struct outside_network* outnet)
 	free(outnet);
 }
 
-void 
+struct pending* 
 pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
 	comm_point_callback_t* callback, void* callback_arg,
@@ -661,9 +716,11 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	pend->addrlen = addrlen;
 	pend->callback = callback;
 	pend->cb_arg = callback_arg;
-	pend->timeout = timeout;
+	pend->timeout = timeout/1000;
 	pend->transport = transport_udp;
 	pend->pkt = NULL;
+	pend->serviced = 0;
+	pend->runtime = runtime;
 	status = ldns_buffer2pkt_wire(&pend->pkt, packet);
 	if(status != LDNS_STATUS_OK) {
 		log_err("ldns error parsing udp output packet: %s",
@@ -674,6 +731,9 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 
 	/* see if it matches the current moment */
 	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
 		find_match(runtime->now->match, pend->pkt, pend->transport)) {
 		log_info("testbound: matched pending to event. "
 			"advance time between events.");
@@ -686,6 +746,165 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	/* add to list */
 	pend->next = runtime->pending_list;
 	runtime->pending_list = pend;
+	return (struct pending*)pend;
+}
+
+struct waiting_tcp* 
+pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
+	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
+	comm_point_callback_t* callback, void* callback_arg,
+	struct ub_randstate* ATTR_UNUSED(rnd))
+{
+	struct replay_runtime* runtime = (struct replay_runtime*)outnet->base;
+	struct fake_pending* pend = (struct fake_pending*)calloc(1,
+		sizeof(struct fake_pending));
+	ldns_status status;
+	log_assert(pend);
+	pend->buffer = ldns_buffer_new(ldns_buffer_capacity(packet));
+	log_assert(pend->buffer);
+	ldns_buffer_write(pend->buffer, ldns_buffer_begin(packet),
+		ldns_buffer_limit(packet));
+	ldns_buffer_flip(pend->buffer);
+	memcpy(&pend->addr, addr, addrlen);
+	pend->addrlen = addrlen;
+	pend->callback = callback;
+	pend->cb_arg = callback_arg;
+	pend->timeout = timeout;
+	pend->transport = transport_tcp;
+	pend->pkt = NULL;
+	pend->runtime = runtime;
+	pend->serviced = 0;
+	status = ldns_buffer2pkt_wire(&pend->pkt, packet);
+	if(status != LDNS_STATUS_OK) {
+		log_err("ldns error parsing tcp output packet: %s",
+			ldns_get_errorstr_by_id(status));
+		fatal_exit("Sending unparseable DNS packets to servers!");
+	}
+	log_pkt("pending tcp pkt: ", pend->pkt);
+
+	/* see if it matches the current moment */
+	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
+		find_match(runtime->now->match, pend->pkt, pend->transport)) {
+		log_info("testbound: matched pending to event. "
+			"advance time between events.");
+		log_info("testbound: do STEP %d %s", runtime->now->time_step,
+			repevt_string(runtime->now->evt_type));
+		advance_moment(runtime);
+		/* still create the pending, because we need it to callback */
+	} 
+	log_info("testbound: created fake pending");
+	/* add to list */
+	pend->next = runtime->pending_list;
+	runtime->pending_list = pend;
+	return (struct waiting_tcp*)pend;
+}
+
+struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
+        uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint16_t flags, int dnssec, struct sockaddr_storage* addr,
+	socklen_t addrlen, comm_point_callback_t* callback,
+	void* callback_arg, ldns_buffer* ATTR_UNUSED(buff),
+	int (*arg_compare)(void*,void*))
+{
+	struct replay_runtime* runtime = (struct replay_runtime*)outnet->base;
+	struct fake_pending* pend = (struct fake_pending*)calloc(1,
+		sizeof(struct fake_pending));
+	ldns_status status;
+	(void)arg_compare;
+	log_assert(pend);
+	log_nametypeclass(VERB_OPS, "pending serviced query", 
+		qname, qtype, qclass);
+	verbose(VERB_OPS, "pending serviced query flags%s%s%s%s", 
+		(flags&BIT_RD)?" RD":"", (flags&BIT_CD)?" CD":"",
+		(flags&~(BIT_RD|BIT_CD))?" MORE":"", (dnssec)?" DO":"");
+
+	/* create packet with EDNS */
+	pend->buffer = ldns_buffer_new(512);
+	log_assert(pend->buffer);
+	ldns_buffer_write_u16(pend->buffer, 0); /* id */
+	ldns_buffer_write_u16(pend->buffer, flags);
+	ldns_buffer_write_u16(pend->buffer, 1); /* qdcount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* ancount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* nscount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* arcount */
+	ldns_buffer_write(pend->buffer, qname, qnamelen);
+	ldns_buffer_write_u16(pend->buffer, qtype);
+	ldns_buffer_write_u16(pend->buffer, qclass);
+	ldns_buffer_flip(pend->buffer);
+	if(1) {
+		/* add edns */
+		struct edns_data edns;
+		edns.edns_present = 1;
+		edns.ext_rcode = 0;
+		edns.edns_version = EDNS_ADVERTISED_VERSION;
+		edns.udp_size = EDNS_ADVERTISED_SIZE;
+		edns.bits = 0;
+		if(dnssec)
+			edns.bits = EDNS_DO;
+		attach_edns_record(pend->buffer, &edns);
+	}
+	memcpy(&pend->addr, addr, addrlen);
+	pend->addrlen = addrlen;
+	pend->callback = callback;
+	pend->cb_arg = callback_arg;
+	pend->timeout = UDP_QUERY_TIMEOUT;
+	pend->transport = transport_udp; /* pretend UDP */
+	pend->pkt = NULL;
+	pend->runtime = runtime;
+	pend->serviced = 1;
+	status = ldns_buffer2pkt_wire(&pend->pkt, pend->buffer);
+	if(status != LDNS_STATUS_OK) {
+		log_err("ldns error parsing serviced output packet: %s",
+			ldns_get_errorstr_by_id(status));
+		fatal_exit("internal error");
+	}
+	/*log_pkt("pending serviced query: ", pend->pkt);*/
+
+	/* see if it matches the current moment */
+	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
+		find_match(runtime->now->match, pend->pkt, pend->transport)) {
+		log_info("testbound: matched pending to event. "
+			"advance time between events.");
+		log_info("testbound: do STEP %d %s", runtime->now->time_step,
+			repevt_string(runtime->now->evt_type));
+		advance_moment(runtime);
+		/* still create the pending, because we need it to callback */
+	} 
+	log_info("testbound: created fake pending");
+	/* add to list */
+	pend->next = runtime->pending_list;
+	runtime->pending_list = pend;
+	return (struct serviced_query*)pend;
+}
+
+void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
+{
+	struct fake_pending* pend = (struct fake_pending*)sq;
+	struct replay_runtime* runtime = pend->runtime;
+	/* delete from the list */
+	struct fake_pending* p = runtime->pending_list, *prev=NULL;
+	while(p) {
+		if(p == pend) {
+			log_assert(p->cb_arg == cb_arg);
+			log_info("serviced pending delete");
+			if(prev)
+				prev->next = p->next;
+			else 	runtime->pending_list = p->next;
+			ldns_buffer_free(p->buffer);
+			ldns_pkt_free(p->pkt);
+			free(p);
+			return;
+		}
+		prev = p;
+		p = p->next;
+	}
+	log_info("double delete of pending serviced query");
 }
 
 struct listen_port* listening_ports_open(struct config_file* ATTR_UNUSED(cfg))
@@ -720,4 +939,24 @@ void comm_point_delete(struct comm_point* c)
 	free(c);
 }
 
+size_t listen_get_mem(struct listen_dnsport* ATTR_UNUSED(listen))
+{
+	return 0;
+}
+
+size_t outnet_get_mem(struct outside_network* ATTR_UNUSED(outnet))
+{
+	return 0;
+}
+
+size_t comm_point_get_mem(struct comm_point* ATTR_UNUSED(c))
+{
+	return 0;
+}
+
+size_t serviced_get_mem(struct serviced_query* ATTR_UNUSED(c))
+{
+	return 0;
+}
+
 /*********** End of Dummy routines ***********/

testcode/ldns-testpkts.c

@@ -31,7 +31,7 @@ struct sockaddr_storage;
 /** string to show in warnings and errors */
 static const char* prog_name = "ldns-testpkts";
 
-/** logging routine, provided by caller. */
+/** logging routine, provided by caller */
 void verbose(int lvl, const char* msg, ...);
 
 /** print error and exit */
@@ -68,7 +68,7 @@ static bool str_keyword(const char** str, const char* keyword)
 	if(strncmp(*str, keyword, len) != 0)
 		return false;
 	*str += len;
-	while(isspace(**str))
+	while(isspace((int)**str))
 		(*str)++;
 	return true;
 }
@@ -106,6 +106,10 @@ static void matchline(const char* line, struct entry* e)
 			e->match_qname = true;
 		} else if(str_keyword(&parse, "all")) {
 			e->match_all = true;
+		} else if(str_keyword(&parse, "ttl")) {
+			e->match_ttl = true;
+		} else if(str_keyword(&parse, "DO")) {
+			e->match_do = true;
 		} else if(str_keyword(&parse, "UDP")) {
 			e->match_transport = transport_udp;
 		} else if(str_keyword(&parse, "TCP")) {
@@ -116,7 +120,7 @@ static void matchline(const char* line, struct entry* e)
 				error("expected = or : in MATCH: %s", line);
 			parse++;
 			e->ixfr_soa_serial = (uint32_t)strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else {
 			error("could not parse MATCH: '%s'", parse);
@@ -153,6 +157,8 @@ static void replyline(const char* line, ldns_pkt *reply)
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_NXDOMAIN);
 		} else if(str_keyword(&parse, "NOTIMPL")) {
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_NOTIMPL);
+		} else if(str_keyword(&parse, "REFUSED")) {
+			ldns_pkt_set_rcode(reply, LDNS_RCODE_REFUSED);
 		} else if(str_keyword(&parse, "YXDOMAIN")) {
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_YXDOMAIN);
 		} else if(str_keyword(&parse, "YXRRSET")) {
@@ -178,6 +184,8 @@ static void replyline(const char* line, ldns_pkt *reply)
 			ldns_pkt_set_ra(reply, true);
 		} else if(str_keyword(&parse, "AD")) {
 			ldns_pkt_set_ad(reply, true);
+		} else if(str_keyword(&parse, "DO")) {
+			ldns_pkt_set_edns_do(reply, true);
 		} else {
 			error("could not parse REPLY: '%s'", parse);
 		}
@@ -194,13 +202,15 @@ static void adjustline(const char* line, struct entry* e,
 			return;
 		if(str_keyword(&parse, "copy_id")) {
 			e->copy_id = true;
+		} else if(str_keyword(&parse, "copy_query")) {
+			e->copy_query = true;
 		} else if(str_keyword(&parse, "sleep=")) {
 			e->sleeptime = (unsigned int) strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else if(str_keyword(&parse, "packet_sleep=")) {
 			pkt->packet_sleep = (unsigned int) strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else {
 			error("could not parse ADJUST: '%s'", parse);
@@ -217,11 +227,14 @@ static struct entry* new_entry()
 	e->match_qtype = false;
 	e->match_qname = false;
 	e->match_all = false;
+	e->match_ttl = false;
+	e->match_do = false;
 	e->match_serial = false;
 	e->ixfr_soa_serial = 0;
 	e->match_transport = transport_any;
 	e->reply_list = NULL;
 	e->copy_id = false;
+	e->copy_query = false;
 	e->sleeptime = 0;
 	e->next = NULL;
 	return e;
@@ -377,7 +390,7 @@ get_origin(const char* name, int lineno, ldns_rdf** origin, char* parse)
 	*origin = NULL;
 
 	end=parse;
-	while(!isspace(*end) && !isendline(*end))
+	while(!isspace((int)*end) && !isendline(*end))
 		end++;
 	store = *end;
 	*end = 0;
@@ -391,7 +404,7 @@ get_origin(const char* name, int lineno, ldns_rdf** origin, char* parse)
 
 /* Reads one entry from file. Returns entry or NULL on error. */
 struct entry*
-read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl, 
+read_entry(FILE* in, const char* name, int *lineno, uint32_t* default_ttl, 
 	ldns_rdf** origin, ldns_rdf** prev_rr)
 {
 	struct entry* current = NULL;
@@ -407,7 +420,7 @@ read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl,
 		parse = line;
 		(*lineno) ++;
 		
-		while(isspace(*parse))
+		while(isspace((int)*parse))
 			parse++;
 		/* test for keywords */
 		if(isendline(*parse))
@@ -425,7 +438,7 @@ read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl,
 			get_origin(name, *lineno, origin, (char*)parse);
 			continue;
 		} else if(str_keyword(&parse, "$TTL")) {
-			*default_ttl = (uint16_t)atoi(parse);
+			*default_ttl = (uint32_t)atoi(parse);
 			continue;
 		}
 
@@ -499,7 +512,7 @@ read_datafile(const char* name)
 	struct entry* current = NULL;
 	FILE *in;
 	int lineno = 0;
-	uint16_t default_ttl = 0;
+	uint32_t default_ttl = 0;
 	ldns_rdf* origin = NULL;
 	ldns_rdf* prev_rr = NULL;
 	int entry_num = 0;
@@ -555,7 +568,7 @@ static uint32_t get_serial(ldns_pkt* p)
 
 /** match two rr lists */
 static int
-match_list(ldns_rr_list* q, ldns_rr_list *p)
+match_list(ldns_rr_list* q, ldns_rr_list *p, bool mttl)
 {
 	size_t i;
 	if(ldns_rr_list_rr_count(q) != ldns_rr_list_rr_count(p))
@@ -567,6 +580,11 @@ match_list(ldns_rr_list* q, ldns_rr_list *p)
 			verbose(3, "rr %d different", i);
 			return 0;
 		}
+		if(mttl && ldns_rr_ttl(ldns_rr_list_rr(q, i)) !=
+			ldns_rr_ttl(ldns_rr_list_rr(p, i))) {
+			verbose(3, "rr %d ttl different", i);
+			return 0;
+		}
 	}
 	return 1;
 }
@@ -583,7 +601,7 @@ cmp_bool(int x, int y)
 
 /** match all of the packet */
 static int
-match_all(ldns_pkt* q, ldns_pkt* p)
+match_all(ldns_pkt* q, ldns_pkt* p, bool mttl)
 {
 	if(ldns_pkt_get_opcode(q) != ldns_pkt_get_opcode(p)) 
 	{ verbose(3, "allmatch: opcode different"); return 0;}
@@ -613,13 +631,13 @@ match_all(ldns_pkt* q, ldns_pkt* p)
 	{ verbose(3, "allmatch: nscount different"); return 0;}
 	if(ldns_pkt_arcount(q) != ldns_pkt_arcount(p))
 	{ verbose(3, "allmatch: arcount different"); return 0;}
-	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p)))
+	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p), mttl))
 	{ verbose(3, "allmatch: qd section different"); return 0;}
-	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p)))
+	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p), mttl))
 	{ verbose(3, "allmatch: an section different"); return 0;}
-	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p)))
+	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p), mttl))
 	{ verbose(3, "allmatch: ar section different"); return 0;}
-	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p))) 
+	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p), mttl)) 
 	{ verbose(3, "allmatch: ns section different"); return 0;}
 	return 1;
 }
@@ -655,11 +673,15 @@ find_match(struct entry* entries, ldns_pkt* query_pkt,
 				verbose(3, "bad serial\n");
 				continue;
 		}
+		if(p->match_do && !ldns_pkt_edns_do(query_pkt)) {
+			verbose(3, "no DO bit set\n");
+			continue;
+		}
 		if(p->match_transport != transport_any && p->match_transport != transport) {
 			verbose(3, "bad transport\n");
 			continue;
 		}
-		if(p->match_all && !match_all(query_pkt, reply)) {
+		if(p->match_all && !match_all(query_pkt, reply, p->match_ttl)) {
 			verbose(3, "bad allmatch\n");
 			continue;
 		}
@@ -675,6 +697,12 @@ adjust_packet(struct entry* match, ldns_pkt* answer_pkt, ldns_pkt* query_pkt)
 	/* copy & adjust packet */
 	if(match->copy_id)
 		ldns_pkt_set_id(answer_pkt, ldns_pkt_id(query_pkt));
+	if(match->copy_query) {
+		ldns_rr_list* list = ldns_pkt_get_section_clone(query_pkt,
+			LDNS_SECTION_QUESTION);
+		ldns_rr_list_deep_free(ldns_pkt_question(answer_pkt));
+		ldns_pkt_set_question(answer_pkt, list);
+	}
 	if(match->sleeptime > 0) {
 		verbose(3, "sleeping for %d seconds\n", match->sleeptime);
 		sleep(match->sleeptime);

testcode/ldns-testpkts.h

@@ -44,19 +44,23 @@
 	; 'qname' makes the query match the qname from the reply
 	; 'serial=1023' makes the query match if ixfr serial is 1023. 
 	; 'all' has to match header byte for byte and all rrs in packet.
-	MATCH [opcode] [qtype] [qname] [serial=<value>] [all]
-	MATCH [UDP|TCP]
+	; 'ttl' used with all, rrs in packet must also have matching TTLs.
+	; 'DO' will match only queries with DO bit set.
+	MATCH [opcode] [qtype] [qname] [serial=<value>] [all] [ttl]
+	MATCH [UDP|TCP] DO
 	MATCH ...
 	; Then the REPLY header is specified.
 	REPLY opcode, rcode or flags.
 		(opcode)  QUERY IQUERY STATUS NOTIFY UPDATE
 		(rcode)   NOERROR FORMERR SERVFAIL NXDOMAIN NOTIMPL YXDOMAIN
 		 		YXRRSET NXRRSET NOTAUTH NOTZONE
-		(flags)   QR AA TC RD CD RA AD
+		(flags)   QR AA TC RD CD RA AD DO
 	REPLY ...
 	; any additional actions to do.
 	; 'copy_id' copies the ID from the query to the answer.
 	ADJUST copy_id
+	; 'copy_query' copies the query name, type and class to the answer.
+	ADJUST copy_query
 	; 'sleep=10' sleeps for 10 seconds before giving the answer (TCP is open)
 	ADJUST [sleep=<num>]    ; sleep before giving any reply
 	ADJUST [packet_sleep=<num>]  ; sleep before this packet in sequence
@@ -157,6 +161,10 @@ struct entry {
 	bool match_serial; 
 	/** match all of the packet */
 	bool match_all;
+	/** match ttls in the packet */
+	bool match_ttl;
+	/** match DO bit */
+	bool match_do;
 	/** match query serial with this value. */
 	uint32_t ixfr_soa_serial; 
 	/** match on UDP/TCP */
@@ -168,6 +176,8 @@ struct entry {
 	/** how to adjust the reply packet */
 	/** copy over the ID from the query into the answer */
 	bool copy_id; 
+	/** copy the query nametypeclass from query into the answer */
+	bool copy_query;
 	/** in seconds */
 	unsigned int sleeptime; 
 
@@ -204,7 +214,7 @@ void delete_entry(struct entry* list);
  * @return: The entry read (malloced) or NULL if no entry could be read.
  */
 struct entry* read_entry(FILE* in, const char* name, int *lineno, 
-	uint16_t* default_ttl, ldns_rdf** origin, ldns_rdf** prev_rr);
+	uint32_t* default_ttl, ldns_rdf** origin, ldns_rdf** prev_rr);
 
 /**
  * finds entry in list, or returns NULL.

testcode/lock_verify.c

@@ -0,0 +1,426 @@
+/*
+ * testcode/lock_verify.c - verifier program for lock traces, checks order.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file checks the lock traces generated by checklock.c.
+ * Checks if locks are consistently locked in the same order.
+ * If not, this can lead to deadlock if threads execute the different
+ * ordering at the same time.
+ * 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/rbtree.h"
+
+/* --- data structures --- */
+struct lock_ref;
+
+/** key for lock lookup */
+struct order_id {
+	/** the thread id that created it */
+	int thr;
+	/** the instance number of creation */
+	int instance;
+};
+
+/** a lock */
+struct order_lock {
+	/** rbnode in all tree */
+	rbnode_t node;
+	/** lock id */
+	struct order_id id;
+	/** the creation file */
+	char* create_file;
+	/** creation line */
+	int create_line;
+	/** set of all locks that are smaller than this one (locked earlier) */
+	rbtree_t* smaller;
+	/** during depthfirstsearch, this is a linked list of the stack 
+	 * of locks. points to the next lock bigger than this one. */
+	struct lock_ref* dfs_next;
+	/** if lock has been visited (all smaller locks have been compared to
+	 * this lock), only need to compare this with all unvisited(bigger) 
+	 * locks */
+	int visited;
+};
+
+/** reference to a lock in a rbtree set */
+struct lock_ref {
+	/** rbnode, key is an order_id ptr */
+	rbnode_t node;
+	/** the lock referenced */
+	struct order_lock* lock;
+	/** why is this ref */
+	char* file;
+	/** line number */
+	int line;
+};
+
+/** count of errors detected */
+static int errors_detected = 0;
+/** verbose? */
+static int verb = 0;
+
+/** print program usage help */
+static void
+usage()
+{
+	printf("lock_verify <trace files>\n");
+}
+
+/** compare two order_ids */
+int order_lock_cmp(const void* e1, const void* e2)
+{
+	struct order_id* o1 = (struct order_id*)e1;
+	struct order_id* o2 = (struct order_id*)e2;
+	if(o1->thr < o2->thr) return -1;
+	if(o1->thr > o2->thr) return 1;
+	if(o1->instance < o2->instance) return -1;
+	if(o1->instance > o2->instance) return 1;
+	return 0;
+}
+
+/** read header entry. 
+ * @param in: file to read header of.
+ * @return: False if it does not belong to the rest. */
+static int 
+read_header(FILE* in)
+{
+	time_t t;
+	pid_t p;
+	int thrno;
+	static int have_values = 0;
+	static time_t the_time;
+	static pid_t the_pid;
+	static int threads[256];
+
+	if(fread(&t, sizeof(t), 1, in) != 1 ||	
+		fread(&thrno, sizeof(thrno), 1, in) != 1 ||
+		fread(&p, sizeof(p), 1, in) != 1) {
+		fatal_exit("fread failed");
+	}
+	/* check these values are sorta OK */
+	if(!have_values) {
+		the_time = t;
+		the_pid = p;
+		memset(threads, 0, 256*sizeof(int));
+		if(thrno >= 256) {
+			fatal_exit("Thread number too big. %d", thrno);
+		}
+		threads[thrno] = 1;
+		have_values = 1;
+		printf(" trace %d from pid %u on %s", thrno, 
+			(unsigned)p, ctime(&t));
+	} else {
+		if(the_pid != p) {
+			printf(" has pid %u, not %u. Skipped.\n",
+				(unsigned)p, (unsigned)the_pid);
+			return 0;
+		}
+		if(threads[thrno])
+			fatal_exit("same threadno in two files");
+		threads[thrno] = 1;
+		if( abs((int)(the_time - t)) > 3600)
+			fatal_exit("input files from different times: %u %u",
+				(unsigned)the_time, (unsigned)t);
+		printf(" trace of thread %u:%d\n", (unsigned)p, thrno);
+	}
+	return 1;
+}
+
+/** max length of strings: filenames and function names. */
+#define STRMAX 1024
+/** read a string from file, false on error */
+static int readup_str(char** str, FILE* in)
+{
+	char buf[STRMAX];
+	int len = 0;
+	int c;
+	/* ends in zero */
+	while( (c = fgetc(in)) != 0) {
+		if(c == EOF)
+			fatal_exit("eof in readstr, file too short");
+		buf[len++] = c;
+		if(len == STRMAX) {
+			fatal_exit("string too long, bad file format");
+		}
+	}
+	buf[len] = 0;
+	*str = strdup(buf);
+	return 1;
+}
+
+/** read creation entry */
+static void read_create(rbtree_t* all, FILE* in)
+{
+	struct order_lock* o = calloc(1, sizeof(struct order_lock));
+	if(!o) fatal_exit("malloc failure");
+	if(fread(&o->id.thr, sizeof(int), 1, in) != 1 ||	
+	   fread(&o->id.instance, sizeof(int), 1, in) != 1 ||	
+	   !readup_str(&o->create_file, in) ||
+	   fread(&o->create_line, sizeof(int), 1, in) != 1)
+		fatal_exit("fread failed");
+	o->smaller = rbtree_create(order_lock_cmp);
+	o->node.key = &o->id;
+	if(!rbtree_insert(all, &o->node)) {
+		/* already inserted */
+		struct order_lock* a = (struct order_lock*)rbtree_search(all, 
+			&o->id);
+		log_assert(a);
+		a->create_file = o->create_file;
+		a->create_line = o->create_line;
+		free(o->smaller);
+		free(o);
+		o = a;
+	}
+	if(verb) printf("read create %u %u %s %d\n", 
+		(unsigned)o->id.thr, (unsigned)o->id.instance,
+		o->create_file, o->create_line);
+}
+
+/** insert lock entry (empty) into list */
+static struct order_lock* 
+insert_lock(rbtree_t* all, struct order_id* id)
+{
+	struct order_lock* o = calloc(1, sizeof(struct order_lock));
+	if(!o) fatal_exit("malloc failure");
+	o->smaller = rbtree_create(order_lock_cmp);
+	o->id = *id;
+	o->node.key = &o->id;
+	if(!rbtree_insert(all, &o->node))
+		fatal_exit("insert fail should not happen");
+	return o;
+}
+
+/** read lock entry */
+static void read_lock(rbtree_t* all, FILE* in, int val)
+{
+	struct order_id prev_id, now_id;
+	struct lock_ref* ref;
+	struct order_lock* prev, *now;
+	ref = (struct lock_ref*)calloc(1, sizeof(struct lock_ref));
+	if(!ref) fatal_exit("malloc failure");
+	prev_id.thr = val;
+	if(fread(&prev_id.instance, sizeof(int), 1, in) != 1 ||	
+	   fread(&now_id.thr, sizeof(int), 1, in) != 1 ||	
+	   fread(&now_id.instance, sizeof(int), 1, in) != 1 ||	
+	   !readup_str(&ref->file, in) ||
+	   fread(&ref->line, sizeof(int), 1, in) != 1)
+		fatal_exit("fread failed");
+	if(verb) printf("read lock %u %u %u %u %s %d\n", 
+		(unsigned)prev_id.thr, (unsigned)prev_id.instance,
+		(unsigned)now_id.thr, (unsigned)now_id.instance,
+		ref->file, ref->line);
+	/* find the two locks involved */
+	prev = (struct order_lock*)rbtree_search(all, &prev_id);
+	now = (struct order_lock*)rbtree_search(all, &now_id);
+	/* if not there - insert 'em */
+	if(!prev) prev = insert_lock(all, &prev_id);
+	if(!now) now = insert_lock(all, &now_id);
+	ref->lock = prev;
+	ref->node.key = &prev->id;
+	if(!rbtree_insert(now->smaller, &ref->node)) {
+		free(ref->file);
+		free(ref);
+	}
+}
+
+/** read input file */
+static void readinput(rbtree_t* all, char* file)
+{
+	FILE *in = fopen(file, "r");
+	int fst;
+	if(!in) {
+		perror(file);
+		exit(1);
+	}
+	printf("file %s", file);
+	if(!read_header(in)) {
+		fclose(in);
+		return;
+	}
+	while(fread(&fst, sizeof(fst), 1, in) == 1) {
+		if(fst == -1)
+			read_create(all, in);
+		else	read_lock(all, in, fst);
+	}
+	fclose(in);
+}
+
+/** print cycle message */
+static void found_cycle(struct lock_ref* visit, int level)
+{
+	struct lock_ref* p;
+	int i = 0;
+	errors_detected++;
+	printf("Found inconsistent locking order of length %d\n", level);
+	printf("for lock %d %d created %s %d\n", 
+		visit->lock->id.thr, visit->lock->id.instance,
+		visit->lock->create_file, visit->lock->create_line);
+	printf("sequence is:\n");
+	p = visit;
+	while(p) {
+		struct order_lock* next = 
+			p->lock->dfs_next?p->lock->dfs_next->lock:visit->lock;
+		printf("[%d] is locked at line %s %d before lock %d %d\n",
+			i, p->file, p->line, next->id.thr, next->id.instance);
+		printf("[%d] lock %d %d is created at %s %d\n",
+			i, next->id.thr, next->id.instance,
+			next->create_file, next->create_line); 
+		i++;
+		p = p->lock->dfs_next;
+		if(p && p->lock == visit->lock)
+			break;
+	}
+}
+
+/** Detect cycle by comparing visited now with all (unvisited) bigger nodes */
+static int detect_cycle(struct lock_ref* visit, struct lock_ref* from)
+{
+	struct lock_ref* p = from;
+	while(p) {
+		if(p->lock == visit->lock)
+			return 1;
+		p = p->lock->dfs_next;
+	}
+	return 0;
+}
+
+/** recursive function to depth first search for cycles.
+ * @param visit: the lock visited at this step.
+ *	its dfs_next pointer gives the visited lock up in recursion.
+ * 	same as lookfor at level 0.
+ * @param level: depth of recursion. 0 is start.
+ * @param from: search for matches from unvisited node upwards.
+ */
+static void search_cycle(struct lock_ref* visit, int level, 
+	struct lock_ref* from)
+{
+	struct lock_ref* ref;
+	/* check for cycle */
+	if(detect_cycle(visit, from) && level != 0) {
+		found_cycle(visit, level);
+		fatal_exit("found lock order cycle");
+	}
+	/* recurse */
+	if(!visit->lock->visited)
+		from = visit;
+	if(verb > 1) fprintf(stderr, "[%d] visit lock %u %u %s %d\n", level,
+			(unsigned)visit->lock->id.thr, 
+			(unsigned)visit->lock->id.instance,
+			visit->lock->create_file, visit->lock->create_line);
+	RBTREE_FOR(ref, struct lock_ref*, visit->lock->smaller) {
+		ref->lock->dfs_next = visit;
+		search_cycle(ref, level+1, from);
+	}
+	visit->lock->visited = 1;
+}
+
+/** Check ordering of one lock */
+static void check_order_lock(struct order_lock* lock)
+{
+	struct lock_ref start;
+	if(lock->visited) return;
+
+	start.node.key = &lock->id;
+	start.lock = lock;
+	start.file = lock->create_file;
+	start.line = lock->create_line;
+
+	if(!lock->create_file)
+		log_err("lock %u %u does not have create info",
+			(unsigned)lock->id.thr, (unsigned)lock->id.instance);
+
+	/* depth first search to find cycle with this lock at head */
+	lock->dfs_next = NULL;
+	search_cycle(&start, 0, &start);
+}
+
+/** Check ordering of locks */
+static void check_order(rbtree_t* all_locks)
+{
+	/* check each lock */
+	struct order_lock* lock;
+	int i=0;
+	RBTREE_FOR(lock, struct order_lock*, all_locks) {
+		if(verb)
+		    printf("[%d/%d] Checking lock %d %d %s %d\n",
+			i, (int)all_locks->count,
+			lock->id.thr, lock->id.instance, 
+			lock->create_file, lock->create_line);
+		else if (i % ((all_locks->count/75)<1?1:all_locks->count/75) 
+			== 0) 
+		    fprintf(stderr, ".");
+		i++;
+		check_order_lock(lock);
+	}
+	fprintf(stderr, "\n");
+}
+
+/** main program to verify all traces passed */
+int
+main(int argc, char* argv[])
+{
+	rbtree_t* all_locks;
+	int i;
+	time_t starttime = time(NULL);
+	if(argc <= 1) {
+		usage();
+		return 1;
+	}
+	log_init(NULL);
+	log_ident_set("lock-verify");
+	/* init */
+	all_locks = rbtree_create(order_lock_cmp);
+	errors_detected = 0;
+
+	/* read the input files */
+	for(i=1; i<argc; i++) {
+		readinput(all_locks, argv[i]);
+	}
+
+	/* check ordering */
+	check_order(all_locks);
+
+	/* do not free a thing, OS will do it */
+	printf("checked %d locks in %d seconds with %d errors.\n", 
+		(int)all_locks->count, (int)(time(NULL)-starttime),
+		errors_detected);
+	if(errors_detected) return 1;
+	return 0;
+}

testcode/memstats.c

@@ -0,0 +1,270 @@
+/*
+ * testcode/memstats.c - debug tool to show memory allocation statistics.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program reads a log file and prints the memory allocation summed
+ * up.
+ */
+#include "config.h"
+#include "util/log.h"
+#include "util/rbtree.h"
+#include <sys/stat.h>
+
+/**
+ * The allocation statistics block
+ */
+struct codeline {
+	/** rbtree node */
+	rbnode_t node;
+	/** the name of the file:linenumber */
+	char* codeline;
+	/** the name of the function */
+	char* func;
+	/** number of bytes allocated */
+	uint64_t alloc;
+	/** number of bytes freed */
+	uint64_t free;
+};
+
+/**
+ * Other allocation stats
+ */
+struct alloc_misc {
+	/** number of region allocs */
+	uint64_t region_alloc;
+};
+
+/** print usage and exit */
+static void
+usage()
+{
+	printf("usage:	memstats <logfile>\n");
+	printf("statistics are printed on stdout.\n");
+	exit(1);
+}
+
+/** compare two codeline structs for rbtree */
+static int
+codeline_cmp(const void* a, const void* b)
+{
+	return strcmp((const char*)a, (const char*)b);
+}
+
+/** match logfile line to see if it needs accounting processing */
+static int
+match(char* line)
+{
+	/* f.e.:
+	 * [1187340064] unbound[24604:0] info: ul/rb.c:81 r_create malloc(12)
+	 * 0123456789 123456789 123456789 123456789
+	 */
+	if(strlen(line) < 36) /* up to 'info: ' */
+		return 0;
+	if(strncmp(line+30, "info: ", 6) != 0)
+		return 0;
+	if(strncmp(line+36, "stat ", 5) == 0)
+		return 0; /* skip the hex dumps */
+	if(strstr(line+36, "malloc("))
+		return 1;
+	else if(strstr(line+36, "calloc("))
+		return 1;
+	/* skip reallocs */
+	return 0;
+}
+
+/** read up the region stats */
+static void
+read_region_stat(char* line, struct alloc_misc* misc)
+{
+	long num = 0;
+	if(sscanf(line+50, "%ld", &num) != 1) {
+		printf("%s\n%s\n", line, line+50);
+		fatal_exit("unhandled region");
+	}
+	misc->region_alloc += num;
+}
+
+/** find or alloc codeline in tree */
+static struct codeline*
+get_codeline(rbtree_t* tree, char* key, char* func)
+{
+	struct codeline* cl = (struct codeline*)rbtree_search(tree, key);
+	if(!cl) {
+		cl = calloc(1, sizeof(*cl));
+		if(!cl) return 0;
+		cl->codeline = strdup(key);
+		if(!cl->codeline) return 0;
+		cl->func = strdup(func);
+		if(!cl->func) return 0;
+		cl->alloc = 0;
+		cl->node.key = cl->codeline;
+		(void)rbtree_insert(tree, &cl->node);
+	}
+	return cl;
+}
+
+/** read up the malloc stats */
+static void
+read_malloc_stat(char* line, rbtree_t* tree)
+{
+	char codeline[10240];
+	char name[10240];
+	int skip = 0;
+	long num = 0;
+	struct codeline* cl = 0;
+	if(sscanf(line+36, "%s %s %n", codeline, name, &skip) != 2) {
+		printf("%s\n%s\n", line, line+36);
+		fatal_exit("unhandled malloc");
+	}
+	if(sscanf(line+36+skip+7, "%ld", &num) != 1) {
+		printf("%s\n%s\n", line, line+36+skip+7);
+		fatal_exit("unhandled malloc");
+	}
+	cl = get_codeline(tree, codeline, name);
+	if(!cl)
+		fatal_exit("alloc failure");
+	cl->alloc += num;
+}
+
+/** read up the calloc stats */
+static void
+read_calloc_stat(char* line, rbtree_t* tree)
+{
+	char codeline[10240];
+	char name[10240];
+	int skip = 0;
+	long num = 0, sz = 0;
+	struct codeline* cl = 0;
+	if(sscanf(line+36, "%s %s %n", codeline, name, &skip) != 2) {
+		printf("%s\n%s\n", line, line+36);
+		fatal_exit("unhandled calloc");
+	}
+	if(sscanf(line+36+skip+7, "%ld, %ld", &num, &sz) != 2) {
+		printf("%s\n%s\n", line, line+36+skip+7);
+		fatal_exit("unhandled calloc");
+	}
+
+	cl = get_codeline(tree, codeline, name);
+	if(!cl)
+		fatal_exit("alloc failure");
+	cl->alloc += num*sz;
+}
+
+/** get size of file */
+static off_t
+get_file_size(const char* fname)
+{
+	struct stat s;
+	if(stat(fname, &s) < 0) {
+		fatal_exit("could not stat %s: %s", fname, strerror(errno));
+	}
+	return s.st_size;
+}
+
+/** read the logfile */
+static void
+readfile(rbtree_t* tree, const char* fname, struct alloc_misc* misc)
+{
+	off_t total = get_file_size(fname);
+	off_t done = (off_t)0;
+	int report = 0;
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	if(!in)
+		fatal_exit("could not open %s: %s", fname, strerror(errno));
+	printf("Reading %s of size %lld\n", fname, (long long)total);
+	while(fgets(buf, 102400, in)) {
+		buf[102400-1] = 0;
+		done += (off_t)strlen(buf);
+		/* progress count */
+		if((int)(((double)done / (double)total)*100.) > report) {
+			report = (int)(((double)done / (double)total)*100.);
+			fprintf(stderr, " %d%%", report);
+		}
+
+		if(!match(buf))
+			continue;
+		if(strncmp(buf+36, "region ", 7) == 0)
+			read_region_stat(buf, misc);
+		else if(strstr(buf+36, "malloc("))
+			read_malloc_stat(buf, tree);
+		else if(strstr(buf+36, "calloc("))
+			read_calloc_stat(buf, tree);
+		else {
+			printf("%s\n", buf);
+			fatal_exit("unhandled input");
+		}
+	}
+	fprintf(stderr, " done\n");
+	fclose(in);
+}
+
+/** print memory stats */
+static void
+printstats(rbtree_t* tree, struct alloc_misc* misc)
+{
+	struct codeline* cl;
+	uint64_t total = 0;
+	printf("%12lld in region alloc\n", (long long)misc->region_alloc);
+	total += misc->region_alloc;
+	RBTREE_FOR(cl, struct codeline*, tree) {
+		printf("%12lld in %s %s\n", (long long)cl->alloc, 
+			cl->codeline, cl->func);
+		total += cl->alloc;
+	}
+	printf("------------\n");
+	printf("%12lld total in %ld code lines\n", (long long)total, 
+		(long)tree->count);
+	printf("\n");
+}
+
+/** main program */
+int main(int argc, const char* argv[])
+{
+	rbtree_t* tree = 0;
+	struct alloc_misc misc;
+	if(argc != 2) {
+		usage();
+	}
+	tree = rbtree_create(codeline_cmp);
+	if(!tree)
+		fatal_exit("alloc failure");
+	memset(&misc, 0, sizeof(misc));
+	readfile(tree, argv[1], &misc);
+	printstats(tree, &misc);
+	return 0;
+}

testcode/mini_tpkg.sh

@@ -0,0 +1,111 @@
+# tpkg that only exes the files.
+args="../.."
+if test "$1" = "-a"; then
+	args=$2
+	shift
+	shift
+fi
+	
+if test "$1" = "clean"; then
+	echo "rm -f result.* .done* .tpkg.var.master .tpkg.var.test"
+	rm -f result.* .done* .tpkg.var.master .tpkg.var.test
+	exit 0
+fi
+if test "$1" = "fake"; then
+	echo "minitpkg fake $2"
+	echo "fake" > .done-`basename $2 .tpkg`
+	exit 0
+fi
+if test "$1" = "report" || test "$2" = "report"; then
+	echo "Minitpkg Report"
+	for result in result.*; do
+		name=`echo $result | sed -e 's/result\.//'`
+		if test -f ".done-$name"; then
+			if test "$1" != "-q"; then
+				echo "** PASSED ** : $name"
+			fi
+		else
+			echo "!! FAILED !! : $name"
+		fi
+	done
+	exit 0
+fi
+
+if test "$1" != 'exe'; then
+	# usage
+	echo "mini tpkg. Reduced functionality for old shells."
+	echo "	tpkg exe <file>"
+	echo "	tpkg fake <file>"
+	echo "	tpkg clean"
+	echo "	tpkg [-q] report"
+	exit 1
+fi
+shift
+
+# do not execute if the disk is too full
+#DISKLIMIT=100000
+# This check is not portable (to Solaris 10).
+#avail=`df . | tail -1 | awk '{print $4}'`
+#if test "$avail" -lt "$DISKLIMIT"; then
+	#echo "minitpkg: The disk is too full! Only $avail."
+	#exit 1
+#fi
+
+name=`basename $1 .tpkg`
+dir=$name.$$
+result=result.$name
+done=.done-$name
+success="no"
+shell="bash"
+
+# check already done
+if test -f .done-$name; then
+	echo "minitpkg .done-$name exists. skip test."
+	exit 0
+fi
+
+# Extract
+echo "minitpkg extract $1 to $dir"
+mkdir $dir
+gzip -cd $name.tpkg | (cd $dir; tar xf -)
+cd $dir
+mv $name.dir/* .
+
+# EXE
+echo "minitpkg exe $name"
+echo "minitpkg exe $name" > $result
+if test -f $name.pre; then
+	echo "minitpkg exe $name.pre"
+	echo "minitpkg exe $name.pre" >> $result
+	$shell $name.pre $args >> $result
+	if test $? -ne 0; then
+		echo "Warning: $name.pre did not exit successfully"
+	fi
+fi
+if test -f $name.test; then
+	echo "minitpkg exe $name.test"
+	echo "minitpkg exe $name.test" >> $result
+	$shell $name.test $args >>$result 2>&1
+	if test $? -ne 0; then
+		echo "$name: FAILED" >> $result
+		echo "$name: FAILED"
+		success="no"
+	else
+		echo "$name: PASSED" >> $result
+		echo "$name: PASSED" > ../.done-$name
+		echo "$name: PASSED"
+		success="yes"
+	fi
+fi
+if test -f $name.post; then
+	echo "minitpkg exe $name.post"
+	echo "minitpkg exe $name.post" >> $result
+	$shell $name.post $args >> $result
+	if test $? -ne 0; then
+		echo "Warning: $name.post did not exit successfully"
+	fi
+fi
+
+mv $result ..
+cd ..
+rm -rf $dir

testcode/pktview.c

@@ -0,0 +1,200 @@
+/*
+ * testcode/pktview.c - debug program to disassemble a DNS packet.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program shows a dns packet wire format.
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/data/dname.h"
+#include "util/data/msgparse.h"
+#include "testcode/unitmain.h"
+#include "testcode/readhex.h"
+
+/** usage information for pktview */
+void usage(char* argv[])
+{
+	printf("usage: %s\n", argv[0]);
+	printf("present hex packet on stdin.\n");
+	exit(1);
+}
+
+/** read hex input */
+void read_input(ldns_buffer* pkt, FILE* in)
+{
+	char buf[102400];
+	char* np = buf;
+	while(fgets(np, (int)sizeof(buf) - (np-buf), in)) {
+		if(buf[0] == ';') /* comment */
+			continue;
+		np = &np[strlen(np)];
+	}
+	hex_to_buf(pkt, buf);
+}
+
+/** analyze domain name in packet, possibly compressed */
+void analyze_dname(ldns_buffer* pkt)
+{
+	size_t oldpos = ldns_buffer_position(pkt);
+	size_t len;
+	printf("[pos %d] dname: ", (int)oldpos);
+	dname_print(stdout, pkt, ldns_buffer_current(pkt));
+	len = pkt_dname_len(pkt);
+	printf(" len=%d", (int)len);
+	if(ldns_buffer_position(pkt)-oldpos != len)
+		printf(" comprlen=%d\n", 
+			(int)(ldns_buffer_position(pkt)-oldpos));
+	else	printf("\n");
+}
+
+/** analyze rdata in packet */
+void analyze_rdata(ldns_buffer*pkt, const ldns_rr_descriptor* desc, 
+	uint16_t rdlen)
+{
+	int rdf = 0;
+	int count = (int)desc->_dname_count;
+	size_t len, oldpos;
+	while(rdlen > 0 && count) {
+		switch(desc->_wireformat[rdf]) {
+		case LDNS_RDF_TYPE_DNAME:
+			oldpos = ldns_buffer_position(pkt);
+			analyze_dname(pkt);
+			rdlen -= ldns_buffer_position(pkt)-oldpos;
+			count --;
+			len = 0;
+			break;
+		case LDNS_RDF_TYPE_STR:
+			len = ldns_buffer_current(pkt)[0] + 1;
+			break;
+		default:
+			len = get_rdf_size(desc->_wireformat[rdf]);
+		}
+		if(len) {
+			printf(" wf[%d]", (int)len);
+			ldns_buffer_skip(pkt, (ssize_t)len);
+			rdlen -= len;
+		}
+		rdf++;
+	}
+	if(rdlen) {
+		size_t i;
+		printf(" remain[%d]\n", (int)rdlen);
+		for(i=0; i<rdlen; i++)
+			printf(" %2.2X", (unsigned)ldns_buffer_current(pkt)[i]);
+		printf("\n");
+	}
+	else	printf("\n");
+	ldns_buffer_skip(pkt, (ssize_t)rdlen);
+}
+
+/** analyze rr in packet */
+void analyze_rr(ldns_buffer* pkt, int q)
+{
+	uint16_t type, dclass, len;
+	uint32_t ttl;
+	analyze_dname(pkt);
+	type = ldns_buffer_read_u16(pkt);
+	dclass = ldns_buffer_read_u16(pkt);
+	printf("type %s(%d)", ldns_rr_descript(type)?  
+		ldns_rr_descript(type)->_name: "??" , (int)type);
+	printf(" class %s(%d) ", ldns_lookup_by_id(ldns_rr_classes, 
+		(int)dclass)?ldns_lookup_by_id( ldns_rr_classes, 
+		(int)dclass)->name:"??", (int)dclass);
+	if(q) {
+		printf("\n");
+	} else {
+		ttl = ldns_buffer_read_u32(pkt);
+		printf(" ttl %d (0x%x)", (int)ttl, (unsigned)ttl);
+		len = ldns_buffer_read_u16(pkt);
+		printf(" rdata len %d:\n", (int)len);
+		if(ldns_rr_descript(type))
+			analyze_rdata(pkt, ldns_rr_descript(type), len);
+		else ldns_buffer_skip(pkt, (ssize_t)len);
+	}
+}
+
+/** analyse pkt */
+void analyze(ldns_buffer* pkt)
+{
+	uint16_t i, f, qd, an, ns, ar;
+	int rrnum = 0;
+	printf("packet length %d\n", (int)ldns_buffer_limit(pkt));
+	if(ldns_buffer_limit(pkt) < 12) return;
+
+	i = ldns_buffer_read_u16(pkt);
+	printf("id (hostorder): %d (0x%x)\n", (int)i, (unsigned)i);
+	f = ldns_buffer_read_u16(pkt);
+	printf("flags: 0x%x\n", (unsigned)f);
+	qd = ldns_buffer_read_u16(pkt);
+	printf("qdcount: %d\n", (int)qd);
+	an = ldns_buffer_read_u16(pkt);
+	printf("ancount: %d\n", (int)an);
+	ns = ldns_buffer_read_u16(pkt);
+	printf("nscount: %d\n", (int)ns);
+	ar = ldns_buffer_read_u16(pkt);
+	printf("arcount: %d\n", (int)ar);
+	
+	printf(";-- query section\n");
+	while(ldns_buffer_remaining(pkt) > 0) {
+		if(rrnum == (int)qd) 
+			printf(";-- answer section\n");
+		if(rrnum == (int)qd+(int)an) 
+			printf(";-- authority section\n");
+		if(rrnum == (int)qd+(int)an+(int)ns) 
+			printf(";-- additional section\n");
+		printf("rr %d ", rrnum);
+		analyze_rr(pkt, rrnum < (int)qd);
+		rrnum++;
+	}
+}
+
+/** main program for pktview */
+int main(int argc, char* argv[]) 
+{
+	ldns_buffer* pkt = ldns_buffer_new(65553);
+	if(argc != 1) {
+		usage(argv);
+	}
+	if(!pkt) fatal_exit("out of memory");
+
+	read_input(pkt, stdin);
+	analyze(pkt);
+
+	ldns_buffer_free(pkt);
+	return 0;
+}

testcode/readhex.c

@@ -0,0 +1,82 @@
+/*
+ * testcode/readhex.c - read hex data.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+#include "config.h"
+#include "testcode/readhex.h"
+#include "util/log.h"
+
+/** skip whitespace */
+static void
+skip_whites(const char** p)
+{
+	while(1) {
+		while(isspace((int)**p))
+			(*p)++;
+		if(**p == ';') {
+			/* comment, skip until newline */
+			while(**p && **p != '\n')
+				(*p)++;
+			if(**p == '\n')
+				(*p)++;
+		} else return;
+	}
+}
+
+/** takes a hex string and puts into buffer */
+void hex_to_buf(ldns_buffer* pkt, const char* hex)
+{
+	const char* p = hex;
+	int val;
+	ldns_buffer_clear(pkt);
+	while(*p) {
+		skip_whites(&p);
+		if(ldns_buffer_position(pkt) == ldns_buffer_limit(pkt))
+			fatal_exit("hex_to_buf: buffer too small");
+		if(!isalnum((int)*p))
+			break;
+		val = ldns_hexdigit_to_int(*p++) << 4;
+		skip_whites(&p);
+		log_assert(*p && isalnum((int)*p));
+		val |= ldns_hexdigit_to_int(*p++);
+		ldns_buffer_write_u8(pkt, (uint8_t)val);
+		skip_whites(&p);
+	}
+	ldns_buffer_flip(pkt);
+}
+

testcode/readhex.h

@@ -0,0 +1,47 @@
+/*
+ * testcode/readhex.h - read hex data.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+
+#ifndef TESTCODE_READHEX_H
+#define TESTCODE_READHEX_H
+
+/** helper to convert hex string to packet buffer */
+void hex_to_buf(ldns_buffer* pkt, const char* hex);
+
+#endif /* TESTCODE_READHEX_H */

testcode/replay.c

@@ -41,6 +41,7 @@
 
 #include "config.h"
 #include "util/log.h"
+#include "util/net_help.h"
 #include "testcode/replay.h"
 #include "testcode/ldns-testpkts.h"
 
@@ -85,6 +86,18 @@ replay_range_delete(struct replay_range* rng)
 	free(rng);
 }
 
+/** strip whitespace from end of string */
+static void
+strip_end_white(char* p)
+{
+	size_t i;
+	for(i = strlen(p); i > 0; i--) {
+		if(isspace((int)p[i-1]))
+			p[i-1] = 0;
+		else return;
+	}
+}
+
 /** 
  * Read a range from file. 
  * @param remain: Rest of line (after RANGE keyword).
@@ -99,7 +112,7 @@ replay_range_delete(struct replay_range* rng)
  */
 static struct replay_range*
 replay_range_read(char* remain, FILE* in, const char* name, int* lineno, 
-	char* line, uint16_t* ttl, ldns_rdf** or, ldns_rdf** prev)
+	char* line, uint32_t* ttl, ldns_rdf** or, ldns_rdf** prev)
 {
 	struct replay_range* rng = (struct replay_range*)malloc(
 		sizeof(struct replay_range));
@@ -120,10 +133,23 @@ replay_range_read(char* remain, FILE* in, const char* name, int* lineno,
 	while(fgets(line, MAX_LINE_LEN-1, in)) {
 		(*lineno)++;
 		parse = line;
-		while(isspace(*parse))
+		while(isspace((int)*parse))
 			parse++;
 		if(!*parse || *parse == ';')
 			continue;
+		if(parse_keyword(&parse, "ADDRESS")) {
+			while(isspace((int)*parse))
+				parse++;
+			strip_end_white(parse);
+			if(!extstrtoaddr(parse, &rng->addr, &rng->addrlen)) {
+				log_err("Line %d: could not read ADDRESS: %s", 
+					*lineno, parse);
+				free(rng);
+				return NULL;
+			}
+			pos = ftello(in);
+			continue;
+		}
 		if(parse_keyword(&parse, "RANGE_END")) {
 			return rng;
 		}
@@ -158,7 +184,7 @@ replay_range_read(char* remain, FILE* in, const char* name, int* lineno,
  */
 static struct replay_moment*
 replay_moment_read(char* remain, FILE* in, const char* name, int* lineno, 
-	uint16_t* ttl, ldns_rdf** or, ldns_rdf** prev)
+	uint32_t* ttl, ldns_rdf** or, ldns_rdf** prev)
 {
 	struct replay_moment* mom = (struct replay_moment*)malloc(
 		sizeof(struct replay_moment));
@@ -173,7 +199,7 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 		return NULL;
 	}
 	remain += skip;
-	while(isspace(*remain))
+	while(isspace((int)*remain))
 		remain++;
 	if(parse_keyword(&remain, "NOTHING")) {
 		mom->evt_type = repevt_nothing;
@@ -198,6 +224,16 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 		free(mom);
 		return NULL;
 	}
+	while(isspace((int)*remain))
+		remain++;
+	if(parse_keyword(&remain, "ADDRESS")) {
+		if(!extstrtoaddr(remain, &mom->addr, &mom->addrlen)) {
+			log_err("line %d: could not parse ADDRESS: %s", 
+				*lineno, remain);
+			free(mom);
+			return NULL;
+		}
+	}
 
 	if(readentry) {
 		mom->match = read_entry(in, name, lineno, ttl, or, prev);
@@ -210,12 +246,12 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 	return mom;
 }
 
-/** makes scenario with title on rest of line. */
+/** makes scenario with title on rest of line */
 static struct replay_scenario*
 make_scenario(char* line)
 {
 	struct replay_scenario* scen;
-	while(isspace(*line))
+	while(isspace((int)*line))
 		line++;
 	if(!*line) {
 		log_err("scenario: no title given");
@@ -234,21 +270,20 @@ make_scenario(char* line)
 }
 
 struct replay_scenario* 
-replay_scenario_read(FILE* in, const char* name)
+replay_scenario_read(FILE* in, const char* name, int* lineno)
 {
 	char line[MAX_LINE_LEN];
 	char *parse;
-	int lineno = 0;
 	struct replay_scenario* scen = NULL;
-	uint16_t ttl = 3600;
+	uint32_t ttl = 3600;
 	ldns_rdf* or = NULL;
 	ldns_rdf* prev = NULL;
 	line[MAX_LINE_LEN-1]=0;
 
 	while(fgets(line, MAX_LINE_LEN-1, in)) {
 		parse=line;
-		lineno++;
-		while(isspace(*parse))
+		(*lineno)++;
+		while(isspace((int)*parse))
 			parse++;
 		if(!*parse) 
 			continue; /* empty line */
@@ -257,26 +292,26 @@ replay_scenario_read(FILE* in, const char* name)
 		if(parse_keyword(&parse, "SCENARIO_BEGIN")) {
 			scen = make_scenario(parse);
 			if(!scen)
-				fatal_exit("%d: could not make scen", lineno);
+				fatal_exit("%d: could not make scen", *lineno);
 			continue;
 		} 
 		if(!scen)
-			fatal_exit("%d: expected SCENARIO", lineno);
+			fatal_exit("%d: expected SCENARIO", *lineno);
 		if(parse_keyword(&parse, "RANGE_BEGIN")) {
 			struct replay_range* newr = replay_range_read(parse, 
-				in, name, &lineno, line, &ttl, &or, &prev);
+				in, name, lineno, line, &ttl, &or, &prev);
 			if(!newr)
-				fatal_exit("%d: bad range", lineno);
+				fatal_exit("%d: bad range", *lineno);
 			newr->next_range = scen->range_list;
 			scen->range_list = newr;
 		} else if(parse_keyword(&parse, "STEP")) {
 			struct replay_moment* mom = replay_moment_read(parse, 
-				in, name, &lineno, &ttl, &or, &prev);
+				in, name, lineno, &ttl, &or, &prev);
 			if(!mom)
-				fatal_exit("%d: bad moment", lineno);
+				fatal_exit("%d: bad moment", *lineno);
 			if(scen->mom_last && 
 				scen->mom_last->time_step >= mom->time_step)
-				fatal_exit("%d: time goes backwards", lineno);
+				fatal_exit("%d: time goes backwards", *lineno);
 			if(scen->mom_last)
 				scen->mom_last->mom_next = mom;
 			else	scen->mom_first = mom;

testcode/replay.h

@@ -44,11 +44,13 @@
  * ; comment line.
  * SCENARIO_BEGIN name_of_scenario
  * RANGE_BEGIN start_time end_time
+ *    ; give ip of the virtual server, it matches any ip if not present.
+ *    ADDRESS ip_address 
  *    match_entries
  * RANGE_END
  * ; more RANGE items.
  * ; go to the next moment
- * STEP time_step event_type
+ * STEP time_step event_type [ADDRESS ip_address]
  * ; event_type can be:
  *	o NOTHING - nothing
  *	o QUERY - followed by entry
@@ -158,6 +160,11 @@ struct replay_moment {
 	/** The sent packet must match this. Incoming events, the data. */
 	struct entry* match;
 
+	/** address that must be matched, or packet remote host address. */
+	struct sockaddr_storage addr;
+	/** length of addr, if 0, then any address will do */
+	socklen_t addrlen;
+
 	/** what pending query should timeout or is answered. or 
 	 * NULL for last sent query. 
 	 * Unused at this time.
@@ -173,6 +180,10 @@ struct replay_range {
 	int start_step;
 	/** end step of time range. */
 	int end_step;
+	/** address of where this range is served. */
+	struct sockaddr_storage addr;
+	/** length of addr, if 0, then any address will do */
+	socklen_t addrlen;
 
 	/** Matching list */
 	struct entry* match;
@@ -245,6 +256,10 @@ struct fake_pending {
 	ldns_pkt* pkt;
 	/** by what transport was the query sent out */
 	enum transport_type transport;
+	/** if this is a serviced query */
+	int serviced;
+	/** the runtime structure this is part of */
+	struct replay_runtime* runtime;
 };
 
 /**
@@ -263,9 +278,11 @@ struct replay_answer {
  * Read a replay scenario from the file.
  * @param in: file to read from.
  * @param name: name to print in errors.
+ * @param lineno: incremented for every line read.
  * @return: Scenario. NULL if no scenario read.
  */
-struct replay_scenario* replay_scenario_read(FILE* in, const char* name);
+struct replay_scenario* replay_scenario_read(FILE* in, const char* name, 
+	int* lineno);
 
 /**
  * Delete scenario.

testcode/signit.c

@@ -0,0 +1,249 @@
+/*
+ * testcode/signit.c - debug tool to sign rrsets with given keys.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program signs rrsets with the given keys. It can be used to 
+ * construct input to test the validator with.
+ */
+#include "config.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+
+/**
+ * Key settings
+ */
+struct keysets {
+	/** signature inception */
+	uint32_t incep;
+	/** signature expiration */
+	uint32_t expi;
+	/** owner name */
+	char* owner;
+	/** keytag */
+	uint16_t keytag;
+	/** DNSKEY flags */
+	uint16_t flags;
+};
+
+/** print usage and exit */
+static void
+usage()
+{
+	printf("usage:	signit expi ince keytag owner keyfile\n");
+	printf("present rrset data on stdin.\n");
+	printf("signed data is printed to stdout.\n");
+	printf("\n");
+	printf("Or use:	signit NSEC3PARAM hash flags iter salt\n");
+	printf("present names on stdin, hashed names are printed to stdout.\n");
+	exit(1);
+}
+
+/** read expi ince keytag owner from cmdline */
+static void
+parse_cmdline(char *argv[], struct keysets* s)
+{
+	s->expi = cfg_convert_timeval(argv[1]);
+	s->incep = cfg_convert_timeval(argv[2]);
+	s->keytag = (uint16_t)atoi(argv[3]);
+	s->owner = argv[4];
+	s->flags = DNSKEY_BIT_ZSK; /* to enforce signing */
+}
+
+/** read all key files, exit on error */
+static ldns_key_list*
+read_keys(int num, char* names[], struct keysets* set)
+{
+	int i;
+	ldns_key_list* keys = ldns_key_list_new();
+	ldns_key* k;
+	ldns_rdf* rdf;
+	ldns_status s;
+	int b;
+	FILE* in;
+
+	if(!keys) fatal_exit("alloc failure");
+	for(i=0; i<num; i++) {
+		printf("read keyfile %s\n", names[i]);
+		in = fopen(names[i], "r");
+		if(!in) fatal_exit("could not open %s: %s", names[i],
+				strerror(errno));
+		s = ldns_key_new_frm_fp(&k, in);
+		fclose(in);
+		if(s != LDNS_STATUS_OK)
+			fatal_exit("bad keyfile %s: %s", names[i],
+				ldns_get_errorstr_by_id(s));
+		ldns_key_set_expiration(k, set->expi);
+		ldns_key_set_inception(k, set->incep);
+		s = ldns_str2rdf_dname(&rdf, set->owner);
+		if(s != LDNS_STATUS_OK)
+			fatal_exit("bad owner name %s: %s", set->owner,
+				ldns_get_errorstr_by_id(s));
+		ldns_key_set_pubkey_owner(k, rdf);
+		ldns_key_set_flags(k, set->flags);
+		ldns_key_set_keytag(k, set->keytag);
+		b = ldns_key_list_push_key(keys, k);
+		log_assert(b);
+	}
+	return keys;
+}
+
+/** read list of rrs from the file */
+static ldns_rr_list*
+read_rrs(FILE* in)
+{
+	uint32_t my_ttl = 3600;
+	ldns_rdf *my_origin = NULL;
+	ldns_rdf *my_prev = NULL;
+	ldns_status s;
+	int line_nr = 1;
+	int b;
+
+	ldns_rr_list* list;
+	ldns_rr *rr;
+
+	list = ldns_rr_list_new();
+	if(!list) fatal_exit("alloc error");
+
+	while(!feof(in)) {
+		s = ldns_rr_new_frm_fp_l(&rr, in, &my_ttl, &my_origin,
+			&my_prev, &line_nr);
+		if(s == LDNS_STATUS_SYNTAX_TTL || 
+			s == LDNS_STATUS_SYNTAX_ORIGIN ||
+			s == LDNS_STATUS_SYNTAX_EMPTY)
+			continue;
+		else if(s != LDNS_STATUS_OK)
+			fatal_exit("parse error in line %d: %s", line_nr,
+				ldns_get_errorstr_by_id(s));
+		b = ldns_rr_list_push_rr(list, rr);
+		log_assert(b);
+	}
+	printf("read %d lines\n", line_nr);
+
+	return list;
+}
+
+/** sign the rrs with the keys */
+static void
+signit(ldns_rr_list* rrs, ldns_key_list* keys)
+{
+	ldns_rr_list* rrset;
+	ldns_rr_list* sigs;
+	
+	while(ldns_rr_list_rr_count(rrs) > 0) {
+		rrset = ldns_rr_list_pop_rrset(rrs);
+		if(!rrset) fatal_exit("copy alloc failure");
+		sigs = ldns_sign_public(rrset, keys);
+		if(!sigs) fatal_exit("failed to sign");
+		ldns_rr_list_print(stdout, rrset);
+		ldns_rr_list_print(stdout, sigs);
+		printf("\n");
+		ldns_rr_list_free(rrset);
+		ldns_rr_list_free(sigs);
+	}
+}
+
+/** process keys and signit */
+static void
+process_keys(int argc, char* argv[])
+{
+	ldns_rr_list* rrs;
+	ldns_key_list* keys;
+	struct keysets settings;
+	log_assert(argc == 6);
+
+	parse_cmdline(argv, &settings);
+	keys = read_keys(1, argv+5, &settings);
+	rrs = read_rrs(stdin);
+	signit(rrs, keys);
+
+	ldns_rr_list_deep_free(rrs);
+	ldns_key_list_free(keys);
+}
+
+/** process nsec3 params and perform hashing */
+static void
+process_nsec3(int argc, char* argv[])
+{
+	char line[10240];
+	ldns_rdf* salt;
+	ldns_rdf* in, *out;
+	ldns_status status;
+	status = ldns_str2rdf_nsec3_salt(&salt, argv[5]);
+	if(status != LDNS_STATUS_OK)
+		fatal_exit("Could not parse salt %s: %s", argv[5],
+			ldns_get_errorstr_by_id(status));
+	log_assert(argc == 6);
+	while(fgets(line, (int)sizeof(line), stdin)) {
+		if(strlen(line) > 0)
+			line[strlen(line)-1] = 0; /* remove trailing newline */
+		if(line[0]==0)
+			continue;
+		status = ldns_str2rdf_dname(&in, line);
+		if(status != LDNS_STATUS_OK)
+			fatal_exit("Could not parse name %s: %s", line,
+				ldns_get_errorstr_by_id(status));
+		ldns_rdf_print(stdout, in);
+		printf(" -> ");
+		/* arg 3 is flags, unused */
+		out = ldns_nsec3_hash_name(in, (uint8_t)atoi(argv[2]), 
+			(uint16_t)atoi(argv[4]),
+			ldns_rdf_data(salt)[0], ldns_rdf_data(salt)+1);
+		if(!out)
+			fatal_exit("Could not hash %s", line);
+		ldns_rdf_print(stdout, out);
+		printf("\n");
+		ldns_rdf_deep_free(in);
+		ldns_rdf_deep_free(out);
+	}
+	ldns_rdf_deep_free(salt);
+}
+
+/** main program */
+int main(int argc, char* argv[])
+{
+	log_init(NULL);
+	if(argc != 6) {
+		usage();
+	}
+	if(strcmp(argv[1], "NSEC3PARAM") == 0) {
+		process_nsec3(argc, argv);
+		return 0;
+	}
+	process_keys(argc, argv);
+	return 0;
+}

testcode/testbed.sh

@@ -77,12 +77,11 @@ EOF
 		# echossh $1 "gtar xzf unbound_ttt.tar.gz && rm unbound_ttt.tar.gz"
 	fi
 	echossh $1 "cd $2; $MAKE_CMD -f makeconf.mak.$$ Makefile"
-	echossh $1 "cd $2; $MAKE_CMD"
+	echossh $1 "cd $2; $MAKE_CMD all tests"
 	echossh $1 "cd $2; $MAKE_CMD doc"
 	if test $RUN_TEST = yes; then
-	echossh $1 "cd $2/testdata; tpkg clean"
 	echossh $1 "cd $2; bash testcode/do-tests.sh"
-	echossh $1 "cd $2/testdata; tpkg -q report" | tee -a $REPORT_FILE
+	echossh $1 "cd $2/testdata; sh ../testcode/mini_tpkg.sh -q report" | tee -a $REPORT_FILE
 	fi
 	echossh $1 "cd $2; rm -f makeconf.mak.$$"
 	rm -f makeconf.mak.$$

testcode/testbound.c

@@ -51,6 +51,11 @@
 #include "daemon/unbound.c"
 #undef main
 
+/** maximum line length for lines in the replay file. */
+#define MAX_LINE_LEN 1024
+/** the config file (removed at exit) */
+static char cfgfile[MAX_LINE_LEN];
+
 /** give commandline usage for testbound. */
 static void
 testbound_usage()
@@ -79,7 +84,7 @@ add_opts(char* optarg, int* pass_argc, char* pass_argv[])
 {
 	char *p = optarg, *np;
 	size_t len;
-	while(p && isspace(*p)) 
+	while(p && isspace((int)*p)) 
 		p++;
 	while(p && *p) {
 		/* find location of next string and length of this one */
@@ -97,35 +102,77 @@ add_opts(char* optarg, int* pass_argc, char* pass_argv[])
 		(*pass_argc)++;
 		/* go to next option */
 	        p = np;
-		while(p && isspace(*p)) 
+		while(p && isspace((int)*p)) 
 			p++;
 	}
 }
 
-/** pretty print commandline for unbound in this test. */
+/** pretty print commandline for unbound in this test */
 static void
 echo_cmdline(int argc, char* argv[])
 {
 	int i;
-	printf("testbound is starting:");
+	fprintf(stderr, "testbound is starting:");
 	for(i=0; i<argc; i++) {
-		printf(" [%s]", argv[i]);
+		fprintf(stderr, " [%s]", argv[i]);
 	}
-	printf("\n");
+	fprintf(stderr, "\n");
+}
+
+/** process config elements */
+static void
+setup_config(FILE* in, char* configfile, int* lineno,
+	int* pass_argc, char* pass_argv[])
+{
+	char line[MAX_LINE_LEN];
+	char* parse;
+	FILE* cfg;
+	snprintf(configfile, MAX_LINE_LEN, "/tmp/testbound_cfg_%u.tmp", 
+		(unsigned)getpid());
+	add_opts("-c", pass_argc, pass_argv);
+	add_opts(configfile, pass_argc, pass_argv);
+	cfg = fopen(configfile, "w");
+	if(!cfg) fatal_exit("could not open %s: %s", 
+			configfile, strerror(errno));
+	line[MAX_LINE_LEN-1] = 0;
+	while(fgets(line, MAX_LINE_LEN-1, in)) {
+		parse = line;
+		(*lineno)++;
+		while(isspace((int)*parse))
+			parse++;
+		if(!*parse || parse[0] == ';')
+			continue;
+		if(strncmp(parse, "COMMANDLINE", 11) == 0) {
+			parse[strlen(parse)-1] = 0; /* strip off \n */
+			add_opts(parse+11, pass_argc, pass_argv);
+			continue;
+		}
+		if(strncmp(parse, "CONFIG_END", 10) == 0) {
+			fclose(cfg);
+			return;
+		}
+		fputs(line, cfg);
+	}
+	fatal_exit("No CONFIG_END in input file");
+
 }
 
 /** read playback file */
 static struct replay_scenario* 
-setup_playback(const char* filename)
+setup_playback(const char* filename, char* configfile,
+	int* pass_argc, char* pass_argv[])
 {
 	struct replay_scenario* scen = NULL;
+	int lineno = 0;
+
 	if(filename) {
 		FILE *in = fopen(filename, "r");
 		if(!in) {
 			perror(filename);
 			exit(1);
 		}
-		scen = replay_scenario_read(in, filename);
+		setup_config(in, configfile, &lineno, pass_argc, pass_argv);
+		scen = replay_scenario_read(in, filename, &lineno);
 		fclose(in);
 		if(!scen)
 			fatal_exit("Could not read: %s", filename);
@@ -134,6 +181,12 @@ setup_playback(const char* filename)
 	log_info("Scenario: %s", scen->title);
 	return scen;
 }
+
+/** remove config file at exit */
+void remove_configfile(void)
+{
+	unlink(cfgfile);
+}
 	
 /**
  * Main fake event test program. Setup, teardown and report errors.
@@ -154,6 +207,7 @@ main(int argc, char* argv[])
 	log_init(NULL);
 	log_info("Start of %s testbound program.", PACKAGE_STRING);
 	/* determine commandline options for the daemon */
+	cfgfile[0] = 0;
 	pass_argc = 1;
 	pass_argv[0] = "unbound";
 	add_opts("-d", &pass_argc, pass_argv);
@@ -178,9 +232,11 @@ main(int argc, char* argv[])
 		testbound_usage();
 		return 1;
 	}
+	if(atexit(&remove_configfile) != 0)
+		fatal_exit("atexit() failed: %s", strerror(errno));
 
 	/* setup test environment */
-	scen = setup_playback(playback_file);
+	scen = setup_playback(playback_file, cfgfile, &pass_argc, pass_argv);
 	/* init fake event backend */
 	fake_event_init(scen);
 
@@ -197,5 +253,7 @@ main(int argc, char* argv[])
 	fake_event_cleanup();
 	for(c=1; c<pass_argc; c++)
 		free(pass_argv[c]);
+	if(res == 0)
+		log_info("Testbound Exit Success");
 	return res;
 }

testcode/unitanchor.c

@@ -0,0 +1,118 @@
+/*
+ * testcode/unitanchor.c - unit test for trust anchor storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls trust anchor unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/data/dname.h"
+#include "testcode/unitmain.h"
+#include "validator/val_anchor.h"
+
+/** test empty set */
+static void
+test_anchor_empty(struct val_anchors* a)
+{
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c) == NULL);
+}
+
+/** test set of one anchor */
+static void
+test_anchor_one(ldns_buffer* buff, struct val_anchors* a)
+{
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchor_store_str(a, buff, 
+		"nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"));
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c) != NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c) != NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c) != NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002oo\000", 4, c) == NULL);
+}
+
+/** test with several anchors */
+static void
+test_anchors(ldns_buffer* buff, struct val_anchors* a)
+{
+	struct trust_anchor* ta;
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchor_store_str(a, buff, 
+	"labs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"));
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(ta = anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c));
+	unit_assert(query_dname_compare(ta->name, (uint8_t*)"\002nl\000")==0);
+	unit_assert(ta = anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c));
+	unit_assert(query_dname_compare(ta->name, 
+		(uint8_t*)"\004labs\002nl\000") == 0);
+	unit_assert(ta = anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c));
+	unit_assert(query_dname_compare(ta->name, 
+		(uint8_t*)"\002nl\000") == 0);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002oo\000", 4, c) == NULL);
+}
+
+void anchors_test()
+{
+	ldns_buffer* buff = ldns_buffer_new(65800);
+	struct val_anchors* a;
+	unit_assert(a = anchors_create());
+	ldns_buffer_flip(buff);
+	test_anchor_empty(a);
+	test_anchor_one(buff, a);
+	test_anchors(buff, a);
+	anchors_delete(a);
+	ldns_buffer_free(buff);
+}

testcode/unitdname.c

@@ -0,0 +1,755 @@
+/*
+ * testcode/unitdname.c - unit test for dname routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls dname unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "util/data/dname.h"
+
+/** put dname into buffer */
+static ldns_buffer*
+dname_to_buf(ldns_buffer* b, const char* str)
+{
+	ldns_rdf* rdf;
+	ldns_status status;
+	ldns_buffer_clear(b);
+	rdf = ldns_dname_new_frm_str(str);	
+	status = ldns_dname2buffer_wire(b, rdf);
+	if(status != LDNS_STATUS_OK)
+		fatal_exit("%s ldns: %s", __func__, 
+			ldns_get_errorstr_by_id(status));
+	ldns_rdf_deep_free(rdf);
+	ldns_buffer_flip(b);
+	return b;
+}
+
+/** test query_dname_len function */
+static void 
+dname_test_qdl(ldns_buffer* buff)
+{
+	unit_assert( query_dname_len(buff) == 0);
+	unit_assert( query_dname_len(dname_to_buf(buff, ".")) == 1 );
+	unit_assert( query_dname_len(dname_to_buf(buff, "bla.foo.")) == 9 );
+	unit_assert( query_dname_len(dname_to_buf(buff, "x.y.z.example.com."
+		)) == 19 );
+}
+
+/** test query_dname_tolower */
+static void
+dname_test_qdtl(ldns_buffer* buff)
+{
+	ldns_buffer_write_at(buff, 0, "\012abCDeaBCde\003cOm\000", 16);
+	query_dname_tolower(ldns_buffer_begin(buff));
+	unit_assert( memcmp(ldns_buffer_begin(buff), 
+		"\012abcdeabcde\003com\000", 16) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\001+\012abC{e-ZYXe\003NET\000", 18);
+	query_dname_tolower(ldns_buffer_begin(buff));
+	unit_assert( memcmp(ldns_buffer_begin(buff), 
+		"\001+\012abc{e-zyxe\003net\000", 18) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\000", 1);
+	query_dname_tolower(ldns_buffer_begin(buff));
+	unit_assert( memcmp(ldns_buffer_begin(buff), "\000", 1) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\002NL\000", 4);
+	query_dname_tolower(ldns_buffer_begin(buff));
+	unit_assert( memcmp(ldns_buffer_begin(buff), "\002nl\000", 4) == 0);
+}
+
+/** test query_dname_compare */
+static void
+dname_test_query_dname_compare()
+{
+	unit_assert(query_dname_compare((uint8_t*)"", (uint8_t*)"") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\001a", 
+					(uint8_t*)"\001a") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc\001a") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003aBc\001a", 
+					(uint8_t*)"\003AbC\001A") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc", 
+					(uint8_t*)"\003abc\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc") == +1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"") == +1);
+	unit_assert(query_dname_compare((uint8_t*)"", 
+					(uint8_t*)"\003abc\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003xxx\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003axx\001a", 
+					(uint8_t*)"\003abc\001a") == 1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc\001Z") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001Z", 
+					(uint8_t*)"\003abc\001a") == 1);
+}
+
+/** test dname_count_labels */
+static void
+dname_test_count_labels()
+{
+	unit_assert(dname_count_labels((uint8_t*)"") == 1);
+	unit_assert(dname_count_labels((uint8_t*)"\003com") == 2);
+	unit_assert(dname_count_labels((uint8_t*)"\003org") == 2);
+	unit_assert(dname_count_labels((uint8_t*)"\007example\003com") == 3);
+	unit_assert(dname_count_labels((uint8_t*)"\003bla\007example\003com") 
+		== 4);
+}
+
+/** test dname_count_size_labels */
+static void
+dname_test_count_size_labels()
+{
+	size_t sz = 0;
+	unit_assert(dname_count_size_labels((uint8_t*)"", &sz) == 1);
+	unit_assert(sz == 1);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003com", &sz) == 2);
+	unit_assert(sz == 5);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003org", &sz) == 2);
+	unit_assert(sz == 5);
+	unit_assert(dname_count_size_labels((uint8_t*)"\007example\003com", 
+		&sz) == 3);
+	unit_assert(sz == 13);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003bla\007example"
+		"\003com", &sz) == 4);
+	unit_assert(sz == 17);
+}
+
+
+/** test pkt_dname_len */
+static void
+dname_test_pkt_dname_len(ldns_buffer* buff)
+{
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\000", 1);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 1 );
+	unit_assert( ldns_buffer_position(buff) == 1);
+
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\003org\000", 5);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 5 );
+	unit_assert( ldns_buffer_position(buff) == 5);
+
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\007example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 16 );
+	unit_assert( ldns_buffer_position(buff) == 16);
+
+	/* invalid compression pointer: to self */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\300\000os\007example\003org\000", 17);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* valid compression pointer */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\003com\000\040\300\000", 8);
+	ldns_buffer_flip(buff);
+	ldns_buffer_set_position(buff, 6);
+	unit_assert( pkt_dname_len(buff) == 5 );
+	unit_assert( ldns_buffer_position(buff) == 8);
+
+	/* unknown label type */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\107example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* label too long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\047example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* label exceeds packet */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\007example\007org\004", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* name very long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, 
+		"\020a1cdef5555544444"
+		"\020a2cdef5555544444"
+		"\020a3cdef5555544444"
+		"\020a4cdef5555544444"
+		"\020a5cdef5555544444"
+		"\020a6cdef5555544444"
+		"\020a7cdef5555544444"
+		"\020a8cdef5555544444"
+		"\020a9cdef5555544444"
+		"\020aAcdef5555544444"
+		"\020aBcdef5555544444"
+		"\020aCcdef5555544444"
+		"\020aDcdef5555544444"
+		"\020aEcdef5555544444"	/* 238 up to here */
+		"\007aabbccd"		/* 246 up to here */
+		"\007example\000"	/* 255 to here */
+		, 255);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 255 );
+	unit_assert( ldns_buffer_position(buff) == 255);
+
+	/* name too long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, 
+		"\020a1cdef5555544444"
+		"\020a2cdef5555544444"
+		"\020a3cdef5555544444"
+		"\020a4cdef5555544444"
+		"\020a5cdef5555544444"
+		"\020a6cdef5555544444"
+		"\020a7cdef5555544444"
+		"\020a8cdef5555544444"
+		"\020a9cdef5555544444"
+		"\020aAcdef5555544444"
+		"\020aBcdef5555544444"
+		"\020aCcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aDcdef5555544444"
+		"\020aEcdef5555544444"	/* 238 up to here */
+		"\007aabbccd"		/* 246 up to here */
+		"\007example\000"	/* 255 to here */
+		, 255);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+}
+
+/** test dname_lab_cmp */
+static void
+dname_test_dname_lab_cmp()
+{
+	int ml = 0; /* number of labels that matched exactly */
+
+	/* test for equality succeeds */
+	unit_assert(dname_lab_cmp((uint8_t*)"", 1, (uint8_t*)"", 1, &ml) == 0);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003net", 2, 
+		(uint8_t*)"\003net", 2, 
+		&ml) == 0);
+	unit_assert(ml == 2);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == 0);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\004test\007example\003net", 4, 
+		(uint8_t*)"\004test\007example\003net", 4, 
+		&ml) == 0);
+	unit_assert(ml == 4);
+
+	/* root is smaller than anything else */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"", 1, 
+		(uint8_t*)"\003net", 2, 
+		&ml) == -1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003net", 2, 
+		(uint8_t*)"", 1, 
+		&ml) == 1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"", 1, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == -1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"", 1, 
+		&ml) == 1);
+	unit_assert(ml == 1);
+
+	/* label length makes a difference */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\004neta", 2, 
+		(uint8_t*)"\003net", 2, 
+		&ml) != 0);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\002ne", 2, 
+		(uint8_t*)"\004neta", 2, 
+		&ml) != 0);
+	unit_assert(ml == 1);
+
+	/* contents follow the zone apex */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == 1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+
+	/* label content makes a difference */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003aag\007example\003net", 4, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003aag\007example\003net", 4, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bla\003aag\007example\003net", 5, 
+		(uint8_t*)"\003aag\003bla\007example\003net", 5, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\02sn\003opt\003aag\007example\003net", 6, 
+		(uint8_t*)"\02sn\003opt\003bla\007example\003net", 6, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+
+	/* but lowercase/uppercase does not make a difference. */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bLa\007examPLe\003net", 4, 
+		(uint8_t*)"\003bla\007eXAmple\003nET", 4, 
+		&ml) == 0);
+	unit_assert(ml == 4);
+}
+
+/** test dname_subdomain_c */
+static void
+dname_test_subdomain()
+{
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"",
+		(uint8_t*)""));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\003com",
+		(uint8_t*)""));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"",
+		(uint8_t*)"\003com"));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\003com"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\007example\003com"));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)""));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003com"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003org"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\007example\003net",
+		(uint8_t*)"\003org"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\007example\003org"));
+}
+
+/** test dname_strict_subdomain */
+static void
+dname_test_strict_subdomain()
+{
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"", 1,
+		(uint8_t*)"", 1));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\003com", 2,
+		(uint8_t*)"", 1));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"", 1,
+		(uint8_t*)"\003com", 2));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\007example\003com", 3,
+		(uint8_t*)"\003com", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003com", 2,
+		(uint8_t*)"\007example\003com", 3));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\007example\003com", 3,
+		(uint8_t*)"", 1));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\003com", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\003org", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\007example\003net", 3,
+		(uint8_t*)"\003org", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\007example\003org", 3));
+}
+
+/** test dname_is_root */
+static void
+dname_test_isroot()
+{
+	unit_assert(dname_is_root((uint8_t*)"\000"));
+	unit_assert(!dname_is_root((uint8_t*)"\001a\000"));
+	unit_assert(!dname_is_root((uint8_t*)"\005abvcd\003com\000"));
+	/* malformed dname in this test, but should work */
+	unit_assert(!dname_is_root((uint8_t*)"\077a\000"));
+	unit_assert(dname_is_root((uint8_t*)"\000"));
+}
+
+/** test dname_remove_label */
+static void
+dname_test_removelabel()
+{
+	uint8_t* orig = (uint8_t*)"\007example\003com\000";
+	uint8_t* n = orig;
+	size_t l = 13;
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+8 );
+	unit_assert( l == 5 );
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+12 );
+	unit_assert( l == 1 );
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+12 );
+	unit_assert( l == 1 );
+}
+
+/** test dname_signame_label_count */
+static void
+dname_test_sigcount()
+{
+	unit_assert(dname_signame_label_count((uint8_t*)"\000") == 0);
+	unit_assert(dname_signame_label_count((uint8_t*)"\001*\000") == 0);
+	unit_assert(dname_signame_label_count((uint8_t*)"\003xom\000") == 1);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\003xom\000") == 1);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\007example\003xom\000") == 2);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\007example\003xom\000") == 2);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\003www\007example\003xom\000") == 3);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\003www\007example\003xom\000") == 3);
+}
+
+/** test dname_is_wild routine */
+static void
+dname_test_iswild()
+{
+	unit_assert( !dname_is_wild((uint8_t*)"\000") );
+	unit_assert( dname_is_wild((uint8_t*)"\001*\000") );
+	unit_assert( !dname_is_wild((uint8_t*)"\003net\000") );
+	unit_assert( dname_is_wild((uint8_t*)"\001*\003net\000") );
+}
+
+/** test dname_canonical_compare */
+static void
+dname_test_canoncmp()
+{
+	/* equality */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net\000",
+		(uint8_t*)"\003net\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003net\000",
+		(uint8_t*)"\007example\003net\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004test\007example\003net\000",
+		(uint8_t*)"\004test\007example\003net\000"
+		) == 0);
+
+	/* subdomains */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\000"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\003com"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\003com"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\007example\003com"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\000"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\007example\003com"
+		) == -1);
+
+	/* compare rightmost label */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003com"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003org"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003net",
+		(uint8_t*)"\003org"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003org",
+		(uint8_t*)"\007example\003net"
+		) == 1);
+
+	/* label length makes a difference; but only if rest is equal */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004neta",
+		(uint8_t*)"\003net"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\002ne",
+		(uint8_t*)"\004neta"
+		) == -1);
+
+	/* label content */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003aag\007example\003net",
+		(uint8_t*)"\003bla\007example\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bla\007example\003net",
+		(uint8_t*)"\003aag\007example\003net"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bla\003aag\007example\003net",
+		(uint8_t*)"\003aag\003bla\007example\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\02sn\003opt\003aag\007example\003net",
+		(uint8_t*)"\02sn\003opt\003bla\007example\003net"
+		) == -1);
+
+	/* lowercase during compare */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bLa\007examPLe\003net",
+		(uint8_t*)"\003bla\007eXAmple\003nET"
+		) == 0);
+
+	/* example from 4034 */
+	/* example a.example yljkjljk.a.example Z.a.example zABC.a.EXAMPLE
+	 z.example \001.z.example *.z.example \200.z.example */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"",
+		(uint8_t*)"\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)"\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\001Z\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001\200\001z\007example"
+		) == -1);
+	/* same example in reverse */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)""
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\001Z\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001z\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\200\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == 1);
+	/* same example for equality */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)"\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\001Z\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\200\001z\007example",
+		(uint8_t*)"\001\200\001z\007example"
+		) == 0);
+}
+
+/** Test dname_get_shared_topdomain */
+static void
+dname_test_topdomain()
+{
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"",
+			(uint8_t*)""), 
+		(uint8_t*)"") == 0);
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"\003www\007example\003com",
+			(uint8_t*)"\003www\007example\003com"), 
+		(uint8_t*)"\003www\007example\003com") == 0);
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"\003www\007example\003com",
+			(uint8_t*)"\003bla\007example\003com"), 
+		(uint8_t*)"\007example\003com") == 0);
+}
+
+void dname_test()
+{
+	ldns_buffer* buff = ldns_buffer_new(65800);
+	ldns_buffer_flip(buff);
+	dname_test_qdl(buff);
+	dname_test_qdtl(buff);
+	dname_test_query_dname_compare();
+	dname_test_count_labels();
+	dname_test_count_size_labels();
+	dname_test_dname_lab_cmp();
+	dname_test_pkt_dname_len(buff);
+	dname_test_strict_subdomain();
+	dname_test_subdomain();
+	dname_test_isroot();
+	dname_test_removelabel();
+	dname_test_sigcount();
+	dname_test_iswild();
+	dname_test_canoncmp();
+	dname_test_topdomain();
+	ldns_buffer_free(buff);
+}

testcode/unitlruhash.c

@@ -0,0 +1,536 @@
+/*
+ * testcode/unitlruhash.c - unit test for lruhash table.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Tests the locking LRU keeping hash table implementation.
+ */
+
+#include "config.h"
+#include "testcode/unitmain.h"
+#include "util/log.h"
+#include "util/storage/lruhash.h"
+
+/* --- test representation --- */
+/** structure contains test key */
+struct testkey {
+	/** the key id */
+	int id;
+	/** the entry */
+	struct lruhash_entry entry;
+};
+/** structure contains test data */
+struct testdata {
+	/** data value */
+	int data;
+};
+
+/** sizefunc for lruhash */
+static size_t test_sizefunc(void*, void*);
+/** comparefunc for lruhash */
+static int test_compfunc(void*, void*);
+/** delkey for lruhash */
+static void test_delkey(void*, void*, int);
+/** deldata for lruhash */
+static void test_deldata(void*, void*);
+/* --- end test representation --- */
+
+/** hash func, very bad to improve collisions */
+static hashvalue_t myhash(int id) {return (hashvalue_t)id & 0x0f;}
+/** allocate new key, fill in hash */
+static struct testkey* newkey(int id) {
+	struct testkey* k = (struct testkey*)calloc(1, sizeof(struct testkey));
+	if(!k) fatal_exit("out of memory");
+	k->id = id;
+	k->entry.hash = myhash(id);
+	k->entry.key = k;
+	lock_rw_init(&k->entry.lock);
+	return k;
+}
+/** new data el */
+static struct testdata* newdata(int val) {
+	struct testdata* d = (struct testdata*)calloc(1, 
+		sizeof(struct testdata));
+	if(!d) fatal_exit("out of memory");
+	d->data = val;
+	return d;
+}
+/** delete key */
+static void delkey(struct testkey* k) {	
+	lock_rw_destroy(&k->entry.lock); free(k);}
+/** delete data */
+static void deldata(struct testdata* d) {free(d);}
+
+/** test bin_find_entry function and bin_overflow_remove */
+static void
+test_bin_find_entry(struct lruhash* table)
+{
+	struct testkey* k = newkey(12);
+	struct testdata* d = newdata(128);
+	struct testkey* k2 = newkey(12 + 1024);
+	struct testkey* k3 = newkey(14);
+	struct testkey* k4 = newkey(12 + 1024*2);
+	hashvalue_t h = myhash(12);
+	struct lruhash_bin bin;
+	memset(&bin, 0, sizeof(bin));
+	bin_init(&bin, 1);
+
+	/* remove from empty list */
+	bin_overflow_remove(&bin, &k->entry);
+
+	/* find in empty list */
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	/* insert */
+	lock_quick_lock(&bin.lock);
+	bin.overflow_list = &k->entry;
+	lock_quick_unlock(&bin.lock);
+
+	/* find, hash not OK. */
+	unit_assert( bin_find_entry(table, &bin, myhash(13), k) == NULL );
+
+	/* find, hash OK, but cmp not */
+	unit_assert( k->entry.hash == k2->entry.hash );
+	unit_assert( bin_find_entry(table, &bin, h, k2) == NULL );
+
+	/* find, hash OK, and cmp too */
+	unit_assert( bin_find_entry(table, &bin, h, k) == &k->entry );
+
+	/* remove the element */
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	/* prepend two different elements; so the list is long */
+	/* one has the same hash, but different cmp */
+	lock_quick_lock(&bin.lock);
+	unit_assert( k->entry.hash == k4->entry.hash );
+	k4->entry.overflow_next = &k->entry;
+	k3->entry.overflow_next = &k4->entry;
+	bin.overflow_list = &k3->entry;
+	lock_quick_unlock(&bin.lock);
+
+	/* find, hash not OK. */
+	unit_assert( bin_find_entry(table, &bin, myhash(13), k) == NULL );
+
+	/* find, hash OK, but cmp not */
+	unit_assert( k->entry.hash == k2->entry.hash );
+	unit_assert( bin_find_entry(table, &bin, h, k2) == NULL );
+
+	/* find, hash OK, and cmp too */
+	unit_assert( bin_find_entry(table, &bin, h, k) == &k->entry );
+
+	/* remove middle element */
+	unit_assert( bin_find_entry(table, &bin, k4->entry.hash, k4) 
+		== &k4->entry );
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k4->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, k4->entry.hash, k4) == NULL);
+
+	/* remove last element */
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	lock_quick_destroy(&bin.lock);
+	delkey(k);
+	delkey(k2);
+	delkey(k3);
+	delkey(k4);
+	deldata(d);
+}
+
+/** test lru_front lru_remove */
+static void test_lru(struct lruhash* table)
+{
+	struct testkey* k = newkey(12);
+	struct testkey* k2 = newkey(14);
+	lock_quick_lock(&table->lock);
+
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+
+	/* add one */
+	lru_front(table, &k->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	/* remove it */
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+
+	/* add two */
+	lru_front(table, &k->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	lru_front(table, &k2->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k->entry);
+	/* remove first in list */
+	lru_remove(table, &k2->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	lru_front(table, &k2->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k->entry);
+	/* remove last in list */
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k2->entry);
+
+	/* empty the list */
+	lru_remove(table, &k2->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+	lock_quick_unlock(&table->lock);
+	delkey(k);
+	delkey(k2);
+}
+
+/** test hashtable using short sequence */
+static void
+test_short_table(struct lruhash* table) 
+{
+	struct testkey* k = newkey(12);
+	struct testkey* k2 = newkey(14);
+	struct testdata* d = newdata(128);
+	struct testdata* d2 = newdata(129);
+	
+	k->entry.data = d;
+	k2->entry.data = d2;
+
+	lruhash_insert(table, myhash(12), &k->entry, d, NULL);
+	lruhash_insert(table, myhash(14), &k2->entry, d2, NULL);
+	
+	unit_assert( lruhash_lookup(table, myhash(12), k, 0) == &k->entry);
+	lock_rw_unlock( &k->entry.lock );
+	unit_assert( lruhash_lookup(table, myhash(14), k2, 0) == &k2->entry);
+	lock_rw_unlock( &k2->entry.lock );
+	lruhash_remove(table, myhash(12), k);
+	lruhash_remove(table, myhash(14), k2);
+}
+
+/** number of hash test max */
+#define HASHTESTMAX 25
+
+/** test adding a random element */
+static void
+testadd(struct lruhash* table, struct testdata* ref[])
+{
+	int numtoadd = random() % HASHTESTMAX;
+	struct testdata* data = newdata(numtoadd);
+	struct testkey* key = newkey(numtoadd);
+	key->entry.data = data;
+	lruhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	ref[numtoadd] = data;
+}
+
+/** test adding a random element */
+static void
+testremove(struct lruhash* table, struct testdata* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	struct testkey* key = newkey(num);
+	lruhash_remove(table, myhash(num), key);
+	ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element */
+static void
+testlookup(struct lruhash* table, struct testdata* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	struct testkey* key = newkey(num);
+	struct lruhash_entry* en = lruhash_lookup(table, myhash(num), key, 0);
+	struct testdata* data = en? (struct testdata*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0) log_info("lookup %d got %d, expect %d", num, en? data->data :-1,
+		ref[num]? ref[num]->data : -1);
+	unit_assert( data == ref[num] );
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** check integrity of hash table */
+static void
+check_table(struct lruhash* table)
+{
+	struct lruhash_entry* p;
+	size_t c = 0;
+	lock_quick_lock(&table->lock);
+	unit_assert( table->num <= table->size);
+	unit_assert( table->size_mask == (int)table->size-1 );
+	unit_assert( (table->lru_start && table->lru_end) ||
+		(!table->lru_start && !table->lru_end) );
+	unit_assert( table->space_used <= table->space_max );
+	/* check lru list integrity */
+	if(table->lru_start)
+		unit_assert(table->lru_start->lru_prev == NULL);
+	if(table->lru_end)
+		unit_assert(table->lru_end->lru_next == NULL);
+	p = table->lru_start;
+	while(p) {
+		if(p->lru_prev) {
+			unit_assert(p->lru_prev->lru_next == p);
+		}
+		if(p->lru_next) {
+			unit_assert(p->lru_next->lru_prev == p);
+		}
+		c++;
+		p = p->lru_next;
+	}
+	unit_assert(c == table->num);
+
+	/* this assertion is specific to the unit test */
+	unit_assert( table->space_used == 
+		table->num * test_sizefunc(NULL, NULL) );
+	lock_quick_unlock(&table->lock);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testadd_unlim(struct lruhash* table, struct testdata** ref)
+{
+	int numtoadd = random() % (HASHTESTMAX * 10);
+	struct testdata* data = newdata(numtoadd);
+	struct testkey* key = newkey(numtoadd);
+	key->entry.data = data;
+	lruhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	if(ref)
+		ref[numtoadd] = data;
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testremove_unlim(struct lruhash* table, struct testdata** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	struct testkey* key = newkey(num);
+	lruhash_remove(table, myhash(num), key);
+	if(ref)
+		ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testlookup_unlim(struct lruhash* table, struct testdata** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	struct testkey* key = newkey(num);
+	struct lruhash_entry* en = lruhash_lookup(table, myhash(num), key, 0);
+	struct testdata* data = en? (struct testdata*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0 && ref) log_info("lookup unlim %d got %d, expect %d", num, en ? 
+		data->data :-1, ref[num] ? ref[num]->data : -1);
+	if(data && ref) {
+		/* its okay for !data, it fell off the lru */
+		unit_assert( data == ref[num] );
+	}
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** test with long sequence of adds, removes and updates, and lookups */
+static void
+test_long_table(struct lruhash* table) 
+{
+	/* assuming it all fits in the hastable, this check will work */
+	struct testdata* ref[HASHTESTMAX * 100];
+	size_t i;
+	memset(ref, 0, sizeof(ref));
+	/* test assumption */
+	if(0) log_info(" size %d x %d < %d", (int)test_sizefunc(NULL, NULL), 
+		(int)HASHTESTMAX, (int)table->space_max);
+	unit_assert( test_sizefunc(NULL, NULL)*HASHTESTMAX < table->space_max);
+	if(0) lruhash_status(table, "unit test", 1);
+	srandom(48);
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd(table, ref);
+				break;
+			case 1:
+				testremove(table, ref);
+				break;
+			case 2:
+				testlookup(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(table, "unit test", 1);
+		check_table(table);
+		unit_assert( table->num <= HASHTESTMAX );
+	}
+
+	/* test more, but 'ref' assumption does not hold anymore */
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(table, ref);
+				break;
+			case 1:
+				testremove_unlim(table, ref);
+				break;
+			case 2:
+				testlookup_unlim(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(table, "unlim", 1);
+		check_table(table);
+	}
+}
+
+/** structure to threaded test the lru hash table */
+struct test_thr {
+	/** thread num, first entry. */
+	int num;
+	/** id */
+	ub_thread_t id;
+	/** hash table */
+	struct lruhash* table;
+};
+
+/** main routine for threaded hash table test */
+static void*
+test_thr_main(void* arg) 
+{
+	struct test_thr* t = (struct test_thr*)arg;
+	int i;
+	log_thread_set(&t->num);
+	for(i=0; i<1000; i++) {
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(t->table, NULL);
+				break;
+			case 1:
+				testremove_unlim(t->table, NULL);
+				break;
+			case 2:
+				testlookup_unlim(t->table, NULL);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(t->table, "hashtest", 1);
+		if(i % 100 == 0) /* because of locking, not all the time */
+			check_table(t->table);
+	}
+	check_table(t->table);
+	return NULL;
+}
+
+/** test hash table access by multiple threads */
+static void
+test_threaded_table(struct lruhash* table)
+{
+	int numth = 10;
+	struct test_thr t[100];
+	int i;
+
+	for(i=1; i<numth; i++) {
+		t[i].num = i;
+		t[i].table = table;
+		ub_thread_create(&t[i].id, test_thr_main, &t[i]);
+	}
+
+	for(i=1; i<numth; i++) {
+		ub_thread_join(t[i].id);
+	}
+	if(0) lruhash_status(table, "hashtest", 1);
+}
+
+void lruhash_test()
+{
+	/* start very very small array, so it can do lots of table_grow() */
+	/* also small in size so that reclaim has to be done quickly. */
+	struct lruhash* table ;
+	printf("lruhash test\n");
+	table = lruhash_create(2, 4096, 
+		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+	test_bin_find_entry(table);
+	test_lru(table);
+	test_short_table(table);
+	test_long_table(table);
+	lruhash_delete(table);
+	table = lruhash_create(2, 4096, 
+		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+	test_threaded_table(table);
+	lruhash_delete(table);
+}
+
+static size_t test_sizefunc(void* ATTR_UNUSED(key), void* ATTR_UNUSED(data))
+{
+	return sizeof(struct testkey) + sizeof(struct testdata);
+}
+
+static int test_compfunc(void* key1, void* key2)
+{
+	struct testkey* k1 = (struct testkey*)key1;
+	struct testkey* k2 = (struct testkey*)key2;
+	if(k1->id == k2->id)
+		return 0;
+	if(k1->id > k2->id)
+		return 1;
+	return -1;
+}
+
+static void test_delkey(void* key, void* ATTR_UNUSED(arg), int l)
+{
+	if(l) { lock_rw_unlock(&((struct testkey*)key)->entry.lock); }
+	delkey((struct testkey*)key);
+}
+
+static void test_deldata(void* data, void* ATTR_UNUSED(arg))
+{
+	deldata((struct testdata*)data);
+}

testcode/unitmain.c

@@ -41,11 +41,10 @@
 
 #include "config.h"
 #include "util/log.h"
+#include "testcode/unitmain.h"
 
 /** number of tests done */
 int testcount = 0;
-/** test bool x, exits on failure, increases testcount. */
-#define unit_assert(x) testcount++; log_assert(x);
 
 #include "util/alloc.h"
 /** test alloc code */
@@ -55,10 +54,9 @@ alloc_test() {
 	struct alloc_cache major, minor1, minor2;
 	int i;
 
-	checklock_start();
-	alloc_init(&major, NULL);
-	alloc_init(&minor1, &major);
-	alloc_init(&minor2, &major);
+	alloc_init(&major, NULL, 0);
+	alloc_init(&minor1, &major, 0);
+	alloc_init(&minor2, &major, 1);
 
 	t1 = alloc_special_obtain(&minor1);
 	alloc_clear(&minor1);
@@ -84,7 +82,6 @@ alloc_test() {
 	alloc_clear(&minor2);
 	unit_assert(major.num_quar == 11);
 	alloc_clear(&major);
-	checklock_stop();
 }
 
 #include "util/net_help.h"
@@ -100,6 +97,104 @@ net_test()
 	unit_assert( !str_is_ip6("213.154.224.12") );
 	unit_assert( !str_is_ip6("213.154.224.255") );
 	unit_assert( !str_is_ip6("255.255.255.0") );
+	unit_assert( is_pow2(0) );
+	unit_assert( is_pow2(1) );
+	unit_assert( is_pow2(2) );
+	unit_assert( is_pow2(4) );
+	unit_assert( is_pow2(8) );
+	unit_assert( is_pow2(16) );
+	unit_assert( is_pow2(1024) );
+	unit_assert( is_pow2(1024*1024) );
+	unit_assert( is_pow2(1024*1024*1024) );
+	unit_assert( !is_pow2(3) );
+	unit_assert( !is_pow2(5) );
+	unit_assert( !is_pow2(6) );
+	unit_assert( !is_pow2(7) );
+	unit_assert( !is_pow2(9) );
+	unit_assert( !is_pow2(10) );
+	unit_assert( !is_pow2(11) );
+	unit_assert( !is_pow2(17) );
+	unit_assert( !is_pow2(23) );
+	unit_assert( !is_pow2(257) );
+	unit_assert( !is_pow2(259) );
+}
+
+#include "util/rtt.h"
+/** test RTT code */
+static void
+rtt_test()
+{
+	int i;
+	struct rtt_info r;
+	rtt_init(&r);
+	/* initial value sensible */
+	unit_assert( rtt_timeout(&r) == 3000 );
+	rtt_lost(&r);
+	unit_assert( rtt_timeout(&r) == 6000 );
+	rtt_lost(&r);
+	unit_assert( rtt_timeout(&r) == 12000 );
+	rtt_update(&r, 4000);
+	unit_assert( rtt_timeout(&r) >= 5000 );
+	rtt_lost(&r);
+	for(i=0; i<100; i++) {
+		rtt_lost(&r); 
+		unit_assert( rtt_timeout(&r) > RTT_MIN_TIMEOUT-1);
+		unit_assert( rtt_timeout(&r) < RTT_MAX_TIMEOUT+1);
+	}
+}
+
+#include "services/cache/infra.h"
+#include "util/config_file.h"
+/** test host cache */
+static void
+infra_test()
+{
+	int one = 1;
+	uint8_t* zone = (uint8_t*)"\007example\003com\000";
+	size_t zonelen = 13;
+	struct infra_cache* slab;
+	struct config_file* cfg = config_create();
+	time_t now = 0;
+	int vs, to;
+	struct infra_host_key* k;
+	struct infra_host_data* d;
+
+	slab = infra_create(cfg);
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == 3000 );
+
+	unit_assert( infra_rtt_update(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), -1, now) );
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == 6000 );
+
+	unit_assert( infra_edns_update(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), -1, now) );
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == -1 && to == 6000 );
+
+	now += cfg->host_ttl + 10;
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == 3000 );
+	
+	unit_assert( infra_set_lame(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), zone, zonelen, now) );
+	unit_assert( (d=infra_lookup_host(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), 0, now, &k)) );
+	unit_assert( d->ttl == now+cfg->host_ttl );
+	unit_assert( d->edns_version == 0 );
+	unit_assert( infra_lookup_lame(d, zone, zonelen, now) );
+	unit_assert( !infra_lookup_lame(d, zone, zonelen, 
+		now+cfg->lame_ttl+10) );
+	unit_assert( !infra_lookup_lame(d, (uint8_t*)"\000", 1, now) );
+	lock_rw_unlock(&k->entry.lock);
+
+	infra_delete(slab);
+	config_delete(cfg);
 }
 
 /**
@@ -117,8 +212,18 @@ main(int argc, char* argv[])
 		return 1;
 	}
 	printf("Start of %s unit test.\n", PACKAGE_STRING);
+	checklock_start();
+	verify_test();
 	net_test();
+	dname_test();
+	anchors_test();
+	rtt_test();
 	alloc_test();
-	printf("%d tests succeeded\n", testcount);
+	lruhash_test();
+	slabhash_test();
+	infra_test();
+	msgparse_test();
+	checklock_stop();
+	printf("%d checks ok.\n", testcount);
 	return 0;
 }

testcode/unitmain.h

@@ -0,0 +1,63 @@
+/*
+ * testcode/unitmain.h - unit test main program for unbound.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+
+#ifndef TESTCODE_UNITMAIN_H
+#define TESTCODE_UNITMAIN_H
+#include "util/log.h"
+
+/** number of tests done */
+extern int testcount;
+/** test bool x, exits on failure, increases testcount. */
+#define unit_assert(x) do {testcount++; log_assert(x);} while(0)
+
+/** unit test lruhashtable implementation */
+void lruhash_test();
+/** unit test slabhashtable implementation */
+void slabhash_test();
+/** unit test for msgreply and msgparse */
+void msgparse_test();
+/** unit test dname handling functions */
+void dname_test();
+/** unit test trust anchor storage functions */
+void anchors_test();
+/** unit test for verification functions */
+void verify_test();
+
+#endif /* TESTCODE_UNITMAIN_H */

testcode/unitmsgparse.c

@@ -0,0 +1,454 @@
+/*
+ * testcode/unitmsgparse.c - unit test for msg parse routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls msg parse unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "util/data/msgparse.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgencode.h"
+#include "util/alloc.h"
+#include "util/region-allocator.h"
+#include "util/net_help.h"
+#include "testcode/readhex.h"
+
+/** verbose message parse unit test */
+static int vbmp = 0;
+/** if matching within a section should disregard the order of RRs. */
+static int matches_nolocation = 0;
+
+/** match two rr lists */
+static int
+match_list(ldns_rr_list* q, ldns_rr_list *p)
+{
+	size_t i;
+	if(ldns_rr_list_rr_count(q) != ldns_rr_list_rr_count(p)) {
+		verbose(3, "rrlistcount different %d %d", 
+			(int)ldns_rr_list_rr_count(q), 
+			(int)ldns_rr_list_rr_count(p));
+		return 0;
+	}
+	for(i=0; i<ldns_rr_list_rr_count(q); i++)
+	{
+		if(matches_nolocation) {
+			if(!ldns_rr_list_contains_rr(p, ldns_rr_list_rr(q, i)))
+			{
+				verbose(3, "rr %u not found", (unsigned)i);
+				return 0;
+			}
+		} else {
+			if(ldns_rr_compare(ldns_rr_list_rr(q, i),
+				ldns_rr_list_rr(p, i)) != 0) {
+				verbose(3, "rr %u different", (unsigned)i);
+				return 0;
+			}
+			/* and check the ttl */
+			if(ldns_rr_ttl(ldns_rr_list_rr(q, i)) !=
+				ldns_rr_ttl(ldns_rr_list_rr(p, i))) {
+				verbose(3, "rr %u ttl different", (unsigned)i);
+				return 0;
+			}
+		}
+	}
+	return 1;
+}
+
+/** match edns sections */
+static int
+match_edns(ldns_pkt* q, ldns_pkt* p)
+{
+	if(ldns_pkt_edns_udp_size(q) != ldns_pkt_edns_udp_size(p))
+		return 0;
+	if(ldns_pkt_edns_extended_rcode(q) != ldns_pkt_edns_extended_rcode(p))
+		return 0;
+	if(ldns_pkt_edns_version(q) != ldns_pkt_edns_version(p))
+		return 0;
+	if(ldns_pkt_edns_do(q) != ldns_pkt_edns_do(p))
+		return 0;
+	if(ldns_pkt_edns_z(q) != ldns_pkt_edns_z(p))
+		return 0;
+	if(ldns_rdf_compare(ldns_pkt_edns_data(q), ldns_pkt_edns_data(p)) != 0)
+		return 0;
+	return 1;
+}
+
+/** compare two booleans */
+static int
+cmp_bool(int x, int y)
+{
+	if(!x && !y) return 0;
+	if(x && y) return 0;
+	if(!x) return -1;
+	return 1;
+}
+
+/** match all of the packet */
+static int
+match_all(ldns_pkt* q, ldns_pkt* p)
+{
+	if(ldns_pkt_get_opcode(q) != ldns_pkt_get_opcode(p))
+	{ verbose(3, "allmatch: opcode different"); return 0;}
+	if(ldns_pkt_get_rcode(q) != ldns_pkt_get_rcode(p))
+	{ verbose(3, "allmatch: rcode different"); return 0;}
+	if(ldns_pkt_id(q) != ldns_pkt_id(p))
+	{ verbose(3, "allmatch: id different"); return 0;}
+	if(cmp_bool(ldns_pkt_qr(q), ldns_pkt_qr(p)) != 0)
+	{ verbose(3, "allmatch: qr different"); return 0;}
+	if(cmp_bool(ldns_pkt_aa(q), ldns_pkt_aa(p)) != 0)
+	{ verbose(3, "allmatch: aa different"); return 0;}
+	if(cmp_bool(ldns_pkt_tc(q), ldns_pkt_tc(p)) != 0)
+	{ verbose(3, "allmatch: tc different"); return 0;}
+	if(cmp_bool(ldns_pkt_rd(q), ldns_pkt_rd(p)) != 0)
+	{ verbose(3, "allmatch: rd different"); return 0;}
+	if(cmp_bool(ldns_pkt_cd(q), ldns_pkt_cd(p)) != 0)
+	{ verbose(3, "allmatch: cd different"); return 0;}
+	if(cmp_bool(ldns_pkt_ra(q), ldns_pkt_ra(p)) != 0)
+	{ verbose(3, "allmatch: ra different"); return 0;}
+	if(cmp_bool(ldns_pkt_ad(q), ldns_pkt_ad(p)) != 0)
+	{ verbose(3, "allmatch: ad different"); return 0;}
+	if(ldns_pkt_qdcount(q) != ldns_pkt_qdcount(p))
+	{ verbose(3, "allmatch: qdcount different"); return 0;}
+	if(ldns_pkt_ancount(q) != ldns_pkt_ancount(p))
+	{ verbose(3, "allmatch: ancount different"); return 0;}
+	if(ldns_pkt_nscount(q) != ldns_pkt_nscount(p))
+	{ verbose(3, "allmatch: nscount different"); return 0;}
+	if(ldns_pkt_arcount(q) != ldns_pkt_arcount(p))
+	{ verbose(3, "allmatch: arcount different"); return 0;}
+	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p)))
+	{ verbose(3, "allmatch: qd section different"); return 0;}
+	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p)))
+	{ verbose(3, "allmatch: an section different"); return 0;}
+	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p)))
+	{ verbose(3, "allmatch: ns section different"); return 0;}
+	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p)))
+	{ verbose(3, "allmatch: ar section different"); return 0;}
+	if(!match_edns(q, p))
+	{ verbose(3, "edns different."); return 0;}
+	return 1;
+}
+
+/** see if buffers contain the same packet */
+static int
+test_buffers(ldns_buffer* pkt, ldns_buffer* out)
+{
+	ldns_pkt* p1=0, *p2=0;
+	ldns_status s1, s2;
+	/* check binary same */
+	if(ldns_buffer_limit(pkt) == ldns_buffer_limit(out) &&
+		memcmp(ldns_buffer_begin(pkt), ldns_buffer_begin(out),
+			ldns_buffer_limit(pkt)) == 0) {
+		if(vbmp) printf("binary the same (length=%u)\n",
+				(unsigned)ldns_buffer_limit(pkt));
+		return 1;
+	}
+
+	if(vbmp) {
+		size_t sz = 16;
+		size_t count;
+		size_t lim = ldns_buffer_limit(out);
+		if(ldns_buffer_limit(pkt) < lim)
+			lim = ldns_buffer_limit(pkt);
+		for(count=0; count<lim; count+=sz) {
+			size_t rem = sz;
+			if(lim-count < sz) rem = lim-count;
+			if(memcmp(ldns_buffer_at(pkt, count), 
+				ldns_buffer_at(out, count), rem) == 0) {
+				log_info("same %d %d", (int)count, (int)rem);
+				log_hex("same: ", ldns_buffer_at(pkt, count),
+					rem);
+			} else {
+				log_info("diff %d %d", (int)count, (int)rem);
+				log_hex("difp: ", ldns_buffer_at(pkt, count),
+					rem);
+				log_hex("difo: ", ldns_buffer_at(out, count),
+					rem);
+			}
+		}
+	}
+	/* check if it 'means the same' */
+	s1 = ldns_buffer2pkt_wire(&p1, pkt);
+	s2 = ldns_buffer2pkt_wire(&p2, out);
+	if(vbmp) {
+		log_hex("orig in hex", ldns_buffer_begin(pkt),
+			ldns_buffer_limit(pkt));
+		log_hex("unbound out in hex", ldns_buffer_begin(out),
+			ldns_buffer_limit(out));
+		printf("\npacket from unbound (%d):\n", 
+			(int)ldns_buffer_limit(out));
+		ldns_pkt_print(stdout, p2);
+
+		printf("\npacket original (%d):\n", 
+			(int)ldns_buffer_limit(pkt));
+		ldns_pkt_print(stdout, p1);
+		printf("\n");
+	}
+	if(s1 != s2) {
+		/* oops! */
+		printf("input ldns parse: %s, output ldns parse: %s.\n",
+			ldns_get_errorstr_by_id(s1), 
+			ldns_get_errorstr_by_id(s2));
+		unit_assert(0);
+	}
+	/* compare packets */
+	unit_assert(match_all(p1, p2));
+	ldns_pkt_free(p1);
+	ldns_pkt_free(p2);
+	return 0;
+}
+
+/** check if unbound formerr equals ldns formerr */
+static void
+checkformerr(ldns_buffer* pkt)
+{
+	ldns_pkt* p;
+	ldns_status status = ldns_buffer2pkt_wire(&p, pkt);
+	if(vbmp) printf("formerr, ldns parse is: %s\n",
+			ldns_get_errorstr_by_id(status));
+	if(status == LDNS_STATUS_OK) {
+		printf("Formerr, but ldns gives packet:\n");
+		ldns_pkt_print(stdout, p);
+		exit(1);
+	}
+	unit_assert(status != LDNS_STATUS_OK);
+}
+
+/** test a packet */
+static void
+testpkt(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out, 
+	const char* hex)
+{
+	struct query_info qi;
+	struct reply_info* rep = 0;
+	int ret;
+	uint16_t id;
+	uint16_t flags;
+	uint32_t timenow = 0;
+	region_type *region = region_create(malloc, free);
+	struct edns_data edns;
+
+	hex_to_buf(pkt, hex);
+	memmove(&id, ldns_buffer_begin(pkt), sizeof(id));
+	if(ldns_buffer_limit(pkt) < 2)
+		flags = 0;
+	else	memmove(&flags, ldns_buffer_at(pkt, 2), sizeof(flags));
+	flags = ntohs(flags);
+	ret = reply_info_parse(pkt, alloc, &qi, &rep, region, &edns);
+	if(ret != 0) {
+		if(vbmp) printf("parse code %d: %s\n", ret, 
+			ldns_lookup_by_id(ldns_rcodes, ret)->name);
+		if(ret == LDNS_RCODE_FORMERR)
+			checkformerr(pkt);
+		unit_assert(ret != LDNS_RCODE_SERVFAIL);
+	} else {
+		const size_t lim = 512;
+		ret = reply_info_encode(&qi, rep, id, flags, out, timenow,
+			region, 65535, (int)(edns.bits & EDNS_DO) );
+		unit_assert(ret != 0); /* udp packets should fit */
+		attach_edns_record(out, &edns);
+		if(vbmp) printf("inlen %u outlen %u\n", 
+			(unsigned)ldns_buffer_limit(pkt),
+			(unsigned)ldns_buffer_limit(out));
+		test_buffers(pkt, out);
+
+		if(ldns_buffer_limit(out) > lim) {
+			ret = reply_info_encode(&qi, rep, id, flags, out, 
+				timenow, region, 
+				lim - calc_edns_field_size(&edns),
+				(int)(edns.bits & EDNS_DO));
+			unit_assert(ret != 0); /* should fit, but with TC */
+			attach_edns_record(out, &edns);
+			if( LDNS_QDCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_QDCOUNT(ldns_buffer_begin(pkt)) ||
+				LDNS_ANCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_ANCOUNT(ldns_buffer_begin(pkt)) ||
+				LDNS_NSCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_NSCOUNT(ldns_buffer_begin(pkt)))
+				unit_assert(
+				LDNS_TC_WIRE(ldns_buffer_begin(out)));
+				/* must set TC bit if shortened */
+			unit_assert(ldns_buffer_limit(out) <= lim);
+		}
+	} 
+
+	query_info_clear(&qi);
+	reply_info_parsedelete(rep, alloc);
+	region_destroy(region);
+}
+
+/** simple test of parsing */
+static void
+simpletest(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out)
+{
+	/* a root query  drill -q - */
+	testpkt(pkt, alloc, out, 
+		" c5 40 01 00 00 01 00 00 00 00 00 00 00 00 02 00 01 ");
+
+	/* very small packet */
+	testpkt(pkt, alloc, out, 
+"; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19\n"
+";-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n"
+"74 0c 85 83 00 01 00 00 00 01 00 00 03 62 6c 61 09 6e 6c 6e    ;          1-  20\n"
+"65 74 6c 61 62 73 02 6e 6c 00 00 0f 00 01 09 6e 6c 6e 65 74    ;         21-  40\n"
+"6c 61 62 73 02 6e 6c 00 00 06 00 01 00 00 46 50 00 40 04 6f    ;         41-  60\n"
+"70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 0a 68 6f    ;         61-  80\n"
+"73 74 6d 61 73 74 65 72 09 6e 6c 6e 65 74 6c 61 62 73 02 6e    ;         81- 100\n"
+"6c 00 77 a1 02 58 00 00 70 80 00 00 1c 20 00 09 3a 80 00 00    ;        101- 120\n"
+"46 50\n");
+	
+	/* a root reply  drill -w - */
+	testpkt(pkt, alloc, out, 
+	" ; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19\n"
+	" ;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n"
+	" 97 3f 81 80 00 01 00 0d 00 00 00 02 00 00 02 00 01 00 00 02    ;          1-  20\n"
+	" 00 01 00 06 6d 38 00 14 01 49 0c 52 4f 4f 54 2d 53 45 52 56    ;         21-  40\n"
+	" 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01    ;         41-  60\n"
+	" 4a 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00    ;         61-  80\n"
+	" 00 02 00 01 00 06 6d 38 00 14 01 4b 0c 52 4f 4f 54 2d 53 45    ;         81- 100\n"
+	" 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00    ;        101- 120\n"
+	" 14 01 4c 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54    ;        121- 140\n"
+	" 00 00 00 02 00 01 00 06 6d 38 00 14 01 4d 0c 52 4f 4f 54 2d    ;        141- 160\n"
+	" 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d    ;        161- 180\n"
+	" 38 00 14 01 41 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e    ;        181- 200\n"
+	" 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 42 0c 52 4f 4f    ;        201- 220\n"
+	" 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00    ;        221- 240\n"
+	" 06 6d 38 00 14 01 43 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53    ;        241- 260\n"
+	" 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 44 0c 52    ;        261- 280\n"
+	" 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00    ;        281- 300\n"
+	" 01 00 06 6d 38 00 14 01 45 0c 52 4f 4f 54 2d 53 45 52 56 45    ;        301- 320\n"
+	" 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 46    ;        321- 340\n"
+	" 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00    ;        341- 360\n"
+	" 02 00 01 00 06 6d 38 00 14 01 47 0c 52 4f 4f 54 2d 53 45 52    ;        361- 380\n"
+	" 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14    ;        381- 400\n"
+	" 01 48 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00    ;        401- 420\n"
+	" 01 41 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00    ;        421- 440\n"
+	" 00 01 00 01 00 02 64 b9 00 04 c6 29 00 04 01 4a 0c 52 4f 4f    ;        441- 460\n"
+	" 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 01 00 01 00 02    ;        461- 480\n"
+	" 64 b9 00 04 c0 3a 80 1e  ");
+}
+
+/** simple test of parsing, pcat file */
+static void
+testfromfile(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
+	const char* fname)
+{
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	int no=0;
+	if(!in) {
+		perror("fname");
+		return;
+	}
+	while(fgets(buf, (int)sizeof(buf), in)) {
+		if(buf[0] == ';') /* comment */
+			continue;
+		if(strlen(buf) < 10) /* skip pcat line numbers. */
+			continue;
+		if(vbmp) {
+			printf("test no %d: %s", no, buf);
+			fflush(stdout);
+		}
+		testpkt(pkt, alloc, out, buf);
+		no++;
+	}
+	fclose(in);
+}
+
+/** simple test of parsing, drill file */
+static void
+testfromdrillfile(ldns_buffer* pkt, struct alloc_cache* alloc, 
+	ldns_buffer* out, const char* fname)
+{
+	/*  ;-- is used to indicate a new message */
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	char* np = buf;
+	buf[0]=0;
+	if(!in) {
+		perror("fname");
+		return;
+	}
+	while(fgets(np, (int)sizeof(buf) - (np-buf), in)) {
+		if(strncmp(np, ";--", 3) == 0) {
+			/* new entry */
+			/* test previous */
+			if(np != buf)
+				testpkt(pkt, alloc, out, buf);
+			/* set for new entry */
+			np = buf;
+			buf[0]=0;
+			continue;
+		}
+		if(np[0] == ';') /* comment */
+			continue;
+		np = &np[strlen(np)];
+	}
+	testpkt(pkt, alloc, out, buf);
+	fclose(in);
+}
+
+void msgparse_test()
+{
+	ldns_buffer* pkt = ldns_buffer_new(65553);
+	ldns_buffer* out = ldns_buffer_new(65553);
+	struct alloc_cache super_a, alloc;
+	/* init */
+	alloc_init(&super_a, NULL, 0);
+	alloc_init(&alloc, &super_a, 2);
+
+	printf("testmsgparse\n");
+	simpletest(pkt, &alloc, out);
+	/* plain hex dumps, like pcat */
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.1");
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.2");
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.3");
+	/* like from drill -w - */
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.4");
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.5");
+
+	matches_nolocation = 1; /* RR order not important for the next test */
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.6");
+	matches_nolocation = 0; 
+
+	/* cleanup */
+	alloc_clear(&alloc);
+	alloc_clear(&super_a);
+	ldns_buffer_free(pkt);
+	ldns_buffer_free(out);
+}

testcode/unitslabhash.c

@@ -0,0 +1,416 @@
+/*
+ * testcode/unitslabhash.c - unit test for slabhash table.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Tests the locking LRU keeping hash table implementation.
+ */
+
+#include "config.h"
+#include "testcode/unitmain.h"
+#include "util/log.h"
+#include "util/storage/slabhash.h"
+
+/* --- test representation --- */
+/** structure contains test key */
+struct slabtestkey {
+	/** the key id */
+	int id;
+	/** the entry */
+	struct lruhash_entry entry;
+};
+/** structure contains test data */
+struct slabtestdata {
+	/** data value */
+	int data;
+};
+
+/** sizefunc for lruhash */
+static size_t test_sizefunc(void*, void*);
+/** comparefunc for lruhash */
+static int test_compfunc(void*, void*);
+/** delkey for lruhash */
+static void test_delkey(void*, void*, int);
+/** deldata for lruhash */
+static void test_deldata(void*, void*);
+/* --- end test representation --- */
+
+/** hash func, very bad to improve collisions, both high and low bits */
+static hashvalue_t myhash(int id) {
+	hashvalue_t h = (hashvalue_t)id & 0x0f;
+	h |= (h << 28);
+	return h;
+}
+
+/** allocate new key, fill in hash */
+static struct slabtestkey* newkey(int id) {
+	struct slabtestkey* k = (struct slabtestkey*)calloc(1, sizeof(struct slabtestkey));
+	if(!k) fatal_exit("out of memory");
+	k->id = id;
+	k->entry.hash = myhash(id);
+	k->entry.key = k;
+	lock_rw_init(&k->entry.lock);
+	return k;
+}
+/** new data el */
+static struct slabtestdata* newdata(int val) {
+	struct slabtestdata* d = (struct slabtestdata*)calloc(1, 
+		sizeof(struct slabtestdata));
+	if(!d) fatal_exit("out of memory");
+	d->data = val;
+	return d;
+}
+/** delete key */
+static void delkey(struct slabtestkey* k) {	
+	lock_rw_destroy(&k->entry.lock); free(k);}
+/** delete data */
+static void deldata(struct slabtestdata* d) {free(d);}
+
+/** test hashtable using short sequence */
+static void
+test_short_table(struct slabhash* table) 
+{
+	struct slabtestkey* k = newkey(12);
+	struct slabtestkey* k2 = newkey(14);
+	struct slabtestdata* d = newdata(128);
+	struct slabtestdata* d2 = newdata(129);
+	
+	k->entry.data = d;
+	k2->entry.data = d2;
+
+	slabhash_insert(table, myhash(12), &k->entry, d, NULL);
+	slabhash_insert(table, myhash(14), &k2->entry, d2, NULL);
+	
+	unit_assert( slabhash_lookup(table, myhash(12), k, 0) == &k->entry);
+	lock_rw_unlock( &k->entry.lock );
+	unit_assert( slabhash_lookup(table, myhash(14), k2, 0) == &k2->entry);
+	lock_rw_unlock( &k2->entry.lock );
+	slabhash_remove(table, myhash(12), k);
+	slabhash_remove(table, myhash(14), k2);
+}
+
+/** number of hash test max */
+#define HASHTESTMAX 32
+
+/** test adding a random element */
+static void
+testadd(struct slabhash* table, struct slabtestdata* ref[])
+{
+	int numtoadd = random() % HASHTESTMAX;
+	struct slabtestdata* data = newdata(numtoadd);
+	struct slabtestkey* key = newkey(numtoadd);
+	key->entry.data = data;
+	slabhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	ref[numtoadd] = data;
+}
+
+/** test adding a random element */
+static void
+testremove(struct slabhash* table, struct slabtestdata* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	struct slabtestkey* key = newkey(num);
+	slabhash_remove(table, myhash(num), key);
+	ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element */
+static void
+testlookup(struct slabhash* table, struct slabtestdata* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	struct slabtestkey* key = newkey(num);
+	struct lruhash_entry* en = slabhash_lookup(table, myhash(num), key, 0);
+	struct slabtestdata* data = en? (struct slabtestdata*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0) log_info("lookup %d got %d, expect %d", num, en? data->data :-1,
+		ref[num]? ref[num]->data : -1);
+	unit_assert( data == ref[num] );
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** check integrity of hash table */
+static void
+check_lru_table(struct lruhash* table)
+{
+	struct lruhash_entry* p;
+	size_t c = 0;
+	lock_quick_lock(&table->lock);
+	unit_assert( table->num <= table->size);
+	unit_assert( table->size_mask == (int)table->size-1 );
+	unit_assert( (table->lru_start && table->lru_end) ||
+		(!table->lru_start && !table->lru_end) );
+	unit_assert( table->space_used <= table->space_max );
+	/* check lru list integrity */
+	if(table->lru_start)
+		unit_assert(table->lru_start->lru_prev == NULL);
+	if(table->lru_end)
+		unit_assert(table->lru_end->lru_next == NULL);
+	p = table->lru_start;
+	while(p) {
+		if(p->lru_prev) {
+			unit_assert(p->lru_prev->lru_next == p);
+		}
+		if(p->lru_next) {
+			unit_assert(p->lru_next->lru_prev == p);
+		}
+		c++;
+		p = p->lru_next;
+	}
+	unit_assert(c == table->num);
+
+	/* this assertion is specific to the unit test */
+	unit_assert( table->space_used == 
+		table->num * test_sizefunc(NULL, NULL) );
+	lock_quick_unlock(&table->lock);
+}
+
+/** check integrity of hash table */
+static void
+check_table(struct slabhash* table)
+{
+	size_t i;
+	for(i=0; i<table->size; i++)
+		check_lru_table(table->array[i]);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testadd_unlim(struct slabhash* table, struct slabtestdata** ref)
+{
+	int numtoadd = random() % (HASHTESTMAX * 10);
+	struct slabtestdata* data = newdata(numtoadd);
+	struct slabtestkey* key = newkey(numtoadd);
+	key->entry.data = data;
+	slabhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	if(ref)
+		ref[numtoadd] = data;
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testremove_unlim(struct slabhash* table, struct slabtestdata** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	struct slabtestkey* key = newkey(num);
+	slabhash_remove(table, myhash(num), key);
+	if(ref)
+		ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testlookup_unlim(struct slabhash* table, struct slabtestdata** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	struct slabtestkey* key = newkey(num);
+	struct lruhash_entry* en = slabhash_lookup(table, myhash(num), key, 0);
+	struct slabtestdata* data = en? (struct slabtestdata*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0 && ref) log_info("lookup unlim %d got %d, expect %d", num, en ? 
+		data->data :-1, ref[num] ? ref[num]->data : -1);
+	if(data && ref) {
+		/* its okay for !data, it fell off the lru */
+		unit_assert( data == ref[num] );
+	}
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** test with long sequence of adds, removes and updates, and lookups */
+static void
+test_long_table(struct slabhash* table) 
+{
+	/* assuming it all fits in the hastable, this check will work */
+	struct slabtestdata* ref[HASHTESTMAX * 100];
+	size_t i;
+	memset(ref, 0, sizeof(ref));
+	/* test assumption */
+	if(0) slabhash_status(table, "unit test", 1);
+	srandom(48);
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd(table, ref);
+				break;
+			case 1:
+				testremove(table, ref);
+				break;
+			case 2:
+				testlookup(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(table, "unit test", 1);
+		check_table(table);
+	}
+
+	/* test more, but 'ref' assumption does not hold anymore */
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(table, ref);
+				break;
+			case 1:
+				testremove_unlim(table, ref);
+				break;
+			case 2:
+				testlookup_unlim(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(table, "unlim", 1);
+		check_table(table);
+	}
+}
+
+/** structure to threaded test the lru hash table */
+struct slab_test_thr {
+	/** thread num, first entry. */
+	int num;
+	/** id */
+	ub_thread_t id;
+	/** hash table */
+	struct slabhash* table;
+};
+
+/** main routine for threaded hash table test */
+static void*
+test_thr_main(void* arg) 
+{
+	struct slab_test_thr* t = (struct slab_test_thr*)arg;
+	int i;
+	log_thread_set(&t->num);
+	for(i=0; i<1000; i++) {
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(t->table, NULL);
+				break;
+			case 1:
+				testremove_unlim(t->table, NULL);
+				break;
+			case 2:
+				testlookup_unlim(t->table, NULL);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(t->table, "hashtest", 1);
+		if(i % 100 == 0) /* because of locking, not all the time */
+			check_table(t->table);
+	}
+	check_table(t->table);
+	return NULL;
+}
+
+/** test hash table access by multiple threads */
+static void
+test_threaded_table(struct slabhash* table)
+{
+	int numth = 10;
+	struct slab_test_thr t[100];
+	int i;
+
+	for(i=1; i<numth; i++) {
+		t[i].num = i;
+		t[i].table = table;
+		ub_thread_create(&t[i].id, test_thr_main, &t[i]);
+	}
+
+	for(i=1; i<numth; i++) {
+		ub_thread_join(t[i].id);
+	}
+	if(0) slabhash_status(table, "hashtest", 1);
+}
+
+void slabhash_test()
+{
+	/* start very very small array, so it can do lots of table_grow() */
+	/* also small in size so that reclaim has to be done quickly. */
+	struct slabhash* table;
+	printf("slabhash test\n");
+	table = slabhash_create(4, 2, 5200, 
+		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+	test_short_table(table);
+	test_long_table(table);
+	slabhash_delete(table);
+	table = slabhash_create(4, 2, 5200, 
+		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+	test_threaded_table(table);
+	slabhash_delete(table);
+}
+
+static size_t test_sizefunc(void* ATTR_UNUSED(key), void* ATTR_UNUSED(data))
+{
+	return sizeof(struct slabtestkey) + sizeof(struct slabtestdata);
+}
+
+static int test_compfunc(void* key1, void* key2)
+{
+	struct slabtestkey* k1 = (struct slabtestkey*)key1;
+	struct slabtestkey* k2 = (struct slabtestkey*)key2;
+	if(k1->id == k2->id)
+		return 0;
+	if(k1->id > k2->id)
+		return 1;
+	return -1;
+}
+
+static void test_delkey(void* key, void* ATTR_UNUSED(arg), int l)
+{
+	if(l) { lock_rw_unlock(&((struct slabtestkey*)key)->entry.lock); }
+	delkey((struct slabtestkey*)key);
+}
+
+static void test_deldata(void* data, void* ATTR_UNUSED(arg))
+{
+	deldata((struct slabtestdata*)data);
+}

testcode/unitverify.c

@@ -0,0 +1,463 @@
+/*
+ * testcode/unitverify.c - unit test for signature verification routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls verification unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "validator/val_sigcrypt.h"
+#include "validator/val_nsec.h"
+#include "validator/val_nsec3.h"
+#include "validator/validator.h"
+#include "testcode/ldns-testpkts.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/region-allocator.h"
+#include "util/alloc.h"
+#include "util/rbtree.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/config_file.h"
+
+/** verbose signature test */
+static int vsig = 0;
+
+/** entry to packet buffer with wireformat */
+static void
+entry_to_buf(struct entry* e, ldns_buffer* pkt)
+{
+	unit_assert(e->reply_list);
+	if(e->reply_list->reply_from_hex) {
+		ldns_buffer_copy(pkt, e->reply_list->reply_from_hex);
+	} else {
+		ldns_status status;
+		size_t answer_size;
+		uint8_t* ans = NULL;
+		status = ldns_pkt2wire(&ans, e->reply_list->reply, 
+			&answer_size);
+		if(status != LDNS_STATUS_OK) {
+			log_err("could not create reply: %s",
+				ldns_get_errorstr_by_id(status));
+			fatal_exit("error in test");
+		}
+		ldns_buffer_clear(pkt);
+		ldns_buffer_write(pkt, ans, answer_size);
+		ldns_buffer_flip(pkt);
+		free(ans);
+	}
+}
+
+/** entry to reply info conversion */
+static void
+entry_to_repinfo(struct entry* e, struct alloc_cache* alloc, struct region* 
+	region, ldns_buffer* pkt, struct query_info* qi, 
+	struct reply_info** rep)
+{
+	int ret;
+	struct edns_data edns;
+	entry_to_buf(e, pkt);
+	/* lock alloc lock to please lock checking software. 
+	 * alloc_special_obtain assumes it is talking to a ub-alloc,
+	 * and does not need to perform locking. Here the alloc is
+	 * the only one, so we lock it here */
+	lock_quick_lock(&alloc->lock);
+	ret = reply_info_parse(pkt, alloc, qi, rep, region, &edns);
+	lock_quick_unlock(&alloc->lock);
+	if(ret != 0) {
+		printf("parse code %d: %s\n", ret,
+			ldns_lookup_by_id(ldns_rcodes, ret)->name);
+		unit_assert(ret != 0);
+	}
+}
+
+/** extract DNSKEY rrset from answer and convert it */
+static struct ub_packed_rrset_key* 
+extract_keys(struct entry* e, struct alloc_cache* alloc, struct region*
+        region, ldns_buffer* pkt)
+{
+	struct ub_packed_rrset_key* dnskey = NULL;
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	size_t i;
+
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+	for(i=0; i<rep->an_numrrsets; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == LDNS_RR_TYPE_DNSKEY) {
+			dnskey = rep->rrsets[i];
+			rep->rrsets[i] = NULL;
+			break;
+		}
+	}
+	unit_assert(dnskey);
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+	return dnskey;
+}
+
+/** return true if answer should be bogus */
+static int
+should_be_bogus(struct ub_packed_rrset_key* rrset)
+{
+	struct packed_rrset_data* d = (struct packed_rrset_data*)rrset->
+		entry.data;
+	if(d->rrsig_count == 0)
+		return 1;
+	return 0;
+}
+
+/** verify and test one rrset against the key rrset */
+static void
+verifytest_rrset(struct module_env* env, struct val_env* ve, 
+	struct ub_packed_rrset_key* rrset, struct ub_packed_rrset_key* dnskey)
+{
+	enum sec_status sec;
+	if(vsig) {
+		log_nametypeclass(VERB_DETAIL, "verify of rrset",
+			rrset->rk.dname, ntohs(rrset->rk.type),
+			ntohs(rrset->rk.rrset_class));
+	}
+	sec = dnskeyset_verify_rrset(env, ve, rrset, dnskey);
+	if(vsig) {
+		printf("verify outcome is: %s\n", sec_status_to_string(sec));
+	}
+	if(should_be_bogus(rrset)) {
+		unit_assert(sec == sec_status_bogus);
+	} else {
+		unit_assert(sec == sec_status_secure);
+	}
+}
+
+/** verify and test an entry - every rr in the message */
+static void
+verifytest_entry(struct entry* e, struct alloc_cache* alloc, struct region*
+        region, ldns_buffer* pkt, struct ub_packed_rrset_key* dnskey,
+	struct module_env* env, struct val_env* ve)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	size_t i;
+
+	region_free_all(region);
+	if(vsig) {
+		printf("verifying pkt:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+
+	for(i=0; i<rep->rrset_count; i++) {
+		verifytest_rrset(env, ve, rep->rrsets[i], dnskey);
+	}
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+/** find RRset in reply by type */
+static struct ub_packed_rrset_key*
+find_rrset_type(struct reply_info* rep, uint16_t type)
+{
+	size_t i;
+	for(i=0; i<rep->rrset_count; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == type)
+			return rep->rrsets[i];
+	}
+	return NULL;
+}
+
+/** DS sig test an entry - get DNSKEY and DS in entry and verify */
+static void
+dstest_entry(struct entry* e, struct alloc_cache* alloc, struct region*
+        region, ldns_buffer* pkt, struct module_env* env)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	struct ub_packed_rrset_key* ds, *dnskey;
+	int ret;
+
+	region_free_all(region);
+	if(vsig) {
+		printf("verifying DS-DNSKEY match:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+	ds = find_rrset_type(rep, LDNS_RR_TYPE_DS);
+	dnskey = find_rrset_type(rep, LDNS_RR_TYPE_DNSKEY);
+	/* check test is OK */
+	unit_assert(ds && dnskey);
+
+	ret = ds_digest_match_dnskey(env, dnskey, 0, ds, 0);
+	if(strncmp((char*)qinfo.qname, "\003yes", 4) == 0) {
+		if(vsig) {
+			printf("result(yes)= %s\n", ret?"yes":"no");
+		}
+		unit_assert(ret);
+	} else if (strncmp((char*)qinfo.qname, "\002no", 3) == 0) {
+		if(vsig) {
+			printf("result(no)= %s\n", ret?"yes":"no");
+		}
+		unit_assert(!ret);
+		verbose(VERB_DETAIL, "DS fail: OK; matched unit test");
+	} else {
+		fatal_exit("Bad qname in DS unit test, yes or no");
+	}
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+/** verify from a file */
+static void
+verifytest_file(const char* fname, const char* at_date)
+{
+	/* 
+	 * The file contains a list of ldns-testpkts entries.
+	 * The first entry must be a query for DNSKEY.
+	 * The answer rrset is the keyset that will be used for verification
+	 */
+	struct ub_packed_rrset_key* dnskey;
+	struct region* region = region_create(malloc, free);
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+	struct module_env env;
+	struct val_env ve;
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	alloc_init(&alloc, NULL, 1);
+	memset(&env, 0, sizeof(env));
+	memset(&ve, 0, sizeof(ve));
+	env.scratch = region;
+	env.scratch_buffer = buf;
+	ve.date_override = cfg_convert_timeval(at_date);
+	unit_assert(region && buf);
+	dnskey = extract_keys(list, &alloc, region, buf);
+	if(vsig) log_nametypeclass(VERB_DETAIL, "test dnskey",
+			dnskey->rk.dname, ntohs(dnskey->rk.type), 
+			ntohs(dnskey->rk.rrset_class));
+	/* ready to go! */
+	for(e = list->next; e; e = e->next) {
+		verifytest_entry(e, &alloc, region, buf, dnskey, &env, &ve);
+	}
+
+	delete_entry(list);
+	region_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+/** verify DS matches DNSKEY from a file */
+static void
+dstest_file(const char* fname)
+{
+	/* 
+	 * The file contains a list of ldns-testpkts entries.
+	 * The first entry must be a query for DNSKEY.
+	 * The answer rrset is the keyset that will be used for verification
+	 */
+	struct region* region = region_create(malloc, free);
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+	struct module_env env;
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	alloc_init(&alloc, NULL, 1);
+	memset(&env, 0, sizeof(env));
+	env.scratch = region;
+	env.scratch_buffer = buf;
+	unit_assert(region && buf);
+
+	/* ready to go! */
+	for(e = list; e; e = e->next) {
+		dstest_entry(e, &alloc, region, buf, &env);
+	}
+
+	delete_entry(list);
+	region_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+/** helper for unittest of NSEC routines */
+static int
+unitest_nsec_has_type_rdata(char* bitmap, size_t len, uint16_t type)
+{
+	return nsecbitmap_has_type_rdata((uint8_t*)bitmap, len, type);
+}
+
+/** Test NSEC type bitmap routine */
+static void
+nsectest()
+{
+	/* bitmap starts at type bitmap rdata field */
+	/* from rfc 4034 example */
+	char* bitmap = "\000\006\100\001\000\000\000\003"
+		"\004\033\000\000\000\000\000\000"
+		"\000\000\000\000\000\000\000\000"
+		"\000\000\000\000\000\000\000\000"
+		"\000\000\000\000\040";
+	size_t len = 37;
+
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 0));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_A));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 2));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 3));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 4));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 5));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 6));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 7));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 8));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 9));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 10));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 11));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 12));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 13));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 14));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_MX));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_RRSIG));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_NSEC));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, 1234));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1233));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1235));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1236));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1237));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1238));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1239));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1240));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 2230));
+}
+
+/** Test hash algo - NSEC3 hash it and compare result */
+static void
+nsec3_hash_test_entry(struct entry* e, rbtree_t* ct,
+	struct alloc_cache* alloc, struct region* region, 
+	ldns_buffer* buf)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	struct ub_packed_rrset_key* answer, *nsec3;
+	struct nsec3_cached_hash* hash;
+	int ret;
+	uint8_t* qname;
+
+	if(vsig) {
+		printf("verifying NSEC3 hash:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, buf, &qinfo, &rep);
+	nsec3 = find_rrset_type(rep, LDNS_RR_TYPE_NSEC3);
+	answer = find_rrset_type(rep, LDNS_RR_TYPE_AAAA);
+	qname = region_alloc_init(region, qinfo.qname, qinfo.qname_len);
+	/* check test is OK */
+	unit_assert(nsec3 && answer && qname);
+
+	ret = nsec3_hash_name(ct, region, buf, nsec3, 0, qname,
+		qinfo.qname_len, &hash);
+	if(ret != 1) {
+		printf("Bad nsec3_hash_name retcode %d\n", ret);
+		unit_assert(ret == 1);
+	}
+	unit_assert(hash->dname && hash->hash && hash->hash_len &&
+		hash->b32 && hash->b32_len);
+	unit_assert(hash->b32_len == (size_t)answer->rk.dname[0]);
+	/* does not do lowercasing. */
+	unit_assert(memcmp(hash->b32, answer->rk.dname+1, hash->b32_len) 
+		== 0);
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+
+/** Read file to test NSEC3 hash algo */
+static void
+nsec3_hash_test(const char* fname)
+{
+	/* 
+	 * The list contains a list of ldns-testpkts entries.
+	 * Every entry is a test.
+	 * 	The qname is hashed.
+	 * 	The answer section AAAA RR name is the required result.
+	 * 	The auth section NSEC3 is used to get hash parameters.
+	 * The hash cache is maintained per file.
+	 *
+	 * The test does not perform canonicalization during the compare.
+	 */
+	rbtree_t ct;
+	struct region* region = region_create(malloc, free);
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	rbtree_init(&ct, &nsec3_hash_cmp);
+	alloc_init(&alloc, NULL, 1);
+	unit_assert(region && buf);
+
+	/* ready to go! */
+	for(e = list; e; e = e->next) {
+		nsec3_hash_test_entry(e, &ct, &alloc, region, buf);
+	}
+
+	delete_entry(list);
+	region_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+void 
+verify_test()
+{
+	printf("verify test\n");
+	verifytest_file("testdata/test_signatures.1", "20070818005004");
+	dstest_file("testdata/test_ds_sig.1");
+	nsectest();
+	nsec3_hash_test("testdata/test_nsec3_hash.1");
+}

testdata/Kexample.com.+003+02854.ds

@@ -0,0 +1 @@
+example.com.	3600	IN	DS	2854 3 1 46e4ffc6e9a4793b488954bd3f0cc6af0dfb201b

testdata/Kexample.com.+003+02854.key

@@ -0,0 +1 @@
+example.com.	3600	IN	DNSKEY	256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}

testdata/Kexample.com.+003+02854.private

@@ -0,0 +1,7 @@
+Private-key-format: v1.2
+Algorithm: 3 (DSA)
+Prime(p): +sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKQ==
+Subprime(q): tctSxapStjclgRHdPhMEkgizvSM=
+Base(g): LGE7N8tUTWCoDQ/B5lHx21jdo1BJGfew+nAmvoyL8+pAoAwytn1yQml/X5tAw46/GDPPZTUZLXxvfJkpyBMwRg==
+Private_value(x): XMs4XYi1oNckzTPvGMkgG5IiuzY=
+Public_value(y): ic3fxmWM4vwTdg80NDLK0sGF37DSxTgW7PDyTFuC2CMXnjnuq1IJFjhewgvQ4f3XbkNUK7CYZmQBBX3egcmFfg==

testdata/Kexample.com.+005+30899.ds

@@ -0,0 +1 @@
+example.com.	3600	IN	DS	30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512

testdata/Kexample.com.+005+30899.key

@@ -0,0 +1 @@
+example.com.	3600	IN	DNSKEY	256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b}

testdata/Kexample.com.+005+30899.private

@@ -0,0 +1,10 @@
+Private-key-format: v1.2
+Algorithm: 5 (RSASHA1)
+Modulus: 0ONXIUfQxB7f2iMyBQKmp2w5UX5SaEbOcs9YxbKESE+3Vn9K/j3g7nsHcZLMzR+sJ1OEC/KXyhUYeTyGFQSytw==
+PublicExponent: Aw==
+PrivateExponent: i0I6Fi/ggr8/5sIhWKxvGkgmNlQ28C80TIo7LncC2t6ar2Q5rpyiDxEHvFLfphRh108ZOqf2tQdHx7tXTx5Gqw==
+Prime1: 9WS85Q92ilTAuGiVi+KesKzrFqF98l2Gpu4003hfmbc=
+Prime2: 2eqsD2jcY4Mgw26A8XFiaLdxx5J4s10Dhd9ur6X3rwE=
+Exponent1: o5h97gpPBuMrJZsOXUG/IHNHZGupTD5ZxJ7N4lA/u88=
+Exponent2: kUcdX5s9l6zAgkmrS6DsRc+hL7b7Ij4CWT+fH8P6dKs=
+Coefficient: PVZrElFmz9tWa4kwu9jArjcocycYu0eBycgguQ03J7w=

testdata/fetch_glue.rpl

@@ -0,0 +1,149 @@
+; config options
+stub-zone:
+	name: "."
+	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
+CONFIG_END
+
+SCENARIO_BEGIN Fetch glue when missing or timed out.
+
+; K.ROOT-SERVERS.NET.
+RANGE_BEGIN 0 100
+	ADDRESS 193.0.14.129 
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+. IN NS
+SECTION ANSWER
+. IN NS	K.ROOT-SERVERS.NET.
+SECTION ADDITIONAL
+K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION AUTHORITY
+com.	IN NS	a.gtld-servers.net.
+SECTION ADDITIONAL
+a.gtld-servers.net.	IN 	A	192.5.6.30
+ENTRY_END
+RANGE_END
+
+; a.gtld-servers.net.
+RANGE_BEGIN 0 100
+	ADDRESS 192.5.6.30
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+; is missing the glue!
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. 	IN 	A
+SECTION ANSWER
+ns.example.com.		IN 	A	1.2.3.4
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. 	IN 	AAAA
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+RANGE_END
+
+; ns.example.com.
+RANGE_BEGIN 0 100
+	ADDRESS 1.2.3.4
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com.		IN 	A	1.2.3.4
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. IN A
+SECTION ANSWER
+ns.example.com. IN A	1.2.3.4
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. IN AAAA
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com. IN A	1.2.3.4
+ENTRY_END
+
+RANGE_END
+
+STEP 1 QUERY
+ENTRY_BEGIN
+REPLY RD
+SECTION QUESTION
+www.example.com. IN A
+ENTRY_END
+
+; recursion happens here.
+STEP 10 CHECK_ANSWER
+ENTRY_BEGIN
+MATCH all
+REPLY QR RD RA NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com.		IN 	A	1.2.3.4
+ENTRY_END
+
+; due to ordering of answer packets, this is still outstanding, remove it
+STEP 21 CHECK_OUT_QUERY
+ENTRY_BEGIN
+ADJUST copy_id
+MATCH qname qtype
+REPLY QR
+SECTION QUESTION
+ns.example.com IN AAAA
+ENTRY_END
+
+SCENARIO_END

testdata/fwd.rpl

@@ -1,10 +1,14 @@
 ; This is a comment.
+; config options go here.
+forward-zone: name: "." forward-addr: 216.0.0.1
+CONFIG_END
 
 SCENARIO_BEGIN Sample of a valid query
 RANGE_BEGIN 0 100
 	ENTRY_BEGIN
 	MATCH opcode qtype qname
 	ADJUST copy_id
+	REPLY QR RD RA NOERROR
 	SECTION QUESTION
 www.example.com. IN A
 	SECTION ANSWER

testdata/fwd_cached.rpl

@@ -0,0 +1,73 @@
+; This is a comment.
+; config options go here.
+forward-zone: name: "." forward-addr: 216.0.0.1
+CONFIG_END
+
+SCENARIO_BEGIN Query receives answer from the cache
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all 
+	REPLY QR RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+; another query, same, so it must be answered from the cache
+STEP 5 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; immediate answer without an OUT_QUERY happening (checked on exit)
+; also, the answer does not have AA set
+STEP 6 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all
+	REPLY QR RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+SCENARIO_END

testdata/fwd_error.rpl

@@ -1,3 +1,6 @@
+; config options go here.
+forward-zone: name: "." forward-addr: 216.0.0.1
+CONFIG_END
 SCENARIO_BEGIN Forwarder and an error happens on server query.
 STEP 1 QUERY
 ENTRY_BEGIN
@@ -11,7 +14,7 @@ STEP 4 CHECK_ANSWER
 ENTRY_BEGIN
 MATCH opcode qname qtype
 SECTION QUESTION
-REPLY SERVFAIL QR
+REPLY SERVFAIL QR RA
 MATCH all
 www.example.com. IN A
 ENTRY_END

testdata/fwd_lrudrop.rpl

@@ -0,0 +1,122 @@
+; This is a comment.
+; config options go here.
+; extremely small cache to force dropping old records.
+server:
+	msg-cache-size: 1 # one whole byte!
+	msg-cache-slabs: 1
+forward-zone: name: "." forward-addr: 216.0.0.1
+CONFIG_END
+
+SCENARIO_BEGIN Old answer is dropped from the cache
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+ENTRY_END
+
+; another query to force the cache to drop the example.com entry.
+STEP 11 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.net. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 12 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.net. IN A
+ENTRY_END
+STEP 13 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.net. IN A
+	SECTION ANSWER
+	www.example.net. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.net. IN NS ns.example.net.
+	SECTION ADDITIONAL
+	ns.example.net. IN A 10.20.30.50
+ENTRY_END
+STEP 14 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.net. IN A
+	SECTION ANSWER
+	www.example.net. IN A 10.20.30.40
+ENTRY_END
+
+
+; query, same as first, but it fell out of the cache.
+STEP 21 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 22 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 23 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 24 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+ENTRY_END
+
+SCENARIO_END

testdata/fwd_notcached.rpl

@@ -0,0 +1,85 @@
+; This is a comment.
+
+; can set commandline options using something like this:
+; COMMANDLINE -v 
+; here config file options:
+server:
+	msg-cache-size: 1024
+forward-zone: 
+	name: "." 
+	forward-addr: 216.0.0.1
+CONFIG_END
+
+SCENARIO_BEGIN Query receives answer not from the cache
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+ENTRY_END
+
+; another query, different, so not from cache.
+STEP 5 QUERY
+ENTRY_BEGIN
+	SECTION QUESTION
+	www.example.net. IN A
+ENTRY_END
+STEP 6 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.net. IN A
+ENTRY_END
+STEP 7 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.net. IN A
+	SECTION ANSWER
+	www.example.net. IN A 10.20.30.60
+	SECTION AUTHORITY
+	www.example.net. IN NS ns.example.net.
+	SECTION ADDITIONAL
+	ns.example.net. IN A 10.20.30.50
+ENTRY_END
+STEP 8 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.net. IN A
+	SECTION ANSWER
+	www.example.net. IN A 10.20.30.60
+ENTRY_END
+
+SCENARIO_END

testdata/fwd_timeout.rpl

@@ -1,3 +1,6 @@
+; config options go here.
+forward-zone: name: "." forward-addr: 216.0.0.1
+CONFIG_END
 SCENARIO_BEGIN Forwarder and a timeout happens on server query.
 STEP 1 QUERY
 ENTRY_BEGIN
@@ -11,7 +14,7 @@ STEP 4 CHECK_ANSWER
 ENTRY_BEGIN
 MATCH opcode qname qtype
 SECTION QUESTION
-REPLY SERVFAIL QR
+REPLY SERVFAIL QR RA
 MATCH all
 www.example.com. IN A
 ENTRY_END

testdata/fwd_two.rpl

@@ -1,3 +1,10 @@
+; config options go here.
+server:
+	num-queries-per-thread: 1
+forward-zone: 
+	name: "." 
+	forward-addr: 216.0.0.1
+CONFIG_END
 SCENARIO_BEGIN Sample of a valid query
 
 ; query responses from authority servers.
@@ -5,6 +12,7 @@ RANGE_BEGIN 0 100
 ENTRY_BEGIN
 	MATCH opcode qtype qname
 	ADJUST copy_id
+	REPLY QR RD RA NOERROR
 	SECTION QUESTION
 www.example.net. IN A
 	SECTION ANSWER
@@ -44,6 +52,7 @@ STEP 5 REPLY
 ENTRY_BEGIN
 	MATCH opcode qtype qname
 	ADJUST copy_id
+	REPLY QR RD RA NOERROR
 	SECTION QUESTION
 www.example.com. IN A
 	SECTION ANSWER
@@ -62,6 +71,15 @@ www.example.com. IN A
 SECTION ANSWER
 www.example.com. IN A 10.20.30.40
 ENTRY_END
+
+STEP 11 CHECK_ANSWER
+ENTRY_BEGIN
+MATCH opcode qname qtype
+SECTION QUESTION
+www.example.net. IN A
+SECTION ANSWER
+www.example.net. IN A 10.20.30.40
+ENTRY_END
 SCENARIO_END
 
 ; testbound checks before exit: