for 0.7.2: fixup building in subdir.

git-svn-id: file:///svn/unbound/branches/beta-0.7@797 be551aaa-1e26-0410-a405-d3ace91eadb9

Makefile.in

@@ -20,18 +20,21 @@ exec_prefix=@exec_prefix@
 bindir=@bindir@
 mandir=@mandir@
 libdir=@libdir@
+# datarootdir is here to please some checkers, use datadir.
 datarootdir=@datarootdir@
 datadir=@datadir@
 includedir=@includedir@
 doxygen=@doxygen@
 libtool=@libtool@
 ldnsdir=@ldnsdir@
+staticexe=@staticexe@
+configfile=@ub_conf_file@
 
 YACC=@YACC@
 LEX=@LEX@
 CC=@CC@
-CPPFLAGS=-I. @CPPFLAGS@ @DEFS@
-CFLAGS=-I. @CFLAGS@
+CPPFLAGS=-I$(srcdir) -I. @CPPFLAGS@ @DEFS@
+CFLAGS=@CFLAGS@
 LDFLAGS=@LDFLAGS@
 LIBS=@LIBS@
 LIBOBJS=@LIBOBJS@
@@ -44,60 +47,109 @@ endif
 BUILD=build/
 
 LINT=splint
-LINTFLAGS=+quiet -weak -warnposix -unrecog -Din_addr_t=uint32_t -Du_int=unsigned -Du_char=uint8_t -preproc -Drlimit=rlimit64
+LINTFLAGS=+quiet -weak -warnposix -unrecog -Din_addr_t=uint32_t -Du_int=unsigned -Du_char=uint8_t -preproc -Drlimit=rlimit64 -D__gnuc_va_list=va_list
 # compat with openssl linux edition.
-LINTFLAGS+="-DBN_ULONG=unsigned long" -Dkrb5_int32=int "-Dkrb5_ui_4=unsigned int" -DPQ_64BIT=uint64_t
+LINTFLAGS+="-DBN_ULONG=unsigned long" -Dkrb5_int32=int "-Dkrb5_ui_4=unsigned int" -DPQ_64BIT=uint64_t -DRC4_INT=unsigned
 
-INSTALL=$(srcdir)/install-sh 
+INSTALL=$(srcdir)/install-sh
 
-COMMON_SRC=$(wildcard services/*.c util/*.c util/data/*.c util/storage/*.c) util/configparser.c util/configlexer.c testcode/checklocks.c
+COMMON_SRC=$(patsubst $(srcdir)/%,%, $(wildcard $(srcdir)/services/*.c \
+	$(srcdir)/services/cache/*.c $(srcdir)/util/*.c \
+	$(srcdir)/util/data/*.c $(srcdir)/util/storage/*.c \
+	$(srcdir)/iterator/*.c $(srcdir)/validator/*.c)) \
+	util/configparser.c util/configlexer.c testcode/checklocks.c
 COMMON_OBJ=$(addprefix $(BUILD),$(COMMON_SRC:.c=.o))
 COMPAT_OBJ=$(addprefix $(BUILD)compat/,$(LIBOBJS))
-UNITTEST_SRC=$(wildcard testcode/unit*.c) testcode/readhex.c $(COMMON_SRC)
+UNITTEST_SRC=$(patsubst $(srcdir)/%,%, \
+	$(wildcard $(srcdir)/testcode/unit*.c)) \
+	testcode/readhex.c testcode/ldns-testpkts.c checkconf/worker_cb.c \
+	$(COMMON_SRC)
 UNITTEST_OBJ=$(addprefix $(BUILD),$(UNITTEST_SRC:.c=.o)) $(COMPAT_OBJ)
-DAEMON_SRC=$(wildcard daemon/*.c) $(COMMON_SRC)
+DAEMON_SRC=$(patsubst $(srcdir)/%,%, $(wildcard $(srcdir)/daemon/*.c)) \
+	$(COMMON_SRC)
 DAEMON_OBJ=$(addprefix $(BUILD),$(DAEMON_SRC:.c=.o)) $(COMPAT_OBJ)
-TESTBOUND_SRC=testcode/testbound.c testcode/ldns-testpkts.c daemon/worker.c daemon/daemon.c daemon/stats.c testcode/replay.c testcode/fake_event.c $(filter-out util/netevent.c services/listen_dnsport.c services/outside_network.c, $(COMMON_SRC))
+CHECKCONF_SRC=checkconf/unbound-checkconf.c checkconf/worker_cb.c $(COMMON_SRC)
+CHECKCONF_OBJ=$(addprefix $(BUILD),$(CHECKCONF_SRC:.c=.o)) $(COMPAT_OBJ)
+TESTBOUND_SRC=testcode/testbound.c testcode/ldns-testpkts.c \
+	daemon/worker.c daemon/daemon.c daemon/stats.c testcode/replay.c \
+	testcode/fake_event.c $(filter-out util/netevent.c \
+	services/listen_dnsport.c services/outside_network.c, $(COMMON_SRC))
 TESTBOUND_OBJ=$(addprefix $(BUILD),$(TESTBOUND_SRC:.c=.o)) $(COMPAT_OBJ)
-LOCKVERIFY_SRC=testcode/lock_verify.c $(COMMON_SRC)
+LOCKVERIFY_SRC=testcode/lock_verify.c checkconf/worker_cb.c $(COMMON_SRC)
 LOCKVERIFY_OBJ=$(addprefix $(BUILD),$(LOCKVERIFY_SRC:.c=.o)) $(COMPAT_OBJ)
-PKTVIEW_SRC=testcode/pktview.c testcode/readhex.c $(COMMON_SRC)
+PKTVIEW_SRC=testcode/pktview.c testcode/readhex.c checkconf/worker_cb.c \
+	$(COMMON_SRC)
 PKTVIEW_OBJ=$(addprefix $(BUILD),$(PKTVIEW_SRC:.c=.o)) $(COMPAT_OBJ)
-ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) $(TESTBOUND_SRC) $(LOCKVERIFY_SRC)
-ALL_OBJ=$(addprefix $(BUILD),$(ALL_SRC:.c=.o) $(addprefix compat/,$(LIBOBJS))) $(COMPAT_OBJ)
+SIGNIT_SRC=testcode/signit.c checkconf/worker_cb.c $(COMMON_SRC)
+SIGNIT_OBJ=$(addprefix $(BUILD),$(SIGNIT_SRC:.c=.o)) $(COMPAT_OBJ)
+MEMSTATS_SRC=testcode/memstats.c checkconf/worker_cb.c $(COMMON_SRC)
+MEMSTATS_OBJ=$(addprefix $(BUILD),$(MEMSTATS_SRC:.c=.o)) $(COMPAT_OBJ)
+ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) \
+	$(TESTBOUND_SRC) $(LOCKVERIFY_SRC) $(PKTVIEW_SRC) $(SIGNIT_SRC) \
+	$(MEMSTATS_SRC) $(CHECKCONF_SRC)
+ALL_OBJ=$(addprefix $(BUILD),$(ALL_SRC:.c=.o) \
+	$(addprefix compat/,$(LIBOBJS))) $(COMPAT_OBJ)
 
 COMPILE=$(LIBTOOL) --tag=CC --mode=compile $(CC) $(CPPFLAGS) $(CFLAGS)
-LINK=$(LIBTOOL) --tag=CC --mode=link $(CC) $(CFLAGS) $(LDFLAGS)
+LINK=$(LIBTOOL) --tag=CC --mode=link $(CC) $(staticexe) $(RUNTIME_PATH) $(CFLAGS) $(LDFLAGS)
 LINK_LIB=$(LIBTOOL) --tag=CC --mode=link $(CC) $(CFLAGS) $(LDFLAGS) -release $(VERSION)
 
+.PHONY:	clean realclean doc lint all install uninstall tests test download_ldns strip
+
 $(BUILD)%.o:    $(srcdir)/%.c 
 	$(INFO) Build $<
+	@if test ! -z "$(ldnsdir)" -a ! -e $(ldnsdir)/include/ldns/ldns.h; \
+		then (cd $(ldnsdir); $(MAKE) copy-headers); fi
 	@if test ! -d $(dir $@); then $(INSTALL) -d $(patsubst %/,%,$(dir $@)); fi
 	$Q$(COMPILE) -c $< -o $@
 
-.PHONY:	clean realclean doc lint all 
+all:	$(COMMON_OBJ) unbound unbound-checkconf
 
-all:	$(COMMON_OBJ) unbound unittest testbound lock-verify pktview
+tests:	all unittest testbound lock-verify pktview signit memstats
 
-unbound:	$(DAEMON_OBJ)
+test:	tests
+	bash testcode/do-tests.sh
+
+ifeq ($(patsubst ldns-src%,ldns-src,$(ldnsdir)),ldns-src)
+ldnslib=$(ldnsdir)/lib/libldns.a
+$(ldnslib):
+	@if test ! -z "$(ldnsdir)"; \
+	then (cd $(ldnsdir) && $(MAKE)); fi
+else
+ldnslib=
+endif
+
+unbound:	$(DAEMON_OBJ) $(ldnslib)
 	$(INFO) Link $@
-	$Q$(LINK) -o $@ $^ $(LIBS)
+	$Q$(LINK) -o $@ $(sort $(DAEMON_OBJ)) $(LIBS)
 
-unittest:	$(UNITTEST_OBJ)
+unbound-checkconf:	$(CHECKCONF_OBJ) $(ldnslib)
 	$(INFO) Link $@
-	$Q$(LINK) -o $@ $^ $(LIBS)
+	$Q$(LINK) -o $@ $(sort $(CHECKCONF_OBJ)) $(LIBS)
 
-testbound:	$(TESTBOUND_OBJ)
+unittest:	$(UNITTEST_OBJ) $(ldnslib)
 	$(INFO) Link $@
-	$Q$(LINK) -o $@ $^ $(LIBS)
+	$Q$(LINK) -o $@ $(sort $(UNITTEST_OBJ)) $(LIBS)
 
-lock-verify:	$(LOCKVERIFY_OBJ)
+testbound:	$(TESTBOUND_OBJ) $(ldnslib)
 	$(INFO) Link $@
-	$Q$(LINK) -o $@ $^ $(LIBS)
+	$Q$(LINK) -o $@ $(sort $(TESTBOUND_OBJ)) $(LIBS)
 
-pktview:	$(PKTVIEW_OBJ)
+lock-verify:	$(LOCKVERIFY_OBJ) $(ldnslib)
 	$(INFO) Link $@
-	$Q$(LINK) -o $@ $^ $(LIBS)
+	$Q$(LINK) -o $@ $(sort $(LOCKVERIFY_OBJ)) $(LIBS)
+
+pktview:	$(PKTVIEW_OBJ) $(ldnslib)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $(sort $(PKTVIEW_OBJ)) $(LIBS)
+
+signit:	$(SIGNIT_OBJ) $(ldnslib)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $(sort $(SIGNIT_OBJ)) $(LIBS)
+
+memstats:	$(MEMSTATS_OBJ) $(ldnslib)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $(sort $(MEMSTATS_OBJ)) $(LIBS)
 
 #testcode/ldns-testpkts.c:	$(ldnsdir)/examples/ldns-testpkts.c \
 #			$(ldnsdir)/examples/ldns-testpkts.h
@@ -107,9 +159,13 @@ pktview:	$(PKTVIEW_OBJ)
 util/config_file.c:	util/configparser.h
 util/configlexer.c:  $(srcdir)/util/configlexer.lex util/configparser.h
 	$(INFO) Lex $<
+ifeq "$(strip $(LEX))" ":"
+	$Qecho "rebuild lexer, but no lex program, skipped"
+else
 	@if test ! -d util; then $(INSTALL) -d util; fi
 	$Qecho "#include \"util/configyyrename.h\"" > $@
-	$Q$(LEX) -i -t $< >> $@
+	$Q$(LEX) -t $< >> $@
+endif
 
 util/configparser.c util/configparser.h:  $(srcdir)/util/configparser.y
 	$(INFO) Yacc $<
@@ -124,6 +180,7 @@ realclean: clean
 	rm -f config.status config.log config.h.in config.h
 	rm -f configure config.sub config.guess ltmain.sh aclocal.m4 libtool
 	rm -f util/configlexer.c util/configparser.c util/configparser.h
+	if test -d ldns-src; then rm -rf ldns-src; fi
 	rm -f Makefile 
 
 $(BUILD)%.lint:    $(srcdir)/%.c 
@@ -142,12 +199,46 @@ ifdef doxygen
 	$(doxygen) $(srcdir)/doc/unbound.doxygen
 endif
 
+strip:
+	strip unbound
+	strip unbound-checkconf
+
+install:
+	$(INSTALL) -d $(bindir)
+	$(INSTALL) -d $(mandir)
+	$(INSTALL) -d $(mandir)/man8
+	$(INSTALL) -d $(mandir)/man5
+	$(LIBTOOL) --mode=install cp unbound $(bindir)/unbound
+	$(LIBTOOL) --mode=install cp unbound-checkconf $(bindir)/unbound-checkconf
+	$(INSTALL) -c -m 644 $(srcdir)/doc/unbound.8 $(mandir)/man8
+	$(INSTALL) -c -m 644 $(srcdir)/doc/unbound-checkconf.8 $(mandir)/man8
+	$(INSTALL) -c -m 644 $(srcdir)/doc/unbound.conf.5 $(mandir)/man5
+	if test ! -e $(configfile); then $(INSTALL) -d `dirname $(configfile)`; $(INSTALL) -c -m 644 $(srcdir)/doc/example.conf $(configfile); fi
+
+uninstall:
+	rm -f -- $(bindir)/unbound $(bindir)/unbound-checkconf
+	rm -f -- $(mandir)/man8/unbound.8 $(mandir)/man8/unbound-checkconf.8 $(mandir)/man5/unbound.conf.5
+	@echo
+	@echo "You still need to remove `dirname $(configfile)` , $(configfile) by hand"
+
+download_ldns:
+	svn export https://www.nlnetlabs.nl/ldns/svn/trunk/makedist.sh ldns_makedist.sh
+	./ldns_makedist.sh -s -d https://www.nlnetlabs.nl/ldns/svn/trunk
+	mv ldns-*_pre_*.tar.gz ldns-src.tar.gz
+	rm ldns-*_pre_*.tar.gz.sha1 ldns_makedist.sh
+
 # Automatic dependencies.
 $(BUILD)%.d: $(srcdir)/%.c
 	$(INFO) Depend $<
+	@if test ! -z "$(ldnsdir)" -a ! -e $(ldnsdir)/include/ldns/ldns.h; \
+		then (cd $(ldnsdir); $(MAKE) copy-headers); fi
 	@if test ! -d $(dir $@); then $(INSTALL) -d $(patsubst %/,%,$(dir $@)); fi
 	$Q$(SHELL) -ec '$(CC) -MM $(CPPFLAGS) $(CFLAGS) $< \
 	              | sed '\''s!\(.*\)\.o[ :]*!$(dir $@)\1.o $@ : !g'\'' > $@; \
 	              [ -s $@ ] || rm -f $@'
 
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(MAKECMDGOALS),realclean)
 -include $(addprefix $(BUILD),$(ALL_SRC:.c=.d))
+endif
+endif

README

@@ -0,0 +1,10 @@
+Unbound README
+* ./configure && make && make install
+* on BSDs and Solaris use gmake (GNU make).
+* If it complains that no libevent can be found; libevent is useful when
+  using many (10000) outgoing ports. By default 16 ports are used and the
+  builtin alternative is equally capable.
+* More detailed README, README.svn, README.tests in doc directory
+* manual pages can be found in doc directory, and are installed, unbound(8).
+* example configuration file doc/example.conf
+

aclocal.m4

@@ -1,7 +1,7 @@
-# generated automatically by aclocal 1.9.6 -*- Autoconf -*-
+# generated automatically by aclocal 1.10 -*- Autoconf -*-
 
 # Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-# 2005  Free Software Foundation, Inc.
+# 2005, 2006  Free Software Foundation, Inc.
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
 # with or without modifications, as long as this notice is preserved.
@@ -1597,7 +1597,7 @@ linux*)
 
   # Append ld.so.conf contents to the search path
   if test -f /etc/ld.so.conf; then
-    lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;s/[:,	]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;/^$/d' | tr '\n' ' '`
+    lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/^[	 ]*//;s/#.*//;/^[^\/]/d;s/[:,	]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;/^$/d' | tr '\n' ' '`
     sys_lib_dlsearch_path_spec="/lib${libsuff} /usr/lib${libsuff} $lt_ld_extra"
   fi
 
@@ -6370,6 +6370,7 @@ do
     done
   done
 done
+IFS=$as_save_IFS
 lt_ac_max=0
 lt_ac_count=0
 # Add /usr/xpg4/bin/sed as it is typically found on Solaris
@@ -6402,6 +6403,7 @@ for lt_ac_sed in $lt_ac_sed_list /usr/xpg4/bin/sed; do
 done
 ])
 SED=$lt_cv_path_SED
+AC_SUBST([SED])
 AC_MSG_RESULT([$SED])
 ])
 

checkconf/unbound-checkconf.c

@@ -0,0 +1,198 @@
+/*
+ * checkconf/unbound-checkconf.c - config file checker for unbound.conf file.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * The config checker checks for syntax and other errors in the unbound.conf
+ * file, and can be used to check for errors before the server is started
+ * or sigHUPped.
+ * Exit status 1 means an error.
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/module.h"
+#include "util/net_help.h"
+#include "util/regional.h"
+#include "iterator/iterator.h"
+#include "validator/validator.h"
+#include <pwd.h>
+
+/** Give checkconf usage, and exit (1). */
+static void
+usage()
+{
+	printf("Usage:	unbound-checkconf [file]\n");
+	printf("	Checks unbound configuration file for errors.\n");
+	printf("file	if omitted %s is used.\n", CONFIGFILE);
+	printf("-h	show this usage help.\n");
+	printf("Version %s\n", PACKAGE_VERSION);
+	printf("BSD licensed, see LICENSE in source package for details.\n");
+	printf("Report bugs to %s\n", PACKAGE_BUGREPORT);
+	exit(1);
+}
+
+/** check if module works with config */
+static void
+check_mod(struct config_file* cfg, struct module_func_block* fb)
+{
+	struct module_env env;
+	memset(&env, 0, sizeof(env));
+	env.cfg = cfg;
+	env.scratch = regional_create();
+	env.scratch_buffer = ldns_buffer_new(BUFSIZ);
+	if(!env.scratch || !env.scratch_buffer)
+		fatal_exit("out of memory");
+	if(!(*fb->init)(&env, 0)) {
+		fatal_exit("bad config for %s module", fb->name);
+	}
+	(*fb->deinit)(&env, 0);
+	ldns_buffer_free(env.scratch_buffer);
+	regional_destroy(env.scratch);
+}
+
+/** check configuration for errors */
+static void
+morechecks(struct config_file* cfg)
+{
+	int i;
+	struct sockaddr_storage a;
+	socklen_t alen;
+	for(i=0; i<cfg->num_ifs; i++) {
+		if(!ipstrtoaddr(cfg->ifs[i], UNBOUND_DNS_PORT, &a, &alen)) {
+			fatal_exit("cannot parse interface specified as '%s'",
+				cfg->ifs[i]);
+		}
+	}
+	for(i=0; i<cfg->num_out_ifs; i++) {
+		if(!ipstrtoaddr(cfg->out_ifs[i], UNBOUND_DNS_PORT, 
+			&a, &alen)) {
+			fatal_exit("cannot parse outgoing-interface "
+				"specified as '%s'", cfg->out_ifs[i]);
+		}
+	}
+
+	if(cfg->verbosity < 0)
+		fatal_exit("verbosity value < 0");
+	if(cfg->num_threads < 0 || cfg->num_threads > 10000)
+		fatal_exit("num_threads value weird");
+	if(!cfg->do_ip4 && !cfg->do_ip6)
+		fatal_exit("ip4 and ip6 are both disabled, pointless");
+	if(!cfg->do_udp && !cfg->do_tcp)
+		fatal_exit("udp and tcp are both disabled, pointless");
+
+	if(cfg->chrootdir && strncmp(cfg->chrootdir, cfg->directory,
+		strlen(cfg->chrootdir)) != 0)
+		fatal_exit("working directory %s not in chrootdir %s",
+			cfg->directory, cfg->chrootdir);
+	if(cfg->chrootdir && cfg->pidfile && cfg->pidfile[0] &&
+		strncmp(cfg->chrootdir, cfg->pidfile,
+			strlen(cfg->chrootdir)) != 0)
+		fatal_exit("pid file %s not in chrootdir %s",
+			cfg->pidfile, cfg->chrootdir);
+	if(cfg->chrootdir && cfg->logfile && cfg->logfile[0] &&
+		strncmp(cfg->chrootdir, cfg->logfile,
+			strlen(cfg->chrootdir)) != 0)
+		fatal_exit("log file %s not in chrootdir %s",
+			cfg->logfile, cfg->chrootdir);
+	
+	if(strcmp(cfg->module_conf, "iterator") != 0 &&
+		strcmp(cfg->module_conf, "validator iterator") != 0) {
+		fatal_exit("module conf %s is not known to work",
+			cfg->module_conf);
+	}
+
+	if(cfg->username && cfg->username[0]) {
+		struct passwd *pwd;
+		if((pwd = getpwnam(cfg->username)) == NULL)
+			fatal_exit("user '%s' does not exist.", cfg->username);
+		endpwent();
+	}
+}
+
+/** check config file */
+static void
+checkconf(char* cfgfile)
+{
+	struct config_file* cfg = config_create();
+	if(!cfg)
+		fatal_exit("out of memory");
+	if(!config_read(cfg, cfgfile)) {
+		/* config_read prints messages to stderr */
+		config_delete(cfg);
+		exit(1);
+	}
+	morechecks(cfg);
+	check_mod(cfg, iter_get_funcblock());
+	check_mod(cfg, val_get_funcblock());
+	config_delete(cfg);
+	printf("unbound-checkconf: no errors in %s\n", cfgfile);
+}
+
+/** getopt global, in case header files fail to declare it. */
+extern int optind;
+/** getopt global, in case header files fail to declare it. */
+extern char* optarg;
+
+/** Main routine for checkconf */
+int main(int argc, char* argv[])
+{
+	int c;
+	char* f;
+	log_ident_set("unbound-checkconf");
+	log_init(NULL, 0, NULL);
+	checklock_start();
+	/* parse the options */
+	while( (c=getopt(argc, argv, "h")) != -1) {
+		switch(c) {
+		case '?':
+		case 'h':
+		default:
+			usage();
+		}
+	}
+	argc -= optind;
+	argv += optind;
+	if(argc != 0 && argc != 1)
+		usage();
+	if(argc == 1)
+		f = argv[0];
+	else	f = CONFIGFILE;
+	checkconf(f);
+	checklock_stop();
+	return 0;
+}

checkconf/worker_cb.c

@@ -0,0 +1,109 @@
+/*
+ * checkconf/worker_cb.c - fake callback routines to make fptr_wlist work
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains fake callback functions, so that the symbols exist
+ * and the fptr_wlist continues to work even if the daemon/worker is not
+ * linked into the resulting program.
+ */
+#include "config.h"
+#include "util/log.h"
+struct comm_reply;
+struct comm_point;
+struct module_qstate;
+
+int worker_handle_control_cmd(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int worker_handle_request(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(repinfo))
+{
+	log_assert(0);
+	return 0;
+}
+
+int worker_handle_reply(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int worker_handle_service_reply(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+void worker_sighandler(int ATTR_UNUSED(sig), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+int worker_send_packet(ldns_buffer* ATTR_UNUSED(pkt), 
+	struct sockaddr_storage* ATTR_UNUSED(addr), 
+	socklen_t ATTR_UNUSED(addrlen), int ATTR_UNUSED(timeout), 
+	struct module_qstate* ATTR_UNUSED(q), int ATTR_UNUSED(use_tcp))
+{
+	log_assert(0);
+	return 0;
+}
+
+struct outbound_entry* worker_send_query(uint8_t* ATTR_UNUSED(qname), 
+	size_t ATTR_UNUSED(qnamelen), uint16_t ATTR_UNUSED(qtype), 
+	uint16_t ATTR_UNUSED(qclass), uint16_t ATTR_UNUSED(flags), 
+	int ATTR_UNUSED(dnssec), struct sockaddr_storage* ATTR_UNUSED(addr), 
+	socklen_t ATTR_UNUSED(addrlen), struct module_qstate* ATTR_UNUSED(q))
+{
+	log_assert(0);
+	return 0;
+}
+
+void 
+worker_alloc_cleanup(void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}

config.h.in

@@ -0,0 +1,426 @@
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Pathname to the Unbound configuration file */
+#undef CONFIGFILE
+
+/* Define if you want to use debug lock checking (slow). */
+#undef ENABLE_LOCK_CHECKS
+
+/* Define to 1 if you have the <arpa/inet.h> header file. */
+#undef HAVE_ARPA_INET_H
+
+/* Whether the C compiler accepts the "format" attribute */
+#undef HAVE_ATTR_FORMAT
+
+/* Whether the C compiler accepts the "unused" attribute */
+#undef HAVE_ATTR_UNUSED
+
+/* Define to 1 if your system has a working `chown' function. */
+#undef HAVE_CHOWN
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
+
+/* Define to 1 if you have the `event_base_free' function. */
+#undef HAVE_EVENT_BASE_FREE
+
+/* Define to 1 if you have the `event_base_once' function. */
+#undef HAVE_EVENT_BASE_ONCE
+
+/* Define to 1 if you have the <event.h> header file. */
+#undef HAVE_EVENT_H
+
+/* Define to 1 if you have the `fork' function. */
+#undef HAVE_FORK
+
+/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */
+#undef HAVE_FSEEKO
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#undef HAVE_GETADDRINFO
+
+/* Define to 1 if you have the <getopt.h> header file. */
+#undef HAVE_GETOPT_H
+
+/* Define to 1 if you have the `inet_aton' function. */
+#undef HAVE_INET_ATON
+
+/* Define to 1 if you have the `inet_ntop' function. */
+#undef HAVE_INET_NTOP
+
+/* Define to 1 if you have the `inet_pton' function. */
+#undef HAVE_INET_PTON
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#undef HAVE_LIBCRYPTO
+
+/* Define to 1 if you have the `nsl' library (-lnsl). */
+#undef HAVE_LIBNSL
+
+/* Define to 1 if you have the `rt' library (-lrt). */
+#undef HAVE_LIBRT
+
+/* Define to 1 if you have the `socket' library (-lsocket). */
+#undef HAVE_LIBSOCKET
+
+/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
+   to 0 otherwise. */
+#undef HAVE_MALLOC
+
+/* Define to 1 if you have the `memmove' function. */
+#undef HAVE_MEMMOVE
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* Define to 1 if you have the <netinet/in.h> header file. */
+#undef HAVE_NETINET_IN_H
+
+/* Define to 1 if you have the <openssl/err.h> header file. */
+#undef HAVE_OPENSSL_ERR_H
+
+/* Define to 1 if you have the <openssl/ssl.h> header file. */
+#undef HAVE_OPENSSL_SSL_H
+
+/* Define if you have POSIX threads libraries and header files. */
+#undef HAVE_PTHREAD
+
+/* Define to 1 if the system has the type `pthread_rwlock_t'. */
+#undef HAVE_PTHREAD_RWLOCK_T
+
+/* Define to 1 if the system has the type `pthread_spinlock_t'. */
+#undef HAVE_PTHREAD_SPINLOCK_T
+
+/* Define to 1 if you have the `snprintf' function. */
+#undef HAVE_SNPRINTF
+
+/* Using Solaris threads */
+#undef HAVE_SOLARIS_THREADS
+
+/* Define if you have the SSL libraries installed. */
+#undef HAVE_SSL
+
+/* Define to 1 if you have the <stdarg.h> header file. */
+#undef HAVE_STDARG_H
+
+/* Define to 1 if you have the <stdbool.h> header file. */
+#undef HAVE_STDBOOL_H
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the `strlcpy' function. */
+#undef HAVE_STRLCPY
+
+/* Define to 1 if you have the <syslog.h> header file. */
+#undef HAVE_SYSLOG_H
+
+/* Define to 1 if you have the <sys/param.h> header file. */
+#undef HAVE_SYS_PARAM_H
+
+/* Define to 1 if you have the <sys/socket.h> header file. */
+#undef HAVE_SYS_SOCKET_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <sys/uio.h> header file. */
+#undef HAVE_SYS_UIO_H
+
+/* Define to 1 if you have the <time.h> header file. */
+#undef HAVE_TIME_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to 1 if you have the `vfork' function. */
+#undef HAVE_VFORK
+
+/* Define to 1 if you have the <vfork.h> header file. */
+#undef HAVE_VFORK_H
+
+/* Define to 1 if `fork' works. */
+#undef HAVE_WORKING_FORK
+
+/* Define to 1 if `vfork' works. */
+#undef HAVE_WORKING_VFORK
+
+/* Define to the maximum message length to pass to syslog. */
+#undef MAXSYSLOGMSGLEN
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to necessary symbol if this constant uses a non-standard name on
+   your system. */
+#undef PTHREAD_CREATE_JOINABLE
+
+/* Define as the return type of signal handlers (`int' or `void'). */
+#undef RETSIGTYPE
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Define to 'int' or type of struct msghdr.msg_iovlen. */
+#undef TYPE_MSGIOVLEN
+
+/* use statistics for allocs and frees, for debug use */
+#undef UNBOUND_ALLOC_STATS
+
+/* Define if you want to use internal select based events */
+#undef USE_MINI_EVENT
+
+/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
+   `char[]'. */
+#undef YYTEXT_POINTER
+
+/* Define to 1 if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+# undef _ALL_SOURCE
+#endif
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+#undef _FILE_OFFSET_BITS
+
+/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */
+#undef _LARGEFILE_SOURCE
+
+/* Define for large files, on AIX-style hosts. */
+#undef _LARGE_FILES
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const
+
+/* Define to `int' if <sys/types.h> doesn't define. */
+#undef gid_t
+
+/* in_addr_t */
+#undef in_addr_t
+
+/* in_port_t */
+#undef in_port_t
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+   calls it, or to nothing if 'inline' is not supported under any name.  */
+#ifndef __cplusplus
+#undef inline
+#endif
+
+/* Define to `short' if <sys/types.h> does not define. */
+#undef int16_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef int32_t
+
+/* Define to `long long' if <sys/types.h> does not define. */
+#undef int64_t
+
+/* Define to `char' if <sys/types.h> does not define. */
+#undef int8_t
+
+/* Define to rpl_malloc if the replacement function should be used. */
+#undef malloc
+
+/* Define to `long int' if <sys/types.h> does not define. */
+#undef off_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef pid_t
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+#undef size_t
+
+/* Define to 'int' if not defined */
+#undef socklen_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef ssize_t
+
+/* Define to `unsigned char' if <sys/types.h> does not define. */
+#undef u_char
+
+/* Define to `int' if <sys/types.h> doesn't define. */
+#undef uid_t
+
+/* Define to `unsigned short' if <sys/types.h> does not define. */
+#undef uint16_t
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+#undef uint32_t
+
+/* Define to `unsigned long long' if <sys/types.h> does not define. */
+#undef uint64_t
+
+/* Define to `unsigned char' if <sys/types.h> does not define. */
+#undef uint8_t
+
+/* Define as `fork' if `vfork' does not work. */
+#undef vfork
+
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#include <stddef.h>
+#endif
+
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#endif
+
+#include <errno.h>
+
+#if HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+
+#ifdef HAVE_OPENSSL_ERR_H
+#include <openssl/err.h>
+#endif
+
+#ifdef HAVE_ATTR_FORMAT
+#  define ATTR_FORMAT(archetype, string_index, first_to_check) \
+    __attribute__ ((format (archetype, string_index, first_to_check)))
+#else /* !HAVE_ATTR_FORMAT */
+#  define ATTR_FORMAT(archetype, string_index, first_to_check) /* empty */
+#endif /* !HAVE_ATTR_FORMAT */
+#if defined(DOXYGEN)
+#  define ATTR_UNUSED(x)  x
+#elif defined(__cplusplus)
+#  define ATTR_UNUSED(x)
+#elif defined(HAVE_ATTR_UNUSED)
+#  define ATTR_UNUSED(x)  x __attribute__((unused))
+#else /* !HAVE_ATTR_UNUSED */
+#  define ATTR_UNUSED(x)  x
+#endif /* !HAVE_ATTR_UNUSED */
+#ifndef HAVE_FSEEKO
+#define fseeko fseek
+#define ftello ftell
+#endif /* HAVE_FSEEKO */
+#if defined(HAVE_EVENT_H) && !defined(HAVE_EVENT_BASE_ONCE) && (defined(HAVE_PTHREAD) || defined(HAVE_SOLARIS_THREADS))
+   /* using version of libevent that is not threadsafe. */
+#  define LIBEVENT_SIGNAL_PROBLEM 1
+#endif
+
+#ifndef RAND_MAX
+#define RAND_MAX	2147483647
+#endif
+
+#ifndef IPV6_MIN_MTU
+#define IPV6_MIN_MTU 1280
+#endif /* IPV6_MIN_MTU */
+
+#ifndef CHECKED_INET6
+#  define CHECKED_INET6
+#  ifdef AF_INET6
+#    define INET6
+#  else
+#    define AF_INET6        28
+#  endif
+#endif /* CHECKED_INET6 */
+
+/* maximum nesting of included files */
+#define MAXINCLUDES 10
+
+#ifndef HAVE_SNPRINTF
+#include <stdarg.h>
+int snprintf (char *str, size_t count, const char *fmt, ...);
+int vsnprintf (char *str, size_t count, const char *fmt, va_list arg);
+#endif /* HAVE_SNPRINTF */
+#ifndef HAVE_INET_PTON
+int inet_pton(int af, const char* src, void* dst);
+#endif /* HAVE_INET_PTON */
+#ifndef HAVE_INET_NTOP
+const char *inet_ntop(int af, const void *src, char *dst, size_t size);
+#endif
+#ifndef HAVE_INET_ATON
+int inet_aton(const char *cp, struct in_addr *addr);
+#endif
+#ifndef HAVE_MEMMOVE
+void *memmove(void *dest, const void *src, size_t n);
+#endif
+#ifndef HAVE_STRLCPY
+size_t strlcpy(char *dst, const char *src, size_t siz);
+#endif
+#ifndef HAVE_GETADDRINFO
+struct sockaddr_storage;
+#include "compat/fake-rfc2553.h"
+#endif
+
+#include "ldns/ldns.h"
+
+#ifdef UNBOUND_ALLOC_STATS
+#  define malloc(s) unbound_stat_malloc_log(s, __FILE__, __LINE__, __func__)
+#  define calloc(n,s) unbound_stat_calloc_log(n, s, __FILE__, __LINE__, __func__)
+#  define free(p) unbound_stat_free_log(p, __FILE__, __LINE__, __func__)
+#  define realloc(p,s) unbound_stat_realloc_log(p, s, __FILE__, __LINE__, __func__)
+void *unbound_stat_malloc(size_t size);
+void *unbound_stat_calloc(size_t nmemb, size_t size);
+void unbound_stat_free(void *ptr);
+void *unbound_stat_realloc(void *ptr, size_t size);
+void *unbound_stat_malloc_log(size_t size, const char* file, int line,
+	const char* func);
+void *unbound_stat_calloc_log(size_t nmemb, size_t size, const char* file,
+	int line, const char* func);
+void unbound_stat_free_log(void *ptr, const char* file, int line,
+	const char* func);
+void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
+	int line, const char* func);
+#endif /* UNBOUND_ALLOC_STATS */
+
+/** default port for DNS traffic. */
+#define UNBOUND_DNS_PORT 53
+

configure.ac

@@ -2,11 +2,40 @@
 # Process this file with autoconf to produce a configure script.
 AC_PREREQ(2.57)
 
-AC_INIT(unbound, 0.3, wouter@nlnetlabs.nl, unbound)
+AC_INIT(unbound, 0.7.2, unbound-bugs@nlnetlabs.nl, unbound)
 
 CFLAGS=
 AC_AIX
 
+dnl
+dnl By default set $sysconfdir to /etc and $localstatedir to /var
+dnl
+case "$prefix" in
+        NONE)
+                case "$sysconfdir" in
+                        '${prefix}/etc')
+                                sysconfdir=/etc
+                        ;;
+                esac
+                case "$localstatedir" in
+                        '${prefix}/var')
+                                localstatedir=/var
+                        ;;
+                esac
+        ;;
+esac
+
+#
+# Determine configuration file
+ub_conf_file=${sysconfdir}/unbound/unbound.conf
+AC_ARG_WITH([conf_file],
+        AC_HELP_STRING([--with-conf-file=path], [Pathname to the Unbound configuration file]),
+	[ub_conf_file=$withval])
+AC_SUBST(ub_conf_file)
+# the eval is to evaluate shell expansion twice, once
+# for $nsd_conf_file and once for the ${prefix} within it.
+AC_DEFINE_UNQUOTED(CONFIGFILE, ["`eval echo $ub_conf_file`"], [Pathname to the Unbound configuration file])
+
 dnl routine to help check for compiler flags.
 AC_DEFUN([CHECK_COMPILER_FLAG],
 [
@@ -251,6 +280,15 @@ if test "$srcdir" != "."; then
 	CPPFLAGS="$CPPFLAGS -I$srcdir";
 fi
 
+# skip these tests, we do not need them.
+AC_DEFUN([AC_PROG_F77], [:])
+AC_DEFUN([AC_PROG_FC], [:])
+AC_DEFUN([AC_PROG_CXX], [:])
+AC_DEFUN([AC_PROG_CXXCPP], [:])
+AC_DEFUN([AC_PROG_OBJC], [:])
+AC_DEFUN([AC_PROG_OBJCCPP], [:])
+AC_DEFUN([AC_LIBTOOL_CXX], [:])
+AC_DEFUN([AC_LIBTOOL_F77], [:])
 # Use libtool 
 AC_CHECK_PROGS(libtool, [glibtool libtool15 libtool], [./libtool])
 AC_PATH_TOOL(AR, ar, [false])
@@ -269,7 +307,7 @@ AC_PROG_LIBTOOL
 
 # Checks for header files.
 AC_HEADER_STDC
-AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/uio.h arpa/inet.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/uio.h arpa/inet.h syslog.h],,, [AC_INCLUDES_DEFAULT])
 
 # check for types
 AC_CHECK_TYPE(int8_t, char)
@@ -336,17 +374,43 @@ cv_msgiovlen_type="int" ],
 AC_MSG_RESULT($cv_msgiovlen_type)
 AC_DEFINE_UNQUOTED([TYPE_MSGIOVLEN], $cv_msgiovlen_type, [Define to 'int' or type of struct msghdr.msg_iovlen.])
 
+# Check for yyunput
+AC_DEFUN([CHECK_YYUNPUT],
+[AC_MSG_CHECKING(whether lex can not generate yyunput)
+AC_CACHE_VAL(ac_cv_c_yy_unput,
+[ac_cv_c_yy_unput=no
+if $LEX --nounput -h 2>&1 | grep "do not generate" 2>&1 >/dev/null; then
+   ac_cv_c_yy_unput="yes"
+else
+   ac_cv_c_yy_unput="no"
+fi
+])
+
+AC_MSG_RESULT($ac_cv_c_yy_unput)
+if test $ac_cv_c_yy_unput = yes; then
+	LEX="$LEX --nounput"
+fi
+])dnl
+
+CHECK_YYUNPUT
+
+# add option to disable the evil rpath
+dnl Check whether to use rpath or not
+AC_ARG_ENABLE(rpath,
+        [  --disable-rpath         disable hardcoded rpath (default=enabled)],
+	enable_rpath=$enableval, enable_rpath=yes)
+
 # Checks for libraries.
 AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
                                     [enable SSL (will check /usr/local/ssl
-                            /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr/sfw /usr)]),[
+                            /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /opt/local /usr/sfw /usr)]),[
         ],[
             withval="yes"
         ])
     if test x_$withval != x_no; then
         AC_MSG_CHECKING(for SSL)
         if test x_$withval = x_ -o x_$withval = x_yes; then
-            withval="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr/sfw /usr"
+            withval="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /opt/local /usr/sfw /usr"
         fi
         for dir in $withval; do
             ssldir="$dir"
@@ -363,7 +427,9 @@ AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
             AC_MSG_RESULT(found in $ssldir)
             HAVE_SSL=yes
             LDFLAGS="$LDFLAGS -L$ssldir/lib";
-	    RUNTIME_PATH="$RUNTIME_PATH -R$ssldir/lib"
+	    if test "x$enable_rpath" = xyes; then
+	        RUNTIME_PATH="$RUNTIME_PATH -R$ssldir/lib"
+	    fi
             AC_CHECK_LIB(crypto, HMAC_CTX_init,, [
                     AC_MSG_ERROR([OpenSSL found in $ssldir, but version 0.9.7 or higher is required])
                 ])
@@ -372,10 +438,11 @@ AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
 	AC_SUBST(RUNTIME_PATH)
     fi
 AC_CHECK_HEADERS([openssl/ssl.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([openssl/err.h],,, [AC_INCLUDES_DEFAULT])
 
 # check for thread library.
 AC_ARG_WITH(pthreads, AC_HELP_STRING([--with-pthreads], 
- [use pthreads library, or --without--pthreads to disable threading support.]), 
+ [use pthreads library, or --without-pthreads to disable threading support.]), 
  [ ],[ withval="yes" ])
 ub_have_pthreads=no
 if test x_$withval != x_no; then
@@ -386,7 +453,7 @@ if test x_$withval != x_no; then
 		CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
 		CC="$PTHREAD_CC"
 		ub_have_pthreads=yes
-		AC_CHECK_TYPES(pthread_spinlock_t,,,[#include <pthread.h>])
+		AC_CHECK_TYPES([pthread_spinlock_t, pthread_rwlock_t],,,[#include <pthread.h>])
 		])
 fi
 
@@ -409,14 +476,16 @@ if test x_$withval != x_no; then
 	fi
 fi
 
+# check for library used by libevent after 1.3c
+AC_CHECK_LIB(rt, clock_gettime)
 # check for libevent
 AC_ARG_WITH(libevent, AC_HELP_STRING([--with-libevent=pathname],
-    [set path to libevent  (will check /usr/local /usr/lib /usr/pkg /usr/sfw /usr)]),
+    [set path to libevent  (will check /usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr)]),
     [ ],[ withval="yes" ])
 if test x_$withval != x_no; then
         AC_MSG_CHECKING(for libevent)
         if test x_$withval = x_ -o x_$withval = x_yes; then
-            withval="/usr/local /usr/lib /usr/pkg /usr/sfw /usr";
+            withval="/usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr";
         fi
         for dir in $withval; do
             thedir="$dir"
@@ -433,12 +502,17 @@ if test x_$withval != x_no; then
                 	CPPFLAGS="$CPPFLAGS -I$thedir";
             		LDFLAGS="$thedir/.libs/*.o $LDFLAGS";
 		else
-            		AC_MSG_ERROR(Cannot find the libevent library in $withval)
+            		AC_MSG_ERROR([Cannot find the libevent library in $withval
+You can restart ./configure --with-libevent=no to use a builtin alternative.
+Please note that this alternative is not as capable as libevent when using
+many outgoing ports.  ])
 		fi
         else
             AC_MSG_RESULT(found in $thedir)
             LDFLAGS="$LDFLAGS -L$thedir/lib";
-	    RUNTIME_PATH="$RUNTIME_PATH -R$thedir/lib"
+	    if test "x$enable_rpath" = xyes; then
+	        RUNTIME_PATH="$RUNTIME_PATH -R$thedir/lib"
+	    fi
         fi
 	AC_SUBST(RUNTIME_PATH)
 	AC_SEARCH_LIBS(event_set, [event])
@@ -449,29 +523,44 @@ else
 	AC_DEFINE(USE_MINI_EVENT, 1, [Define if you want to use internal select based events])
 fi
 
+# set static linking if requested
+AC_SUBST(staticexe)
+staticexe=""
+AC_ARG_ENABLE(staticexe, AC_HELP_STRING([--enable-static-exe],
+	[ enable to compile executables statically against event, ldns libs, for debug purposes ]), 
+	, )
+if test x_$enable_static_exe = x_yes; then
+	staticexe="-static"
+fi
+
+# set lock checking if requested
+AC_ARG_ENABLE(lock_checks, AC_HELP_STRING([--enable-lock-checks],
+	[ enable to check lock and unlock calls, for debug purposes ]), 
+	, )
+if test x_$enable_lock_checks = x_yes; then
+	AC_DEFINE(ENABLE_LOCK_CHECKS, 1, [Define if you want to use debug lock checking (slow).])
+fi
+
+# set memory allocation checking if requested
+AC_ARG_ENABLE(alloc-checks, AC_HELP_STRING([--enable-alloc-checks],
+	[ enable to check memory allocation, for debug purposes ]), 
+	, )
+if test x_$enable_alloc_checks = x_yes; then
+	AC_DEFINE(UNBOUND_ALLOC_STATS, 1, [use statistics for allocs and frees, for debug use])
+else
+	AC_FUNC_MALLOC
+fi
+
 # check to see if libraries are needed for these functions.
 AC_CHECK_LIB(socket, socket)
 AC_CHECK_LIB(nsl, inet_pton)
 
 AC_FUNC_CHOWN
 AC_FUNC_FORK
-AC_FUNC_MALLOC
 AC_TYPE_SIGNAL
 AC_FUNC_FSEEKO
 AC_SYS_LARGEFILE
 
-AC_REPLACE_FUNCS(inet_aton)
-AC_REPLACE_FUNCS(inet_pton)
-AC_REPLACE_FUNCS(inet_ntop)
-AC_REPLACE_FUNCS(snprintf)
-AC_REPLACE_FUNCS(strlcpy)
-AC_REPLACE_FUNCS(memmove)
-
-AC_CHECK_FUNCS([getaddrinfo])
-if test $ac_cv_func_getaddrinfo = no; then
-AC_LIBOBJ([fake-rfc2553])
-fi
-
 # check this after all other compilation checks, since the linking of the lib
 # may break checks after this.
 AC_ARG_WITH(ldns,
@@ -483,9 +572,52 @@ AC_ARG_WITH(ldns,
 	AC_SUBST(ldnsdir)
 ])
 
-AC_CHECK_LIB(ldns, ldns_rr_new,, [AC_MSG_ERROR([Can't find ldns library])])
+AC_CHECK_LIB(ldns, ldns_rr_new, [
+dnl ldns was found, check compat functions
+AC_REPLACE_FUNCS(inet_aton)
+AC_REPLACE_FUNCS(inet_pton)
+AC_REPLACE_FUNCS(inet_ntop)
+AC_REPLACE_FUNCS(snprintf)
+AC_REPLACE_FUNCS(strlcpy)
+AC_REPLACE_FUNCS(memmove)
 
-AC_DEFINE_UNQUOTED([MAXSYSLOGMSGLEN], [512], [Define to the maximum message length to pass to syslog.])
+AC_CHECK_FUNCS([getaddrinfo])
+if test $ac_cv_func_getaddrinfo = no; then
+AC_LIBOBJ([fake-rfc2553])
+fi
+
+] , [
+dnl use the builtin ldns-src.tar.gz file to build ldns.
+if test ! -f $srcdir/ldns-src.tar.gz; then
+	AC_MSG_ERROR([No ldns library found and no ldns-src.tar.gz, use --with-ldns=path.])
+fi
+echo "****************************************************************"
+echo "***                                                          ***"
+echo "*** ldns library not found (--with-ldns= to specify by hand) ***"
+echo "*** Building ldns library from package included in this one. ***"
+echo "***                                                          ***"
+echo "****************************************************************"
+if test -d ldns-src; then rm -rf ldns-src; fi
+mkdir ldns-src; cp $srcdir/ldns-src.tar.gz ldns-src/ldns-src.tar.gz; (cd ldns-src; gzip -cd ldns-src.tar.gz | tar xf -); rm -f ldns-src/ldns-src.tar.gz
+ldnsdir=`ls -d ldns-src/ldns*`
+AC_MSG_NOTICE([Configure $ldnsdir])
+echo "(cd $ldnsdir; ./configure)"
+(cd $ldnsdir; ./configure)
+AC_MSG_NOTICE([Finished $ldnsdir configure])
+CPPFLAGS="$CPPFLAGS -I$ldnsdir/include"
+LDFLAGS="$LDFLAGS $ldnsdir/lib/*.o"
+AC_SUBST(ldnsdir)
+dnl use ldns compat functions
+AC_DEFINE(HAVE_INET_ATON)
+AC_DEFINE(HAVE_INET_PTON)
+AC_DEFINE(HAVE_INET_NTOP)
+AC_DEFINE(HAVE_SNPRINTF)
+AC_DEFINE(HAVE_STRLCPY)
+AC_DEFINE(HAVE_MEMMOVE)
+AC_DEFINE(HAVE_GETADDRINFO)
+])
+
+AC_DEFINE_UNQUOTED([MAXSYSLOGMSGLEN], [5120], [Define to the maximum message length to pass to syslog.])
 
 AH_BOTTOM([
 #include <stdio.h>
@@ -528,6 +660,10 @@ AH_BOTTOM([
 #include <getopt.h>
 #endif
 
+#ifdef HAVE_OPENSSL_ERR_H
+#include <openssl/err.h>
+#endif
+
 #ifdef HAVE_ATTR_FORMAT
 #  define ATTR_FORMAT(archetype, string_index, first_to_check) \
     __attribute__ ((format (archetype, string_index, first_to_check)))
@@ -599,6 +735,25 @@ struct sockaddr_storage;
 
 #include "ldns/ldns.h"
 
+#ifdef UNBOUND_ALLOC_STATS
+#  define malloc(s) unbound_stat_malloc_log(s, __FILE__, __LINE__, __func__)
+#  define calloc(n,s) unbound_stat_calloc_log(n, s, __FILE__, __LINE__, __func__)
+#  define free(p) unbound_stat_free_log(p, __FILE__, __LINE__, __func__)
+#  define realloc(p,s) unbound_stat_realloc_log(p, s, __FILE__, __LINE__, __func__)
+void *unbound_stat_malloc(size_t size);
+void *unbound_stat_calloc(size_t nmemb, size_t size);
+void unbound_stat_free(void *ptr);
+void *unbound_stat_realloc(void *ptr, size_t size);
+void *unbound_stat_malloc_log(size_t size, const char* file, int line,
+	const char* func);
+void *unbound_stat_calloc_log(size_t nmemb, size_t size, const char* file,
+	int line, const char* func);
+void unbound_stat_free_log(void *ptr, const char* file, int line,
+	const char* func);
+void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
+	int line, const char* func);
+#endif /* UNBOUND_ALLOC_STATS */
+
 /** default port for DNS traffic. */
 #define UNBOUND_DNS_PORT 53
 ])

daemon/daemon.c

@@ -39,9 +39,6 @@
  * The daemon consists of global settings and a number of workers.
  */
 
-/** buffer size for network connections */
-#define BUFSZ 65552
-
 #include "config.h"
 #include "daemon/daemon.h"
 #include "daemon/worker.h"
@@ -50,6 +47,12 @@
 #include "util/data/msgreply.h"
 #include "util/storage/slabhash.h"
 #include "services/listen_dnsport.h"
+#include "services/cache/rrset.h"
+#include "services/cache/infra.h"
+#include "util/module.h"
+#include "iterator/iterator.h"
+#include "validator/validator.h"
+#include "util/fptr_wlist.h"
 #include <signal.h>
 
 /** How many quit requests happened. */
@@ -58,10 +61,15 @@ static int sig_record_quit = 0;
 static int sig_record_reload = 0;
 
 /** used when no other sighandling happens, so we don't die
-  * when multiple signals in quick succession are sent to us. */
+  * when multiple signals in quick succession are sent to us. 
+  * @param sig: signal number.
+  * @return signal handler return type (void or int).
+  */
 static RETSIGTYPE record_sigh(int sig)
 {
 #ifdef LIBEVENT_SIGNAL_PROBLEM
+	verbose(VERB_OPS, "quit on signal, no cleanup and statistics, "
+		"because installed libevent version is not threadsafe");
 	exit(0);
 #endif 
 	switch(sig)
@@ -100,10 +108,10 @@ signal_handling_record()
 static void
 signal_handling_playback(struct worker* wrk)
 {
-	if(sig_record_quit)
-		worker_sighandler(SIGTERM, wrk);
 	if(sig_record_reload)
 		worker_sighandler(SIGHUP, wrk);
+	if(sig_record_quit)
+		worker_sighandler(SIGTERM, wrk);
 	sig_record_quit = 0;
 	sig_record_reload = 0;
 }
@@ -117,21 +125,11 @@ daemon_init()
 		return NULL;
 	signal_handling_record();
 	checklock_start();
+	ERR_load_crypto_strings();
 	daemon->need_to_exit = 0;
-	daemon->msg_cache = slabhash_create(HASH_DEFAULT_SLABS, 
-		HASH_DEFAULT_STARTARRAY, HASH_DEFAULT_MAXMEM, 
-		msgreply_sizefunc, query_info_compare,
-		query_entry_delete, reply_info_delete, NULL);
-	if(!daemon->msg_cache) {
-		free(daemon);
-		return NULL;
-	}
-	daemon->rrset_cache = slabhash_create(HASH_DEFAULT_SLABS, 
-		HASH_DEFAULT_STARTARRAY, HASH_DEFAULT_MAXMEM, 
-		ub_rrset_sizefunc, ub_rrset_compare,
-		ub_rrset_key_delete, rrset_data_delete, &daemon->superalloc);
-	if(!daemon->rrset_cache) {
-		slabhash_delete(daemon->msg_cache);
+	daemon->num_modules = 0;
+	if(!(daemon->env = (struct module_env*)calloc(1, 
+		sizeof(*daemon->env)))) {
 		free(daemon);
 		return NULL;
 	}
@@ -152,6 +150,138 @@ daemon_open_shared_ports(struct daemon* daemon)
 	return 1;
 }
 
+/** count number of modules (words) in the string */
+static int
+count_modules(const char* s)
+{
+	int num = 0;
+	if(!s)
+		return 0;
+	while(*s) {
+		/* skip whitespace */
+		while(*s && isspace((int)*s))
+			s++;
+		if(*s && !isspace((int)*s)) {
+			/* skip identifier */
+			num++;
+			while(*s && !isspace((int)*s))
+				s++;
+		}
+	}
+	return num;
+}
+
+/**
+ * Get funcblock for module name
+ * @param str: string with module name. Advanced to next value on success.
+ * @return funcblock or NULL on error.
+ */
+static struct module_func_block*
+daemon_module_factory(const char** str)
+{
+	/* these are the modules available */
+	int num = 2;
+	const char* names[] = {"iterator", "validator", NULL};
+	struct module_func_block* (*fb[])(void) = 
+		{&iter_get_funcblock, &val_get_funcblock, NULL};
+
+	int i;
+	const char* s = *str;
+	while(*s && isspace((int)*s))
+		s++;
+	for(i=0; i<num; i++) {
+		if(strncmp(names[i], s, strlen(names[i])) == 0) {
+			s += strlen(names[i]);
+			*str = s;
+			return (*fb[i])();
+		}
+	}
+	return NULL;
+}
+
+/**
+ * Read config file module settings and set up the modfunc block
+ * @param daemon: the daemon.
+ * @return false on error
+ */
+static int
+daemon_config_modules(struct daemon* daemon)
+{
+	const char* str = daemon->cfg->module_conf;
+	int i;
+	verbose(VERB_DETAIL, "module config: \"%s\"", str);
+	daemon->num_modules = count_modules(str);
+	if(daemon->num_modules == 0) {
+		log_err("error: no modules specified");
+		return 0;
+	}
+	if(daemon->num_modules > MAX_MODULE) {
+		log_err("error: too many modules (%d max %d)",
+			daemon->num_modules, MAX_MODULE);
+		return 0;
+	}
+	daemon->modfunc = (struct module_func_block**)calloc((size_t)
+		daemon->num_modules, sizeof(struct module_func_block*));
+	if(!daemon->modfunc) {
+		log_err("out of memory");
+		return 0;
+	}
+	for(i=0; i<daemon->num_modules; i++) {
+		daemon->modfunc[i] = daemon_module_factory(&str);
+		if(!daemon->modfunc[i]) {
+			log_err("Unknown value for first module in: '%s'",
+				str);
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/**
+ * Desetup the modules, deinit, delete.
+ * @param daemon: the daemon.
+ */
+static void
+daemon_desetup_modules(struct daemon* daemon)
+{
+	int i;
+	for(i=0; i<daemon->num_modules; i++) {
+		log_assert(fptr_whitelist_mod_deinit(
+			daemon->modfunc[i]->deinit));
+		(*daemon->modfunc[i]->deinit)(daemon->env, i);
+	}
+	daemon->num_modules = 0;
+	free(daemon->modfunc);
+	daemon->modfunc = 0;
+}
+
+/**
+ * Setup modules. Assigns ids and calls module_init.
+ * @param daemon: the daemon
+ */
+static void daemon_setup_modules(struct daemon* daemon)
+{
+	int i;
+	if(daemon->num_modules != 0)
+		daemon_desetup_modules(daemon);
+	/* fixed setup of the modules */
+	if(!daemon_config_modules(daemon)) {
+		fatal_exit("failed to setup modules");
+	}
+	daemon->env->cfg = daemon->cfg;
+	daemon->env->alloc = &daemon->superalloc;
+	daemon->env->worker = NULL;
+	daemon->env->need_to_validate = 0; /* set by module init below */
+	for(i=0; i<daemon->num_modules; i++) {
+		log_info("init module %d: %s", i, daemon->modfunc[i]->name);
+		log_assert(fptr_whitelist_mod_init(daemon->modfunc[i]->init));
+		if(!(*daemon->modfunc[i]->init)(daemon->env, i)) {
+			fatal_exit("module init for module %s failed",
+				daemon->modfunc[i]->name);
+		}
+	}
+}
+
 /**
  * Allocate empty worker structures. With backptr and thread-number,
  * from 0..numthread initialised. Used as user arguments to new threads.
@@ -209,8 +339,7 @@ thread_start(void* arg)
 	worker->cmd_send_fd = -1;
 	close_other_pipes(worker->daemon, worker->thread_num);
 #endif
-	if(!worker_init(worker, worker->daemon->cfg, worker->daemon->ports,
-		BUFSZ, 0))
+	if(!worker_init(worker, worker->daemon->cfg, worker->daemon->ports, 0))
 		fatal_exit("Could not initialize thread");
 
 	worker_work(worker);
@@ -255,7 +384,7 @@ daemon_stop_others(struct daemon* daemon)
 		worker_send_cmd(daemon->workers[i], 
 			daemon->workers[0]->front->udp_buff, worker_cmd_quit);
 	}
-	/** wait for them to quit */
+	/* wait for them to quit */
 	for(i=1; i<daemon->num; i++) {
 		/* join it to make sure its dead */
 		verbose(VERB_ALGO, "join %d", i);
@@ -268,6 +397,10 @@ void
 daemon_fork(struct daemon* daemon)
 {
 	log_assert(daemon);
+
+	/* setup modules */
+	daemon_setup_modules(daemon);
+
 	/* first create all the worker structures, so we can pass
 	 * them to the newly created threads. 
 	 */
@@ -281,8 +414,7 @@ daemon_fork(struct daemon* daemon)
 	/* Special handling for the main thread. This is the thread
 	 * that handles signals.
 	 */
-	if(!worker_init(daemon->workers[0], daemon->cfg, daemon->ports,
-		BUFSZ, 1))
+	if(!worker_init(daemon->workers[0], daemon->cfg, daemon->ports, 1))
 		fatal_exit("Could not initialize main thread");
 	signal_handling_playback(daemon->workers[0]);
 
@@ -293,9 +425,7 @@ daemon_fork(struct daemon* daemon)
 	/* we exited! a signal happened! Stop other threads */
 	daemon_stop_others(daemon);
 
-	if(daemon->workers[0]->need_to_restart)
-		daemon->need_to_exit = 0;
-	else	daemon->need_to_exit = 1;
+	daemon->need_to_exit = daemon->workers[0]->need_to_exit;
 }
 
 void 
@@ -307,6 +437,13 @@ daemon_cleanup(struct daemon* daemon)
 	   don't die on multiple reload signals for example. */
 	signal_handling_record();
 	log_thread_set(NULL);
+	/* clean up caches because
+	 * a) RRset IDs will be recycled after a reload, causing collisions
+	 * b) validation config can change, thus rrset, msg, keycache clear 
+	 * The infra cache is kept, the timing and edns info is still valid */
+	slabhash_clear(&daemon->env->rrset_cache->table);
+	slabhash_clear(daemon->env->msg_cache);
+	/* key cache is cleared by module desetup during next daemon_init() */
 	for(i=0; i<daemon->num; i++)
 		worker_delete(daemon->workers[i]);
 	free(daemon->workers);
@@ -320,12 +457,23 @@ daemon_delete(struct daemon* daemon)
 {
 	if(!daemon)
 		return;
+	daemon_desetup_modules(daemon);
 	listening_ports_free(daemon->ports);
-	slabhash_delete(daemon->msg_cache);
-	slabhash_delete(daemon->rrset_cache);
+	if(daemon->env) {
+		slabhash_delete(daemon->env->msg_cache);
+		rrset_cache_delete(daemon->env->rrset_cache);
+		infra_delete(daemon->env->infra_cache);
+	}
 	alloc_clear(&daemon->superalloc);
-	free(daemon->cwd);
 	free(daemon->pidfile);
+	free(daemon->env);
 	free(daemon);
+	/* libcrypto cleanup */
+	/* CONF_modules_unload(1); */
+	EVP_cleanup();
+	/* ENGINE_cleanup(); */
+	CRYPTO_cleanup_all_ex_data(); /* safe, no more threads right now */
+	ERR_remove_state(0);
+	ERR_free_strings();
 	checklock_stop();
 }

daemon/daemon.h

@@ -48,6 +48,8 @@ struct config_file;
 struct worker;
 struct listen_port;
 struct slabhash;
+struct module_env;
+struct rrset_cache;
 
 /**
  * Structure holding worker list.
@@ -56,8 +58,6 @@ struct slabhash;
 struct daemon {
 	/** The config settings */
 	struct config_file* cfg;
-	/** current working directory */
-	char* cwd;
 	/** pidfile that is used */
 	char* pidfile;
 	/** port number that has ports opened. */
@@ -72,10 +72,12 @@ struct daemon {
 	int need_to_exit;
 	/** master allocation cache */
 	struct alloc_cache superalloc;
-	/** the message cache, content is struct msgreply_entry* */
-	struct slabhash* msg_cache;
-	/** the rrset cache, content is struct ub_packed_rrset_key* */
-	struct slabhash* rrset_cache;
+	/** the module environment master value, copied and changed by threads*/
+	struct module_env* env;
+	/** number of modules active, ids from 0 to num-1. */
+	int num_modules;
+	/** the module callbacks, array of num_modules length */
+	struct module_func_block** modfunc;
 };
 
 /**

daemon/stats.c

@@ -42,6 +42,7 @@
 #include "config.h"
 #include "daemon/stats.h"
 #include "daemon/worker.h"
+#include "services/mesh.h"
 
 void server_stats_init(struct server_stats* stats)
 {
@@ -51,9 +52,9 @@ void server_stats_init(struct server_stats* stats)
 void server_stats_querymiss(struct server_stats* stats, struct worker* worker)
 {
 	stats->num_queries_missed_cache++;
-	stats->sum_query_list_size += worker->num_requests;
-	if(worker->num_requests > stats->max_query_list_size)
-		stats->max_query_list_size = worker->num_requests;
+	stats->sum_query_list_size += worker->env.mesh->all.count;
+	if(worker->env.mesh->all.count > stats->max_query_list_size)
+		stats->max_query_list_size = worker->env.mesh->all.count;
 }
 
 void server_stats_log(struct server_stats* stats, int threadnum)
@@ -65,7 +66,7 @@ void server_stats_log(struct server_stats* stats, int threadnum)
 	log_info("server stats for thread %d: requestlist max %u avg %g "
 		"exceeded %u", threadnum, (unsigned)stats->max_query_list_size,
 		stats->num_queries_missed_cache?
-		(double)stats->sum_query_list_size/
-		stats->num_queries_missed_cache:0.0,
+			(double)stats->sum_query_list_size/
+			stats->num_queries_missed_cache : 0.0,
 		(unsigned)stats->num_query_list_exceeded);
 }

daemon/unbound.c

@@ -46,21 +46,28 @@
 #include "util/config_file.h"
 #include "util/storage/slabhash.h"
 #include "services/listen_dnsport.h"
+#include "services/cache/rrset.h"
+#include "services/cache/infra.h"
 #include "util/data/msgreply.h"
+#include "util/module.h"
 #include <signal.h>
 #include <fcntl.h>
 #include <pwd.h>
 #include <sys/resource.h>
 
+/** global debug value to keep track of heap memory allocation */
+void* unbound_start_brk = 0;
+
 /** print usage. */
 static void usage()
 {
-	printf("usage: unbound [options]\n");
+	printf("usage:  unbound [options]\n");
 	printf("	start unbound daemon DNS resolver.\n");
 	printf("-h	this help\n");
-	printf("-c file	config file to read, unbound.conf(5).\n");
+	printf("-c file	config file to read instead of %s\n", CONFIGFILE);
+	printf("	file format is described in unbound.conf(5).\n");
 	printf("-d	do not fork into the background.\n");
-	printf("-v	verbose (multiple times increase verbosity)\n");
+	printf("-v	verbose (more times to increase verbosity)\n");
 	printf("Version %s\n", PACKAGE_VERSION);
 	printf("BSD licensed, see LICENSE in source package for details.\n");
 	printf("Report bugs to %s\n", PACKAGE_BUGREPORT);
@@ -71,10 +78,12 @@ static void
 checkrlimits(struct config_file* cfg)
 {
 	int list = ((cfg->do_ip4?1:0) + (cfg->do_ip6?1:0)) * 
-		((cfg->do_udp?1:0) + (cfg->do_tcp?1 + TCP_ACCEPT_COUNT:0));
+		((cfg->do_udp?1:0) + (cfg->do_tcp?1 + 
+			(int)cfg->incoming_num_tcp:0));
 	size_t ifs = (size_t)(cfg->num_ifs==0?1:cfg->num_ifs);
 	size_t listen_num = list*ifs;
-	size_t outnum = cfg->outgoing_num_ports*ifs + cfg->outgoing_num_tcp;
+	size_t out_ifs = (size_t)(cfg->num_out_ifs==0?1:cfg->num_out_ifs);
+	size_t outnum = cfg->outgoing_num_ports*out_ifs + cfg->outgoing_num_tcp;
 	size_t misc = 4; /* logfile, pidfile, stdout... */
 	size_t perthread = listen_num + outnum + 2/*cmdpipe*/ + 2/*libevent*/
 		+ misc; 
@@ -89,6 +98,8 @@ checkrlimits(struct config_file* cfg)
 		log_warn("getrlimit: %s", strerror(errno));
 		return;
 	}
+	if(rlim.rlim_cur == (rlim_t)RLIM_INFINITY)
+		return;
 	if((size_t)rlim.rlim_cur < total) {
 		log_err("Not enough sockets available. Increase "
 			"ulimit(open files).");
@@ -100,51 +111,40 @@ checkrlimits(struct config_file* cfg)
 	}
 }
 
-/** to changedir, logfile */
+/** set verbosity, check rlimits, cache settings */
 static void
-apply_dir(struct daemon* daemon, struct config_file* cfg, int cmdline_verbose)
+apply_settings(struct daemon* daemon, struct config_file* cfg, 
+	int cmdline_verbose)
 {
 	/* apply if they have changed */
 	daemon->cfg = cfg;
 	verbosity = cmdline_verbose + cfg->verbosity;
-	if(cfg->directory && cfg->directory[0]) {
-		if(!daemon->cwd || strcmp(daemon->cwd, cfg->directory) != 0) {
-			if(chdir(cfg->directory)) {
-				log_err("Could not chdir to %s: %s",
-					cfg->directory, strerror(errno));
-			}
-			free(daemon->cwd);
-			if(!(daemon->cwd = strdup(cfg->directory)))
-				log_err("cwd: malloc failed");
-		}
-	}
-	if(cfg->msg_cache_size != slabhash_get_size(daemon->msg_cache) ||
-		cfg->msg_cache_slabs != daemon->msg_cache->size) {
-		slabhash_delete(daemon->msg_cache);
-		daemon->msg_cache = slabhash_create(cfg->msg_cache_slabs, 
+	config_apply(cfg);
+	if(!daemon->env->msg_cache ||
+	   cfg->msg_cache_size != slabhash_get_size(daemon->env->msg_cache) ||
+	   cfg->msg_cache_slabs != daemon->env->msg_cache->size) {
+		slabhash_delete(daemon->env->msg_cache);
+		daemon->env->msg_cache = slabhash_create(cfg->msg_cache_slabs, 
 			HASH_DEFAULT_STARTARRAY, cfg->msg_cache_size, 
 			msgreply_sizefunc, query_info_compare,
 			query_entry_delete, reply_info_delete, NULL);
-		if(!daemon->msg_cache) {
-			fatal_exit("malloc failure updating config settings");
-		}
-	}
-	if(cfg->rrset_cache_size != slabhash_get_size(daemon->rrset_cache) ||
-		cfg->rrset_cache_slabs != daemon->rrset_cache->size) {
-		slabhash_delete(daemon->rrset_cache);
-		daemon->rrset_cache = slabhash_create(cfg->rrset_cache_slabs, 
-			HASH_DEFAULT_STARTARRAY, cfg->rrset_cache_size, 
-			ub_rrset_sizefunc, ub_rrset_compare,
-			ub_rrset_key_delete, rrset_data_delete, 
-			&daemon->superalloc);
-		if(!daemon->rrset_cache) {
+		if(!daemon->env->msg_cache) {
 			fatal_exit("malloc failure updating config settings");
 		}
 	}
+	if((daemon->env->rrset_cache = rrset_cache_adjust(
+		daemon->env->rrset_cache, cfg, &daemon->superalloc)) == 0)
+		fatal_exit("malloc failure updating config settings");
+	if((daemon->env->infra_cache = infra_adjust(daemon->env->infra_cache, 
+		cfg))==0)
+		fatal_exit("malloc failure updating config settings");
 	checkrlimits(cfg);
 }
 
-/** Read existing pid from pidfile. */
+/** Read existing pid from pidfile. 
+ * @param file: file name of pid file.
+ * @return: the pid from the file or -1 if none.
+ */
 static pid_t
 readpid (const char* file)
 {
@@ -176,7 +176,8 @@ readpid (const char* file)
 		return -1;
 	}
 
-	pid = strtol(pidbuf, &t, 10);
+	pidbuf[sizeof(pidbuf)-1] = 0;
+	pid = (pid_t)strtol(pidbuf, &t, 10);
 	
 	if (*t && *t != '\n') {
 		return -1;
@@ -184,7 +185,10 @@ readpid (const char* file)
 	return pid;
 }
 
-/** write pid to file. */
+/** write pid to file. 
+ * @param pidfile: file name of pid file.
+ * @param pid: pid to write to file.
+ */
 static void
 writepid (const char* pidfile, pid_t pid)
 {
@@ -210,7 +214,12 @@ static void
 checkoldpid(struct config_file* cfg)
 {
 	pid_t old;
-	if((old = readpid(cfg->pidfile)) != -1) {
+	char* file = cfg->pidfile;
+	if(cfg->chrootdir && cfg->chrootdir[0] &&
+		strncmp(file, cfg->chrootdir, strlen(cfg->chrootdir))==0) {
+		file += strlen(cfg->chrootdir);
+	}
+	if((old = readpid(file)) != -1) {
 		/* see if it is still alive */
 		if(kill(old, 0) == 0 || errno == EPERM)
 			log_warn("unbound is already running as pid %u.", 
@@ -252,35 +261,62 @@ detach(struct config_file* cfg)
 static void
 do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
 {
+	uid_t uid;
+	gid_t gid;
+	/* initialize, but not to 0 (root) */
+	memset(&uid, -12, sizeof(uid));
+	memset(&gid, -12, sizeof(gid));
 	log_assert(cfg);
 
 	/* daemonize last to be able to print error to user */
-	if(cfg->chrootdir && cfg->chrootdir[0])
-		if(chroot(cfg->chrootdir))
-			fatal_exit("unable to chroot: %s", strerror(errno));
+	if(cfg->directory && cfg->directory[0]) {
+		if(chdir(cfg->directory)) {
+			fatal_exit("Could not chdir to %s: %s",
+				cfg->directory, strerror(errno));
+		}
+		verbose(VERB_DETAIL, "chdir to %s", cfg->directory);
+	}
 	if(cfg->username && cfg->username[0]) {
 		struct passwd *pwd;
 		if((pwd = getpwnam(cfg->username)) == NULL)
 			fatal_exit("user '%s' does not exist.", cfg->username);
-		if(setgid(pwd->pw_gid) != 0)
-			fatal_exit("unable to set group id: %s", strerror(errno));
-		if(setuid(pwd->pw_uid) != 0)
-			fatal_exit("unable to set user id: %s", strerror(errno));
+		uid = pwd->pw_uid;
+		gid = pwd->pw_gid;
 		endpwent();
 	}
+	if(cfg->chrootdir && cfg->chrootdir[0]) {
+		if(chroot(cfg->chrootdir))
+			fatal_exit("unable to chroot to %s: %s", 
+				cfg->chrootdir, strerror(errno));
+		verbose(VERB_DETAIL, "chroot to %s", cfg->chrootdir);
+	}
+	if(cfg->username && cfg->username[0]) {
+		if(setgid(gid) != 0)
+			fatal_exit("unable to set group id of %s: %s", 
+				cfg->username, strerror(errno));
+		if(setuid(uid) != 0)
+			fatal_exit("unable to set user id of %s: %s", 
+				cfg->username, strerror(errno));
+		verbose(VERB_DETAIL, "drop user privileges, run as %s", 
+			cfg->username);
+	}
 	/* check old pid file before forking */
 	if(cfg->pidfile && cfg->pidfile[0]) {
 		checkoldpid(cfg);
 	}
 
 	/* init logfile just before fork */
-	log_init(cfg->logfile);
+	log_init(cfg->logfile, cfg->use_syslog, cfg->chrootdir);
 	if(!debug_mode && cfg->do_daemonize) {
 		detach(cfg);
 	}
 	if(cfg->pidfile && cfg->pidfile[0]) {
-		writepid(cfg->pidfile, getpid());
-		if(!(daemon->pidfile = strdup(cfg->pidfile)))
+		char* pf = cfg->pidfile;
+		if(cfg->chrootdir && cfg->chrootdir[0] &&
+			strncmp(pf, cfg->chrootdir, strlen(cfg->chrootdir))==0)
+			pf += strlen(cfg->chrootdir);
+		writepid(pf, getpid());
+		if(!(daemon->pidfile = strdup(pf)))
 			log_err("pidf: malloc failed");
 	}
 }
@@ -293,7 +329,7 @@ do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
  * @param debug_mode: if set, do not daemonize.
  */
 static void 
-run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode)
+run_daemon(char* cfgfile, int cmdline_verbose, int debug_mode)
 {
 	struct config_file* cfg = NULL;
 	struct daemon* daemon = NULL;
@@ -311,7 +347,7 @@ run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode)
 			fatal_exit("Could not alloc config defaults");
 		if(!config_read(cfg, cfgfile))
 			fatal_exit("Could not read config file: %s", cfgfile);
-		apply_dir(daemon, cfg, cmdline_verbose);
+		apply_settings(daemon, cfg, cmdline_verbose);
 	
 		/* prepare */
 		if(!daemon_open_shared_ports(daemon))
@@ -319,7 +355,7 @@ run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode)
 		if(!done_chroot) { 
 			do_chroot(daemon, cfg, debug_mode); 
 			done_chroot = 1; 
-		}
+		} else log_init(cfg->logfile, cfg->use_syslog, cfg->chrootdir);
 		/* work */
 		daemon_fork(daemon);
 
@@ -349,11 +385,14 @@ int
 main(int argc, char* argv[])
 {
 	int c;
-	const char* cfgfile = NULL;
+	char* cfgfile = CONFIGFILE;
 	int cmdline_verbose = 0;
 	int debug_mode = 0;
 
-	log_init(NULL);
+	/* take debug snapshot of heap */
+	unbound_start_brk = sbrk(0);
+
+	log_init(NULL, 0, NULL);
 	/* parse the options */
 	while( (c=getopt(argc, argv, "c:dhv")) != -1) {
 		switch(c) {
@@ -383,5 +422,6 @@ main(int argc, char* argv[])
 	}
 
 	run_daemon(cfgfile, cmdline_verbose, debug_mode);
+	log_init(NULL, 0, NULL); /* close logfile */
 	return 0;
 }

daemon/worker.h

@@ -50,13 +50,14 @@
 #include "util/data/msgreply.h"
 #include "util/data/msgparse.h"
 #include "daemon/stats.h"
+#include "util/module.h"
 struct listen_dnsport;
 struct outside_network;
 struct config_file;
 struct daemon;
 struct listen_port;
 struct ub_randstate;
-struct region;
+struct regional;
 
 /** size of table used for random numbers. large to be more secure. */
 #define RND_STATE_SIZE 256
@@ -67,28 +68,6 @@ enum worker_commands {
 	worker_cmd_quit
 };
 
-/** information per query that is in processing */
-struct work_query {
-	/** next query in freelist */
-	struct work_query* next;
-	/** the worker for this query */
-	struct worker* worker;
-	/** the query reply destination, packet buffer and where to send. */
-	struct comm_reply query_reply;
-	/** the query_info structure from the query */
-	struct query_info qinfo;
-	/** hash value of the query qinfo */
-	hashvalue_t query_hash;
-	/** next query in all-list */
-	struct work_query* all_next;
-	/** id of query, in network byteorder. */
-	uint16_t query_id;
-	/** flags uint16 from query */
-	uint16_t query_flags;
-	/** edns data from the query */
-	struct edns_data edns;
-};
-
 /**
  * Structure holding working information for unbound.
  * Holds globally visible information.
@@ -115,30 +94,22 @@ struct worker {
 	/** commpoint to listen to commands. */
 	struct comm_point* cmd_com;
 
-	/** number of requests currently active */
-	size_t num_requests;
 	/** number of requests that can be handled by this worker */
 	size_t request_size;
-	/** the free working queries */
-	struct work_query* free_queries;
-	/** list of all working queries */
-	struct work_query* all_queries;
-
-	/** address to forward to */
-	struct sockaddr_storage fwd_addr;
-	/** length of fwd_addr */
-	socklen_t fwd_addrlen;
 
 	/** random() table for this worker. */
 	struct ub_randstate* rndstate;
-	/** do we need to restart (instead of exit) ? */
-	int need_to_restart;
+	/** do we need to restart or quit (on signal) */
+	int need_to_exit;
 	/** allocation cache for this thread */
 	struct alloc_cache alloc;
 	/** per thread statistics */
 	struct server_stats stats;
-	/** thread scratch region */
-	struct region* scratchpad;
+	/** thread scratch regional */
+	struct regional* scratchpad;
+
+	/** module environment passed to modules, changed for this thread */
+	struct module_env env;
 };
 
 /**
@@ -156,12 +127,11 @@ struct worker* worker_create(struct daemon* daemon, int id);
  * @param worker: worker to initialize, created with worker_create.
  * @param cfg: configuration settings.
  * @param ports: list of shared query ports.
- * @param buffer_size: size of datagram buffer.
  * @param do_sigs: if true, worker installs signal handlers.
  * @return: false on error.
  */
 int worker_init(struct worker* worker, struct config_file *cfg, 
-	struct listen_port* ports, size_t buffer_size, int do_sigs);
+	struct listen_port* ports, int do_sigs);
 
 /**
  * Make worker work.
@@ -173,15 +143,6 @@ void worker_work(struct worker* worker);
  */
 void worker_delete(struct worker* worker);
 
-/**
- * Set forwarder
- * @param worker: the worker to modify.
- * @param ip: the server name.
- * @param port: port on server or NULL for default 53.
- * @return: false on error.
- */
-int worker_set_fwd(struct worker* worker, const char* ip, int port);
-
 /**
  * Send a command to a worker. Uses blocking writes.
  * @param worker: worker to send command to.
@@ -198,4 +159,62 @@ void worker_send_cmd(struct worker* worker, ldns_buffer* buffer,
  */
 void worker_sighandler(int sig, void* arg);
 
+/**
+ * Worker service routine to send udp messages for modules.
+ * @param pkt: packet to send.
+ * @param addr: where to.
+ * @param addrlen: length of addr.
+ * @param timeout: seconds to wait until timeout.
+ * @param q: wich query state to reactivate upon return.
+ * @param use_tcp: true to use TCP, false for UDP.
+ * @return: false on failure (memory or socket related). no query was
+ *      sent.
+ */
+int worker_send_packet(ldns_buffer* pkt, struct sockaddr_storage* addr,
+	socklen_t addrlen, int timeout, struct module_qstate* q, int use_tcp);
+
+/**
+ * Worker service routine to send serviced queries to authoritative servers.
+ * @param qname: query name. (host order)
+ * @param qnamelen: length in bytes of qname, including trailing 0.
+ * @param qtype: query type. (host order)
+ * @param qclass: query class. (host order)
+ * @param flags: host order flags word, with opcode and CD bit.
+ * @param dnssec: if set, EDNS record will have DO bit set.
+ * @param addr: where to.
+ * @param addrlen: length of addr.
+ * @param q: wich query state to reactivate upon return.
+ * @return: false on failure (memory or socket related). no query was
+ *      sent.
+ */
+struct outbound_entry* worker_send_query(uint8_t* qname, size_t qnamelen, 
+	uint16_t qtype, uint16_t qclass, uint16_t flags, int dnssec, 
+	struct sockaddr_storage* addr, socklen_t addrlen,
+	struct module_qstate* q);
+
+/** 
+ * process control messages from the main thread. 
+ * @param c: comm point to read from.
+ * @param arg: worker.
+ * @param error: error status of comm point.
+ * @param reply_info: not used.
+ */
+int worker_handle_control_cmd(struct comm_point* c, void* arg, int error, 
+	struct comm_reply* reply_info);
+
+/** handles callbacks from listening event interface */
+int worker_handle_request(struct comm_point* c, void* arg, int error,
+	struct comm_reply* repinfo);
+
+/** process incoming replies from the network */
+int worker_handle_reply(struct comm_point* c, void* arg, int error, 
+	struct comm_reply* reply_info);
+
+/** process incoming serviced query replies from the network */
+int worker_handle_service_reply(struct comm_point* c, void* arg, int error, 
+	struct comm_reply* reply_info);
+
+/** cleanup the cache to remove all rrset IDs from it, arg is worker */
+void worker_alloc_cleanup(void* arg);
+
 #endif /* DAEMON_WORKER_H */

doc/CREDITS

@@ -0,0 +1,9 @@
+Unbound was developed at NLnet Labs by Wouter Wijngaards.
+
+Unbound was devised by Bill Manning, David Blacka, and Matt Larson 
+from the University of California and from Verisign. The Java prototype
+was made in further cooperation with Geoff Sisson and Roy Arends from Nominet.
+
+At NLnet Labs, Jelte Jansen and Mark Santcroos reviewed the unbound C sources.
+
+Jakob Schlyter - for advice on secure settings, random numbers and blacklists.

doc/Changelog

@@ -1,3 +1,894 @@
+3 December 2007: Wouter
+	- Fixup building in a subdirectory. (for 0.7.2)
+
+19 November 2007: Wouter
+	- version 0.7.1:
+		* includes tpkg fixes to kill daemons at end of test
+		* includes NSEC/RRSIG not downcasing fixup from
+		dnssec-bis-updates draft-06.
+		* README file updated.
+		* nice libevent not found error.
+		* README talks about gnu make.
+	- for 0.7.2:
+		* fixup for donotq matching.
+
+15 November 2007: Wouter
+	- review fixups from Jelte.
+
+14 November 2007: Wouter
+	- testbed script does not recreate configure, since its in svn now.
+	- fixup checkconf test so that it does not test 
+	  /etc/unbound/unbound.conf.
+	- tag 0.6.
+
+13 November 2007: Wouter
+	- remove debug print.
+	- fixup testbound exit when LIBEVENT_SIGNAL_PROBLEM exists.
+
+12 November 2007: Wouter
+	- fixup signal handling where SIGTERM could be ignored if a SIGHUP
+	  arrives later on.
+	- bugreports to unbound-bugs@nlnetlabs.nl
+	- fixup testbound so it exits cleanly.
+	- cleanup the caches on a reload, so that rrsetID numbers won't clash.
+
+9 November 2007: Wouter
+	- took ldns snapshot in repo.
+	- default config file is /etc/unbound/unbound.conf.
+	  If it doesn't exist, it is installed with the doc/example.conf file.
+	  The file is not deleted on uninstall.
+	- default listening is not all, but localhost interfaces.
+	
+8 November 2007: Wouter
+	- Fixup chroot and drop user privileges.
+	- new L root ip address in default hints.
+
+1 November 2007: Wouter
+	- Fixup of crash on reload, due to anchors in env not NULLed after
+	  dealloc during deinit.
+	- Fixup of chroot call. Happens after privileges are dropped, so
+	  that checking the passwd entry still works.
+	- minor touch up of clear() hashtable function.
+	- VERB_DETAIL prints out what chdir, username, chroot is being done.
+	- when id numbers run out, caches are cleared, as in design notes.
+	  Tested with a mock setup with very few bits in id, it worked.
+	- harden-dnssec-stripped: yes is now default. It insists on dnssec
+	  data for trust anchors. Included tests for the feature.
+
+31 October 2007: Wouter
+	- cache-max-ttl config option.
+	- building outside sourcedir works again.
+	- defaults more secure:
+		username: "unbound"
+		chroot: "/etc/unbound"
+	  The operator can override them to be less secure ("") if necessary.
+	- fix horrible oversight in sorting rrset references in a message,
+	  sort per reference key pointer, not on referencepointer itself.
+	- pidfile: "/etc/unbound/unbound.pid" is now the default.
+	- tests changed to reflect the updated default.
+	- created hashtable clear() function that respects locks.
+
+30 October 2007: Wouter
+	- fixup assertion failure that relied on compressed names to be
+	  smaller than uncompressed names. A packet from comrite.com was seen
+	  to be compressed to a larger size. Added it as unit test.
+	- quieter logging at low verbosity level for common tcp messages.
+	- no greedy TTL update.
+
+23 October 2007: Wouter
+	- fixup (grand-)parent problem for dnssec-lameness detection.
+	- fixup tests to do additional section processing for lame replies,
+	  since the detection needs that.
+	- no longer trust in query section in reply during dnssec lame detect.
+	- dnssec lameness does not make the server never ever queried, but
+	  non-preferred. If no other servers exist or answer, the dnssec lame
+	  server is used; the fastest dnssec lame server is chosen.
+	- added test then when trust anchor cannot be primed (nodata), the
+	  insecure mode from unbound works.
+	- Fixup max queries per thread, any more are dropped.
+
+22 October 2007: Wouter
+	- added donotquerylocalhost config option. Can be turned off for
+	  out test cases.
+	- ISO C compat changes.
+	- detect RA-no-AA lameness, as LAME.
+	- DNSSEC-lameness detection, as LAME.
+	  See notes in requirements.txt for choices made.
+	- tests for lameness detection.
+	- added all to make test target; need unbound for fwd tests.
+	- testbound does not pollute /etc/unbound.
+
+19 October 2007: Wouter
+	- added configure (and its files) to svn, so that the trunk is easier
+	  to use. ./configure, config.guess, config.sub, ltmain.sh,
+	  and config.h.in.
+	- added yacc/lex generated files, util/configlexer.c, 
+	  util/configparser.c util/configparser.h, to svn. 
+	- without lex no attempt to use it.
+	- unsecure response validation collated into one block.
+	- remove warning about const cast of cfgfile name.
+	- outgoing-interfaces can be different from service interfaces.
+	- ldns-src configure is done during unbound configure and
+	  ldns-src make is done during unbound make, and so inherits the
+	  make arguments from the unbound make invocation.
+	- nicer error when libevent problem causes instant exit on signal.
+	- read root hints from a root hint file (like BIND does).
+	  
+18 October 2007: Wouter
+	- addresses are logged with errors.
+	- fixup testcode fake event to remove pending before callback
+	  since the callback may create new pending items.
+	- tests updated because retries are now in iterator module.
+	- ldns-testpkts code is checked for differences between unbound
+	  and ldns by makedist.sh.
+	- ldns trunk from today added in svn repo for fallback in case
+	  no ldns is installed on the system.
+	  make download_ldns refreshes the tarball with ldns svn trunk.
+	- ldns-src.tar.gz is used if no ldns is found on the system, and
+	  statically linked into unbound.
+	- start of regional allocator code.
+	- regional uses less memory and variables, simplified code.
+	- remove of region-allocator.
+	- alloc cache keeps a cache of recently released regional blocks,
+	  up to a maximum.
+	- make unit test cleanly free memory.
+
+17 October 2007: Wouter
+	- fixup another cycle detect and ns-addr timeout resolution bug.
+	  This time by refusing delegations from the cache without addresses
+	  when resolving a mandatory-glue nameserver-address for that zone.
+	  We're going to have to ask a TLD server anyway; might as well be
+	  the TLD server for this name. And this resolves a lot of cases where
+	  the other nameserver names lead to cycles or are not available.
+	- changed random generator from random(3) clone to arc4random wrapped
+	  for thread safety. The random generator is initialised with
+	  entropy from the system.
+	- fix crash where failure to prime DNSKEY tried to print null pointer
+	  in the log message.
+	- removed some debug prints, only verb_algo (4) enables them.
+	- fixup test; new random generator took new paths; such as one 
+	  where no scripted answer was available.
+	- mark insecure RRs as insecure.
+	- fixup removal of nonsecure items from the additional.
+	- reduced timeout values to more realistic, 376 msec (262 msec has
+	  90% of roundtrip times, 512 msec has 99% of roundtrip times.)
+	- server selection failover to next server after timeout (376 msec).
+
+16 October 2007: Wouter
+	- no malloc in log_hex.
+	- assertions around system calls.
+	- protect against gethostname without ending zero.
+	- ntop output is null terminated by unbound.
+	- pidfile content null termination
+	- various snprintf use sizeof(stringbuf) instead of fixed constant.
+	- changed loopdetect % 8 with & 0x7 since % can become negative for
+	  weird negative input and particular interpretation of integer math.
+	- dname_pkt_copy checks length of result, to protect result buffers.
+	  prints an error, this should not happen. Bad strings should have
+	  been rejected earlier in the program.
+	- remove a size_t underflow from msgreply size func.
+
+15 October 2007: Wouter
+	- nicer warning.
+	- fix IP6 TCP, wrong definition check. With test package.
+	- fixup the fact that the query section was not compressed to,
+	  the code was there but was called by value instead of by reference.
+	  And test for the case, uses xxd and nc.
+	- more portable ip6 check for sockaddr types.
+
+8 October 2007: Wouter
+	- --disable-rpath option in configure for 64bit systems with
+	  several dynamic lib dirs.
+
+7 October 2007: Wouter
+	- fixup tests for no AD bit in non-DO queries.
+	- test that makes sure AD bit is not set on non-DO query.
+
+6 October 2007: Wouter
+	- removed logfile open early. It did not have the proper permissions;
+	  it was opened as root instead of the user. And we cannot change user
+	  id yet, since chroot and bind ports need to be done.
+	- callback checks for event callbacks done from mini_event. Because
+	  of deletions cannot do this from netevent. This means when using
+	  libevent the protection does not work on event-callbacks.
+	- fixup too small reply (did not zero counts).
+	- fixup reply no longer AD bit when query without DO bit.
+
+5 October 2007: Wouter
+	- function pointer whitelist.
+
+4 October 2007: Wouter
+	- overwrite sensitive random seed value after use.
+	- switch to logfile very soon if not -d (console attached).
+	- error messages do not reveal the trustanchor contents.
+	- start work on function pointer whitelists.
+
+3 October 2007: Wouter
+	- fix for multiple empty nonterminals, after multiple DSes in the
+	  chain of trust.
+	- mesh checks if modules are looping, and stops them.
+	- refetch with CNAMEd nameserver address regression test added.
+	- fixup line count bug in testcode, so testbound prints correct line
+	  number with parse errors.
+	- unit test for multiple ENT case.
+	- fix for cname out of validated unsec zone.
+	- fixup nasty id=0 reuse. Also added assertions to detect its
+	  return (the assertion catches in the existing test cases).
+
+1 October 2007: Wouter
+	- skip F77, CXX, objC tests in configure step.
+	- fixup crash in refetch glue after a CNAME.
+	  and protection against similar failures (with error print).
+
+28 September 2007: Wouter
+	- test case for unbound-checkconf, fixed so it also checks the
+	  interface: statements.
+
+26 September 2007: Wouter
+	- SIGHUP will reopen the log file.
+	- Option to log to syslog.
+	- please lint, fixup tests (that went to syslog on open, oops).
+	- config check program.
+
+25 September 2007: Wouter
+	- tests for NSEC3. Fixup bitmap checks for NSEC3.
+	- positive ANY response needs to check if wildcard expansion, and
+	  check that original data did not exist.
+	- tests for NSEC3 that wrong use of OPTOUT is bad. For insecure
+	  delegation, for abuse of child zone apex nsec3.
+	- create 0.5 release tag.
+
+24 September 2007: Wouter
+	- do not make test programs by default.
+	- But 'make test' will perform all of the tests.
+	- Advertise builtin select libevent alternative when no libevent
+	  is found.
+	- signit can generate NSEC3 hashes, for generating tests.
+	- multiple nsec3 paramaters in message test.
+	- too high nsec3 iterations becomes insecure test.
+
+21 September 2007: Wouter
+	- fixup empty_DS_name allocated in wrong region (port DEC Alpha).
+	- fixup testcode lock safety (port FreeBSD).
+	- removes subscript has type char warnings (port Solaris 9).
+	- fixup of field with format type to int (port MacOS/X intel).
+	- added test for infinite loop case in nonRD answer validation.
+	  It was a more general problem, but hard to reproduce. When an
+	  unsigned rrset is being validated and the key fetched, the DS
+	  sequence is followed, but if the final name has no DS, then no
+	  proof is possible - the signature has been stripped off.
+
+20 September 2007: Wouter
+	- fixup and test for NSEC wildcard with empty nonterminals. 
+	- makedist.sh fixup for svn info.
+	- acl features request in plan.
+	- improved DS empty nonterminal handling.
+	- compat with ANS nxdomain for empty nonterminals. Attempts the nodata
+	  proof anyway, which succeeds in ANS failure case.
+	- striplab protection in case it becomes -1.
+	- plans for static and blacklist config.
+
+19 September 2007: Wouter
+	- comments about non-packed usage.
+	- plan for overload support in 0.6.
+	- added testbound tests for a failed resolution from the logs
+	  and for failed prime when missing glue.
+	- fixup so useless delegation points are not returned from the
+	  cache. Also the safety belt is used if priming fails to complete.
+	- fixup NSEC rdata not to be lowercased, bind compat.
+
+18 September 2007: Wouter
+	- wildcard nsec3 testcases, and fixup to get correct wildcard name.
+	- validator prints subtype classification for debug.
+
+17 September 2007: Wouter
+	- NSEC3 hash cache unit test.
+	- validator nsec3 nameerror test.
+
+14 September 2007: Wouter
+	- nsec3 nodata proof, nods proof, wildcard proof.
+	- nsec3 support for cname chain ending in noerror or nodata.
+	- validator calls nsec3 proof routines if no NSECs prove anything.
+	- fixup iterator bug where it stored the answer to a cname under
+	  the wrong qname into the cache. When prepending the cnames, the
+	  qname has to be reset to the original qname.
+
+13 September 2007: Wouter
+	- nsec3 find matching and covering, ce proof, prove namerror msg.
+
+12 September 2007: Wouter
+	- fixup of manual page warnings, like for NSD bugreport.
+	- nsec3 work, config, max iterations, filter, and hash cache. 
+
+6 September 2007: Wouter
+	- fixup to find libevent on mac port install.
+	- fixup size_t vs unsigned portability in validator/sigcrypt.
+	- please compiler on different platforms, for unreachable code.
+	- val_nsec3 file.
+	- pthread_rwlock type is optional, in case of old pthread libs.
+
+5 September 2007: Wouter
+	- cname, name error validator tests.
+	- logging of qtype ANY works.
+	- ANY type answers get RRSIG in answer section of replies (but not
+	  in other sections, unless DO bit is on).
+	- testbound can replay a TCP query (set MATCH TCP in the QUERY).
+	- DS and noDS referral validation test.
+	- if you configure many trust anchors, parent trust anchors can
+	  securely deny existance of child trust anchors, if validated.
+	- not all *.name NSECs are present because a wildcard was matched,
+	  and *.name NSECs can prove nodata for empty nonterminals.
+	  Also, for wildcard name NSECs, check they are not from the parent
+	  zone (for wildcarded zone cuts), and check absence of CNAME bit,
+	  for a nodata proof.
+	- configure option for memory allocation debugging.
+	- port configure option for memory allocation to solaris10.
+
+4 September 2007: Wouter
+	- fixup of Leakage warning when serviced queries processed multiple
+	  callbacks for the same query from the same server.
+	- testbound removes config file from /tmp on failed exit.
+	- fixup for referral cleanup of the additional section.
+	- tests for cname, referral validation.
+	- neater testbound tpkg output.
+	- DNAMEs no longer match their apex when synthesized from the cache.
+	- find correct signer name for DNAME responses.
+	- wildcarded DNAME test and fixup code to detect.
+	- prepend NSEC and NSEC3 rrsets in the iterator while chasing CNAMEs.
+	  So that wildcarded CNAMEs get their NSEC with them to the answer.
+	- test for a CNAME to a DNAME to a CNAME to an answer, all from
+	  different domains, for key fetching and signature checking of
+	  CNAME'd messages.
+
+3 September 2007: Wouter
+	- Fixed error in iterator that would cause assertion failure in 
+	  validator. CNAME to a NXDOMAIN response was collated into a response
+	  with both a CNAME and the NXDOMAIN rcode. Added a test that the
+	  rcode is changed to NOERROR (because of the CNAME).
+	- timeout on tcp does not lead to spurious leakage detect.
+	- account memory for name of lame zones, so that memory leakages does
+	  not show lame cache growth as a leakage growth.
+	- config setting for lameness cache expressed in bytes, instead of
+	  number of entries.
+	- tool too summarize allocations per code line.
+
+31 August 2007: Wouter
+	- can read bind trusted-keys { ... }; files, in a compatibility mode. 
+	- iterator should not detach target queries that it still could need.
+	  the protection against multiple outstanding queries is moved to a
+	  current_query num check.
+	- validator nodata, positive, referral tests.
+	- dname print can print '*' wildcard.
+
+30 August 2007: Wouter
+	- fixup override date config option.
+	- config options to control memory usage.
+	- caught bad free of un-alloced data in worker_send error case.
+	- memory accounting for key cache (trust anchors and temporary cache).
+	- memory accounting fixup for outside network tcp pending waits.
+	- memory accounting fixup for outside network tcp callbacks.
+	- memory accounting for iterator fixed storage.
+	- key cache size and slabs config options.
+	- lib crypto cleanups at exit. 
+
+29 August 2007: Wouter
+	- test tool to sign rrsets for testing validator with.
+	- added RSA and DSA test keys, public and private pairs, 512 bits.
+	- default configuration is with validation enabled.
+	  Only a trust-anchor needs to be configured for DNSSEC to work.
+	- do not convert to DER for DSA signature verification.
+	- validator replay test file, for a DS to DNSKEY DSA key prime and
+	  positive response.
+
+28 August 2007: Wouter
+	- removed double use for udp buffers, that could fail,
+	  instead performs a malloc to do the backup.
+	- validator validates referral messages, by validating all the rrsets
+	  and stores the rrsets in the cache. Further referral (nonRD queries)
+	  replies are made from the rrset cache directly. Unless unchecked
+	  rrsets are encountered, there are then validated.
+	- enforce that signing is done by a parent domain (or same domain).
+	- adjust TTL downwards if rrset TTL bigger than signature allows.
+	- permissive mode feature, sets AD bit for secure, but bogus does
+	  not give servfail (bogus is changed into indeterminate).
+	- optimization of rrset verification. rr canonical sorting is reused,
+	  for the same rrset. canonical rrset image in buffer is reused for
+	  the same signature.
+	- if the rrset is too big (64k exactly + large owner name) the
+	  canonicalization routine will fail if it does not fit in buffer.
+	- faster verification for large sigsets.
+	- verb_detail mode reports validation failures, but not the entire
+	  algorithm for validation. Key prime failures are reported as 
+	  verb_ops level.
+
+27 August 2007: Wouter
+	- do not garble the edns if a cache answer fails.
+	- answer norecursive from cache if possible.
+	- honor clean_additional setting when returning secure non-recursive
+	  referrals.
+	- do not store referral in msg cache for nonRD queries.
+	- store verification status in the rrset cache to speed up future
+	  verification.
+	- mark rrsets indeterminate and insecure if they are found to be so.
+	  and store this in the cache.
+
+24 August 2007: Wouter
+	- message is bogus if unsecure authority rrsets are present.
+	- val-clean-additional option, so you can turn it off.
+	- move rrset verification out of the specific proof types into one
+	  routine. This makes the proof routines prettier.
+	- fixup cname handling in validator, cname-to-positive and cname-to-
+	  nodata work.
+	- Do not synthesize DNSKEY and DS responses from the rrset cache if
+	  the rrset is from the additional section. Signatures may have
+	  fallen off the packet, and cause validation failure.
+	- more verbose signature date errors (with the date attached).
+	- increased default infrastructure cache size. It is important for
+	  performance, and 1000 entries are only 212k (or a 400 k total cache
+	  size). To 10000 entries (for 2M entries, 4M cache size).
+
+23 August 2007: Wouter
+	- CNAME handling - move needs_validation to before val_new().
+	  val_new() setups the chase-reply to be an edited copy of the msg.
+	  new classification, and find signer can find for it. 
+	  removal of unsigned crap from additional, and query restart for
+	  cname.
+	- refuse to follow wildcarded DNAMEs when validating.
+	  But you can query for qtype ANY, or qtype DNAME and validate that.
+
+22 August 2007: Wouter
+	- bogus TTL.
+	- review - use val_error().
+
+21 August 2007: Wouter
+	- ANY response validation.
+	- store security status in cache.
+	- check cache security status and either send the query to be
+	  validated, return the query to client, or send servfail to client.
+	  Sets AD bit on validated replies.
+	- do not examine security status on an error reply in mesh_done.
+	- construct DS, DNSKEY messages from rrset cache.
+	- manual page entry for override-date.
+
+20 August 2007: Wouter
+	- validate and positive validation, positive wildcard NSEC validation.
+	- nodata validation, nxdomain validation.
+
+18 August 2007: Wouter
+	- process DNSKEY response in FINDKEY state.
+
+17 August 2007: Wouter
+	- work on DS2KE routine.
+	- val_nsec.c for validator NSEC proofs.
+	- unit test for NSEC bitmap reading.
+	- dname iswild and canonical_compare with unit tests.
+
+16 August 2007: Wouter
+	- DS sig unit test.
+	- latest release libevent 1.3c and 1.3d have threading fixed.
+	- key entry fixup data pointer and ttl absolute.
+	- This makes a key-prime succeed in validator, with DS or DNSKEY as
+	  trust-anchor.
+	- fixup canonical compare byfield routine, fix bug and also neater.
+	- fixed iterator response type classification for queries of type
+	  ANY and NS.
+	  dig ANY gives sometimes NS rrset in AN and NS section, and parser
+	  removes the NS section duplicate. dig NS gives sometimes the NS
+	  in the answer section, as referral.
+	- validator FINDKEY state.
+
+15 August 2007: Wouter
+	- crypto calls to verify signatures.
+	- unit test for rrsig verification.
+
+14 August 2007: Wouter
+	- default outgoing ports changed to avoid port 2049 by default.
+	  This port is widely blocked by firewalls.
+	- count infra lameness cache in memory size.
+	- accounting of memory improved
+	- outbound entries are allocated in the query region they are for.
+	- extensive debugging for memory allocations.
+	- --enable-lock-checks can be used to enable lock checking.
+	- protect undefs in config.h from autoheaders ministrations.
+	- print all received udp packets. log hex will print on multiple
+	  lines if needed.
+	- fixed error in parser with backwards rrsig references.
+	- mark cycle targets for iterator did not have CD flag so failed
+	  its task.
+
+13 August 2007: Wouter
+	- fixup makefile, if lexer is missing give nice error and do not
+	  mess up the dependencies.
+	- canonical compare routine updated.
+	- canonical hinfo compare.
+	- printout list of the queries that the mesh is working on.
+
+10 August 2007: Wouter
+	- malloc and free overrides that track total allocation and frees.
+	  for memory debugging.
+	- work on canonical sort.
+
+9 August 2007: Wouter
+	- canonicalization, signature checks
+	- dname signature label count and unit test.
+	- added debug heap size print to memory printout.
+	- typo fixup in worker.c
+	- -R needed on solaris.
+	- validator override option for date check testing.
+
+8 August 2007: Wouter
+	- ldns _raw routines created (in ldns trunk).
+	- sigcrypt DS digest routines
+	- val_utils uses sigcrypt to perform signature cryptography.
+	- sigcrypt keyset processing
+
+7 August 2007: Wouter
+	- security status type.
+	- security status is copied when rdata is equal for rrsets.
+	- rrset id is updated to invalidate all the message cache entries
+	  that refer to NSEC, NSEC3, DNAME rrsets that have changed.
+	- val_util work
+	- val_sigcrypt file for validator signature checks.
+
+6 August 2007: Wouter
+	- key cache for validator.
+	- moved isroot and dellabel to own dname routines, with unit test.
+
+3 August 2007: Wouter
+	- replanning.
+	- scrubber check section of lame NS set.
+	- trust anchors can be in config file or read from zone file,
+	  DS and DNSKEY entries.
+	- unit test trust anchor storage.
+	- trust anchors converted to packed rrsets.
+	- key entry definition.
+
+2 August 2007: Wouter
+	- configure change for latest libevent trunk version (needs -lrt).
+	- query_done and walk_supers are moved out of module interface.
+	- fixup delegation point duplicates.
+	- fixup iterator scrubber; lame NS set is let through the scrubber
+	  so that the classification is lame.
+	- validator module exists, and does nothing but pass through,
+	  with calling of next module and return.
+	- validator work.
+
+1 August 2007: Wouter
+	- set version to 0.5
+	- module work for module to module interconnections.
+	- config of modules.
+	- detect cycle takes flags.
+
+31 July 2007: Wouter
+	- updated plan
+	- release 0.4 tag.
+
+30 July 2007: Wouter
+	- changed random state init, so that sequential process IDs are not
+	  cancelled out by sequential thread-ids in the random number seed.
+	- the fwd_three test, which sends three queries to unbound, and 
+	  unbound is kept waiting by ldns-testns for 3 seconds, failed
+	  because the retry timeout for default by unbound is 3 seconds too,
+	  it would hit that timeout and fail the test. Changed so that unbound
+	  is kept waiting for 2 seconds instead.
+
+27 July 2007: Wouter
+	- removed useless -C debug option. It did not work.
+	- text edit of documentation.
+	- added doc/CREDITS file, referred to by the manpages.
+	- updated planning.
+
+26 July 2007: Wouter
+	- cycle detection, for query state dependencies. Will attempt to
+	  circumvent the cycle, but if no other targets available fails.
+	- unit test for AXFR, IXFR response.
+	- test for cycle detection.
+
+25 July 2007: Wouter
+	- testbound read ADDRESS and check it.
+	- test for version.bind and friends.
+	- test for iterator chaining through several referrals.
+	- test and fixup for refetch for glue. Refetch fails if glue
+	  is still not provided.
+
+24 July 2007: Wouter
+	- Example section in config manual.
+	- Addr stored for range and moment in replay.
+
+20 July 2007: Wouter
+	- Check CNAME chain before returning cache entry with CNAMEs.
+	- Option harden-glue, default is on. It will discard out of zone
+	  data. If disabled, performance is faster, but spoofing attempts
+	  become a possibility. Note that still normalize scrubbing is done,
+	  and that the potentially spoofed data is used for infrastructure
+	  and not returned to the client.
+	- if glue times out, refetch by asking parent of delegation again.
+	  Much like asking for DS at the parent side.
+	- TODO items from forgery-resilience draft.
+	  and on memory handling improvements.
+	- renamed module_event_timeout to module_event_noreply.
+	- memory reporting code; reports on memory usage after handling
+	  a network packet (not on cache replies).
+
+19 July 2007: Wouter
+	- shuffle NS selection when getting nameserver target addresses.
+	- fixup of deadlock warnings, yield cpu in checklock code so that
+	  freebsd scheduler selects correct process to run.
+	- added identity and version config options and replies.
+	- store cname messages complete answers.
+
+18 July 2007: Wouter
+	- do not query addresses, 127.0.0.1, and ::1 by default.
+
+17 July 2007: Wouter
+	- forward zone options in config file.
+	- forward per zone in iterator. takes precendence over stubs.
+	- fixup commithooks.
+	- removed forward-to and forward-to-port features, subsumed by
+	  new forward zones.
+	- fix parser to handle absent server: clause.
+	- change untrusted rrset test to account for scrubber that is now
+	  applied during the test (which removes the poison, by the way).
+	- feature, addresses can be specified with @portnumber, like nsd.conf.
+	- test config files changed over to new forwarder syntax.
+
+27 June 2007: Wouter
+	- delete of mesh does a postorder traverse of the tree.
+	- found and fixed a memory leak. For TTL=0 messages, that would
+	  not be cached, instead the msg-replyinfo structure was leaked.
+	- changed server selection so it will filter out hosts that are
+	  unresponsive. This is defined as a host with the maximum rto value.
+	  This means that unbound tried the host for retries up to 120 secs.
+	  The rto value will time out after host-ttl seconds from the cache.
+	  This keeps such unresolvable queries from taking up resources.
+	- utility for keeping histogram.
+
+26 June 2007: Wouter
+	- mesh is called by worker, and iterator uses it.
+	  This removes the hierarchical code.
+	  QueryTargets state and Finished state are merged for iterator.
+	- forwarder mode no longer sets AA bit on first reply.
+	- rcode in walk_supers is not needed.
+
+25 June 2007: Wouter
+	- more mesh work.
+	- error encode routine for ease.
+
+22 June 2007: Wouter
+	- removed unused _node iterator value from rbtree_t. Takes up space.
+	- iterator can handle querytargets state without a delegation point
+	  set, so that a priming(stub) subquery error can be handled.
+	- iterator stores if it is priming or not.
+	- log_query_info() neater logging.
+	- changed iterator so that it does not alter module_qstate.qinfo
+	  but keeps a chase query info. Also query_flags are not altered,
+	  the iterator uses chase_flags.
+	- fixup crash in case no ports for the family exist.
+
+21 June 2007: Wouter
+	- Fixup secondary buffer in case of error callback.
+	- cleanup slumber list of runnable states.
+	- module_subreq_depth fails to work in slumber list.
+	- fixup query release for cached results to sub targets.
+	- neater error for tcp connection failure, shows addr in verbose.
+	- rbtree_init so that it can be used with preallocated memory.
+
+20 June 2007: Wouter
+	- new -C option to enable coredumps after forking away.
+	- doc update.
+	- fixup CNAME generation by scrubber, and memory allocation of it.
+	- fixup deletion of serviced queries when all callbacks delete too.
+	- set num target queries to 0 when you move them to slumber list.
+	- typo in check caused subquery errors to be ignored, fixed.
+	- make lint happy about rlim_t.
+	- freeup of modules after freeup of module-states.
+	- duplicate replies work, this uses secondary udp buffer in outnet.
+
+19 June 2007: Wouter
+	- nicer layout in stats.c, review 0.3 change.
+	- spelling improvement, review 0.3 change.
+	- uncapped timeout for server selection, so that very fast or slow
+	  servers will stand out from the rest.
+	- target-fetch-policy: "3 2 1 0 0" config setting.
+	- fixup queries answered without RD bit (for root prime results).
+	- refuse AXFR and IXFR requests.
+	- fixup RD flag in error reply from iterator. fixup RA flag from
+	  worker error reply.
+	- fixup encoding of very short edns buffer sizes, now sets TC bit.
+	- config options harden-short-bufsize and harden-large-queries.
+
+18 June 2007: Wouter
+	- same, move subqueries to slumber list when first has resolved.
+	- fixup last fix for duplicate callbacks.
+	- another offbyone in targetcounter. Also in Java prototype by the way.
+
+15 June 2007: Wouter
+	- if a query asks to be notified of the same serviced query result
+	  multiple times, this will succeed. Only one callback will happen;
+	  multiple outbound-list entries result (but the double cleanup of it
+	  will not matter).
+	- when iterator moves on due to CNAME or referral, it will remove
+	  the subqueries (for other targets). These are put on the slumber
+	  list.
+	- state module wait subq is OK with no new subqs, an old one may have
+	  stopped, with an error, and it is still waiting for other ones.
+	- if a query loops, halt entire query (easy way to clean up properly).
+
+14 June 2007: Wouter
+	- num query targets was > 0 , not >= 0 compared, so that fetch
+	  policy of 0 did nothing.
+
+13 June 2007: Wouter
+	- debug option: configure --enable-static-exe for compile where
+	  ldns and libevent are linked statically. Default is off.
+	- make install and make uninstall. Works with static-exe and without.
+	  installation of unbound binary and manual pages.
+	- alignement problem fix on solaris 64.
+	- fixup address in case of TCP error.
+
+12 June 2007: Wouter
+	- num target queries was set to 0 at a bad time. Default it to 0 and
+	  increase as target queries are done.
+	- synthesize CNAME and DNAME responses from the cache.
+	- Updated doxygen config for doxygen 1.5.
+	- aclocal newer version.
+	- doxygen 1.5 fixes for comments (for the strict check on docs).
+
+11 June 2007: Wouter
+	- replies on TCP queries have the address field set in replyinfo,
+	  for serviced queries, because the initiator does not know that
+	  a TCP fallback has occured.
+	- omit DNSSEC types from nonDO replies, except if qtype is ANY or
+	  if qtype directly queries for the type (and then only show that
+	  'unknown type' in the answer section).
+	- fixed message parsing where rrsigs on their own would be put
+	  in the signature list over the rrsig type.
+
+7 June 2007: Wouter
+	- fixup error in double linked list insertion for subqueries and
+	  for outbound list of serviced queries for iterator module.
+	- nicer printout of outgoing port selection. 
+	- fixup cname target readout.
+	- nicer debug output.
+	- fixup rrset counts when prepending CNAMEs to the answer.
+	- fixup rrset TTL for prepended CNAMEs.
+	- process better check for looping modules, and which submodule to
+	  run next.
+	- subreq insertion code fixup for slumber list.
+	- VERB_DETAIL, verbosity: 2 level gives short but readable output.
+	  VERB_ALGO, verbosity: 3 gives extensive output.
+	- fixup RA bit in cached replies.
+	- fixup CNAME responses from the cache no longer partial response.
+	- error in network send handled without leakage.
+	- enable ip6 from config, and try ip6 addresses if available,
+	  if ip6 is not connected, skips to next server.
+
+5 June 2007: Wouter
+	- iterator state finished.
+	- subrequests without parent store in cache and stop.
+	- worker slumber list for ongoing promiscuous queries.
+	- subrequest error handling.
+	- priming failure returns SERVFAIL.
+	- priming gives LAME result, returns SERVFAIL.
+	- debug routine to print dns_msg as handled by iterator.
+	- memleak in config file stubs fixup.
+	- more small bugs, in scrubber, query compare no ID for lookup,
+	  in dname validation for NS targets.
+	- sets entry.key for new special allocs.
+	- lognametypeclass can display unknown types and classes.
+
+4 June 2007: Wouter
+	- random selection of equally preferred nameserver targets.
+	- reply info copy routine. Reuses existing code.
+	- cache lameness in response handling.
+	- do not touch qstate after worker_process_query because it may have
+	  been deleted by that routine.
+	- Prime response state.
+	- Process target response state.
+	- some memcmp changed to dname_compare for case preservation.
+
+1 June 2007: Wouter
+	- normalize incoming messages. Like unbound-java, with CNAME chain
+	  checked, DNAME checked, CNAME's synthesized, glue checked.
+	- sanitize incoming messages.
+	- split msgreply encode functions into own file msgencode.c.
+	- msg_parse to queryinfo/replyinfo conversion more versatile.
+	- process_response, classify response, delegpt_from_message. 
+
+31 May 2007: Wouter
+	- querytargets state.
+	- dname_subdomain_c() routine.
+	- server selection, based on RTT. ip6 is filtered out if not available,
+	  and lameness is checked too.
+	- delegation point copy routine.
+
+30 May 2007: Wouter
+	- removed FLAG_CD from message and rrset caches. This was useful for
+	  an agnostic forwarder, but not for a sophisticated (trust value per
+	  rrset enabled) cache.
+	- iterator reponse typing.
+	- iterator cname handle.
+	- iterator prime start.
+	- subquery work.
+	- processInitRequest and processInitRequest2.
+	- cache synthesizes referral messages, with DS and NSEC.
+	- processInitRequest3.
+	- if a request creates multiple subrequests these are all activated.
+
+29 May 2007: Wouter
+	- routines to lock and unlock array of rrsets moved to cache/rrset.
+	- lookup message from msg cache (and copy to region).
+	- fixed cast error in dns msg lookup.
+	- message with duplicate rrset does not increase its TTLs twice.
+	- 'qnamesize' changed to 'qname_len' for similar naming scheme.
+
+25 May 2007: Wouter
+	- Acknowledge use of unbound-java code in iterator. Nicer readme.
+	- services/cache/dns.c DNS Cache. Hybrid cache uses msgcache and
+	  rrset cache from module environment.
+	- packed rrset key has type and class as easily accessable struct
+	  members. They are still kept in network format for fast msg encode.
+	- dns cache find_delegation routine.
+	- iterator main functions setup.
+	- dns cache lookup setup.
+
+24 May 2007: Wouter
+	- small changes to prepare for subqueries.
+	- iterator forwarder feature separated out.
+	- iterator hints stub code, config file stub code, so that first
+	  testing can proceed locally.
+	- replay tests now have config option to enable forwarding mode.
+
+23 May 2007: Wouter
+	- outside network does precise timers for roundtrip estimates for rtt
+	  and for setting timeout for UDP. Pending_udp takes milliseconds.
+	- cleaner iterator sockaddr conversion of forwarder address.
+	- iterator/iter_utils and iter_delegpt setup.
+	- root hints.
+
+22 May 2007: Wouter
+	- outbound query list for modules and support to callback with the
+	  outbound entry to the module.
+	- testbound support for new serviced queries.
+	- test for retry to TCP cannot use testbound any longer.
+	- testns test for EDNS fallback, test for TCP fallback already exists.
+	- fixes for no-locking compile.
+	- mini_event timer precision and fix for change in timeouts during
+	  timeout callback. Fix for fwd_three tests, performed nonexit query.
+
+21 May 2007: Wouter
+	- small comment on hash table locking.
+	- outside network serviced queries, contain edns and tcp fallback,
+	  and udp retries and rtt timing.
+
+16 May 2007: Wouter
+	- lruhash_touch() would cause locking order problems. Fixup in 
+	  lock-verify in case locking cycle is found.
+	- services/cache/rrset.c for rrset cache code.
+	- special rrset_cache LRU updating function that uses the rrset id.
+	- no dependencies calculation when make clean is called.
+	- config settings for infra cache.
+	- daemon code slightly cleaner, only creates caches once.
+
+15 May 2007: Wouter
+	- host cache code.
+	- unit test for host cache.
+
+14 May 2007: Wouter
+	- Port to OS/X and Dec Alpha. Printf format and alignment fixes.
+	- extensive lock debug report on join timeout.
+	- proper RTT calculation, in utility code.
+	- setup of services/cache/infra, host cache.
+
+11 May 2007: Wouter
+	- iterator/iterator.c module.
+	- fixup to pass reply_info in testcode and in netevent.
+
+10 May 2007: Wouter
+	- created release-0.3 svn tag.
+	- util/module.h
+	- fixed compression - no longer compresses root name.
+
 9 May 2007: Wouter
 	- outside network cleans up waiting tcp queries on exit.
 	- fallback to TCP.

doc/README

@@ -4,25 +4,63 @@ http://unbound.net
 
 This software is under BSD license, see LICENSE for details.
 
-* Needs the following libraries
+* Download the latest version of this software from 
+  	http://unbound.net 
+  or get a beta version from the svn repository at 
+  	http://unbound.net/svn/
+
+* Uses the following libraries; 
   * ldns	http://www.nlnetlabs.nl/ldns/			(BSD license)
   * libevent	http://www.monkey.org/~provos/libevent/		(BSD license)
 
-* Make and install: ./configure; make
+* Create build environment 
+  * run libtoolize -c if config.sub is missing, or run glibtoolize.
+  * autoreconf (autoheader && autoconf), if ./configure is missing.
+
+* Make and install: ./configure; make; make install
+  * Use GNU make; default on linux, often called 'gmake' on BSD and Solaris.
   * --with-ldns=/path/to/ldns
+  	If ldns is not detected on the system, a prepackaged tarball
+	of the ldns library is used to compile and statically link against.
+	If detected on the system, it will dynamically link against it.
   * --with-libevent=/path/to/libevent
   	Can be set to either the system install or the build directory.
-	--with-libevent=no gives an alternative to libevent (based on select).
+	--with-libevent=no gives a builtin alternative implementation.
+	libevent is useful when having many (thousands) of outgoing ports.
+	This improves randomization and spoof resistance. For the default 
+	of 16 ports the builtin alternative works well.
+  * --without-pthreads 
+	This disables pthreads, and uses Solaris thr library or no threading. 
+	Without this option the pthreads library is detected automatically.
+  * --enable-static-exe
+	This enables a debug option to statically link, against ldns and 
+	libevent libraries.
+  * --enable-lock-checks
+  	This enables a debug option to check lock and unlock calls. It needs
+	a recent pthreads library to work.
+  * --enable-alloc-checks
+	This enables a debug option to check malloc (calloc, realloc, free).
+	The server periodically checks if the amount of memory used fits with
+	the amount of memory it thinks it should be using, and reports 
+	memory usage in detail.
 
-* Create build environment 
-  * run libtoolize -c if config.sub is missing.
-  * autoreconf (autoheader && autoconf), if ./configure is missing.
+* 'make test' attempts to run a series of tests, depending on the support
+  programs that are installed.
 
 Known issues
 ------------
-o If libevent is older (1.3 and before), unbound will exit instead of reload
+o If libevent is older (before 1.3c), unbound will exit instead of reload
   on sighup. On a restart 'did not exit gracefully last time' warning is 
   printed. Perform ./configure --with-libevent=no or update libevent, rerun 
   configure and recompile unbound to make sighup work correctly.
 
-* mailto:wouter@nlnetlabs.nl
+Acknowledgements
+----------------
+o Thanks to David Blacka and Matt Larson (Verisign) for the unbound-java
+  prototype. Design and code from that prototype has been used to create
+  this program. Such as the iterator state machine and the cache design.
+o Other code origins are from the NSD (NLnet Labs) and LDNS (NLnet Labs)
+  projects. Such as buffer, region-allocator and red-black tree code.
+o See Credits file for contributors.
+
+* mailto:unbound-bugs@nlnetlabs.nl

doc/README.tests

@@ -1,6 +1,9 @@
 README unbound tests
 
-There is a test setup for unbound.
+There is a test setup for unbound. Use
+	make test
+To make and run the tests. The results are summarized at the end.
+You can also run ./unittest by hand, if the system lacks other debug tools.
 
 testdata/ contains the data for tests. 
 testcode/ contains scripts and c code for the tests.
@@ -8,4 +11,4 @@ testcode/ contains scripts and c code for the tests.
 do-tests.sh : runs all the tests in the testdata directory.
 testbed.sh : compiles on a set of (user specific) hosts and runs do-tests.
 
-Tests are run using tpkg.
+Tests are run using testcode/mini_tpkg.sh.

doc/TODO

@@ -5,16 +5,54 @@ o profile memory allocation, and if performance issues, use special memory
   allocator. For example, with caches per thread.
 o #define BIT_... different on bigendian and smallendian systems so that
   the htons on flags is not needed to send a message from the cache.
-o possible optimization with delayed malloc of msgreply after parse
-  (reuse parse structures and ptrs to packet in meantime).
-o possible optimization, so that precious id number resource is not depleted
-  by parsing of messages. Delay malloc, as above, or try to reverse release
-  special id numbers, and if you release the next_id number for the thread
-  it reuses that id number.
-o use rbtree to compress domain names in messages, sorted AXFRs of 16Kb
-  and so on. For current packets, even worstcase, list, is ok. Only if we
-  start to send out compressed AXFRs then it becomes a problem.
-  We can cap AXFR packets on 256 or 1024 domain names, for example.
 o speed up pkt domain name decompression loop detection using counter perhaps.
 o detect OS/400 pthreads implementation that allows upgrading to writelock
   on pthreads rwlocks and use it to examine-rd before storing-wr rrset cache.
+o understand synthesized DNAMEs, so those TTL=0 packets are cached properly.
+o understand NSEC/NSEC3, aggressive negative caching, so that updates to
+  NSEC/NSEC3 will result in proper negative responses.
+o fallback without EDNS if result is NOTIMPL, now only on FORMERR like in java.
+o scrubber has slow pkt_subdomain and pkt_strict_subdomain functions.
+o get serverselection algorithm out of local optimum.
+  make subtargets to get rtt info for a couple of targets, like fetch-policy.
+  or send out multiple queries to multiple servers.
+o donotqueryaddresses allow specification of subnets, use trie to store.
+  extend default list to contain illegal ip4 and ip6 addresses.
+o memory profile; and if needed put serviced queries under qstate->region
+  and special purpose region code, reuse blocks, shrink if too big.
+o check query, option to enforce presence of qdsection in noerror answers.
+o configuration option where port 53 is used for send and receive, no other
+  ports are used.
+o (option) to not send replies to clients after a timeout of (say 5 secs) has
+  passed, but keep task active for later retries by client.
+o private TTL feature
+o pretend-dnssec-unaware, and pretend-edns-unaware modes for debug/workshops.
+o delegpt use rbtree for ns-list, to avoid slowdown for very large NS sets.
+o be able to have different listen and query-to addresses to bind to,
+  so you can listen to localhost and query-to to the internet.
+o reprime and refresh oft used data before timeout.
+o retain prime results in a overlaid roothints file.
+o store primed key data in a overlaid keyhints file (sort of like drafttimers).
+o windows version, auto update feature, a query to check for the version.
+o autoreport of problems
+o logrotation, syslog
+o command the server with TSIG inband. get-config, clearcache, 
+	get stats, get memstats, get ..., reload, clear one zone from cache
+o watch for spoof nearmisses.
+o improve compression of DNS packets by first puttig uncompressible rrs, then
+  compress to their rdata.
+o if one server is not responsive do not spend 75 secs on that server, but
+  try other servers with lower rtt.
+o NSID rfc 5001 support.
+o timers rfc 5011 support.
+o Treat YXDOMAIN from a DNAME properly, in iterator (not throwaway), validator.
+o grab ports nonconsequtive and change the set after a while (change within
+  a given range). Could be bad for OS if wrong port. unsure if it helps secure.
+o make timeout backoffs randomized (a couple percent random) to spread traffic.
+o inspect date on executable, then warn user in log if its more than 1 year.
+o proactively prime root, stubs and trust anchors, feature.
+  early failure, faster on first query, but more traffic.
+o use privilege separation, to change privilege options during reload securely
+  not needed.
+o check if for PowerDNS(2.9.21) CNAME in Answer section & rcode=NXDOMAIN needs
+  to be fixed up to be rcode=NOERROR?

doc/example.conf

@@ -13,14 +13,16 @@ server:
 	# whitespace is not necessary, but looks cleaner.
 
 	# verbosity number, 0 is least verbose. 1 is default.
-	verbosity: 2
-	
+	verbosity: 1
+
 	# number of threads to create. 1 disables threading.
 	# num-threads: 1
 
 	# specify the interfaces to answer queries from by ip-address.
-	# If you give none the default (all) interface is used.
+	# The default is to listen to localhost (127.0.0.1 and ::1).
+	# specify 0.0.0.0 and ::0 to bind to all available interfaces.
 	# specify every interface on a new 'interface:' labelled line.
+	# The listen interfaces are not changed on reload, only on restart.
 	# interface: 192.0.2.153
 	# interface: 192.0.2.154
 	# interface: 2001:DB8::5
@@ -28,6 +30,13 @@ server:
 	# port to answer queries from
 	# port: 53
 
+	# specify the interfaces to send outgoing queries to authoritative
+	# server from by ip-address. If none, the default (all) interface
+	# is used. Specify every interface on a 'outgoing-interface:' line.
+	# outgoing-interface: 192.0.2.153
+	# outgoing-interface: 2001:DB8::5
+	# outgoing-interface: 2001:DB8::6
+
 	# unbound needs to send packets to authoritative nameservers.
 	# it uses a range of ports for that.
 	# the start number of the port range
@@ -42,6 +51,13 @@ server:
 	# number of outgoing simultaneous tcp buffers to hold per thread.
 	# outgoing-num-tcp: 10
 
+	# number of incoming simultaneous tcp buffers to hold per thread.
+	# incoming-num-tcp: 10
+	
+	# buffer size for handling DNS data. No messages larger than this
+	# size can be sent or received, by UDP or TCP. In bytes.
+	# msg-buffer-size: 65552
+
 	# the amount of memory to use for the message cache.
 	# in bytes. default is 4 Mb
 	# msg-cache-size: 4194304
@@ -62,6 +78,28 @@ server:
 	# the number of slabs must be a power of 2.
 	# more slabs reduce lock contention, but fragment memory usage.
 	# rrset-cache-slabs: 4
+	
+	# the time to live (TTL) value cap for RRsets and messages in the
+	# cache. Items are not cached for longer. In seconds.
+	# cache-max-ttl: 864000
+
+	# the time to live (TTL) value for cached roundtrip times and
+	# EDNS version information for hosts. In seconds.
+	# infra-host-ttl: 900
+
+	# the time to live (TTL) value for cached lame delegations. In sec.
+	# infra-lame-ttl: 900
+
+	# the number of slabs to use for the Infrastructure cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# infra-cache-slabs: 4
+
+	# the maximum number of hosts that are cached (roundtrip times, EDNS).
+	# infra-cache-numhosts: 10000
+
+	# the maximum size of the lame zones cached per host. in bytes.
+	# infra-cache-lame-size: 10240
 
 	# Enable IPv4, "yes" or "no".
 	# do-ip4: yes
@@ -75,28 +113,160 @@ server:
 	# Enable TCP, "yes" or "no".
 	# do-tcp: yes
 
-	# Set this to configure unbound to act as a forwarder. All queries are
-	# sent to the remote nameserver that will resolve them.
-	# Set to "" to disable forwarding, or give ip-address to enable.
-	# forward-to: ""
-
-	# The port number to send forwarded queries to.
-	# forward-to-port: 53
-
 	# if given, a chroot(2) is done to the given directory.
 	# i.e. you can chroot to the working directory, for example,
 	# for extra security, but make sure all files are in that directory.
-	# chroot: "/some/directory"
+	# If you give "" no chroot is performed.
+	# chroot: "/etc/unbound"
 
 	# if given, user privileges are dropped (after binding port),
-	# and the given username is assumed. Default is nothing "".
+	# and the given username is assumed. Default is user "unbound".
+	# If you give "" no priviliges are dropped.
 	# username: "unbound"
 
 	# the working directory.
 	# directory: "/etc/unbound"
 
-	# the log file, "" means log to stderr.
+	# the log file, "" means log to stderr. 
+	# Use of this option sets use-syslog to "no".
 	# logfile: ""
+	
+	# Log to syslog(3) if yes. The log facility LOG_DAEMON is used to 
+	# log to, with identity "unbound". If yes, it overrides the logfile.
+	# use-syslog: yes 
 
 	# the pid file.
-	# pidfile: "unbound.pid"
+	# pidfile: "/etc/unbound/unbound.pid"
+	
+	# file to read root hints from.
+	# get one from ftp://FTP.INTERNIC.NET/domain/named.cache
+	# root-hints: ""
+	
+	# enable to not answer id.server and hostname.bind queries.
+	# hide-identity: no
+	
+	# enable to not answer version.server and version.bind queries.
+	# hide-version: no
+	
+	# the identity to report. Leave "" or default to return hostname.
+	# identity: ""
+	
+	# the version to report. Leave "" or default to return package version.
+	# version: ""
+	
+	# the target fetch policy.
+	# series of integers describing the policy per dependency depth. 
+	# The number of values in the list determines the maximum dependency 
+	# depth the recursor will pursue before giving up. Each integer means:
+	# 	-1 : fetch all targets opportunistically,
+	# 	0: fetch on demand,
+	#	positive value: fetch that many targets opportunistically.
+	# Enclose the list of numbers between quotes ("").
+	# target-fetch-policy: "3 2 1 0 0"
+	
+	# Harden against very small EDNS buffer sizes. 
+	# harden-short-bufsize: no
+	
+	# Harden against unseemly large queries.
+	# harden-large-queries: no
+	
+	# Harden against out of zone rrsets, to avoid spoofing attempts. 
+	# harden-glue: yes
+	
+	# Harden against receiving dnssec-stripped data. If you turn it
+	# off, failing to validate dnskey data for a trustanchor will 
+	# trigger insecure mode for that zone (like without a trustanchor).
+	# Default on, which insists on dnssec data for trust-anchored zones.
+	# harden-dnssec-stripped: yes
+	
+	# Do not query the following addresses. No DNS queries are sent there.
+	# List one address per entry. List classless netblocks with /size,
+	# do-not-query-address: 127.0.0.1/8
+	# do-not-query-address: ::1
+	
+	# if yes, the above default do-not-query-address entries are present.
+	# if no, localhost can be queried (for testing and debugging).
+	# do-not-query-localhost: yes
+	
+	# module configuration of the server. A string with identifiers
+	# separated by spaces. "iterator" or "validator iterator"
+	# module-config: "validator iterator"
+	
+	# File with trusted keys for validation. Specify more than one file
+	# with several entries, one file per entry.
+	# Zone file format, with DS and DNSKEY entries.
+	# trust-anchor-file: ""
+	
+	# Trusted key for validation. DS or DNSKEY. specify the RR on a
+	# single line, surrounded by "". TTL is ignored. class is IN default.
+	# (These examples are from August 2007 and may not be valid anymore).
+	# trust-anchor: "nlnetlabs.nl. DNSKEY 257 3 5 AQPzzTWMz8qSWIQlfRnPckx2BiVmkVN6LPupO3mbz7FhLSnm26n6iG9N Lby97Ji453aWZY3M5/xJBSOS2vWtco2t8C0+xeO1bc/d6ZTy32DHchpW 6rDH1vp86Ll+ha0tmwyy9QP7y2bVw5zSbFCrefk8qCUBgfHm9bHzMG1U BYtEIQ=="
+	# trust-anchor: "jelte.nlnetlabs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"
+
+	# File with trusted keys for validation. Specify more than one file
+	# with several entries, one file per entry. Like trust-anchor-file
+	# but has a different file format. Format is BIND-9 style format, 
+	# the trusted-keys { name flag proto algo "key"; }; clauses are read.
+	# trusted-keys-file: ""
+
+	# Override the date for validation with a specific fixed date.
+	# Do not set this unless you are debugging signature inception
+	# and expiration. "" or "0" turns the feature off. 
+	# val-override-date: ""
+	
+	# The time to live for bogus data, rrsets and messages. This avoids
+	# some of the revalidation, until the time interval expires. in secs.
+	# val-bogus-ttl: 900
+	
+	# Should additional section of secure message also be kept clean of
+	# unsecure data. Useful to shield the users of this validator from
+	# potential bogus data in the additional section. All unsigned data 
+	# in the additional section is removed from secure messages.
+	# val-clean-additional: yes
+	
+	# Turn permissive mode on to permit bogus messages. Thus, messages
+	# for which security checks failed will be returned to clients,
+	# instead of SERVFAIL. It still performs the security checks, which
+	# result in interesting log files and possibly the AD bit in
+	# replies if the message is found secure. The default is off.
+	# val-permissive-mode: no
+	
+	# It is possible to configure NSEC3 maximum iteration counts per
+	# keysize. Keep this table very short, as linear search is done.
+	# A message with an NSEC3 with larger count is marked insecure.
+	# List in ascending order the keysize and count values.
+	# val-nsec3-keysize-iterations: "1024 150 2048 500 4096 2500"
+
+	# the amount of memory to use for the key cache.
+	# in bytes. default is 4 Mb
+	# key-cache-size: 4194304
+
+	# the number of slabs to use for the key cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# key-cache-slabs: 4
+
+
+# Stub zones.
+# Create entries like below, to make all queries for 'example.com' and 
+# 'example.org' go to the given list of nameservers. list zero or more 
+# nameservers by hostname or by ipaddress.
+# stub-zone:
+#	name: "example.com"
+#	stub-addr: 192.0.2.68
+# stub-zone:
+#	name: "example.org"
+#	stub-host: ns.example.com.
+
+# Forward zones
+# Create entries like below, to make all queries for 'example.com' and
+# 'example.org' go to the given list of servers. These servers have to handle
+# recursion to other nameservers. List zero or more nameservers by hostname
+# or by ipaddress. Use an entry with name "." to forward all queries.
+# forward-zone:
+# 	name: "example.com"
+# 	forward-addr: 192.0.2.68
+# 	forward-addr: 192.0.2.73@5355  # forward to port 5355.
+# forward-zone:
+# 	name: "example.org"
+# 	forward-host: fwd.example.com

doc/plan

@@ -29,14 +29,26 @@ Roughly the boxes are as follows:
     This stage takes longer, due to complexity in the iterator module.
     Twice as long; one box for module layout, one box for iterator module.
 0.5 Validator - validator module.
-0.6 Bigger and better - good syslog, compressed packets, port ranges,
-    config file format with config checker. 
-0.7 Local zones feature - localzones stubzones fwdzones, no leak rfc1918.
-0.8 Library use - resolver validator lib (and test apps)
-0.9 Corner cases - be able to resolve in the wild. Run fuzzers.
+0.6 Bigger and better - Operational useful features (config, log, memory)
+0.7 Put to a limited audience.
+    gamma/alpha core functionality test release, to a small audience.
+    partial functionality. For more extensive use and testing.
+0.8 Local zones feature - localzones stubzones fwdzones, no leak rfc1918.
+    views support; for selective recursive service.
+0.9 Library use - resolver validator lib (and test apps)
+0.10 Corner cases - be able to resolve in the wild. Run fuzzers.
     Run as many tests as we can think of.
-0.10 Beta release. Run shadow for a resolver in production for several
+    Go through logs and check for long, unresolved cases
+    Use profiler.
+0.11 Beta release. Run shadow for a resolver in production for several
     weeks.
+0.12 Features features
+    aggressive negative caching for NSEC, NSEC3.
+    multiple queries per question, server exploration, server selection.
+    option to use real entropy for randomness (mix it in once in a while).
+    check query, option to enforce qdsection checking (forgery-resilience).
+    NSID support.
+    Be able to prime roots using several queries (only NS on first).
 
 For boxes 0.5-1.0 the planning is to be revised, at the 0.5 stage external
 coders are welcome.  Since the project is bigger, there is room for them.
@@ -115,7 +127,7 @@ Styleguide:
 * Speed test of caching forwarder.
 
 *** Basic Resolver
-* Create module interfacce and module caller algorithm.
+* Create module interface and module caller algorithm.
 * Daemon config to use modules. Test the module caller.
 * Create basic iterator and scrubber modules.
 	* Test every state of the iterator by passing test data into
@@ -133,20 +145,63 @@ Styleguide:
 	* Test resolver.
 	* Speed test.
 
+*** Put to a limited audience
+* The alpha/gamma core functionality, svn access to limited audience.
+* Support features and requests as they arise.
+* Provide real-world experiences.
+
 *** Bigger and Better
-* DNS packet compression of output.
-	* test packet compression (that it compresses.)
-	* config option to compress yes/no
-	* speed test with and without compression.
 * Config file syntax checker program. Tests on checker.
-* Use a huge number of random outgoing ports.
 * Logging first class feature with config options.
+	X with logfile turnover to avoid Gbs of logs.
+	* use syslog optional.
+* donotqueryaddresses with trie for blocking entire netblocks.
+* Memory overhaul, special allocators for hashtable caches, and mesh qstates.
+	* keep a preallocated list of region-chunks per worker thread.
+	* allocate region struct and cleanup list in region itself; use
+	  linked list cleanup list. unit test on this. do not call region
+	  to avoid name-collision with nsd regions, 'regional'.
+* read root hints from file.
+* failover to next server in 1 second, instead of 100 seconds on one server.
+X failure to return answer, w. reason (donotq, noanswer servers, cannot
+  find servers, validationfail w.classification, error), 
+  with threadno, starttime and endtime and qname/type/class, prime/qflags, 
+  from-clients, from-internal, has-subrequests, a nice error report,
+  so that an excerpt from those times can be made from the logs.
+  logfileparsing tool that makes these excerpts and emails them.
+  Not done; user can change verbosity and kill -HUP.
+* clear cache as a callback from the new-rrset-id routine.
+X make overload mode work; phase 0 all ok, phase 1 some threads close ports,
+  to let other threads pick up work. phase 2, all threads closed, so all open
+  the ports again and drop all non-cache-reply queries.
+  Keep mutexed num-overloaded-threads counter. thread incs it when it hits
+  max number of user queries serviced in mesh. threads decs it when it
+  falls below 90% of the max. if incs, and not all threads closed, phase 1,
+  else, phase 2 start is broadcast over command pipes. if decs, open ports
+  if phase 1, start servicing, phase is 0 again. Make robust against delays.
+  readme: max about 1 second worth of incoming queries, 10k perhaps,
+	or 1/number of seconds it takes start up of 10k.
+  Not done. Implement drop when full. 
+* the source includes a copy of the ldns lib for ease of building by 
+  new users. Detect system installed ldns, if installed ldns is OK; use 
+  dynamic linking against it, otherwise static linking against packaged ldns.
+* no greedy TTL algo (and test). 
+* maximum TTL, cap incoming values, and config option.
 
 *** Local zones feature.
 * Build in local zone features. First the total stop for1912.
 * Then 'local content' for minimal serving of localhost.localdomain,
   and so on. 
-* Remember jakob's diagram.
+* Remember jakob's diagram. views support, selective recursive service:
+	* acl for allowed recursion (RD=1), then drop or refused query.
+	  like 10.0.0.0/8 allow, 0.0.0.0/0 refuse, ... in-order.
+	  perhaps also, same list to disallow RD=0 access, like;
+	  allow_recursion, drop_recursion, refuse_recursion, drop_all
+	* static answers for queries, fixed RRs from cfg, option
+	  query for that RR returns answer with that RR.
+	* blacklist (return fixed nxdomain for domain and below), option
+	  can be used to block AS112 traffic, option to unblock a zone.
+	* after checking acl, do iter: static, blacklist, forwards, recurse.
 * Forward-local-zone to NSD.
 	- include in package, autoforkexec on localhost to do so.
 * forward local zone to remote server.
@@ -169,6 +224,31 @@ Styleguide:
 * Try to setup corner cases of (mis)configured DNS service/websites.
 * Resolve msoft, google, yahoo, etc weird websites.
 * Try to resolve many many different queries, perhaps compared with bind.
+* create module testers, specific for the modules
+	* read a file with cache contents and settings, provide fake
+	  environment for module-handle-state-X functions, then check
+	  resulting module state structure to correct answer.
+* speed test cache responses.
+* using two servers, compare answer differences between bind and unbound.
+  this gives false differences due to changes in the rest of internet.
+
+*** Beta release. 
+* Run shadow for a resolver in production for several weeks.
+* Check logs for errors, long queries.
+* Run in valgrind, speed profiling (as production shadow).
+
+*** Features features
+* aggressive negative caching for NSEC, NSEC3.
+* multiple queries per question, server exploration, server selection.
+* option to use real entropy for randomness (mix it in once in a while).
+* check query, enforce qdsection, checking for forgery-resilience.
+* NSID support.
+* support TSIG on queries, for validating resolver deployment.
+* Nicer statistics
+* private TTL, dTLS features.
+* retry-mode, where a bogus result triggers a retry-mode query, where a list
+  of responses over a time interval is collected, and each is validated.
+* draft-timers, DLV features.
 
 treeshrew/
 	validator/ *.c *.h
@@ -225,10 +305,10 @@ treeshrew/
 		unbound.c for validating caching recursive dns server.
 		scheduler.c for the modules.
 
-	libunbound-all/
+	libunbound/
 		app linkable. Can be configged to do whatever,
 		validator, iterator, validating iterator, forwarding stub.
-	libunbound-fwd/
+	libforwardbound/
 		app linkable forwarding stub. Small lib.
 
 	ask_cachor/ *.c *.h

doc/requirements.txt

@@ -79,3 +79,99 @@ o An authoritative name server.
 o Too many Features.
 
 
+5. Choices
+----------
+o rfc2181 decourages duplicates RRs in RRsets. unbound does not create
+  duplicates, but when presented with duplicates on the wire from the
+  authoritative servers, does not perform duplicate removal.
+  It does do some rrsig duplicate removal, in the msgparser, for dnssec qtype
+  rrsig and any, because of special rrsig processing in the msgparser.
+o The harden-glue feature, when yes all out of zone glue is deleted, when
+  no out of zone glue is used for further resolving, is more complicated 
+  than that, see below.
+  Main points:
+  	* rfc2182 trust handling is used. 
+	* data is let through only in very specific cases
+	* spoofability remains possible.
+  Not all glue is let through (despite the name of the option). Only glue 
+  which is present in a delegation, of type A and AAAA, where the name is
+  present in the NS record in the authority section is let through.
+  The glue that is let through is stored in the cache (marked as 'from the
+  additional section'). And will then be used for sending queries to. It
+  will not be present in the reply to the client (if RD is off).
+  A direct query for that name will attempt to get a msg into the message
+  cache. Since A and AAAA queries are not synthesized by the unbound cache,
+  this query will be (eventually) sent to the authoritative server and its
+  answer will be put in the cache, marked as 'from the answer section' and
+  thus remove the 'from the additional section' data, and this record is 
+  returned to the client.
+  The message has a TTL smaller or equal to the TTL of the answer RR.
+  If the cache memory is low; the answer RR may be dropped, and a glue
+  RR may be inserted, within the message TTL time, and thus return the
+  spoofed glue to a client. When the message expires, it is refetched and
+  the cached RR is updated with the correct content.
+  The server can be spoofed by getting it to visit a especially prepared 
+  domain. This domain then inserts an address for another authoritative 
+  server into the cache, when visiting that other domain, this address may
+  then be used to send queries to. And fake answers may be returned.
+  If the other domain is signed by DNSSEC, the fakes will be detected.
+
+  In summary, the harden glue feature presents a security risk if
+  disabled. Disabling the feature leads to possible better performance
+  as more glue is present for the recursive service to use. The feature
+  is implemented so as to minimise the security risk, while trying to 
+  keep this performance gain.
+o The method by which dnssec-lameness is detected is not secure. DNSSEC lame
+  is when a server has the zone in question, but lacks dnssec data, such as
+  signatures. The method to detect dnssec lameness looks at nonvalidated 
+  data from the parent of a zone. This can be used, by spoofing the parent,
+  to create a false sense of dnssec-lameness in the child, or a false sense
+  or dnssec-non-lameness in the child. The first results in the server marked
+  lame, and not used for 900 seconds, and the second will result in a 
+  validator failure (SERVFAIL again), when the query is validated later on.
+
+  Concluding, a spoof of the parent delegation can be used for many cases
+  of denial of service. I.e. a completely different NS set could be returned,
+  or the information withheld. All of these alterations can be caught by
+  the validator if the parent is signed, and result in 900 seconds bogus. 
+  The dnssec-lameness detection is used to detect operator failures, 
+  before the validator will properly verify the messages.
+
+  Also for zones for which no chain of trust exists, but a DS is given by the
+  parent, dnssec-lameness detection enables. This delivers dnnsec to our 
+  clients when possible (for client validators).
+
+  The following issue needs to be resolved:
+	a server that serves both a parent and child zone, where
+	parent is signed, but child is not. The server must not be marked 
+	lame for the parent zone, because the child answer is not signed. 
+  Instead of a false positive, we want false negatives; failure to 
+  detect dnssec-lameness is less of a problem than marking honest 
+  servers lame. dnssec-lameness is a config error and deserves the trouble.
+  So, only messages that identify the zone are used to mark the zone
+  lame. The zone is identified by SOA or NS RRsets in the answer/auth.
+  That includes almost all negative responses and also A, AAAA qtypes.
+  That would be most responses from servers.
+  For referrals, delegations that add a single label can be checked to be
+  from their zone, this covers most delegation-centric zones.
+
+  So possibly, for complicated setups, with multiple (parent-child) zones 
+  on a server, dnssec-lameness detection does not work - no dnssec-lameness 
+  is detected. Instead the zone that is dnssec-lame becomes bogus.
+
+o authority features.
+  This is a recursive server, and authority features are out of scope.
+  However, some authority features are expected in a recursor. Things like
+  localhost, reverse lookup for 127.0.0.1, or blocking AS112 traffic.
+  Also redirection of domain names with fixed data is needed by service
+  providers. Limited support is added specifically to address this.
+
+  Adding full authority support, requires much more code, and more complex
+  maintenance.
+
+  The limited support allows adding some static data (for localhost and so),
+  and to respond with a fixed rcode (NXDOMAIN) for domains (such as AS112).
+
+  You can put authority data on a separate server, and set the server in 
+  unbound.conf as stub for those zones, this allows clients to access data 
+  from the server without making unbound authoritative for the zones.

doc/unbound-checkconf.8

@@ -0,0 +1,45 @@
+.\"
+.\" unbound-checkconf.8 -- unbound configuration checker manual
+.\"
+.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
+.\"
+.\" See LICENSE for the license.
+.\"
+.\"
+.Dd @date@
+.Dt unbound-checkconf 8
+.Sh NAME
+unbound-checkconf
+.Nd Check unbound configuration file for errors.
+.Sh SYNOPSIS
+.Nm unbound-checkconf
+.Op Fl h
+.Op cfgfile
+.Sh DESCRIPTION
+.Ic Unbound-checkconf
+checks the configuration file for the
+.Xr unbound 8
+DNS resolver for syntax and other errors. 
+The config file syntax is
+described in 
+.Xr unbound.conf 5 .
+.Pp
+The available options are:
+.Bl -tag -width indent
+.It Fl h
+Show the version and commandline option help.
+.It cfgfile
+The config file to read with settings for unbound. It is checked.
+If omitted, the config file at the default location is checked.
+.Sh EXIT CODE
+The unbound-checkconf program exits with status code 1 on error, 
+0 for a correct config file.
+.El
+.Sh FILES
+.Bl -tag -width indent
+.It Pa /etc/unbound/unbound.conf
+unbound configuration file.
+.El
+.Sh SEE ALSO
+.Xr unbound.conf 5 , 
+.Xr unbound 8 .

doc/unbound.8

@@ -1,38 +1,11 @@
-.ig
-unbound.8 -- unbound manual
-
-Copyright (c) 2007, NLnet Labs. All rights reserved.
-
-This software is open source.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-Redistributions of source code must retain the above copyright notice,
-this list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-Neither the name of the NLNET LABS nor the names of its contributors may
-be used to endorse or promote products derived from this software without
-specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-..
+.\"
+.\" unbound.8 -- unbound manual
+.\"
+.\" Copyright (c) 2007, NLnet Labs. All rights reserved.
+.\"
+.\" See LICENSE for the license.
+.\"
+.\"
 .Dd @date@
 .Dt unbound 8
 .Sh NAME
@@ -44,7 +17,6 @@ unbound
 .Op Fl d
 .Op Fl v
 .Op Fl c Ar cfgfile
-
 .Sh DESCRIPTION
 .Ic Unbound 
 is an implementation of a DNS resolver, that does caching and 
@@ -52,27 +24,24 @@ DNSSEC validation.
 .Pp
 The available options are:
 .Bl -tag -width indent
-
 .It Fl h
 Show the version and commandline option help.
-
 .It Fl c Ar cfgfile
-Set the config file to read with settings for unbound. The syntax is
+Set the config file with settings for unbound to read instead of the 
+file at default location /etc/unbound/unbound.conf. The syntax is
 described in 
 .Xr unbound.conf 5 .
-
 .It Fl d
 Debug flag, do not fork into the background, but stay attached to the
-console.
-
+console. This flag will also delay writing to the logfile until the
+thread-spawn time. So that most config and setup errors appear on stderr.
 .It Fl v
 Increase verbosity. If given multiple times, more information is logged.
 This is in addition to the verbosity (if any) from the config file.
-
 .El
 .Sh SEE ALSO
-.Xr unbound.conf 5 .
-
+.Xr unbound.conf 5 , 
+.Xr unbound-checkconf 8 .
 .Sh AUTHORS
 .Ic Unbound
 developers are mentioned in the CREDITS file in the distribution.

doc/unbound.conf.5

@@ -20,24 +20,46 @@ is used to configure
 .Xr unbound 8 .
 The file format has attributes and values. Some attributes have attributes inside them.
 The notation is: attribute: value.
-
+.Pp
 Comments start with # and last to the end of line. Empty lines are
 ignored as is whitespace at the beginning of a line.
-
-.El
+.Pp
+The utility 
+.Xr unbound-checkconf 8
+can be used to check unbound.conf prior to usage.
+.Sh EXAMPLE
+An example config file is shown below. Copy this to /etc/unbound/unbound.conf
+and start the server with:
+.nf
+	$ unbound -c /etc/unbound/unbound.conf 
+.fi
+Most settings are the defaults. Stop the server with:
+.nf
+	$ kill `cat /etc/unbound/unbound.pid`
+.fi
+Below is a minimal config file. The source distribution contains an extensive
+example.conf file with all the options.
+.nf
+# unbound.conf(5) config file for unbound(8).
+server:
+	directory: "/etc/unbound"
+	username: unbound 	# make sure it can write to pidfile.
+	chroot: "/etc/unbound"
+	# logfile: "/etc/unbound/unbound.log"  #uncomment to use logfile.
+	pidfile: "/etc/unbound/unbound.pid"
+	# verbosity: 1		# uncomment and increase to get more logging.
+.fi
 .Sh FILE FORMAT
 There must be whitespace between keywords. Attribute keywords end with a colon ':'. An attribute
 is followed by its containing attributes, or a value.
-
 .Pp
 Files can be included using the
 .Ic include:
 directive. It can appear anywhere, and takes a single filename as an argument.
 Processing continues as if the text from the included file was copied into
 the config file at that point.
-
 .Ss Server Options
-There may only be one 
+These options are part of the
 .Ic server:
 clause.
 .Bl -tag -width indent
@@ -53,8 +75,21 @@ The number of threads to create to serve clients. Use 1 for no threading.
 .It \fBport:\fR <port number>
 The port number, default 53, on which the server responds to queries.
 .It \fBinterface:\fR <ip address>
-Interface to use to connect to the network. Can be given multiple times to
-work on several interfaces. If none are given the default (all) is used.
+Interface to use to connect to the network. This interface is listened to
+for queries from clients, and answers to clients are given from it.
+Can be given multiple times to work on several interfaces. If none are 
+given the default is to listen to localhost.
+The interfaces are not changed on a reload (kill -HUP) but only on restart.
+.It \fBoutgoing-interface:\fR <ip address>
+Interface to use to connect to the network. This interface is used to send
+queries to authoritative servers and receive their replies. Can be given 
+multiple times to work on several interfaces. If none are given the 
+default (all) is used. You can specify the same interfaces in 
+.Ic interface:
+and
+.Ic outgoing-interface:
+lines, the interfaces are then used for both purposes. Outgoing queries are 
+sent via a random outgoing interface to counter spoofing.
 .It \fBoutgoing-port:\fR <port number>
 The starting port number where the outgoing query port range is allocated.
 Default is 1053.
@@ -66,6 +101,15 @@ extra resources from the operating system.
 .It \fBoutgoing-num-tcp:\fR <number>
 Number of outgoing TCP buffers to allocate per thread. Default is 10. If set
 to 0, or if do_tcp is "no", no TCP queries to authoritative servers are done.
+.It \fBincoming-num-tcp:\fR <number>
+Number of incoming TCP buffers to allocate per thread. Default is 10. If set
+to 0, or if do_tcp is "no", no TCP queries from clients are accepted.
+.It \fBmsg-buffer-size:\fR <number>
+Number of bytes size of the message buffers. Default is 65552 bytes, enough
+for 64 Kb packets, the maximum DNS message size. No message larger than this
+can be sent or received. Can be reduced to use less memory, but some requests
+for DNS data, such as for huge resource records, will result in a SERVFAIL 
+reply to the client.
 .It \fBmsg-cache-size:\fR <number>
 Number of bytes size of the message cache. Default is 4 megabytes.
 .It \fBmsg-cache-slabs:\fR <number>
@@ -74,11 +118,35 @@ Must be set to a power of 2. Setting (close) to the number of cpus is a
 reasonable guess.
 .It \fBnum-queries-per-thread:\fR <number>
 The number of queries that every thread will service simultaneously.
+If more queries arrive that need servicing, they are dropped. This forces
+the client to resend after a timeout; allowing the server time to work on
+the existing queries. Default 1024.
 .It \fBrrset-cache-size:\fR <number>
 Number of bytes size of the RRset cache. Default is 4 megabytes.
 .It \fBrrset-cache-slabs:\fR <number>
 Number of slabs in the RRset cache. Slabs reduce lock contention by threads.
 Must be set to a power of 2. 
+.It \fBcache-max-ttl:\fR <seconds>
+Time to live maximum for RRsets and messages in the cache. Default is 
+864000 seconds (10 days). If the maximum kicks in, responses to clients 
+still get decrementing TTLs based on the original (larger) values. 
+When the internal TTL expires, the cache item has expired.
+Can be set lower to force the resolver to query for data often, and not
+trust (very large) TTL values.
+.It \fBinfra-host-ttl:\fR <seconds>
+Time to live for entries in the host cache. The host cache contains 
+roundtrip timing and EDNS support information. Default is 900.
+.It \fBinfra-lame-ttl:\fR <seconds>
+The time to live when a delegation is discovered to be lame. Default is 900.
+.It \fBinfra-cache-slabs:\fR <number>
+Number of slabs in the infrastructure cache. Slabs reduce lock contention 
+by threads. Must be set to a power of 2. 
+.It \fBinfra-cache-numhosts:\fR <number>
+Number of hosts for which information is cached. Default is 10000.
+.It \fBinfra-cache-lame-size:\fR <number>
+Number of bytes that the lameness cache per host is allowed to use. Default
+is 10 kb, which gives maximum storage for a couple score zones, depending on 
+the lame zone name lengths.
 .It \fBdo-ip4:\fR <yes or no>
 Enable or disable whether ip4 queries are answered. Default is yes.
 .It \fBdo-ip6:\fR <yes or no>
@@ -87,38 +155,238 @@ Enable or disable whether ip6 queries are answered. Default is yes.
 Enable or disable whether UDP queries are answered. Default is yes.
 .It \fBdo-tcp:\fR <yes or no>
 Enable or disable whether TCP queries are answered. Default is yes.
-.It \fBforward-to:\fR <ip address>
-If set (not "") then forwarder mode is enabled. Default is "" (disabled).
-The ip address is used to forward all DNS queries to.
-.It \fBforward-to-port:\fR <port number>
-The port on which the remote server is running that answers forwarded queries.
-Default is 53.
 .It \fBchroot:\fR <directory>
-If given a chroot is done to the given directory. The default is none ("").
+If given a chroot is done to the given directory. The default is 
+"/etc/unbound". If you give "" no chroot is performed.
 .It \fBusername:\fR <name>
 If given, after binding the port the user privileges are dropped. Default is
-not to change user, username: "". If this user is not capable of binding the
+"unbound". If you give username: "" no user change is performed. 
+.Pp
+If this user is not capable of binding the
 port, reloads (by signal HUP) will still retain the opened ports.
 If you change the port number in the config file, and that new port number 
-requires privileges, then a restart is needed.
+requires privileges, then a reload will fail; a restart is needed.
 .It \fBdirectory:\fR <directory>
 Sets the working directory for the program.
 .It \fBlogfile:\fR <filename>
 If "" is given, logging goes to stderr, or nowhere once daemonized.
 The logfile is appended to, in the following format: 
 [seconds since 1970] unbound[pid:tid]: type: message. 
+If this option is given, the use-syslog is option is set to "no".
+The logfile is reopened (for append) when the config file is reread, on 
+SIGHUP.
+.It \fBuse-syslog:\fR <yes or no>
+Sets unbound to send log messages to the syslogd, using 
+.Xr syslog 3 .
+The log facility LOG_DAEMON is used, with identity "unbound".
+The logfile setting is overridden when use-syslog is turned on.
+The default is to log to syslog.
 .It \fBpidfile:\fR <filename>
-The process id is written to the file. Default is "unbound.pid". So,
-kill -HUP `cat /etc/unbound/unbound.pid` will trigger a reload,
-kill -QUIT `cat /etc/unbound/unbound.pid` will gracefully terminate.
-
+The process id is written to the file. Default is "/etc/unbound/unbound.pid". 
+So,
+.nf
+kill -HUP `cat /etc/unbound/unbound.pid` 
+.fi
+triggers a reload,
+.nf
+kill -QUIT `cat /etc/unbound/unbound.pid` 
+.fi
+gracefully terminates.
+.It \fBroot-hints:\fR <filename>
+Read the root hints from this file. Default is nothing, using builtin hints
+for the IN class. The file has the format of zone files, with root 
+nameserver names and addresses only. The default may become outdated,
+when servers change, therefore it is good practice to use a root-hints file.
+.It \fBhide-identity:\fR <yes or no>
+If enabled id.server and hostname.bind queries are refused.
+.It \fBidentity:\fR <string>
+Set the identity to report. If set to "", the default, then the hostname
+of the server is returned.
+.It \fBhide-version:\fR <yes or no>
+If enabled version.server and version.bind queries are refused.
+.It \fBversion:\fR <string>
+Set the version to report. If set to "", the default, then the package
+version is returned.
+.It \fBtarget-fetch-policy:\fR <"list of numbers">
+Set the target fetch policy used by unbound to determine if it should fetch
+nameserver target addresses opportunistically. The policy is described per
+dependency depth. 
+.Pp
+The number of values determines the maximum dependency depth
+that unbound will pursue in answering a query.  
+A value of -1 means to fetch all targets opportunistically for that dependency
+depth. A value of 0 means to fetch on demand only. A positive value fetches
+that many targets opportunistically. 
+.Pp
+Enclose the list between quotes ("") and put spaces between numbers.
+The default is "3 2 1 0 0". Setting all zeroes, "0 0 0 0 0" gives behaviour
+closer to that of BIND 9, while setting "-1 -1 -1 -1 -1" gives behaviour 
+rumoured to be closer to that of BIND 8.
+.It \fBharden-short-bufsize:\fR <yes or no>
+Very small EDNS buffer sizes from queries are ignored. Default is off, since
+it is legal protocol wise to send these, and unbound tries to give very 
+small answers to these queries, where possible.
+.It \fBharden-large-queries:\fR <yes or no>
+Very large queries are ignored. Default is off, since it is legal protocol
+wise to send these, and could be necessary for operation if TSIG or EDNS
+payload is very large.
+.It \fBharden-glue:\fR <yes or no>
+Will trust glue only if it is within the servers authority. Default is on.
+.It \fBharden-dnssec-stripped:\fR <yes or no>
+Require DNSSEC data for trust-anchored zones, if such data is absent,
+the zone becomes bogus. If turned off, and no DNSSEC data is received
+(or the DNSKEY data fails to validate), then the zone is made insecure, 
+this behaves like there is no trust anchor. You could turn this off if 
+you are sometimes behind an intrusive firewall (of some sort) that 
+removes DNSSEC data from packets, or a zone changes from signed to 
+unsigned to badly signed often. If turned off you run the risk of a 
+downgrade attack that disables security for a zone. Default is on.
+.It \fBdo-not-query-address:\fR <IP address>
+Do not query the given IP address. Can be IP4 or IP6. Append /num to 
+indicate a classless delegation netblock, for example like
+10.2.3.4/24 or 2001::11/64.
+.It \fBdo-not-query-localhost:\fR <yes or no>
+If yes, localhost is added to the do-not-query-address entries, both
+IP6 ::1 and IP4 127.0.0.1/8. If no, then localhost can be used to send
+queries to. Default is yes.
+.It \fBmodule-config:\fR <"module names">
+Module configuration, a list of module names separated by spaces, surround
+the string with quotes (""). The modules can be validator, iterator.
+Setting this to "iterator" will result in a non-validating server.
+Setting this to "validator iterator" will turn on DNSSEC validation.
+You must also set trust-anchors for validation to be useful.
+.It \fBtrust-anchor-file:\fR <filename>
+File with trusted keys for validation. Both DS and DNSKEY entries can appear
+in the file. The format of the file is the standard DNS Zone file format.
+Default is "", or no trust anchor file.
+.It \fBtrust-anchor:\fR <"Resource Record">
+A DS or DNSKEY RR for a key to use for validation. Multiple entries can be
+given to specify multiple trusted keys, in addition to the trust-anchor-files.
+The resource record is entered in the same format as 'dig' or 'drill' prints
+them, the same format as in the zone file. Has to be on a single line, with
+"" around it. A TTL can be specified for ease of cut and paste, but is ignored. 
+A class can be specified, but class IN is default.
+.It \fBtrusted-keys-file:\fR <filename>
+File with trusted keys for validation. Specify more than one file
+with several entries, one file per entry. Like \fBtrust-anchor-file\fR
+but has a different file format. Format is BIND-9 style format, 
+the trusted-keys { name flag proto algo "key"; }; clauses are read.
+.It \fBval-override-date:\fR <rrsig-style date spec>
+Default is "" or "0", which disables this debugging feature. If enabled by
+giving a RRSIG style date, that date is used for verifying RRSIG inception
+and expiration dates, instead of the current date. Do not set this unless 
+you are debugging signature inception and expiration.
+.It \fBval-bogus-ttl:\fR <number>
+The time to live for bogus data. This is data that has failed validation;
+due to invalid signatures or other checks. The TTL from that data cannot be
+trusted, and this value is used instead. The value is in seconds, default 900.
+The time interval prevents repeated revalidation of bogus data.
+.It \fBval-clean-additional:\fR <yes or no>
+Instruct the validator to remove data from the additional section of secure
+messages that are not signed properly. Messages that are insecure, bogus,
+indeterminate or unchecked are not affected. Default is yes. Use this setting
+to protect the users that rely on this validator for authentication from 
+protentially bad data in the additional section.
+.It \fBval-permissive-mode:\fR <yes or no>
+Instruct the validator to mark bogus messages as indeterminate. The security
+checks are performed, but if the result is bogus (failed security), the
+reply is not withheld from the client with SERVFAIL as usual. The client 
+receives the bogus data. For messages that are found to be secure the AD bit 
+is set in replies. Also logging is performed as for full validation.
+The default value is "no". 
+.It \fBval-nsec3-keysize-iterations:\fR <"list of values">
+List of keysize and iteration count values, separated by spaces, surrounded
+by quotes. Default is "1024 150 2048 500 4096 2500". This determines the
+maximum allowed NSEC3 iteration count before a message is simply marked
+insecure instead of performing the many hashing iterations. The list must
+be in ascending order and have at least one entry. If you set it to 
+"1024 65535" there is no restriction to NSEC3 iteration values.
+This table must be kept short; a very long list could cause slower operation.
+.It \fBkey-cache-size:\fR <number>
+Number of bytes size of the key cache. Default is 4 megabytes.
+.It \fBkey-cache-slabs:\fR <number>
+Number of slabs in the key cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. Setting (close) to the number of cpus is a 
+reasonable guess.
+.El
+.Ss Stub Zone Options
+There may be multiple
+.Ic stub-zone:
+clauses. Each with a name: and zero or more hostnames or IP addresses.
+For the stub zone this list of nameservers is used. Class IN is assumed.
+.Bl -tag -width indent
+.It \fBname:\fR <domain name>
+Name of the stub zone.
+.It \fBstub-host:\fR <domain name>
+Name of stub zone nameserver. Is itself resolved before it is used.
+.It \fBstub-addr:\fR <IP address>
+IP address of stub zone nameserver. Can be IP 4 or IP 6.
+To use a nondefault port for DNS communication append '@' with the port number.
+.El
+.Ss Forward Zone Options
+There may be multiple
+.Ic forward-zone:
+clauses. Each with a name: and zero or more hostnames or IP addresses.
+For the forward zone this list of nameservers is used to forward the queries
+to. The servers have to handle further recursion for the query. Class IN is 
+assumed. A forward-zone entry with name "." and a forward-addr target will 
+forward all queries to that other server (unless it can answer from the cache).
+.Bl -tag -width indent
+.It \fBname:\fR <domain name>
+Name of the forward zone.
+.It \fBforward-host:\fR <domain name>
+Name of server to forward to. Is itself resolved before it is used.
+.It \fBforward-addr:\fR <IP address>
+IP address of server to forward to. Can be IP 4 or IP 6.
+To use a nondefault port for DNS communication append '@' with the port number.
+.El
+.Sh MEMORY CONTROL EXAMPLE
+In the example config settings below memory usage is reduced. Some service
+levels are lower, notable very large data and a high TCP load are no longer
+supported. Very large data and high TCP loads are exceptional for the DNS.
+DNSSEC validation is enabled, just add trust anchors.
+If you do not have to worry about programs using more than 1 meg of memory,
+the below example is not for you. Use the defaults to receive full service.
+.Pp
+.nf
+# example settings that reduce memory usage
+server:
+	num-threads: 1
+	outgoing-num-tcp: 1	# this limits TCP service, uses less buffers.
+	incoming-num-tcp: 1
+	outgoing-range: 1	# uses less memory, but less port randomness.
+	msg-buffer-size: 8192   # note this limits service, 'no huge stuff'.
+	msg-cache-size: 102400  # 100 Kb.
+	msg-cache-slabs: 1
+	rrset-cache-size: 102400  # 100 Kb.
+	rrset-cache-slabs: 1
+	infra-cache-numhosts: 200
+	infra-cache-numlame: 10
+	key-cache-size: 102400  # 100 Kb.
+	key-cache-slabs: 1
+	num-queries-per-thread: 30
+	target-fetch-policy: "2 1 0 0 0 0"
+	harden-large-queries: "yes"
+	harden-short-bufsize: "yes"
+	do-ip6: no	# save a bit of memory if not used.
+.fi
 .Sh FILES
 .Bl -tag -width indent
+.It Pa /etc/unbound
+default unbound working directory and default 
+.Xr chroot 2 
+location.
 .It Pa unbound.conf
 unbound configuration file.
+.It Pa unbound.pid
+default unbound pidfile with process ID of the running daemon.
+.It Pa unbound.log
+unbound log file. default is to log to 
+.Xr syslog 3 .
 .El
 .Sh SEE ALSO
-.Xr unbound 8
+.Xr unbound 8 , 
+.Xr unbound-checkconf 8 .
 .Sh AUTHORS
 .Ic Unbound 
 was written by NLnet Labs. Please see CREDITS file

doc/unbound.doxygen

@@ -1,4 +1,4 @@
-# Doxyfile 1.4.6
+# Doxyfile 1.5.2
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project
@@ -14,6 +14,14 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 
+# This tag specifies the encoding used for all characters in the config file that 
+# follow. The default is UTF-8 which is also the encoding used for all text before 
+# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into 
+# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of 
+# possible encodings.
+
+# DOXYFILE_ENCODING      = UTF-8
+
 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded 
 # by quotes) that should identify the project.
 
@@ -45,24 +53,14 @@ CREATE_SUBDIRS         = NO
 # documentation generated by doxygen is written. Doxygen will use this 
 # information to generate all constant output in the proper language. 
 # The default language is English, other supported languages are: 
-# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, 
-# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, 
-# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, 
-# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, 
-# Swedish, and Ukrainian.
+# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, 
+# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, 
+# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, 
+# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, 
+# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian.
 
 OUTPUT_LANGUAGE        = English
 
-# This tag can be used to specify the encoding used in the generated output. 
-# The encoding is not always determined by the language that is chosen, 
-# but also whether or not the output is meant for Windows or non-Windows users. 
-# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES 
-# forces the Windows encoding (this is the default for the Windows binary), 
-# whereas setting the tag to NO uses a Unix-style encoding (the default for 
-# all platforms other than Windows).
-
-USE_WINDOWS_ENCODING   = NO
-
 # If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will 
 # include brief member descriptions after the members that are listed in 
 # the file and class documentation (similar to JavaDoc). 
@@ -204,6 +202,11 @@ OPTIMIZE_OUTPUT_JAVA   = NO
 
 BUILTIN_STL_SUPPORT    = NO
 
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+
+# CPP_CLI_SUPPORT        = NO
+
 # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC 
 # tag is set to YES, then doxygen will reuse the documentation of the first 
 # member in the group (if any) for the other members of the group. By default 
@@ -459,7 +462,14 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories 
 # with spaces.
 
-INPUT                  =  .
+INPUT                  = .
+
+# This tag can be used to specify the character encoding of the source files that 
+# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default 
+# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. 
+# See http://www.gnu.org/software/libiconv for the list of possible encodings.
+
+# INPUT_ENCODING         = UTF-8
 
 # If the value of the INPUT tag contains directories, you can use the 
 # FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
@@ -468,7 +478,7 @@ INPUT                  =  .
 # *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx 
 # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py
 
-#FILE_PATTERNS          = *.h *.c *.dox
+FILE_PATTERNS          = 
 
 # The RECURSIVE tag can be used to turn specify whether or not subdirectories 
 # should be searched for input files as well. Possible values are YES and NO. 
@@ -480,7 +490,13 @@ RECURSIVE              = YES
 # excluded from the INPUT source files. This way you can easily exclude a 
 # subdirectory from a directory tree whose root is specified with the INPUT tag.
 
-EXCLUDE                =  ./build ./compat util/configparser.c util/configparser.h util/configlexer.c util/locks.h
+EXCLUDE                = ./build \
+                         ./compat \
+                         util/configparser.c \
+                         util/configparser.h \
+                         util/configlexer.c \
+                         util/locks.h \
+			 ./ldns-src
 
 # The EXCLUDE_SYMLINKS tag can be used select whether or not files or 
 # directories that are symbolic links (a Unix filesystem feature) are excluded 
@@ -496,6 +512,13 @@ EXCLUDE_SYMLINKS       = NO
 
 EXCLUDE_PATTERNS       = 
 
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names 
+# (namespaces, classes, functions, etc.) that should be excluded from the output. 
+# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, 
+# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test
+
+# EXCLUDE_SYMBOLS        = 
+
 # The EXAMPLE_PATH tag can be used to specify one or more files or 
 # directories that contain example code fragments that are included (see 
 # the \include command).
@@ -581,6 +604,13 @@ REFERENCED_BY_RELATION = YES
 
 REFERENCES_RELATION    = YES
 
+# If the REFERENCES_LINK_SOURCE tag is set to YES (the default)
+# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from
+# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will
+# link to the source code.  Otherwise they will link to the documentstion.
+
+# REFERENCES_LINK_SOURCE = YES
+
 # If the USE_HTAGS tag is set to YES then the references to source code 
 # will point to the HTML generated by the htags(1) tool instead of doxygen 
 # built-in source browser. The htags tool is part of GNU's global source 
@@ -1078,6 +1108,14 @@ PERL_PATH              = /usr/bin/perl
 
 CLASS_DIAGRAMS         = YES
 
+# You can define message sequence charts within doxygen comments using the \msc 
+# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to 
+# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to 
+# specify the directory where the mscgen tool resides. If left empty the tool is assumed to 
+# be found in the default search path.
+
+# MSCGEN_PATH            = 
+
 # If set to YES, the inheritance and collaboration graphs will hide 
 # inheritance and usage relations if the target is undocumented 
 # or is not a class.
@@ -1143,6 +1181,14 @@ INCLUDED_BY_GRAPH      = YES
 
 CALL_GRAPH             = NO
 
+# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will 
+# generate a caller dependency graph for every global function or class method. 
+# Note that enabling this option will significantly increase the time of a run. 
+# So in most cases it will be better to enable caller graphs for selected 
+# functions only using the \callergraph command.
+
+# CALLER_GRAPH           = NO
+
 # If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen 
 # will graphical hierarchy of all classes instead of a textual one.
 
@@ -1172,33 +1218,13 @@ DOT_PATH               =
 
 DOTFILE_DIRS           = 
 
-# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width 
-# (in pixels) of the graphs generated by dot. If a graph becomes larger than 
-# this value, doxygen will try to truncate the graph, so that it fits within 
-# the specified constraint. Beware that most browsers cannot cope with very 
-# large images.
+# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of 
+# nodes that will be shown in the graph. If the number of nodes in a graph 
+# becomes larger than this value, doxygen will truncate the graph, which is 
+# visualized by representing a node as a red box. Note that doxygen will always 
+# show the root nodes and its direct children regardless of this setting.
 
-MAX_DOT_GRAPH_WIDTH    = 1024
-
-# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height 
-# (in pixels) of the graphs generated by dot. If a graph becomes larger than 
-# this value, doxygen will try to truncate the graph, so that it fits within 
-# the specified constraint. Beware that most browsers cannot cope with very 
-# large images.
-
-MAX_DOT_GRAPH_HEIGHT   = 1024
-
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the 
-# graphs generated by dot. A depth value of 3 means that only nodes reachable 
-# from the root by following a path via at most 3 edges will be shown. Nodes 
-# that lay further from the root node will be omitted. Note that setting this 
-# option to 1 or 2 may greatly reduce the computation time needed for large 
-# code bases. Also note that a graph may be further truncated if the graph's 
-# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH 
-# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), 
-# the graph is not depth-constrained.
-
-MAX_DOT_GRAPH_DEPTH    = 0
+# DOT_GRAPH_MAX_NODES    = 50
 
 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent 
 # background. This is disabled by default, which results in a white background. 

iterator/iter_delegpt.c

@@ -0,0 +1,387 @@
+/*
+ * iterator/iter_delegpt.c - delegation point with NS and address information.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file implements the Delegation Point. It contains a list of name servers
+ * and their addresses if known.
+ */
+#include "config.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "util/regional.h"
+#include "util/data/dname.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/msgreply.h"
+#include "util/net_help.h"
+
+struct delegpt* 
+delegpt_create(struct regional* region)
+{
+	struct delegpt* dp=(struct delegpt*)regional_alloc(
+		region, sizeof(*dp));
+	if(!dp)
+		return NULL;
+	memset(dp, 0, sizeof(*dp));
+	return dp;
+}
+
+struct delegpt* delegpt_copy(struct delegpt* dp, struct regional* region)
+{
+	struct delegpt* copy = delegpt_create(region);
+	struct delegpt_ns* ns;
+	struct delegpt_addr* a;
+	if(!copy) 
+		return NULL;
+	if(!delegpt_set_name(copy, region, dp->name))
+		return NULL;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(!delegpt_add_ns(copy, region, ns->name))
+			return NULL;
+		copy->nslist->resolved = ns->resolved;
+	}
+	for(a = dp->target_list; a; a = a->next_target) {
+		if(!delegpt_add_addr(copy, region, &a->addr, a->addrlen))
+			return NULL;
+	}
+	return copy;
+}
+
+int 
+delegpt_set_name(struct delegpt* dp, struct regional* region, uint8_t* name)
+{
+	dp->namelabs = dname_count_size_labels(name, &dp->namelen);
+	dp->name = regional_alloc_init(region, name, dp->namelen);
+	return dp->name != 0;
+}
+
+int 
+delegpt_add_ns(struct delegpt* dp, struct regional* region, uint8_t* name)
+{
+	struct delegpt_ns* ns;
+	size_t len;
+	(void)dname_count_size_labels(name, &len);
+	/* slow check for duplicates to avoid counting failures when
+	 * adding the same server as a dependency twice */
+	if(delegpt_find_ns(dp, name, len))
+		return 1;
+	ns = (struct delegpt_ns*)regional_alloc(region,
+		sizeof(struct delegpt_ns));
+	if(!ns)
+		return 0;
+	ns->next = dp->nslist;
+	ns->namelen = len;
+	dp->nslist = ns;
+	ns->name = regional_alloc_init(region, name, ns->namelen);
+	ns->resolved = 0;
+	return 1;
+}
+
+struct delegpt_ns*
+delegpt_find_ns(struct delegpt* dp, uint8_t* name, size_t namelen)
+{
+	struct delegpt_ns* p = dp->nslist;
+	while(p) {
+		if(namelen == p->namelen && 
+			query_dname_compare(name, p->name) == 0) {
+			return p;
+		}
+		p = p->next;
+	}
+	return NULL;
+}
+
+int 
+delegpt_add_target(struct delegpt* dp, struct regional* region, 
+	uint8_t* name, size_t namelen, struct sockaddr_storage* addr, 
+	socklen_t addrlen)
+{
+	struct delegpt_ns* ns = delegpt_find_ns(dp, name, namelen);
+	if(!ns) {
+		/* ignore it */
+		return 1;
+	}
+	ns->resolved = 1;
+	return delegpt_add_addr(dp, region, addr, addrlen);
+}
+
+int 
+delegpt_add_addr(struct delegpt* dp, struct regional* region, 
+	struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	struct delegpt_addr* a = (struct delegpt_addr*)regional_alloc(region,
+		sizeof(struct delegpt_addr));
+	if(!a)
+		return 0;
+	a->next_target = dp->target_list;
+	dp->target_list = a;
+	a->next_result = 0;
+	a->next_usable = dp->usable_list;
+	dp->usable_list = a;
+	memcpy(&a->addr, addr, addrlen);
+	a->addrlen = addrlen;
+	a->attempts = 0;
+	return 1;
+}
+
+/** count NS and number missing */
+static void
+delegpt_count_ns(struct delegpt* dp, size_t* numns, size_t* missing)
+{
+	struct delegpt_ns* ns;
+	*numns = 0;
+	*missing = 0;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		(*numns)++;
+		if(!ns->resolved)
+			(*missing)++;
+	}
+}
+
+/** count addresses, and number in result and available lists */
+static void
+delegpt_count_addr(struct delegpt* dp, size_t* numaddr, size_t* numres, 
+	size_t* numavail)
+{
+	struct delegpt_addr* a;
+	*numaddr = 0;
+	*numres = 0;
+	*numavail = 0;
+	for(a = dp->target_list; a; a = a->next_target) {
+		(*numaddr)++;
+	}
+	for(a = dp->result_list; a; a = a->next_result) {
+		(*numres)++;
+	}
+	for(a = dp->usable_list; a; a = a->next_usable) {
+		(*numavail)++;
+	}
+}
+
+void delegpt_log(enum verbosity_value v, struct delegpt* dp)
+{
+	char buf[LDNS_MAX_DOMAINLEN+1];
+	struct delegpt_ns* ns;
+	struct delegpt_addr* a;
+	size_t missing=0, numns=0, numaddr=0, numres=0, numavail=0;
+	if(verbosity < v)
+		return;
+	dname_str(dp->name, buf);
+	if(dp->nslist == NULL && dp->target_list == NULL) {
+		log_info("DelegationPoint<%s>: empty", buf);
+		return;
+	}
+	delegpt_count_ns(dp, &numns, &missing);
+	delegpt_count_addr(dp, &numaddr, &numres, &numavail);
+	log_info("DelegationPoint<%s>: %u names (%u missing), "
+		"%u addrs (%u result, %u avail)", 
+		buf, (unsigned)numns, (unsigned)missing, 
+		(unsigned)numaddr, (unsigned)numres, (unsigned)numavail);
+	if(verbosity >= VERB_ALGO) {
+		for(ns = dp->nslist; ns; ns = ns->next) {
+			dname_str(ns->name, buf);
+			log_info("  %s%s", buf, (ns->resolved?"*":""));
+		}
+		for(a = dp->target_list; a; a = a->next_target) {
+			log_addr(VERB_ALGO, "  ", &a->addr, a->addrlen);
+		}
+	}
+}
+
+void 
+delegpt_add_unused_targets(struct delegpt* dp)
+{
+	struct delegpt_addr* usa = dp->usable_list;
+	dp->usable_list = NULL;
+	while(usa) {
+		usa->next_result = dp->result_list;
+		dp->result_list = usa;
+		usa = usa->next_usable;
+	}
+}
+
+size_t 
+delegpt_count_missing_targets(struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	size_t n = 0;
+	for(ns = dp->nslist; ns; ns = ns->next)
+		if(!ns->resolved)
+			n++;
+	return n;
+}
+
+/** find NS rrset in given list */
+static struct ub_packed_rrset_key*
+find_NS(struct reply_info* rep, size_t from, size_t to)
+{
+	size_t i;
+	for(i=from; i<to; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == LDNS_RR_TYPE_NS)
+			return rep->rrsets[i];
+	}
+	return NULL;
+}
+
+struct delegpt* 
+delegpt_from_message(struct dns_msg* msg, struct regional* region)
+{
+	struct ub_packed_rrset_key* ns_rrset = NULL;
+	struct delegpt* dp;
+	size_t i;
+	/* look for NS records in the authority section... */
+	ns_rrset = find_NS(msg->rep, msg->rep->an_numrrsets, 
+		msg->rep->an_numrrsets+msg->rep->ns_numrrsets);
+
+	/* In some cases (even legitimate, perfectly legal cases), the 
+	 * NS set for the "referral" might be in the answer section. */
+	if(!ns_rrset)
+		ns_rrset = find_NS(msg->rep, 0, msg->rep->an_numrrsets);
+	
+	/* If there was no NS rrset in the authority section, then this 
+	 * wasn't a referral message. (It might not actually be a 
+	 * referral message anyway) */
+	if(!ns_rrset)
+		return NULL;
+	
+	/* If we found any, then Yay! we have a delegation point. */
+	dp = delegpt_create(region);
+	if(!dp)
+		return NULL;
+	if(!delegpt_set_name(dp, region, ns_rrset->rk.dname))
+		return NULL;
+	if(!delegpt_rrset_add_ns(dp, region, ns_rrset))
+		return NULL;
+
+	/* add glue, A and AAAA in answer and additional section */
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+		/* skip auth section. FIXME really needed?*/
+		if(msg->rep->an_numrrsets <= i && 
+			i < (msg->rep->an_numrrsets+msg->rep->ns_numrrsets))
+			continue;
+
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_A) {
+			if(!delegpt_add_rrset_A(dp, region, s))
+				return NULL;
+		} else if(ntohs(s->rk.type) == LDNS_RR_TYPE_AAAA) {
+			if(!delegpt_add_rrset_AAAA(dp, region, s))
+				return NULL;
+		}
+	}
+	return dp;
+}
+
+int 
+delegpt_rrset_add_ns(struct delegpt* dp, struct regional* region,
+        struct ub_packed_rrset_key* ns_rrset)
+{
+	struct packed_rrset_data* nsdata = (struct packed_rrset_data*)
+		ns_rrset->entry.data;
+	size_t i;
+	for(i=0; i<nsdata->count; i++) {
+		if(nsdata->rr_len[i] < 2+1) continue; /* len + root label */
+		if(dname_valid(nsdata->rr_data[i]+2, nsdata->rr_len[i]-2) !=
+			(size_t)ldns_read_uint16(nsdata->rr_data[i]))
+			continue; /* bad format */
+		/* add rdata of NS (= wirefmt dname), skip rdatalen bytes */
+		if(!delegpt_add_ns(dp, region, nsdata->rr_data[i]+2))
+			return 0;
+	}
+	return 1;
+}
+
+int 
+delegpt_add_rrset_A(struct delegpt* dp, struct regional* region,
+	struct ub_packed_rrset_key* ak)
+{
+        struct packed_rrset_data* d=(struct packed_rrset_data*)ak->entry.data;
+        size_t i;
+        struct sockaddr_in sa;
+        socklen_t len = (socklen_t)sizeof(sa);
+        memset(&sa, 0, len);
+        sa.sin_family = AF_INET;
+        sa.sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+        for(i=0; i<d->count; i++) {
+                if(d->rr_len[i] != 2 + INET_SIZE)
+                        continue;
+                memmove(&sa.sin_addr, d->rr_data[i]+2, INET_SIZE);
+                if(!delegpt_add_target(dp, region, ak->rk.dname,
+                        ak->rk.dname_len, (struct sockaddr_storage*)&sa,
+                        len))
+                        return 0;
+        }
+        return 1;
+}
+
+int 
+delegpt_add_rrset_AAAA(struct delegpt* dp, struct regional* region,
+	struct ub_packed_rrset_key* ak)
+{
+        struct packed_rrset_data* d=(struct packed_rrset_data*)ak->entry.data;
+        size_t i;
+        struct sockaddr_in6 sa;
+        socklen_t len = (socklen_t)sizeof(sa);
+        memset(&sa, 0, len);
+        sa.sin6_family = AF_INET6;
+        sa.sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+        for(i=0; i<d->count; i++) {
+                if(d->rr_len[i] != 2 + INET6_SIZE) /* rdatalen + len of IP6 */
+                        continue;
+                memmove(&sa.sin6_addr, d->rr_data[i]+2, INET6_SIZE);
+                if(!delegpt_add_target(dp, region, ak->rk.dname,
+                        ak->rk.dname_len, (struct sockaddr_storage*)&sa,
+                        len))
+                        return 0;
+        }
+        return 1;
+}
+
+int 
+delegpt_add_rrset(struct delegpt* dp, struct regional* region,
+        struct ub_packed_rrset_key* rrset)
+{
+	if(!rrset)
+		return 1;
+	if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_NS)
+		return delegpt_rrset_add_ns(dp, region, rrset);
+	else if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_A)
+		return delegpt_add_rrset_A(dp, region, rrset);
+	else if(ntohs(rrset->rk.type) == LDNS_RR_TYPE_AAAA)
+		return delegpt_add_rrset_AAAA(dp, region, rrset);
+	log_warn("Unknown rrset type added to delegpt");
+	return 1;
+}

iterator/iter_delegpt.h

@@ -0,0 +1,263 @@
+/*
+ * iterator/iter_delegpt.h - delegation point with NS and address information.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file implements the Delegation Point. It contains a list of name servers
+ * and their addresses if known.
+ */
+
+#ifndef ITERATOR_ITER_DELEGPT_H
+#define ITERATOR_ITER_DELEGPT_H
+#include "util/log.h"
+struct regional;
+struct delegpt_ns;
+struct delegpt_addr;
+struct dns_msg;
+struct ub_packed_rrset_key;
+
+/**
+ * Delegation Point.
+ * For a domain name, the NS rrset, and the A and AAAA records for those.
+ */
+struct delegpt {
+	/** the domain name of the delegation point. */
+	uint8_t* name;
+	/** length of the delegation point name */
+	size_t namelen;
+	/** number of labels in delegation point */
+	int namelabs;
+
+	/** the nameservers, names from the NS RRset rdata. */
+	struct delegpt_ns* nslist;
+	/** the target addresses for delegation */
+	struct delegpt_addr* target_list;
+	/** the list of usable targets; subset of target_list 
+	 * the items in this list are not part of the result list.  */
+	struct delegpt_addr* usable_list;
+	/** the list of returned targets; subset of target_list */
+	struct delegpt_addr* result_list;
+};
+
+/**
+ * Nameservers for a delegation point.
+ */
+struct delegpt_ns {
+	/** next in list */
+	struct delegpt_ns* next;
+	/** name of nameserver */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** 
+	 * If the name has been resolved. false if not queried for yet.
+	 * true if the address is known, or marked true if failed.
+	 */
+	int resolved;
+};
+
+/**
+ * Address of target nameserver in delegation point.
+ */
+struct delegpt_addr {
+	/** next delegation point in results */
+	struct delegpt_addr* next_result;
+	/** next delegation point in usable list */
+	struct delegpt_addr* next_usable;
+	/** next delegation point in all targets list */
+	struct delegpt_addr* next_target;
+
+	/** delegation point address */
+	struct sockaddr_storage addr;
+	/** length of addr */
+	socklen_t addrlen;
+	/** number of attempts for this addr */
+	int attempts;
+};
+
+/**
+ * Create new delegation point.
+ * @param regional: where to allocate it.
+ * @return new delegation point or NULL on error.
+ */
+struct delegpt* delegpt_create(struct regional* regional);
+
+/**
+ * Create a copy of a delegation point.
+ * @param dp: delegation point to copy.
+ * @param regional: where to allocate it.
+ * @return new delegation point or NULL on error.
+ */
+struct delegpt* delegpt_copy(struct delegpt* dp, struct regional* regional);
+
+/**
+ * Set name of delegation point.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the name copy.
+ * @param name: name to use.
+ * @return false on error.
+ */
+int delegpt_set_name(struct delegpt* dp, struct regional* regional, 
+	uint8_t* name);
+
+/**
+ * Add a name to the delegation point.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param name: domain name in wire format.
+ * @return false on error.
+ */
+int delegpt_add_ns(struct delegpt* dp, struct regional* regional, 
+	uint8_t* name);
+
+/**
+ * Add NS rrset; calls add_ns repeatedly.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param ns_rrset: NS rrset.
+ * return 0 on alloc error.
+ */
+int delegpt_rrset_add_ns(struct delegpt* dp, struct regional* regional,
+	struct ub_packed_rrset_key* ns_rrset);
+
+/**
+ * Add target address to the delegation point.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param name: name for which target was found (must be in nslist).
+ *	This name is marked resolved.
+ * @param namelen: length of name.
+ * @param addr: the address.
+ * @param addrlen: the length of addr.
+ * @return false on error.
+ */
+int delegpt_add_target(struct delegpt* dp, struct regional* regional, 
+	uint8_t* name, size_t namelen, struct sockaddr_storage* addr, 
+	socklen_t addrlen);
+
+/**
+ * Add A RRset to delegpt.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param rrset: RRset A to add.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset_A(struct delegpt* dp, struct regional* regional, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add AAAA RRset to delegpt.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param rrset: RRset AAAA to add.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset_AAAA(struct delegpt* dp, struct regional* regional, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add any RRset to delegpt.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param rrset: RRset to add, NS, A, AAAA.
+ * @return 0 on alloc error.
+ */
+int delegpt_add_rrset(struct delegpt* dp, struct regional* regional, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Add address to the delegation point. No servername is associated or checked.
+ * @param dp: delegation point.
+ * @param regional: where to allocate the info.
+ * @param addr: the address.
+ * @param addrlen: the length of addr.
+ * @return false on error.
+ */
+int delegpt_add_addr(struct delegpt* dp, struct regional* regional, 
+	struct sockaddr_storage* addr, socklen_t addrlen);
+
+/** 
+ * Find NS record in name list of delegation point.
+ * @param dp: delegation point.
+ * @param name: name of nameserver to look for, uncompressed wireformat.
+ * @param namelen: length of name.
+ * @return the ns structure or NULL if not found.
+ */
+struct delegpt_ns* delegpt_find_ns(struct delegpt* dp, uint8_t* name, 
+	size_t namelen);
+
+/**
+ * Print the delegation point to the log. For debugging.
+ * @param v: verbosity value that is needed to emit to log.
+ * @param dp: delegation point.
+ */
+void delegpt_log(enum verbosity_value v, struct delegpt* dp);
+
+/**
+ * Add all usable targets to the result list.
+ * @param dp: delegation point.
+ */
+void delegpt_add_unused_targets(struct delegpt* dp);
+
+/**
+ * Count number of missing targets. These are ns names with no resolved flag.
+ * @param dp: delegation point.
+ * @return number of missing targets (or 0).
+ */
+size_t delegpt_count_missing_targets(struct delegpt* dp);
+
+/**
+ * Create new delegation point from a dns message
+ *
+ * Note that this method does not actually test to see if the message is an
+ * actual referral. It really is just checking to see if it can construct a
+ * delegation point, so the message could be of some other type (some ANSWER
+ * messages, some CNAME messages, generally.) Note that the resulting
+ * DelegationPoint will contain targets for all "relevant" glue (i.e.,
+ * address records whose ownernames match the target of one of the NS
+ * records), so if policy dictates that some glue should be discarded beyond
+ * that, discard it before calling this method. Note that this method will
+ * find "glue" in either the ADDITIONAL section or the ANSWER section.
+ *
+ * @param msg: the dns message, referral.
+ * @param regional: where to allocate delegation point.
+ * @return new delegation point or NULL on alloc error, or if the
+ *         message was not appropriate.
+ */
+struct delegpt* delegpt_from_message(struct dns_msg* msg, 
+	struct regional* regional);
+
+#endif /* ITERATOR_ITER_DELEGPT_H */

iterator/iter_donotq.c

@@ -0,0 +1,309 @@
+/*
+ * iterator/iter_donotq.c - iterative resolver donotqueryaddresses storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * The donotqueryaddresses are stored and looked up. These addresses
+ * (like 127.0.0.1) must not be used to send queries to, and can be
+ * discarded immediately from the server selection.
+ */
+#include "config.h"
+#include "iterator/iter_donotq.h"
+#include "util/regional.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+
+int
+donotq_cmp(const void* k1, const void* k2)
+{
+	struct iter_donotq_addr* n1 = (struct iter_donotq_addr*)k1;
+	struct iter_donotq_addr* n2 = (struct iter_donotq_addr*)k2;
+	int r = sockaddr_cmp_addr(&n1->addr, n1->addrlen, &n2->addr, 
+		n2->addrlen);
+	if(r != 0) return r;
+	if(n1->net < n2->net)
+		return -1;
+	if(n1->net > n2->net)
+		return 1;
+	return 0;
+}
+
+struct iter_donotq* 
+donotq_create()
+{
+	struct iter_donotq* dq = (struct iter_donotq*)calloc(1,
+		sizeof(struct iter_donotq));
+	if(!dq)
+		return NULL;
+	dq->region = regional_create();
+	if(!dq->region) {
+		donotq_delete(dq);
+		return NULL;
+	}
+	return dq;
+}
+
+void 
+donotq_delete(struct iter_donotq* dq)
+{
+	if(!dq) 
+		return;
+	regional_destroy(dq->region);
+	free(dq->tree);
+	free(dq);
+}
+
+/** insert new address into donotq structure */
+static int
+donotq_insert(struct iter_donotq* dq, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int net)
+{
+	struct iter_donotq_addr* node = regional_alloc(dq->region,
+		sizeof(struct iter_donotq_addr));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	memcpy(&node->addr, addr, addrlen);
+	node->addrlen = addrlen;
+	node->net = net;
+	node->parent = NULL;
+	if(!rbtree_insert(dq->tree, &node->node)) {
+		verbose(VERB_DETAIL, "duplicate donotquery address ignored.");
+	}
+	return 1;
+}
+
+/** make sure the netblock ends in zeroes for compare in tree */
+static void
+mask_block(int ip6, struct sockaddr_storage* addr, int net)
+{
+	uint8_t mask[8] = {0x0, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	int i, max;
+	uint8_t* s;
+	if(ip6) {
+		s = (uint8_t*)&((struct sockaddr_in6*)addr)->sin6_addr;
+		max = 128;
+	} else {
+		s = (uint8_t*)&((struct sockaddr_in*)addr)->sin_addr;
+		max = 32;
+	}
+	if(net >= max)
+		return;
+	for(i=net/8+1; i<max/8; i++) {
+		s[i] = 0;
+	}
+	s[net/8] &= mask[net&0x7];
+}
+
+/** apply donotq string */
+static int
+donotq_str_cfg(struct iter_donotq* dq, const char* str)
+{
+	struct sockaddr_storage addr;
+	int net;
+	char* s = NULL;
+	socklen_t addrlen;
+	net = (str_is_ip6(str)?128:32);
+	if((s=strchr(str, '/'))) {
+		if(atoi(s+1) > net) {
+			log_err("netblock too large: %s", str);
+			return 0;
+		}
+		net = atoi(s+1);
+		if(net == 0 && strcmp(s+1, "0") != 0) {
+			log_err("cannot parse donotquery netblock:"
+				" '%s'", str);
+			return 0;
+		}
+		if(!(s = strdup(str))) {
+			log_err("out of memory");
+			return 0;
+		}
+		*strchr(s, '/') = '\0';
+	}
+	if(!ipstrtoaddr(s?s:str, UNBOUND_DNS_PORT, &addr, &addrlen)) {
+		free(s);
+		log_err("cannot parse donotquery ip address: '%s'", str);
+		return 0;
+	}
+	if(s) {
+		free(s);
+		mask_block(str_is_ip6(str), &addr, net);
+	}
+	if(!donotq_insert(dq, &addr, addrlen, net)) {
+		log_err("out of memory");
+		return 0;
+	}
+	return 1;
+}
+
+/** read donotq config */
+static int 
+read_donotq(struct iter_donotq* dq, struct config_file* cfg)
+{
+	struct config_strlist* p;
+	for(p = cfg->donotqueryaddrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!donotq_str_cfg(dq, p->str))
+			return 0;
+	}
+	return 1;
+}
+
+/** number of bits that two addrs share (are equal) */
+static int
+addr_in_common(struct sockaddr_storage* addr1, int net1, 
+	struct sockaddr_storage* addr2, int net2, socklen_t addrlen)
+{
+	int min = (net1<net2)?net1:net2;
+	int i, to;
+	int match = 0;
+	uint8_t* s1, *s2;
+	if(addr_is_ip6(addr1, addrlen)) {
+		s1 = (uint8_t*)&((struct sockaddr_in6*)addr1)->sin6_addr;
+		s2 = (uint8_t*)&((struct sockaddr_in6*)addr2)->sin6_addr;
+		to = 16;
+	} else {
+		s1 = (uint8_t*)&((struct sockaddr_in*)addr1)->sin_addr;
+		s2 = (uint8_t*)&((struct sockaddr_in*)addr2)->sin_addr;
+		to = 4;
+	}
+	/* match = bits_in_common(s1, s2, to); */
+	for(i=0; i<to; i++) {
+		if(s1[i] == s2[i]) {
+			match += 8;
+		} else {
+			uint8_t z = s1[i]^s2[i];
+			log_assert(z);
+			while(!(z&0x80)) {
+				match++;
+				z<<=1;
+			}
+			break;
+		}
+	}
+	if(match > min) match = min;
+	return match;
+}
+
+/** initialise parent pointers in the tree */
+static void
+donotq_init_parents(struct iter_donotq* donotq)
+{
+	struct iter_donotq_addr* node, *prev = NULL, *p;
+	int m;
+	RBTREE_FOR(node, struct iter_donotq_addr*, donotq->tree) {
+		node->parent = NULL;
+		if(!prev || prev->addrlen != node->addrlen) {
+			prev = node;
+			continue;
+		}
+		m = addr_in_common(&prev->addr, prev->net, &node->addr, 
+			node->net, node->addrlen);
+		/* sort order like: ::/0, 1::/2, 1::/4, ... 2::/2 */
+		/* find the previous, or parent-parent-parent */
+		for(p = prev; p; p = p->parent)
+			if(p->net <= m) {
+				/* ==: since prev matched m, this is closest*/
+				/* <: prev matches more, but is not a parent,
+				* this one is a (grand)parent */
+				node->parent = p;
+				break;
+			}
+		prev = node;
+	}
+}
+
+int 
+donotq_apply_cfg(struct iter_donotq* dq, struct config_file* cfg)
+{
+	free(dq->tree);
+	dq->tree = rbtree_create(donotq_cmp);
+	if(!dq->tree)
+		return 0;
+	if(!read_donotq(dq, cfg))
+		return 0;
+	if(cfg->donotquery_localhost) {
+		if(!donotq_str_cfg(dq, "127.0.0.0/8"))
+			return 0;
+		if(!donotq_str_cfg(dq, "::1"))
+			return 0;
+	}
+	donotq_init_parents(dq);
+	return 1;
+}
+
+int 
+donotq_lookup(struct iter_donotq* donotq, struct sockaddr_storage* addr,
+        socklen_t addrlen)
+{
+	/* lookup in the tree */
+	rbnode_t* res = NULL;
+	struct iter_donotq_addr* result;
+	struct iter_donotq_addr key;
+	key.node.key = &key;
+	memcpy(&key.addr, addr, addrlen);
+	key.addrlen = addrlen;
+	key.net = (addr_is_ip6(addr, addrlen)?128:32);
+	if(rbtree_find_less_equal(donotq->tree, &key, &res)) {
+		/* exact */
+		return 1;
+	} else {
+		/* smaller element (or no element) */
+		int m;
+		result = (struct iter_donotq_addr*)res;
+		if(!result || result->addrlen != addrlen)
+			return 0;
+		/* count number of bits matched */
+		m = addr_in_common(&result->addr, result->net, addr, 
+			key.net, addrlen);
+		while(result) { /* go up until addr is inside netblock */
+			if(result->net <= m)
+				return 1;
+			result = result->parent;
+		}
+	}
+	return 0;
+}
+
+size_t 
+donotq_get_mem(struct iter_donotq* donotq)
+{
+	if(!donotq) return 0;
+	return sizeof(*donotq) + regional_get_mem(donotq->region);
+}

iterator/iter_donotq.h

@@ -0,0 +1,120 @@
+/*
+ * iterator/iter_donotq.h - iterative resolver donotqueryaddresses storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of the donotquery addresses and lookup fast.
+ */
+
+#ifndef ITERATOR_ITER_DONOTQ_H
+#define ITERATOR_ITER_DONOTQ_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct regional;
+
+/**
+ * Iterator donotqueryaddresses structure
+ */
+struct iter_donotq {
+	/** regional for allocation */
+	struct regional* region;
+	/** 
+	 * Tree of the single addresses that are blocked.
+	 * contents of type iter_donotq_addr.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator donotquery address.
+ * A single address that must not be used to send queries to.
+ */
+struct iter_donotq_addr {
+	/** redblacktree node, key is this structure: addr and addrlen */
+	rbnode_t node;
+	/** address */
+	struct sockaddr_storage addr;
+	/** length of addr */
+	socklen_t addrlen;
+	/** netblock size */
+	int net;
+	/** parent node in donotq tree that encompasses this entry */
+	struct iter_donotq_addr* parent;
+};
+
+/**
+ * Create donotqueryaddresses structure 
+ * @return new structure or NULL on error.
+ */
+struct iter_donotq* donotq_create();
+
+/**
+ * Delete donotqueryaddresses structure.
+ * @param donotq: to delete.
+ */
+void donotq_delete(struct iter_donotq* donotq);
+
+/**
+ * Process donotqueryaddresses config.
+ * @param donotq: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int donotq_apply_cfg(struct iter_donotq* donotq, struct config_file* cfg);
+
+/**
+ * See if an address is blocked.
+ * @param donotq: structure for address storage.
+ * @param addr: address to check
+ * @param addrlen: length of addr.
+ * @return: true if the address must not be queried. false if unlisted.
+ */
+int donotq_lookup(struct iter_donotq* donotq, struct sockaddr_storage* addr,
+	socklen_t addrlen);
+
+/**
+ * Get memory used by donotqueryaddresses structure.
+ * @param donotq: structure for address storage.
+ * @return bytes in use.
+ */
+size_t donotq_get_mem(struct iter_donotq* donotq);
+
+/** compare two donotq entries */
+int donotq_cmp(const void* k1, const void* k2);
+
+#endif /* ITERATOR_ITER_DONOTQ_H */

iterator/iter_fwd.c

@@ -0,0 +1,292 @@
+/*
+ * iterator/iter_fwd.c - iterative resolver module forward zones.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of forward zones and config settings.
+ */
+#include "config.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_delegpt.h"
+#include "util/regional.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+int
+fwd_cmp(const void* k1, const void* k2)
+{
+	int m;
+	struct iter_forward_zone* n1 = (struct iter_forward_zone*)k1;
+	struct iter_forward_zone* n2 = (struct iter_forward_zone*)k2;
+	if(n1->dclass != n2->dclass) {
+		if(n1->dclass < n2->dclass)
+			return -1;
+		return 1;
+	}
+	return dname_lab_cmp(n1->name, n1->namelabs, n2->name, n2->namelabs, 
+		&m);
+}
+
+struct iter_forwards* 
+forwards_create()
+{
+	struct iter_forwards* fwd = (struct iter_forwards*)calloc(1,
+		sizeof(struct iter_forwards));
+	if(!fwd)
+		return NULL;
+	fwd->region = regional_create();
+	if(!fwd->region) {
+		forwards_delete(fwd);
+		return NULL;
+	}
+	return fwd;
+}
+
+void 
+forwards_delete(struct iter_forwards* fwd)
+{
+	if(!fwd) 
+		return;
+	regional_destroy(fwd->region);
+	free(fwd->tree);
+	free(fwd);
+}
+
+/** insert new info into forward structure */
+static int
+forwards_insert(struct iter_forwards* fwd, uint16_t c, struct delegpt* dp)
+{
+	struct iter_forward_zone* node = regional_alloc(fwd->region,
+		sizeof(struct iter_forward_zone));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	node->dclass = c;
+	node->name = regional_alloc_init(fwd->region, dp->name, dp->namelen);
+	if(!node->name)
+		return 0;
+	node->namelen = dp->namelen;
+	node->namelabs = dp->namelabs;
+	node->dp = dp;
+	if(!rbtree_insert(fwd->tree, &node->node)) {
+		log_err("duplicate forward zone ignored.");
+	}
+	return 1;
+}
+
+/** initialise parent pointers in the tree */
+static void
+fwd_init_parents(struct iter_forwards* fwd)
+{
+	struct iter_forward_zone* node, *prev = NULL, *p;
+	int m;
+	RBTREE_FOR(node, struct iter_forward_zone*, fwd->tree) {
+		node->parent = NULL;
+		if(!prev || prev->dclass != node->dclass) {
+			prev = node;
+			continue;
+		}
+		(void)dname_lab_cmp(prev->name, prev->namelabs, node->name,
+			node->namelabs, &m); /* we know prev is smaller */
+		/* sort order like: . com. bla.com. zwb.com. net. */
+		/* find the previous, or parent-parent-parent */
+		for(p = prev; p; p = p->parent)
+			/* looking for name with few labels, a parent */
+			if(p->namelabs <= m) {
+				/* ==: since prev matched m, this is closest*/
+				/* <: prev matches more, but is not a parent,
+				 * this one is a (grand)parent */
+				node->parent = p;
+				break;
+			}
+		prev = node;
+	}
+}
+
+/** set zone name */
+static int 
+read_fwds_name(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	ldns_rdf* rdf;
+	if(!s->name) {
+		log_err("forward zone without a name (use name \".\" to forward everything)");
+		return 0;
+	}
+	rdf = ldns_dname_new_frm_str(s->name);
+	if(!rdf) {
+		log_err("cannot parse forward zone name %s", s->name);
+		return 0;
+	}
+	if(!delegpt_set_name(dp, fwd->region, ldns_rdf_data(rdf))) {
+		ldns_rdf_deep_free(rdf);
+		log_err("out of memory");
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** set fwd host names */
+static int 
+read_fwds_host(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	ldns_rdf* rdf;
+	for(p = s->hosts; p; p = p->next) {
+		log_assert(p->str);
+		rdf = ldns_dname_new_frm_str(p->str);
+		if(!rdf) {
+			log_err("cannot parse forward %s server name: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_ns(dp, fwd->region, ldns_rdf_data(rdf))) {
+			ldns_rdf_deep_free(rdf);
+			log_err("out of memory");
+			return 0;
+		}
+		ldns_rdf_deep_free(rdf);
+	}
+	return 1;
+}
+
+/** set fwd server addresses */
+static int 
+read_fwds_addr(struct iter_forwards* fwd, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	for(p = s->addrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!extstrtoaddr(p->str, &addr, &addrlen)) {
+			log_err("cannot parse forward %s ip address: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_addr(dp, fwd->region, &addr, addrlen)) {
+			log_err("out of memory");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/** read forwards config */
+static int 
+read_forwards(struct iter_forwards* fwd, struct config_file* cfg)
+{
+	struct config_stub* s;
+	for(s = cfg->forwards; s; s = s->next) {
+		struct delegpt* dp = delegpt_create(fwd->region);
+		if(!dp) {
+			log_err("out of memory");
+			return 0;
+		}
+		if(!read_fwds_name(fwd, s, dp) ||
+			!read_fwds_host(fwd, s, dp) ||
+			!read_fwds_addr(fwd, s, dp))
+			return 0;
+		if(!forwards_insert(fwd, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		verbose(VERB_DETAIL, "Forward zone server list:");
+		delegpt_log(VERB_DETAIL, dp);
+	}
+	return 1;
+}
+
+int 
+forwards_apply_cfg(struct iter_forwards* fwd, struct config_file* cfg)
+{
+	free(fwd->tree);
+	fwd->tree = rbtree_create(fwd_cmp);
+	if(!fwd->tree)
+		return 0;
+
+	/* read forward zones */
+	if(!read_forwards(fwd, cfg))
+		return 0;
+	fwd_init_parents(fwd);
+	return 1;
+}
+
+struct delegpt* 
+forwards_lookup(struct iter_forwards* fwd, uint8_t* qname, uint16_t qclass)
+{
+	/* lookup the forward zone in the tree */
+	rbnode_t* res = NULL;
+	struct iter_forward_zone *result;
+	struct iter_forward_zone key;
+	key.node.key = &key;
+	key.dclass = qclass;
+	key.name = qname;
+	key.namelabs = dname_count_size_labels(qname, &key.namelen);
+	if(rbtree_find_less_equal(fwd->tree, &key, &res)) {
+		/* exact */
+		result = (struct iter_forward_zone*)res;
+	} else {
+		/* smaller element (or no element) */
+		int m;
+		result = (struct iter_forward_zone*)res;
+		if(!result || result->dclass != qclass)
+			return NULL;
+		/* count number of labels matched */
+		(void)dname_lab_cmp(result->name, result->namelabs, key.name,
+			key.namelabs, &m);
+		while(result) { /* go up until qname is subdomain of stub */
+			if(result->namelabs <= m)
+				break;
+			result = result->parent;
+		}
+	}
+	if(result)
+		return result->dp;
+	return NULL;
+}
+
+size_t 
+forwards_get_mem(struct iter_forwards* fwd)
+{
+	if(!fwd)
+		return 0;
+	return sizeof(*fwd) + regional_get_mem(fwd->region);
+}

iterator/iter_fwd.h

@@ -0,0 +1,131 @@
+/*
+ * iterator/iter_fwd.h - iterative resolver module forward zones.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of forward zones, and read those from config.
+ */
+
+#ifndef ITERATOR_ITER_FWD_H
+#define ITERATOR_ITER_FWD_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct delegpt;
+struct regional;
+
+/**
+ * Iterator forward zones structure
+ */
+struct iter_forwards {
+	/** regional where forward zone server addresses are allocated */
+	struct regional* region;
+	/** 
+	 * Zones are stored in this tree. Sort order is specially chosen.
+	 * first sorted on qtype. Then on dname in nsec-like order, so that
+	 * a lookup on class, name will return an exact match or the closest
+	 * match which gives the ancestor needed.
+	 * contents of type iter_forward_zone.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator forward servers for a particular zone.
+ */
+struct iter_forward_zone {
+	/** redblacktree node, key is this structure: class and name */
+	rbnode_t node;
+	/** name */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** number of labels in name */
+	int namelabs;
+	/** delegation point with forward server information for this zone. */
+	struct delegpt* dp;
+	/** pointer to parent in tree (or NULL if none) */
+	struct iter_forward_zone* parent;
+	/** class. host order. */
+	uint16_t dclass;
+};
+
+/**
+ * Create forwards 
+ * @return new forwards or NULL on error.
+ */
+struct iter_forwards* forwards_create();
+
+/**
+ * Delete forwards.
+ * @param fwd: to delete.
+ */
+void forwards_delete(struct iter_forwards* fwd);
+
+/**
+ * Process forwards config.
+ * @param fwd: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int forwards_apply_cfg(struct iter_forwards* fwd, struct config_file* cfg);
+
+/**
+ * Find forward zone information
+ * For this qname/qclass find forward zone information, returns delegation
+ * point with server names and addresses, or NULL if no forwarding is needed.
+ *
+ * @param fwd: forward storage.
+ * @param qname: The qname of the query.
+ * @param qclass: The qclass of the query.
+ * @return: A delegation point if the query has to be forwarded to that list,
+ *         otherwise null.
+ */
+struct delegpt* forwards_lookup(struct iter_forwards* fwd, 
+	uint8_t* qname, uint16_t qclass);
+
+/**
+ * Get memory in use by forward storage
+ * @param fwd: forward storage.
+ * @return bytes in use
+ */
+size_t forwards_get_mem(struct iter_forwards* fwd);
+
+/** compare two fwd entries */
+int fwd_cmp(const void* k1, const void* k2);
+
+#endif /* ITERATOR_ITER_FWD_H */

iterator/iter_hints.c

@@ -0,0 +1,506 @@
+/*
+ * iterator/iter_hints.c - iterative resolver module stub and root hints.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of stub and root hints, and read those from config.
+ */
+#include "config.h"
+#include "iterator/iter_hints.h"
+#include "iterator/iter_delegpt.h"
+#include "util/regional.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+int
+stub_cmp(const void* k1, const void* k2)
+{
+	int m;
+	struct iter_hints_stub* n1 = (struct iter_hints_stub*)k1;
+	struct iter_hints_stub* n2 = (struct iter_hints_stub*)k2;
+	if(n1->hint_class != n2->hint_class) {
+		if(n1->hint_class < n2->hint_class)
+			return -1;
+		return 1;
+	}
+	return dname_lab_cmp(n1->name, n1->namelabs, n2->name, n2->namelabs, 
+		&m);
+}
+
+struct iter_hints* 
+hints_create()
+{
+	struct iter_hints* hints = (struct iter_hints*)calloc(1,
+		sizeof(struct iter_hints));
+	if(!hints)
+		return NULL;
+	hints->region = regional_create();
+	if(!hints->region) {
+		hints_delete(hints);
+		return NULL;
+	}
+	return hints;
+}
+
+void 
+hints_delete(struct iter_hints* hints)
+{
+	if(!hints) 
+		return;
+	regional_destroy(hints->region);
+	free(hints->tree);
+	free(hints);
+}
+
+/** add hint to delegation hints */
+static int
+ah(struct delegpt* dp, struct regional* r, const char* sv, const char* ip)
+{
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	ldns_rdf* rdf = ldns_dname_new_frm_str(sv);
+	if(!rdf) {
+		log_err("could not parse %s", sv);
+		return 0;
+	}
+	if(!delegpt_add_ns(dp, r, ldns_rdf_data(rdf)) ||
+	   !extstrtoaddr(ip, &addr, &addrlen) ||
+	   !delegpt_add_target(dp, r, ldns_rdf_data(rdf), ldns_rdf_size(rdf),
+		&addr, addrlen)) {
+		ldns_rdf_deep_free(rdf);
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** obtain compiletime provided root hints */
+static struct delegpt* 
+compile_time_root_prime(struct regional* r)
+{
+	/* from:
+	 ;       This file is made available by InterNIC
+	 ;       under anonymous FTP as
+	 ;           file                /domain/named.cache
+	 ;           on server           FTP.INTERNIC.NET
+	 ;       -OR-                    RS.INTERNIC.NET
+	 ;
+	 ;       last update:    Jan 29, 2004
+	 ;       related version of root zone:   2004012900
+	 */
+	struct delegpt* dp = delegpt_create(r);
+	if(!dp)
+		return NULL;
+	if(!delegpt_set_name(dp, r, (uint8_t*)"\000"))
+		return NULL;
+	if(!ah(dp, r, "A.ROOT-SERVERS.NET.", "198.41.0.4"))	return 0;
+	if(!ah(dp, r, "B.ROOT-SERVERS.NET.", "192.228.79.201")) return 0;
+	if(!ah(dp, r, "C.ROOT-SERVERS.NET.", "192.33.4.12"))	return 0;
+	if(!ah(dp, r, "D.ROOT-SERVERS.NET.", "128.8.10.90"))	return 0;
+	if(!ah(dp, r, "E.ROOT-SERVERS.NET.", "192.203.230.10")) return 0;
+	if(!ah(dp, r, "F.ROOT-SERVERS.NET.", "192.5.5.241"))	return 0;
+	if(!ah(dp, r, "G.ROOT-SERVERS.NET.", "192.112.36.4"))	return 0;
+	if(!ah(dp, r, "H.ROOT-SERVERS.NET.", "128.63.2.53"))	return 0;
+	if(!ah(dp, r, "I.ROOT-SERVERS.NET.", "192.36.148.17"))	return 0;
+	if(!ah(dp, r, "J.ROOT-SERVERS.NET.", "192.58.128.30"))	return 0;
+	if(!ah(dp, r, "K.ROOT-SERVERS.NET.", "193.0.14.129"))	return 0;
+	if(!ah(dp, r, "L.ROOT-SERVERS.NET.", "199.7.83.42"))	return 0;
+	if(!ah(dp, r, "M.ROOT-SERVERS.NET.", "202.12.27.33"))	return 0;
+	return dp;
+}
+
+/** insert new hint info into hint structure */
+static int
+hints_insert(struct iter_hints* hints, uint16_t c, struct delegpt* dp)
+{
+	struct iter_hints_stub* node = regional_alloc(hints->region,
+		sizeof(struct iter_hints_stub));
+	if(!node)
+		return 0;
+	node->node.key = node;
+	node->hint_class = c;
+	node->name = regional_alloc_init(hints->region, dp->name, dp->namelen);
+	if(!node->name)
+		return 0;
+	node->namelen = dp->namelen;
+	node->namelabs = dp->namelabs;
+	node->dp = dp;
+	if(!rbtree_insert(hints->tree, &node->node)) {
+		log_err("second hints ignored.");
+	}
+	return 1;
+}
+
+/** initialise parent pointers in the tree */
+static void
+init_parents(struct iter_hints* hints)
+{
+	struct iter_hints_stub* node, *prev = NULL, *p;
+	int m;
+	RBTREE_FOR(node, struct iter_hints_stub*, hints->tree) {
+		node->parent = NULL;
+		if(!prev || prev->hint_class != node->hint_class) {
+			prev = node;
+			continue;
+		}
+		(void)dname_lab_cmp(prev->name, prev->namelabs, node->name,
+			node->namelabs, &m); /* we know prev is smaller */
+		/* sort order like: . com. bla.com. zwb.com. net. */
+		/* find the previous, or parent-parent-parent */
+		for(p = prev; p; p = p->parent)
+			/* looking for name with few labels, a parent */
+			if(p->namelabs <= m) {
+				/* ==: since prev matched m, this is closest*/
+				/* <: prev matches more, but is not a parent,
+				 * this one is a (grand)parent */
+				node->parent = p;
+				break;
+			}
+		prev = node;
+	}
+}
+
+/** set stub name */
+static int 
+read_stubs_name(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	ldns_rdf* rdf;
+	if(!s->name) {
+		log_err("stub zone without a name");
+		return 0;
+	}
+	rdf = ldns_dname_new_frm_str(s->name);
+	if(!rdf) {
+		log_err("cannot parse stub zone name %s", s->name);
+		return 0;
+	}
+	if(!delegpt_set_name(dp, hints->region, ldns_rdf_data(rdf))) {
+		ldns_rdf_deep_free(rdf);
+		log_err("out of memory");
+		return 0;
+	}
+	ldns_rdf_deep_free(rdf);
+	return 1;
+}
+
+/** set stub host names */
+static int 
+read_stubs_host(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	ldns_rdf* rdf;
+	for(p = s->hosts; p; p = p->next) {
+		log_assert(p->str);
+		rdf = ldns_dname_new_frm_str(p->str);
+		if(!rdf) {
+			log_err("cannot parse stub %s nameserver name: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_ns(dp, hints->region, ldns_rdf_data(rdf))) {
+			ldns_rdf_deep_free(rdf);
+			log_err("out of memory");
+			return 0;
+		}
+		ldns_rdf_deep_free(rdf);
+	}
+	return 1;
+}
+
+/** set stub server addresses */
+static int 
+read_stubs_addr(struct iter_hints* hints, struct config_stub* s, 
+	struct delegpt* dp)
+{
+	struct config_strlist* p;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	for(p = s->addrs; p; p = p->next) {
+		log_assert(p->str);
+		if(!extstrtoaddr(p->str, &addr, &addrlen)) {
+			log_err("cannot parse stub %s ip address: '%s'", 
+				s->name, p->str);
+			return 0;
+		}
+		if(!delegpt_add_addr(dp, hints->region, &addr, addrlen)) {
+			log_err("out of memory");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+/** read stubs config */
+static int 
+read_stubs(struct iter_hints* hints, struct config_file* cfg)
+{
+	struct config_stub* s;
+	for(s = cfg->stubs; s; s = s->next) {
+		struct delegpt* dp = delegpt_create(hints->region);
+		if(!dp) {
+			log_err("out of memory");
+			return 0;
+		}
+		if(!read_stubs_name(hints, s, dp) ||
+			!read_stubs_host(hints, s, dp) ||
+			!read_stubs_addr(hints, s, dp))
+			return 0;
+		if(!hints_insert(hints, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		delegpt_log(VERB_DETAIL, dp);
+	}
+	return 1;
+}
+
+/** read root hints from file */
+static int 
+read_root_hints(struct iter_hints* hints, char* fname)
+{
+	int lineno = 0;
+	uint32_t default_ttl = 0;
+	ldns_rdf* origin = NULL;
+	ldns_rdf* prev_rr = NULL;
+	struct delegpt* dp;
+	ldns_rr* rr = NULL;
+	ldns_status status;
+	uint16_t c = LDNS_RR_CLASS_IN;
+	FILE* f = fopen(fname, "r");
+	if(!f) {
+		log_err("could not read root hints %s: %s",
+			fname, strerror(errno));
+		return 0;
+	}
+	dp = delegpt_create(hints->region);
+	if(!dp) {
+		log_err("out of memory reading root hints");
+		fclose(f);
+		return 0;
+	}
+	verbose(VERB_DETAIL, "Reading root hints from %s", fname);
+	while(!feof(f)) {
+		status = ldns_rr_new_frm_fp_l(&rr, f, 
+			&default_ttl, &origin, &prev_rr, &lineno);
+		if(status == LDNS_STATUS_SYNTAX_EMPTY ||
+			status == LDNS_STATUS_SYNTAX_TTL ||
+			status == LDNS_STATUS_SYNTAX_ORIGIN)
+			continue;
+		if(status != LDNS_STATUS_OK) {
+			log_err("reading root hints %s %d: %s", fname,
+				lineno, ldns_get_errorstr_by_id(status));
+			fclose(f);
+			return 0;
+		}
+		if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_NS) {
+			if(!delegpt_add_ns(dp, hints->region,
+				ldns_rdf_data(ldns_rr_rdf(rr, 0)))) {
+				log_err("out of memory reading root hints");
+				fclose(f);
+				return 0;
+			}
+			c = ldns_rr_get_class(rr);
+			if(!dp->name) {
+				if(!delegpt_set_name(dp, hints->region, 
+					ldns_rdf_data(ldns_rr_owner(rr)))){
+					log_err("out of memory.");
+					fclose(f);
+					return 0;
+				}
+			}
+		} else if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_A) {
+			struct sockaddr_in sa;
+			socklen_t len = (socklen_t)sizeof(sa);
+			memset(&sa, 0, len);
+			sa.sin_family = AF_INET;
+			sa.sin_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+			memmove(&sa.sin_addr, 
+				ldns_rdf_data(ldns_rr_rdf(rr, 0)), INET_SIZE);
+			if(!delegpt_add_target(dp, hints->region,
+					ldns_rdf_data(ldns_rr_owner(rr)),
+					ldns_rdf_size(ldns_rr_owner(rr)),
+					(struct sockaddr_storage*)&sa, len)) {
+				log_err("out of memory reading root hints");
+				fclose(f);
+				return 0;
+			}
+		} else if(ldns_rr_get_type(rr) == LDNS_RR_TYPE_AAAA) {
+			struct sockaddr_in6 sa;
+			socklen_t len = (socklen_t)sizeof(sa);
+			memset(&sa, 0, len);
+			sa.sin6_family = AF_INET6;
+			sa.sin6_port = (in_port_t)htons(UNBOUND_DNS_PORT);
+			memmove(&sa.sin6_addr, 
+				ldns_rdf_data(ldns_rr_rdf(rr, 0)), INET6_SIZE);
+			if(!delegpt_add_target(dp, hints->region,
+					ldns_rdf_data(ldns_rr_owner(rr)),
+					ldns_rdf_size(ldns_rr_owner(rr)),
+					(struct sockaddr_storage*)&sa, len)) {
+				log_err("out of memory reading root hints");
+				fclose(f);
+				return 0;
+			}
+		} else {
+			log_warn("root hints %s:%d skipping type %d",
+				fname, lineno, ldns_rr_get_type(rr));
+		}
+
+		ldns_rr_free(rr);
+	}
+	fclose(f);
+	if(!dp->name) {
+		log_warn("root hints %s: no NS content", fname);
+		return 1;
+	}
+	if(!hints_insert(hints, c, dp)) {
+		return 0;
+	}
+	delegpt_log(VERB_DETAIL, dp);
+	return 1;
+}
+
+/** read root hints list */
+static int 
+read_root_hints_list(struct iter_hints* hints, struct config_file* cfg)
+{
+	struct config_strlist* p;
+	for(p = cfg->root_hints; p; p = p->next) {
+		log_assert(p->str);
+		if(p->str && p->str[0]) {
+			if(!read_root_hints(hints, p->str))
+				return 0;
+		}
+	}
+	return 1;
+}
+
+int 
+hints_apply_cfg(struct iter_hints* hints, struct config_file* cfg)
+{
+	free(hints->tree);
+	hints->tree = rbtree_create(stub_cmp);
+	if(!hints->tree)
+		return 0;
+	
+	/* read root hints */
+	if(!read_root_hints_list(hints, cfg))
+		return 0;
+
+	/* read stub hints */
+	if(!read_stubs(hints, cfg))
+		return 0;
+
+	/* use fallback compiletime root hints */
+	if(!hints_lookup_root(hints, LDNS_RR_CLASS_IN)) {
+		struct delegpt* dp = compile_time_root_prime(hints->region);
+		verbose(VERB_ALGO, "no config, using builtin root hints.");
+		if(!dp) 
+			return 0;
+		if(!hints_insert(hints, LDNS_RR_CLASS_IN, dp))
+			return 0;
+		delegpt_log(VERB_DETAIL, dp);
+	}
+
+	init_parents(hints);
+	return 1;
+}
+
+struct delegpt* 
+hints_lookup_root(struct iter_hints* hints, uint16_t qclass)
+{
+	uint8_t rootlab = 0;
+	struct iter_hints_stub key, *stub;
+	key.node.key = &key;
+	key.hint_class = qclass;
+	key.name = &rootlab;
+	key.namelen = 1;
+	key.namelabs = 1;
+	stub = (struct iter_hints_stub*)rbtree_search(hints->tree, &key);
+	if(!stub)
+		return NULL;
+	return stub->dp;
+}
+
+struct delegpt* 
+hints_lookup_stub(struct iter_hints* hints, uint8_t* qname, 
+	uint16_t qclass, struct delegpt* cache_dp)
+{
+	/* first lookup the stub */
+	rbnode_t* res = NULL;
+	struct iter_hints_stub *result;
+	struct iter_hints_stub key;
+	key.node.key = &key;
+	key.hint_class = qclass;
+	key.name = qname;
+	key.namelabs = dname_count_size_labels(qname, &key.namelen);
+	if(rbtree_find_less_equal(hints->tree, &key, &res)) {
+		/* exact */
+		result = (struct iter_hints_stub*)res;
+	} else {
+		/* smaller element (or no element) */
+		int m;
+		result = (struct iter_hints_stub*)res;
+		if(!result || result->hint_class != qclass)
+			return NULL;
+		/* count number of labels matched */
+		(void)dname_lab_cmp(result->name, result->namelabs, key.name,
+			key.namelabs, &m);
+		while(result) { /* go up until qname is subdomain of stub */
+			if(result->namelabs <= m)
+				break;
+			result = result->parent;
+		}
+		if(!result)
+			return NULL;
+	}
+	/* 
+	 * If our cached delegation point is above the hint, we need to prime.
+	 */
+	if(dname_strict_subdomain(result->dp->name, result->dp->namelabs,
+		cache_dp->name, cache_dp->namelabs))
+		return result->dp; /* need to prime this stub */
+	return NULL;
+}
+
+size_t 
+hints_get_mem(struct iter_hints* hints)
+{
+	if(!hints) return 0;
+	return sizeof(*hints) + regional_get_mem(hints->region);
+}

iterator/iter_hints.h

@@ -0,0 +1,140 @@
+/*
+ * iterator/iter_hints.h - iterative resolver module stub and root hints.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Keep track of stub and root hints, and read those from config.
+ */
+
+#ifndef ITERATOR_ITER_HINTS_H
+#define ITERATOR_ITER_HINTS_H
+#include "util/rbtree.h"
+struct iter_env;
+struct config_file;
+struct delegpt;
+struct regional;
+
+/**
+ * Iterator hints structure
+ */
+struct iter_hints {
+	/** regional where hints are allocated */
+	struct regional* region;
+	/** 
+	 * Hints are stored in this tree. Sort order is specially chosen.
+	 * first sorted on qtype. Then on dname in nsec-like order, so that
+	 * a lookup on class, name will return an exact match or the closest
+	 * match which gives the ancestor needed.
+	 * contents of type iter_hints_stub. The class IN root is in here.
+	 */
+	rbtree_t* tree;
+};
+
+/**
+ * Iterator hints for a particular stub.
+ */
+struct iter_hints_stub {
+	/** redblacktree node, key is this structure: class and name */
+	rbnode_t node;
+	/** name */
+	uint8_t* name;
+	/** length of name */
+	size_t namelen;
+	/** number of labels in name */
+	int namelabs;
+	/** delegation point with hint information for this stub. */
+	struct delegpt* dp;
+	/** pointer to parent in stub hint tree (or NULL if none) */
+	struct iter_hints_stub* parent;
+	/** class of hints. host order. */
+	uint16_t hint_class;
+};
+
+/**
+ * Create hints 
+ * @return new hints or NULL on error.
+ */
+struct iter_hints* hints_create();
+
+/**
+ * Delete hints.
+ * @param hints: to delete.
+ */
+void hints_delete(struct iter_hints* hints);
+
+/**
+ * Process hints config. Sets default values for root hints if no config.
+ * @param hints: where to store.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int hints_apply_cfg(struct iter_hints* hints, struct config_file* cfg);
+
+/**
+ * Find root hints for the given class.
+ * @param hints: hint storage.
+ * @param qclass: class for which root hints are requested. host order.
+ * @return: NULL if no hints, or a ptr to stored hints.
+ */
+struct delegpt* hints_lookup_root(struct iter_hints* hints, uint16_t qclass);
+
+/**
+ * Given a qname/qclass combination, and the delegation point from the cache
+ * for this qname/qclass, determine if this combination indicates that a
+ * stub hint exists and must be primed.
+ *
+ * @param hints: hint storage.
+ * @param qname: The qname that generated the delegation point.
+ * @param qclass: The qclass that generated the delegation point.
+ * @param dp: The cache generated delegation point.
+ * @return: A priming delegation point if there is a stub hint that must
+ *         be primed, otherwise null.
+ */
+struct delegpt* hints_lookup_stub(struct iter_hints* hints, 
+	uint8_t* qname, uint16_t qclass, struct delegpt* dp);
+
+/**
+ * Get memory in use by hints
+ * @param hints: hint storage.
+ * @return bytes in use
+ */
+size_t hints_get_mem(struct iter_hints* hints);
+
+/** compare two hint entries */
+int stub_cmp(const void* k1, const void* k2);
+
+#endif /* ITERATOR_ITER_HINTS_H */

iterator/iter_resptype.c

@@ -0,0 +1,252 @@
+/*
+ * iterator/iter_resptype.c - response type information and classification.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file defines the response type. DNS Responses can be classified as
+ * one of the response types.
+ */
+#include "config.h"
+#include "iterator/iter_resptype.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+
+enum response_type 
+response_type_from_cache(struct dns_msg* msg, 
+	struct query_info* request)
+{
+	/* If the message is NXDOMAIN, then it is an ANSWER. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_NXDOMAIN)
+		return RESPONSE_TYPE_ANSWER;
+	
+	/* First we look at the answer section. This can tell us if this is
+	 * CNAME or positive ANSWER. */
+	if(msg->rep->an_numrrsets > 0) {
+		/* Now look at the answer section first. 3 states: 
+		 *	o our answer is there directly,
+		 *	o our answer is there after a cname,
+		 *	o or there is just a cname. */
+		uint8_t* mname = request->qname;
+		size_t mname_len = request->qname_len;
+		size_t i;
+		for(i=0; i<msg->rep->an_numrrsets; i++) {
+			struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+
+			/* If we have encountered an answer (before or 
+			 * after a CNAME), then we are done! Note that 
+			 * if qtype == CNAME then this will be noted as 
+			 * an ANSWER before it gets treated as a CNAME, 
+			 * as it should */
+			if(ntohs(s->rk.type) == request->qtype &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				return RESPONSE_TYPE_ANSWER;
+			}
+
+			/* If we have encountered a CNAME, make sure that 
+			 * it is relevant. */
+			if(ntohs(s->rk.type) == LDNS_RR_TYPE_CNAME &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				get_cname_target(s, &mname, &mname_len);
+			}
+		}
+
+		/* if we encountered a CNAME (or a bunch of CNAMEs), and 
+		 * still got to here, then it is a CNAME response. (i.e., 
+		 * the CNAME chain didn't terminate in an answer rrset.) */
+		if(mname != request->qname) {
+			return RESPONSE_TYPE_CNAME;
+		}
+	}
+
+	/* At this point, since we don't need to detect REFERRAL or LAME 
+	 * messages, it can only be an ANSWER. */
+	return RESPONSE_TYPE_ANSWER;
+}
+
+enum response_type 
+response_type_from_server(int rdset,
+	struct dns_msg* msg, struct query_info* request, struct delegpt* dp)
+{
+	uint8_t* origzone = (uint8_t*)"\000"; /* the default */
+	size_t origzonelen = 1;
+	size_t i;
+
+	if(!msg || !request)
+		return RESPONSE_TYPE_THROWAWAY;
+	
+	/* If the message is NXDOMAIN, then it answers the question. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) == LDNS_RCODE_NXDOMAIN) {
+		/* make sure its not recursive when we don't want it to */
+		if( (msg->rep->flags&BIT_RA) &&
+			!(msg->rep->flags&BIT_AA) && !rdset)
+				return RESPONSE_TYPE_LAME;
+		return RESPONSE_TYPE_ANSWER;
+	}
+	
+	/* Other response codes mean (so far) to throw the response away as
+	 * meaningless and move on to the next nameserver. */
+	if(FLAGS_GET_RCODE(msg->rep->flags) != LDNS_RCODE_NOERROR)
+		return RESPONSE_TYPE_THROWAWAY;
+
+	/* Note: TC bit has already been handled */
+
+	if(dp) {
+		origzone = dp->name;
+		origzonelen = dp->namelen;
+	}
+
+	/* First we look at the answer section. This can tell us if this is a
+	 * CNAME or ANSWER or (provisional) ANSWER. */
+	if(msg->rep->an_numrrsets > 0) {
+		uint8_t* mname = request->qname;
+		size_t mname_len = request->qname_len;
+
+		/* Now look at the answer section first. 3 states: our 
+		 * answer is there directly, our answer is there after 
+		 * a cname, or there is just a cname. */
+		for(i=0; i<msg->rep->an_numrrsets; i++) {
+			struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+			
+			/* if the answer section has NS rrset, and qtype ANY 
+		 	 * and the delegation is lower, and no CNAMEs followed,
+		 	 * this is a referral where the NS went to AN section */
+			if((request->qtype == LDNS_RR_TYPE_ANY ||
+				request->qtype == LDNS_RR_TYPE_NS) &&
+				ntohs(s->rk.type) == LDNS_RR_TYPE_NS &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				dname_strict_subdomain_c(s->rk.dname, 
+				origzone)) {
+				return RESPONSE_TYPE_REFERRAL;
+			}
+
+			/* If we have encountered an answer (before or 
+			 * after a CNAME), then we are done! Note that 
+			 * if qtype == CNAME then this will be noted as an
+			 * ANSWER before it gets treated as a CNAME, as 
+			 * it should. */
+			if(ntohs(s->rk.type) == request->qtype &&
+				ntohs(s->rk.rrset_class) == request->qclass &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				if((msg->rep->flags&BIT_AA))
+					return RESPONSE_TYPE_ANSWER;
+				/* If the AA bit isn't on, and we've seen 
+				 * the answer, we only provisionally say 
+				 * 'ANSWER' -- it very well could be a 
+				 * REFERRAL. */
+				break;
+			}
+
+			/* If we have encountered a CNAME, make sure that 
+			 * it is relevant. */
+			if(ntohs(s->rk.type) == LDNS_RR_TYPE_CNAME &&
+				query_dname_compare(mname, s->rk.dname) == 0) {
+				get_cname_target(s, &mname, &mname_len);
+			}
+		}
+		/* if we encountered a CNAME (or a bunch of CNAMEs), and 
+		 * still got to here, then it is a CNAME response. 
+		 * (This is regardless of the AA bit at this point) */
+		if(mname != request->qname) {
+			return RESPONSE_TYPE_CNAME;
+		}
+	}
+
+	/* Looking at the authority section, we just look and see if 
+	 * there is a delegation NS set, turning it into a delegation. 
+	 * Otherwise, we will have to conclude ANSWER (either it is 
+	 * NOERROR/NODATA, or an non-authoritative answer). */
+	for(i = msg->rep->an_numrrsets; i < (msg->rep->an_numrrsets +
+		msg->rep->ns_numrrsets); i++) {
+		struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+
+		/* The normal way of detecting NOERROR/NODATA. */
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_SOA &&
+			dname_subdomain_c(request->qname, s->rk.dname)) {
+			/* we do our own recursion, thank you */
+			if( (msg->rep->flags&BIT_RA) &&
+				!(msg->rep->flags&BIT_AA) && !rdset)
+				return RESPONSE_TYPE_LAME;
+			return RESPONSE_TYPE_ANSWER;
+		}
+
+		/* Detect REFERRAL/LAME/ANSWER based on the relationship 
+		 * of the NS set to the originating zone name. */
+		if(ntohs(s->rk.type) == LDNS_RR_TYPE_NS) {
+			/* If we are getting an NS set for the zone we 
+			 * thought we were contacting, then it is an answer.*/
+			/* FIXME: is this correct? */
+			if(query_dname_compare(s->rk.dname, origzone) == 0) {
+				/* see if mistakenly a recursive server was
+				 * deployed and is responding nonAA */
+				if( (msg->rep->flags&BIT_RA) &&
+					!(msg->rep->flags&BIT_AA) && !rdset)
+					return RESPONSE_TYPE_LAME;
+				return RESPONSE_TYPE_ANSWER;
+			}
+			/* If we are getting a referral upwards (or to 
+			 * the same zone), then the server is 'lame'. */
+			if(dname_subdomain_c(origzone, s->rk.dname)) {
+				return RESPONSE_TYPE_LAME;
+			}
+			/* If the NS set is below the delegation point we 
+			 * are on, and it is non-authoritative, then it is 
+			 * a referral, otherwise it is an answer. */
+			if(dname_subdomain_c(s->rk.dname, origzone)) {
+				/* NOTE: I no longer remember in what case 
+				 * we would like this to be an answer. 
+				 * NODATA should have a SOA or nothing, 
+				 * not an NS rrset. 
+				 * True, referrals should not have the AA 
+				 * bit set, but... */
+				 
+				/* if((msg->rep->flags&BIT_AA))
+					return RESPONSE_TYPE_ANSWER; */
+				return RESPONSE_TYPE_REFERRAL;
+			}
+			/* Otherwise, the NS set is irrelevant. */
+		}
+	}
+
+	/* If we've gotten this far, this is NOERROR/NODATA (which could 
+	 * be an entirely empty message) */
+	/* check if recursive answer; saying it has empty cache */
+	if( (msg->rep->flags&BIT_RA) && !(msg->rep->flags&BIT_AA) && !rdset)
+		return RESPONSE_TYPE_LAME;
+	return RESPONSE_TYPE_ANSWER;
+}

iterator/iter_resptype.h

@@ -0,0 +1,120 @@
+/*
+ * iterator/iter_resptype.h - response type information and classification.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file defines the response type. DNS Responses can be classified as
+ * one of the response types.
+ */
+
+#ifndef ITERATOR_ITER_RESPTYPE_H
+#define ITERATOR_ITER_RESPTYPE_H
+struct dns_msg;
+struct query_info;
+struct delegpt;
+
+/**
+ * The response type is used to interpret the response.
+ */
+enum response_type {
+	/** 
+	 * 'untyped' means that the type of this response hasn't been 
+	 * assigned. 
+	 */
+	RESPONSE_TYPE_UNTYPED   = 0,
+
+	/** 
+	 * 'answer' means that the response terminates the resolution 
+	 * process. 
+	 */
+	RESPONSE_TYPE_ANSWER,
+
+	/** 'delegation' means that the response is a delegation. */
+	RESPONSE_TYPE_REFERRAL,
+
+	/**
+	 * 'cname' means that the response is a cname without the final 
+	 * answer, and thus must be restarted.
+	 */
+	RESPONSE_TYPE_CNAME,
+
+	/**
+	 * 'throwaway' means that this particular response should be 
+	 * discarded and the next nameserver should be contacted
+	 */
+	RESPONSE_TYPE_THROWAWAY,
+
+	/**
+	 * 'lame' means that this particular response indicates that 
+	 * the nameserver knew nothing about the question.
+	 */
+	RESPONSE_TYPE_LAME
+};
+
+/**
+ * Classifies a response message from cache based on the current request.
+ * Note that this routine assumes that THROWAWAY or LAME responses will not
+ * occur. Also, it will not detect REFERRAL type messages, since those are
+ * (currently) automatically classified based on how they came from the
+ * cache (findDelegation() instead of lookup()).
+ *
+ * @param msg: the message from the cache.
+ * @param request: the request that generated the response.
+ * @return the response type (CNAME or ANSWER).
+ */
+enum response_type response_type_from_cache(struct dns_msg* msg, 
+	struct query_info* request);
+
+/**
+ * Classifies a response message (from the wire) based on the current
+ * request.
+ *
+ * NOTE: currently this routine uses the AA bit in the response to help
+ * distinguish between some non-standard referrals and answers. It also
+ * relies somewhat on the originating zone to be accurate (for lameness
+ * detection, mostly).
+ *
+ * @param rdset: if RD bit was sent in query sent by unbound.
+ * @param msg: the message from the cache.
+ * @param request: the request that generated the response.
+ * @param dp: The delegation point that was being queried
+ *          when the response was returned.
+ * @return the response type (CNAME or ANSWER).
+ */
+enum response_type response_type_from_server(int rdset, 
+	struct dns_msg* msg, struct query_info* request, struct delegpt* dp);
+
+#endif /* ITERATOR_ITER_RESPTYPE_H */

iterator/iter_scrub.c

@@ -0,0 +1,575 @@
+/*
+ * iterator/iter_scrub.c - scrubbing, normalization, sanitization of DNS msgs.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file has routine(s) for cleaning up incoming DNS messages from 
+ * possible useless or malicious junk in it.
+ */
+#include "config.h"
+#include "iterator/iter_scrub.h"
+#include "services/cache/rrset.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/regional.h"
+#include "util/config_file.h"
+#include "util/module.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/data/msgreply.h"
+#include "util/alloc.h"
+
+/** RRset flag used during scrubbing. The RRset is OK. */
+#define RRSET_SCRUB_OK	0x80
+
+/** remove rrset, update loop variables */
+static void
+remove_rrset(const char* str, ldns_buffer* pkt, struct msg_parse* msg, 
+	struct rrset_parse* prev, struct rrset_parse** rrset)
+{
+	if(verbosity >= VERB_DETAIL 
+		&& (*rrset)->dname_len <= LDNS_MAX_DOMAINLEN) {
+		uint8_t buf[LDNS_MAX_DOMAINLEN+1];
+		dname_pkt_copy(pkt, buf, (*rrset)->dname);
+		log_nametypeclass(VERB_DETAIL, str, buf, 
+			(*rrset)->type, ntohs((*rrset)->rrset_class));
+	}
+	if(prev)
+		prev->rrset_all_next = (*rrset)->rrset_all_next;
+	else	msg->rrset_first = (*rrset)->rrset_all_next;
+	if(msg->rrset_last == *rrset)
+		msg->rrset_last = prev;
+	msg->rrset_count --;
+	switch((*rrset)->section) {
+		case LDNS_SECTION_ANSWER: msg->an_rrsets--; break;
+		case LDNS_SECTION_AUTHORITY: msg->ns_rrsets--; break;
+		case LDNS_SECTION_ADDITIONAL: msg->ar_rrsets--; break;
+		default: log_assert(0);
+	}
+	msgparse_bucket_remove(msg, *rrset);
+	*rrset = (*rrset)->rrset_all_next;
+}
+
+/** return true if rr type has additional names in it */
+static int
+has_additional(uint16_t t)
+{
+	switch(t) {
+		case LDNS_RR_TYPE_MB:
+		case LDNS_RR_TYPE_MD:
+		case LDNS_RR_TYPE_MF:
+		case LDNS_RR_TYPE_NS:
+		case LDNS_RR_TYPE_MX:
+		case LDNS_RR_TYPE_KX:
+		case LDNS_RR_TYPE_SRV:
+			return 1;
+		case LDNS_RR_TYPE_NAPTR:
+			/* TODO: NAPTR not supported, glue stripped off */
+			return 0;
+		default:
+			return 0;
+	}
+	return 0;
+}
+
+/** get additional name from rrset RR, return false if no name present */
+static int
+get_additional_name(struct rrset_parse* rrset, struct rr_parse* rr, 
+	uint8_t** nm, size_t* nmlen, ldns_buffer* pkt) 
+{
+	size_t offset = 0;
+	size_t len, oldpos;
+	switch(rrset->type) {
+		case LDNS_RR_TYPE_MB:
+		case LDNS_RR_TYPE_MD:
+		case LDNS_RR_TYPE_MF:
+		case LDNS_RR_TYPE_NS:
+			offset = 0;
+			break;
+		case LDNS_RR_TYPE_MX:
+		case LDNS_RR_TYPE_KX:
+			offset = 2;
+			break;
+		case LDNS_RR_TYPE_SRV:
+			offset = 6;
+			break;
+		case LDNS_RR_TYPE_NAPTR:
+			/* TODO: NAPTR not supported, glue stripped off */
+			return 0;
+		default:
+			return 0;
+	}
+	len = ldns_read_uint16(rr->ttl_data+sizeof(uint32_t));
+	if(len < offset+1)
+		return 0; /* rdata field too small */
+	*nm = rr->ttl_data+sizeof(uint32_t)+sizeof(uint16_t)+offset;
+	oldpos = ldns_buffer_position(pkt);
+	ldns_buffer_set_position(pkt, (size_t)(*nm - ldns_buffer_begin(pkt)));
+	*nmlen = pkt_dname_len(pkt);
+	ldns_buffer_set_position(pkt, oldpos);
+	if(*nmlen == 0)
+		return 0;
+	return 1;
+}
+
+/** Place mark on rrsets in additional section they are OK */
+static void
+mark_additional_rrset(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct rrset_parse* rrset)
+{
+	/* Mark A and AAAA for NS as appropriate additional section info. */
+	uint8_t* nm = NULL;
+	size_t nmlen = 0;
+	struct rr_parse* rr;
+
+	if(!has_additional(rrset->type))
+		return;
+	for(rr = rrset->rr_first; rr; rr = rr->next) {
+		if(get_additional_name(rrset, rr, &nm, &nmlen, pkt)) {
+			/* mark A */
+			hashvalue_t h = pkt_hash_rrset(pkt, nm, LDNS_RR_TYPE_A, 
+				rrset->rrset_class, 0);
+			struct rrset_parse* r = msgparse_hashtable_lookup(
+				msg, pkt, h, 0, nm, nmlen, 
+				LDNS_RR_TYPE_A, rrset->rrset_class);
+			if(r && r->section == LDNS_SECTION_ADDITIONAL) {
+				r->flags |= RRSET_SCRUB_OK;
+			}
+			
+			/* mark AAAA */
+			h = pkt_hash_rrset(pkt, nm, LDNS_RR_TYPE_AAAA, 
+				rrset->rrset_class, 0);
+			r = msgparse_hashtable_lookup(msg, pkt, h, 0, nm, 
+				nmlen, LDNS_RR_TYPE_AAAA, rrset->rrset_class);
+			if(r && r->section == LDNS_SECTION_ADDITIONAL) {
+				r->flags |= RRSET_SCRUB_OK;
+			}
+		}
+	}
+}
+
+/** Get target name of a CNAME */
+static int
+parse_get_cname_target(struct rrset_parse* rrset, uint8_t** sname, 
+	size_t* snamelen)
+{
+	if(rrset->rr_count != 1) {
+		verbose(VERB_ALGO, "Found CNAME rrset with "
+			"size > 1: %u", (unsigned)rrset->rr_count);
+		return 0;
+	}
+	if(rrset->rr_first->size < sizeof(uint16_t)+1)
+		return 0; /* CNAME rdata too small */
+	*sname = rrset->rr_first->ttl_data + sizeof(uint32_t)
+		+ sizeof(uint16_t); /* skip ttl, rdatalen */
+	*snamelen = rrset->rr_first->size - sizeof(uint16_t);
+	return 1;
+}
+
+/** Synthesize CNAME from DNAME, false if too long */
+static int 
+synth_cname(uint8_t* qname, size_t qnamelen, struct rrset_parse* dname_rrset, 
+	uint8_t* alias, size_t* aliaslen, ldns_buffer* pkt)
+{
+	/* we already know that sname is a strict subdomain of DNAME owner */
+	uint8_t* dtarg = NULL;
+	size_t dtarglen;
+	if(!parse_get_cname_target(dname_rrset, &dtarg, &dtarglen))
+		return 0; 
+	log_assert(qnamelen > dname_rrset->dname_len);
+	/* DNAME from com. to net. with qname example.com. -> example.net. */
+	/* so: \3com\0 to \3net\0 and qname \7example\3com\0 */
+	*aliaslen = qnamelen + dtarglen - dname_rrset->dname_len;
+	if(*aliaslen > LDNS_MAX_DOMAINLEN)
+		return 0; /* should have been RCODE YXDOMAIN */
+	/* decompress dnames into buffer, we know it fits */
+	dname_pkt_copy(pkt, alias, qname);
+	dname_pkt_copy(pkt, alias+(qnamelen-dname_rrset->dname_len), dtarg);
+	return 1;
+}
+
+/** synthesize a CNAME rrset */
+static struct rrset_parse*
+synth_cname_rrset(uint8_t** sname, size_t* snamelen, uint8_t* alias, 
+	size_t aliaslen, struct regional* region, struct msg_parse* msg, 
+	struct rrset_parse* rrset, struct rrset_parse* prev,
+	struct rrset_parse* nx, ldns_buffer* pkt)
+{
+	struct rrset_parse* cn = (struct rrset_parse*)regional_alloc(region,
+		sizeof(struct rrset_parse));
+	if(!cn)
+		return NULL;
+	memset(cn, 0, sizeof(*cn));
+	cn->rr_first = (struct rr_parse*)regional_alloc(region, 
+		sizeof(struct rr_parse));
+	if(!cn->rr_first)
+		return NULL;
+	cn->rr_last = cn->rr_first;
+	/* CNAME from sname to alias */
+	cn->dname = (uint8_t*)regional_alloc(region, *snamelen);
+	if(!cn->dname)
+		return NULL;
+	dname_pkt_copy(pkt, cn->dname, *sname);
+	cn->dname_len = *snamelen;
+	cn->type = LDNS_RR_TYPE_CNAME;
+	cn->section = rrset->section;
+	cn->rrset_class = rrset->rrset_class;
+	cn->rr_count = 1;
+	cn->size = sizeof(uint16_t) + aliaslen;
+	cn->hash=pkt_hash_rrset(pkt, cn->dname, cn->type, cn->rrset_class, 0);
+	/* allocate TTL + rdatalen + uncompressed dname */
+	memset(cn->rr_first, 0, sizeof(struct rr_parse));
+	cn->rr_first->outside_packet = 1;
+	cn->rr_first->ttl_data = (uint8_t*)regional_alloc(region, 
+		sizeof(uint32_t)+sizeof(uint16_t)+aliaslen);
+	if(!cn->rr_first->ttl_data)
+		return NULL;
+	ldns_write_uint32(cn->rr_first->ttl_data, 0); /* TTL = 0 */
+	ldns_write_uint16(cn->rr_first->ttl_data+4, aliaslen);
+	memmove(cn->rr_first->ttl_data+6, alias, aliaslen);
+	cn->rr_first->size = sizeof(uint16_t)+aliaslen;
+
+	/* link it in */
+	cn->rrset_all_next = nx;
+	if(prev)
+		prev->rrset_all_next = cn;
+	else	msg->rrset_first = cn;
+	if(nx == NULL)
+		msg->rrset_last = cn;
+	msg->rrset_count ++;
+	msg->an_rrsets++;
+	/* it is not inserted in the msg hashtable. */
+
+	*sname = cn->rr_first->ttl_data + sizeof(uint32_t)+sizeof(uint16_t);
+	*snamelen = aliaslen;
+	return cn;
+}
+
+/** check if DNAME applies to a name */
+static int
+pkt_strict_sub(ldns_buffer* pkt, uint8_t* sname, uint8_t* dr)
+{
+	uint8_t buf1[LDNS_MAX_DOMAINLEN+1];
+	uint8_t buf2[LDNS_MAX_DOMAINLEN+1];
+	/* decompress names */
+	dname_pkt_copy(pkt, buf1, sname);
+	dname_pkt_copy(pkt, buf2, dr);
+	return dname_strict_subdomain_c(buf1, buf2);
+}
+
+/** check subdomain with decompression */
+static int
+pkt_sub(ldns_buffer* pkt, uint8_t* comprname, uint8_t* zone)
+{
+	uint8_t buf[LDNS_MAX_DOMAINLEN+1];
+	dname_pkt_copy(pkt, buf, comprname);
+	return dname_subdomain_c(buf, zone);
+}
+
+/**
+ * This routine normalizes a response. This includes removing "irrelevant"
+ * records from the answer and additional sections and (re)synthesizing
+ * CNAMEs from DNAMEs, if present.
+ *
+ * @param pkt: packet.
+ * @param msg: msg to normalize.
+ * @param qinfo: original query.
+ * @param region: where to allocate synthesized CNAMEs.
+ * @return 0 on error.
+ */
+static int
+scrub_normalize(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, struct regional* region)
+{
+	uint8_t* sname = qinfo->qname;
+	size_t snamelen = qinfo->qname_len;
+	struct rrset_parse* rrset, *prev;
+
+	if(FLAGS_GET_RCODE(msg->flags) != LDNS_RCODE_NOERROR &&
+		FLAGS_GET_RCODE(msg->flags) != LDNS_RCODE_NXDOMAIN)
+		return 1;
+
+	/* For the ANSWER section, remove all "irrelevant" records and add
+	 * synthesized CNAMEs from DNAMEs
+	 * This will strip out-of-order CNAMEs as well. */
+
+	/* walk through the parse packet rrset list, keep track of previous
+	 * for insert and delete ease, and examine every RRset */
+	prev = NULL;
+	rrset = msg->rrset_first;
+	while(rrset && rrset->section == LDNS_SECTION_ANSWER) {
+		if(rrset->type == LDNS_RR_TYPE_DNAME && 
+			pkt_strict_sub(pkt, sname, rrset->dname)) {
+			/* check if next rrset is correct CNAME. else,
+			 * synthesize a CNAME */
+			struct rrset_parse* nx = rrset->rrset_all_next;
+			uint8_t alias[LDNS_MAX_DOMAINLEN+1];
+			size_t aliaslen = 0;
+			if(rrset->rr_count != 1) {
+				verbose(VERB_ALGO, "Found DNAME rrset with "
+					"size > 1: %u", 
+					(unsigned)rrset->rr_count);
+				return 0;
+			}
+			if(!synth_cname(sname, snamelen, rrset, alias, 
+				&aliaslen, pkt)) {
+				verbose(VERB_ALGO, "synthesized CNAME "
+					"too long");
+				return 0;
+			}
+			if(nx && nx->type == LDNS_RR_TYPE_CNAME && 
+			   dname_pkt_compare(pkt, sname, nx->dname) == 0) {
+				/* check next cname */
+				uint8_t* t = NULL;
+				size_t tlen = 0;
+				if(!parse_get_cname_target(rrset, &t, &tlen))
+					return 0;
+				if(dname_pkt_compare(pkt, alias, t) == 0) {
+					/* it's OK and better capitalized */
+					prev = rrset;
+					rrset = nx;
+					continue;
+				}
+				/* synth ourselves */
+			}
+			/* synth a CNAME rrset */
+			prev = synth_cname_rrset(&sname, &snamelen, alias, 
+				aliaslen, region, msg, rrset, rrset, nx, pkt);
+			if(!prev) {
+				log_err("out of memory synthesizing CNAME");
+				return 0;
+			}
+			/* FIXME: resolve the conflict between synthesized 
+			 * CNAME ttls and the cache. */
+			rrset = nx;
+			continue;
+
+		}
+
+		/* The only records in the ANSWER section not allowed to */
+		if(dname_pkt_compare(pkt, sname, rrset->dname) != 0) {
+			remove_rrset("normalize: removing irrelevant RRset:", 
+				pkt, msg, prev, &rrset);
+			continue;
+		}
+
+		/* Follow the CNAME chain. */
+		if(rrset->type == LDNS_RR_TYPE_CNAME) {
+			if(!parse_get_cname_target(rrset, &sname, &snamelen))
+				return 0;
+			prev = rrset;
+			rrset = rrset->rrset_all_next;
+			continue;
+		}
+
+		/* Otherwise, make sure that the RRset matches the qtype. */
+		if(qinfo->qtype != LDNS_RR_TYPE_ANY && 
+			qinfo->qtype != rrset->type) {
+			remove_rrset("normalize: removing irrelevant RRset:", 
+				pkt, msg, prev, &rrset);
+			continue;
+		}
+
+		/* Mark the additional names from relevant rrset as OK. */
+		mark_additional_rrset(pkt, msg, rrset);
+		
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+
+	/* Mark additional names from AUTHORITY */
+	while(rrset && rrset->section == LDNS_SECTION_AUTHORITY) {
+		mark_additional_rrset(pkt, msg, rrset);
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+
+	/* For each record in the additional section, remove it if it is an
+	 * address record and not in the collection of additional names 
+	 * found in ANSWER and AUTHORITY. */
+	/* These records have not been marked OK previously */
+	while(rrset && rrset->section == LDNS_SECTION_ADDITIONAL) {
+		/* FIXME: what about other types? */
+		if(rrset->type==LDNS_RR_TYPE_A || 
+			rrset->type==LDNS_RR_TYPE_AAAA) 
+		{
+			if((rrset->flags & RRSET_SCRUB_OK)) {
+				/* remove flag to clean up flags variable */
+				rrset->flags &= ~RRSET_SCRUB_OK;
+			} else {
+				remove_rrset("normalize: removing irrelevant "
+					"RRset:", pkt, msg, prev, &rrset);
+				continue;
+			}
+		}
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+	
+	return 1;
+}
+
+/**
+ * Store potential poison in the cache (only if hardening disabled).
+ * The rrset is stored in the cache but removed from the message.
+ * So that it will be used for infrastructure purposes, but not be 
+ * returned to the client.
+ * @param pkt: packet
+ * @param msg: message parsed
+ * @param env: environment with cache
+ * @param rrset: to store.
+ */
+static void
+store_rrset(ldns_buffer* pkt, struct msg_parse* msg, struct module_env* env,
+	struct rrset_parse* rrset)
+{
+	struct ub_packed_rrset_key* k;
+	struct packed_rrset_data* d;
+	struct rrset_ref ref;
+	uint32_t now = time(NULL);
+
+	k = alloc_special_obtain(env->alloc);
+	if(!k)
+		return;
+	k->entry.data = NULL;
+	if(!parse_copy_decompress_rrset(pkt, msg, rrset, NULL, k)) {
+		alloc_special_release(env->alloc, k);
+		return;
+	}
+	d = (struct packed_rrset_data*)k->entry.data;
+	packed_rrset_ttl_add(d, now);
+	ref.key = k;
+	ref.id = k->id;
+	/*ignore ret: it was in the cache, ref updated */
+	(void)rrset_cache_update(env->rrset_cache, &ref,
+		env->alloc, now);
+}
+
+/**
+ * Given a response event, remove suspect RRsets from the response.
+ * "Suspect" rrsets are potentially poison. Note that this routine expects
+ * the response to be in a "normalized" state -- that is, all "irrelevant"
+ * RRsets have already been removed, CNAMEs are in order, etc.
+ *
+ * @param pkt: packet.
+ * @param msg: msg to normalize.
+ * @param zonename: name of server zone.
+ * @param env: module environment with config and cache.
+ * @return 0 on error.
+ */
+static int
+scrub_sanitize(ldns_buffer* pkt, struct msg_parse* msg, uint8_t* zonename,
+	struct module_env* env)
+{
+	struct rrset_parse* rrset, *prev;
+	prev = NULL;
+	rrset = msg->rrset_first;
+
+	/* At this point, we brutally remove ALL rrsets that aren't 
+	 * children of the originating zone. The idea here is that, 
+	 * as far as we know, the server that we contacted is ONLY 
+	 * authoritative for the originating zone. It, of course, MAY 
+	 * be authoriative for any other zones, and of course, MAY 
+	 * NOT be authoritative for some subdomains of the originating 
+	 * zone. */
+	while(rrset) {
+		
+		/* skip DNAME records -- they will always be followed by a 
+		 * synthesized CNAME, which will be relevant.
+		 * FIXME: should this do something differently with DNAME 
+		 * rrsets NOT in Section.ANSWER? */
+		/* But since DNAME records are also subdomains of the zone,
+		 * same check can be used */
+
+		if(!pkt_sub(pkt, rrset->dname, zonename)) {
+			if(msg->an_rrsets == 0 && 
+				rrset->type == LDNS_RR_TYPE_NS && 
+				rrset->section == LDNS_SECTION_AUTHORITY &&
+				FLAGS_GET_RCODE(msg->flags) == 
+				LDNS_RCODE_NOERROR) {
+				/* noerror, nodata and this NS rrset is above
+				 * the zone. This is LAME! 
+				 * Leave in the NS for lame classification. */
+			} else if(!env->cfg->harden_glue) {
+				/* store in cache! Since it is relevant
+				 * (from normalize) it will be picked up 
+				 * from the cache to be used later */
+				store_rrset(pkt, msg, env, rrset);
+				remove_rrset("sanitize: storing potential "
+				"poison RRset:", pkt, msg, prev, &rrset);
+				continue;
+			} else {
+				remove_rrset("sanitize: removing potential "
+				"poison RRset:", pkt, msg, prev, &rrset);
+				continue;
+			}
+		}
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+	return 1;
+}
+
+int 
+scrub_message(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, uint8_t* zonename, struct regional* region,
+	struct module_env* env)
+{
+	/* basic sanity checks */
+	log_nametypeclass(VERB_ALGO, "scrub for", zonename, LDNS_RR_TYPE_NS, 
+		qinfo->qclass);
+	if(msg->qdcount > 1)
+		return 0;
+	if( !(msg->flags&BIT_QR) )
+		return 0;
+	
+	/* if a query is echoed back, make sure it is correct. Otherwise,
+	 * this may be not a reply to our query. */
+	if(msg->qdcount == 1) {
+		if(dname_pkt_compare(pkt, msg->qname, qinfo->qname) != 0)
+			return 0;
+		if(msg->qtype != qinfo->qtype || msg->qclass != qinfo->qclass)
+			return 0;
+	}
+
+	/* normalize the response, this cleans up the additional.  */
+	if(!scrub_normalize(pkt, msg, qinfo, region))
+		return 0;
+	/* delete all out-of-zone information */
+	if(!scrub_sanitize(pkt, msg, zonename, env))
+		return 0;
+	return 1;
+}

iterator/iter_scrub.h

@@ -0,0 +1,66 @@
+/*
+ * iterator/iter_scrub.h - scrubbing, normalization, sanitization of DNS msgs.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file has routine(s) for cleaning up incoming DNS messages from 
+ * possible useless or malicious junk in it.
+ */
+
+#ifndef ITERATOR_ITER_SCRUB_H
+#define ITERATOR_ITER_SCRUB_H
+struct msg_parse;
+struct query_info;
+struct regional;
+struct module_env;
+
+/**
+ * Cleanup the passed dns message.
+ * @param pkt: the packet itself, for resolving name compression pointers.
+ *	the packet buffer is unaltered.
+ * @param msg: the parsed packet, this structure is cleaned up.
+ * @param qinfo: the query info that was sent to the server. Checked.
+ * @param zonename: the name of the last delegation point.
+ *	Used to determine out of bailiwick information.
+ * @param regional: where to allocate (new) parts of the message.
+ * @param env: module environment with config settings and cache. 
+ * @return: false if the message is total waste. true if scrubbed with success.
+ */
+int scrub_message(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct query_info* qinfo, uint8_t* zonename, struct regional* regional,
+	struct module_env* env);
+
+#endif /* ITERATOR_ITER_SCRUB_H */

iterator/iter_utils.c

@@ -0,0 +1,455 @@
+/*
+ * iterator/iter_utils.c - iterative resolver module utility functions.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Configuration options. Forward zones. 
+ */
+#include "config.h"
+#include "iterator/iter_utils.h"
+#include "iterator/iterator.h"
+#include "iterator/iter_hints.h"
+#include "iterator/iter_fwd.h"
+#include "iterator/iter_donotq.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/infra.h"
+#include "services/cache/dns.h"
+#include "services/cache/rrset.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/regional.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/random.h"
+#include "util/fptr_wlist.h"
+#include "validator/val_anchor.h"
+
+/** fillup fetch policy array */
+static void
+fetch_fill(struct iter_env* ie, const char* str)
+{
+	char* s = (char*)str, *e;
+	int i;
+	for(i=0; i<ie->max_dependency_depth+1; i++) {
+		ie->target_fetch_policy[i] = strtol(s, &e, 10);
+		if(s == e)
+			fatal_exit("cannot parse fetch policy number %s", s);
+		s = e;
+	}
+}
+
+/** Read config string that represents the target fetch policy */
+static int
+read_fetch_policy(struct iter_env* ie, const char* str)
+{
+	int count = cfg_count_numbers(str);
+	if(count < 1) {
+		log_err("Cannot parse target fetch policy: \"%s\"", str);
+		return 0;
+	}
+	ie->max_dependency_depth = count - 1;
+	ie->target_fetch_policy = (int*)calloc(
+		(size_t)ie->max_dependency_depth+1, sizeof(int));
+	if(!ie->target_fetch_policy) {
+		log_err("alloc fetch policy: out of memory");
+		return 0;
+	}
+	fetch_fill(ie, str);
+	return 1;
+}
+
+int 
+iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg)
+{
+	int i;
+	/* target fetch policy */
+	if(!read_fetch_policy(iter_env, cfg->target_fetch_policy))
+		return 0;
+	for(i=0; i<iter_env->max_dependency_depth+1; i++)
+		verbose(VERB_DETAIL, "target fetch policy for level %d is %d",
+			i, iter_env->target_fetch_policy[i]);
+	
+	if(!iter_env->hints)
+		iter_env->hints = hints_create();
+	if(!iter_env->hints || !hints_apply_cfg(iter_env->hints, cfg)) {
+		log_err("Could not set root or stub hints");
+		return 0;
+	}
+	if(!iter_env->fwds)
+		iter_env->fwds = forwards_create();
+	if(!iter_env->fwds || !forwards_apply_cfg(iter_env->fwds, cfg)) {
+		log_err("Could not set forward zones");
+		return 0;
+	}
+	if(!iter_env->donotq)
+		iter_env->donotq = donotq_create();
+	if(!iter_env->donotq || !donotq_apply_cfg(iter_env->donotq, cfg)) {
+		log_err("Could not set donotqueryaddresses");
+		return 0;
+	}
+	iter_env->supports_ipv6 = cfg->do_ip6;
+	return 1;
+}
+
+/** filter out unsuitable targets, return rtt or -1 */
+static int
+iter_filter_unsuitable(struct iter_env* iter_env, struct module_env* env,
+	uint8_t* name, size_t namelen, time_t now, struct delegpt_addr* a)
+{
+	int rtt;
+	int lame;
+	int dnsseclame;
+	if(donotq_lookup(iter_env->donotq, &a->addr, a->addrlen)) {
+		return -1; /* server is on the donotquery list */
+	}
+	if(!iter_env->supports_ipv6 && addr_is_ip6(&a->addr, a->addrlen)) {
+		return -1; /* there is no ip6 available */
+	}
+	/* check lameness - need zone , class info */
+	if(infra_get_lame_rtt(env->infra_cache, &a->addr, a->addrlen, 
+		name, namelen, &lame, &dnsseclame, &rtt, now)) {
+		if(lame)
+			return -1; /* server is lame */
+		else if(rtt >= USEFUL_SERVER_TOP_TIMEOUT)
+			return -1; /* server is unresponsive */
+		else if(dnsseclame)
+			return rtt+USEFUL_SERVER_TOP_TIMEOUT; /* nonpref */
+		else	return rtt;
+	}
+	/* no server information present */
+	return UNKNOWN_SERVER_NICENESS;
+}
+
+/** filter the addres list, putting best targets at front,
+ * returns number of best targets (or 0, no suitable targets) */
+static int
+iter_filter_order(struct iter_env* iter_env, struct module_env* env,
+	uint8_t* name, size_t namelen, time_t now, struct delegpt* dp,
+	int* best_rtt)
+{
+	int got_num = 0, got_rtt = 0, thisrtt, swap_to_front;
+	struct delegpt_addr* a, *n, *prev=NULL;
+
+	a = dp->result_list;
+	while(a) {
+		/* filter out unsuitable targets */
+		thisrtt = iter_filter_unsuitable(iter_env, env, name, namelen, 
+			now, a);
+		if(thisrtt == -1) {
+			prev = a;
+			a = a->next_result;
+			continue;
+		}
+		/* classify the server address and determine what to do */
+		swap_to_front = 0;
+		if(got_num == 0) {
+			got_rtt = thisrtt;
+			got_num = 1;
+			swap_to_front = 1;
+		} else if(thisrtt == got_rtt) {
+			got_num++;
+			swap_to_front = 1;
+		} else if(thisrtt < got_rtt) {
+			got_rtt = thisrtt;
+			got_num = 1; /* start back at count of 1 */
+			swap_to_front = 1;
+		}
+		/* swap to front if necessary, or move to next result */
+		if(swap_to_front && prev) {
+			n = a->next_result;
+			prev->next_result = n;
+			a->next_result = dp->result_list;
+			dp->result_list = a;
+			a = n;
+		} else {
+			prev = a;
+			a = a->next_result;
+		}
+	}
+	*best_rtt = got_rtt;
+	return got_num;
+}
+
+struct delegpt_addr* 
+iter_server_selection(struct iter_env* iter_env, 
+	struct module_env* env, struct delegpt* dp, 
+	uint8_t* name, size_t namelen, int* dnssec_expected)
+{
+	time_t now = time(NULL);
+	int sel;
+	int selrtt;
+	struct delegpt_addr* a, *prev;
+	int num = iter_filter_order(iter_env, env, name, namelen, now, dp,
+		&selrtt);
+
+	if(num == 0)
+		return NULL;
+	if(selrtt >= USEFUL_SERVER_TOP_TIMEOUT)
+		*dnssec_expected = 0;
+	if(num == 1) {
+		a = dp->result_list;
+		if(++a->attempts < OUTBOUND_MSG_RETRY)
+			return a;
+		dp->result_list = a->next_result;
+		return a;
+	}
+	/* randomly select a target from the list */
+	log_assert(num > 1);
+	/* we do not need secure random numbers here, but
+	 * we do need it to be threadsafe, so we use this */
+	sel = ub_random(env->rnd) % num; 
+	a = dp->result_list;
+	prev = NULL;
+	while(sel > 0 && a) {
+		prev = a;
+		a = a->next_result;
+		sel--;
+	}
+	if(!a)  /* robustness */
+		return NULL;
+	if(++a->attempts < OUTBOUND_MSG_RETRY)
+		return a;
+	/* remove it from the delegation point result list */
+	if(prev)
+		prev->next_result = a->next_result;
+	else	dp->result_list = a->next_result;
+	return a;
+}
+
+struct dns_msg* 
+dns_alloc_msg(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct regional* region)
+{
+	struct dns_msg* m = (struct dns_msg*)regional_alloc(region,
+		sizeof(struct dns_msg));
+	if(!m)
+		return NULL;
+	memset(m, 0, sizeof(*m));
+	if(!parse_create_msg(pkt, msg, NULL, &m->qinfo, &m->rep, region)) {
+		log_err("malloc failure: allocating incoming dns_msg");
+		return NULL;
+	}
+	return m;
+}
+
+struct dns_msg* 
+dns_copy_msg(struct dns_msg* from, struct regional* region)
+{
+	struct dns_msg* m = (struct dns_msg*)regional_alloc(region,
+		sizeof(struct dns_msg));
+	if(!m)
+		return NULL;
+	m->qinfo = from->qinfo;
+	if(!(m->qinfo.qname = regional_alloc_init(region, from->qinfo.qname,
+		from->qinfo.qname_len)))
+		return NULL;
+	if(!(m->rep = reply_info_copy(from->rep, NULL, region)))
+		return NULL;
+	return m;
+}
+
+int 
+iter_dns_store(struct module_env* env, struct query_info* msgqinf,
+	struct reply_info* msgrep, int is_referral)
+{
+	return dns_cache_store(env, msgqinf, msgrep, is_referral);
+}
+
+int 
+iter_ns_probability(struct ub_randstate* rnd, int n, int m)
+{
+	int sel;
+	if(n == m) /* 100% chance */
+		return 1;
+	/* we do not need secure random numbers here, but
+	 * we do need it to be threadsafe, so we use this */
+	sel = ub_random(rnd) % m; 
+	return (sel < n);
+}
+
+/** detect dependency cycle for query and target */
+static int
+causes_cycle(struct module_qstate* qstate, uint8_t* name, size_t namelen,
+	uint16_t t, uint16_t c)
+{
+	struct query_info qinf;
+	qinf.qname = name;
+	qinf.qname_len = namelen;
+	qinf.qtype = t;
+	qinf.qclass = c;
+	log_assert(fptr_whitelist_modenv_detect_cycle(
+		qstate->env->detect_cycle));
+	return (*qstate->env->detect_cycle)(qstate, &qinf, 
+		(uint16_t)(BIT_RD|BIT_CD), qstate->is_priming);
+}
+
+void 
+iter_mark_cycle_targets(struct module_qstate* qstate, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved)
+			continue;
+		/* see if this ns as target causes dependency cycle */
+		if(causes_cycle(qstate, ns->name, ns->namelen, 
+			LDNS_RR_TYPE_AAAA, qstate->qinfo.qclass) ||
+		   causes_cycle(qstate, ns->name, ns->namelen, 
+			LDNS_RR_TYPE_A, qstate->qinfo.qclass)) {
+			log_nametypeclass(VERB_DETAIL, "skipping target due "
+			 	"to dependency cycle (harden-glue: no may "
+				"fix some of the cycles)", 
+				ns->name, LDNS_RR_TYPE_A, 
+				qstate->qinfo.qclass);
+			ns->resolved = 1;
+		}
+	}
+}
+
+int 
+iter_dp_is_useless(struct module_qstate* qstate, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	/* check:
+	 *      o all NS items are required glue.
+	 *      o no addresses are provided.
+	 *      o RD qflag is on.
+	 * OR
+	 *      o no addresses are provided.
+	 *      o RD qflag is on.
+	 *      o the query is for one of the nameservers in dp,
+	 *        and that nameserver is a glue-name for this dp.
+	 */
+	if(!(qstate->query_flags&BIT_RD))
+		return 0;
+	/* either available or unused targets */
+	if(dp->usable_list || dp->result_list) 
+		return 0;
+	
+	/* see if query is for one of the nameservers, which is glue */
+	if( (qstate->qinfo.qtype == LDNS_RR_TYPE_A ||
+		qstate->qinfo.qtype == LDNS_RR_TYPE_AAAA) &&
+		dname_subdomain_c(qstate->qinfo.qname, dp->name) &&
+		delegpt_find_ns(dp, qstate->qinfo.qname, 
+			qstate->qinfo.qname_len))
+		return 1;
+	
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved) /* skip failed targets */
+			continue;
+		if(!dname_subdomain_c(ns->name, dp->name))
+			return 0; /* one address is not required glue */
+	}
+	return 1;
+}
+
+int 
+iter_indicates_dnssec(struct module_env* env, struct delegpt* dp,
+        struct dns_msg* msg, uint16_t dclass)
+{
+	/* information not available, !env->anchors can be common */
+	if(!env || !env->anchors || !dp || !dp->name || !msg || !msg->rep)
+		return 0;
+	/* a trust anchor exists with this name, RRSIGs expected */
+	if(anchor_find(env->anchors, dp->name, dp->namelabs, dp->namelen,
+		dclass))
+		return 1;
+	/* see if DS rrset was given, in AUTH section */
+	if(reply_find_rrset_section_ns(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_DS, dclass))
+		return 1;
+	return 0;
+}
+
+int 
+iter_msg_has_dnssec(struct dns_msg* msg)
+{
+	size_t i;
+	if(!msg || !msg->rep)
+		return 0;
+	for(i=0; i<msg->rep->an_numrrsets + msg->rep->ns_numrrsets; i++) {
+		if(((struct packed_rrset_data*)msg->rep->rrsets[i]->
+			entry.data)->rrsig_count > 0)
+			return 1;
+	}
+	/* empty message has no DNSSEC info, with DNSSEC the reply is
+	 * not empty (NSEC) */
+	return 0;
+}
+
+int iter_msg_from_zone(struct dns_msg* msg, struct delegpt* dp,
+        enum response_type type, uint16_t dclass)
+{
+	if(!msg || !dp || !msg->rep || !dp->name)
+		return 0;
+	/* SOA RRset - always from reply zone */
+	if(reply_find_rrset_section_an(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_SOA, dclass) ||
+	   reply_find_rrset_section_ns(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_SOA, dclass))
+		return 1;
+	if(type == RESPONSE_TYPE_REFERRAL) {
+		size_t i;
+		/* if it adds a single label, i.e. we expect .com,
+		 * and referral to example.com. NS ... , then origin zone
+		 * is .com. For a referral to sub.example.com. NS ... then
+		 * we do not know, since example.com. may be in between. */
+		for(i=0; i<msg->rep->an_numrrsets+msg->rep->ns_numrrsets; 
+			i++) {
+			struct ub_packed_rrset_key* s = msg->rep->rrsets[i];
+			if(ntohs(s->rk.type) == LDNS_RR_TYPE_NS &&
+				ntohs(s->rk.rrset_class) == dclass) {
+				int l = dname_count_labels(s->rk.dname);
+				if(l == dp->namelabs + 1 &&
+					dname_strict_subdomain(s->rk.dname,
+					l, dp->name, dp->namelabs))
+					return 1;
+			}
+		}
+		return 0;
+	}
+	log_assert(type==RESPONSE_TYPE_ANSWER || type==RESPONSE_TYPE_CNAME);
+	/* not a referral, and not lame delegation (upwards), so, 
+	 * any NS rrset must be from the zone itself */
+	if(reply_find_rrset_section_an(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_NS, dclass) ||
+	   reply_find_rrset_section_ns(msg->rep, dp->name, dp->namelen,
+		LDNS_RR_TYPE_NS, dclass))
+		return 1;
+	return 0;
+}

iterator/iter_utils.h

@@ -0,0 +1,180 @@
+/*
+ * iterator/iter_utils.h - iterative resolver module utility functions.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist the iterator module.
+ * Configuration options. Forward zones. 
+ */
+
+#ifndef ITERATOR_ITER_UTILS_H
+#define ITERATOR_ITER_UTILS_H
+#include "iterator/iter_resptype.h"
+struct iter_env;
+struct config_file;
+struct module_env;
+struct delegpt_addr;
+struct delegpt;
+struct regional;
+struct msg_parse;
+struct ub_randstate;
+struct query_info;
+struct reply_info;
+struct module_qstate;
+
+/**
+ * Process config options and set iterator module state.
+ * Sets default values if no config is found.
+ * @param iter_env: iterator module state.
+ * @param cfg: config options.
+ * @return 0 on error.
+ */
+int iter_apply_cfg(struct iter_env* iter_env, struct config_file* cfg);
+
+/**
+ * Select a valid, nice target to send query to.
+ * Sorting and removing unsuitable targets is combined.
+ *
+ * @param iter_env: iterator module global state, with ip6 enabled and 
+ *	do-not-query-addresses.
+ * @param env: environment with infra cache (lameness, rtt info).
+ * @param dp: delegation point with result list.
+ * @param name: zone name (for lameness check).
+ * @param namelen: length of name.
+ * @param dnssec_expected: set to 0, if a known dnssec-lame server is selected
+ *	these are not preferred, but are used as a last resort.
+ * @return best target or NULL if no target.
+ *	if not null, that target is removed from the result list in the dp.
+ */
+struct delegpt_addr* iter_server_selection(struct iter_env* iter_env, 
+	struct module_env* env, struct delegpt* dp, uint8_t* name, 
+	size_t namelen, int* dnssec_expected);
+
+/**
+ * Allocate dns_msg from parsed msg, in regional.
+ * @param pkt: packet.
+ * @param msg: parsed message (cleaned and ready for regional allocation).
+ * @param regional: regional to use for allocation.
+ * @return newly allocated dns_msg, or NULL on memory error.
+ */
+struct dns_msg* dns_alloc_msg(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct regional* regional);
+
+/**
+ * Copy a dns_msg to this regional.
+ * @param from: dns message, also in regional.
+ * @param regional: regional to use for allocation.
+ * @return newly allocated dns_msg, or NULL on memory error.
+ */
+struct dns_msg* dns_copy_msg(struct dns_msg* from, struct regional* regional);
+
+/**
+ * Allocate a dns_msg with malloc/alloc structure and store in dns cache.
+ * @param env: environment, with alloc structure and dns cache.
+ * @param qinf: query info, the query for which answer is stored.
+ * @param rep: reply in dns_msg from dns_alloc_msg for example.
+ * @param is_referral: If true, then the given message to be stored is a
+ *	referral. The cache implementation may use this as a hint.
+ * @return 0 on alloc error (out of memory).
+ */
+int iter_dns_store(struct module_env* env, struct query_info* qinf,
+	struct reply_info* rep, int is_referral);
+
+/**
+ * Select randomly with n/m probability.
+ * For shuffle NS records for address fetching.
+ * @param rnd: random table
+ * @param n: probability.
+ * @param m: divisor for probability.
+ * @return true with n/m probability.
+ */
+int iter_ns_probability(struct ub_randstate* rnd, int n, int m);
+
+/**
+ * Mark targets that result in a dependency cycle as done, so they
+ * will not get selected as targets.
+ * @param qstate: query state.
+ * @param dp: delegpt to mark ns in.
+ */
+void iter_mark_cycle_targets(struct module_qstate* qstate, struct delegpt* dp);
+
+/**
+ * See if delegation is useful or offers immediately no targets for 
+ * further recursion.
+ * @param qstate: query state with RD flag and query name.
+ * @param dp: delegpt to check.
+ */
+int iter_dp_is_useless(struct module_qstate* qstate, struct delegpt* dp);
+
+/**
+ * See if delegation is expected to have DNSSEC information (RRSIGs) in 
+ * its answers, or not. Inspects delegation point (name), trust anchors,
+ * and delegation message (DS RRset) to determine this.
+ * @param env: module env with trust anchors.
+ * @param dp: delegation point.
+ * @param msg: delegation message, with DS if a secure referral.
+ * @param dclass: class of query.
+ * @return 1 if dnssec is expected, 0 if not.
+ */
+int iter_indicates_dnssec(struct module_env* env, struct delegpt* dp,
+	struct dns_msg* msg, uint16_t dclass);
+
+/**
+ * See if a message contains DNSSEC.
+ * This is examined by looking for RRSIGs. With DNSSEC a valid answer, 
+ * nxdomain, nodata, referral or cname reply has RRSIGs in answer or auth 
+ * sections, sigs on answer data, SOA, DS, or NSEC/NSEC3 records.
+ * @param msg: message to examine.
+ * @return true if DNSSEC information was found.
+ */
+int iter_msg_has_dnssec(struct dns_msg* msg);
+
+/**
+ * See if a message is known to be from a certain zone.
+ * This looks for SOA or NS rrsets, for answers.
+ * For referrals, when one label is delegated, the zone is detected.
+ * Does not look at signatures.
+ * @param msg: the message to inspect.
+ * @param dp: delegation point with zone name to look for.
+ * @param type: type of message.
+ * @param dclass: class of query.
+ * @return true if message is certain to be from zone in dp->name.
+ *	false if not sure (empty msg), or not from the zone.
+ */
+int iter_msg_from_zone(struct dns_msg* msg, struct delegpt* dp, 
+	enum response_type type, uint16_t dclass);
+
+#endif /* ITERATOR_ITER_UTILS_H */

iterator/iterator.h

@@ -0,0 +1,314 @@
+/*
+ * iterator/iterator.h - iterative resolver DNS query response module
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains a module that performs recusive iterative DNS query
+ * processing.
+ */
+
+#ifndef ITERATOR_ITERATOR_H
+#define ITERATOR_ITERATOR_H
+#include "services/outbound_list.h"
+#include "util/data/msgreply.h"
+#include "util/module.h"
+struct delegpt;
+struct iter_hints;
+struct iter_forwards;
+struct iter_donotq;
+struct iter_prep_list;
+
+/** max number of query restarts. Determines max number of CNAME chain. */
+#define MAX_RESTART_COUNT       8
+/** max number of referrals. Makes sure resolver does not run away */
+#define MAX_REFERRAL_COUNT	30
+/** how nice is a server without further information, in msec 
+ * Equals rtt initial timeout value.
+ */
+#define UNKNOWN_SERVER_NICENESS 376
+/** maximum timeout before a host is deemed unsuitable, in msec. 
+ * After host_ttl this will be timed out and the host will be tried again. 
+ * Equals RTT_MAX_TIMEOUT
+ */
+#define USEFUL_SERVER_TOP_TIMEOUT	120000
+/** number of retries on outgoing queries */
+#define OUTBOUND_MSG_RETRY 4
+
+/**
+ * Global state for the iterator. 
+ */
+struct iter_env {
+	/** 
+	 * The hints -- these aren't stored in the cache because they don't 
+	 * expire. The hints are always used to "prime" the cache. Note 
+	 * that both root hints and stub zone "hints" are stored in this 
+	 * data structure.
+	 */
+	struct iter_hints* hints;
+
+	/** A flag to indicate whether or not we have an IPv6 route */
+	int supports_ipv6;
+
+	/** Mapping of forwarding zones to targets. */
+	struct iter_forwards* fwds;
+
+	/** A set of inetaddrs that should never be queried. */
+	struct iter_donotq* donotq;
+
+	/** The maximum dependency depth that this resolver will pursue. */
+	int max_dependency_depth;
+
+	/**
+	 * The target fetch policy for each dependency level. This is 
+	 * described as a simple number (per dependency level): 
+	 *	negative numbers (usually just -1) mean fetch-all, 
+	 *	0 means only fetch on demand, and 
+	 *	positive numbers mean to fetch at most that many targets.
+	 * array of max_dependency_depth+1 size.
+	 */
+	int* target_fetch_policy;
+};
+
+/**
+ * State of the iterator for a query.
+ */
+enum iter_state {
+	/**
+	 * Externally generated queries start at this state. Query restarts are
+	 * reset to this state.
+	 */
+	INIT_REQUEST_STATE = 0,
+
+	/**
+	 * Root priming events reactivate here, most other events pass 
+	 * through this naturally as the 2nd part of the INIT_REQUEST_STATE.
+	 */
+	INIT_REQUEST_2_STATE,
+
+	/**
+	 * Stub priming events reactivate here, most other events pass 
+	 * through this naturally as the 3rd part of the INIT_REQUEST_STATE.
+	 */
+	INIT_REQUEST_3_STATE,
+
+	/**
+	 * Each time a delegation point changes for a given query or a 
+	 * query times out and/or wakes up, this state is (re)visited. 
+	 * This state is reponsible for iterating through a list of 
+	 * nameserver targets.
+	 */
+	QUERYTARGETS_STATE,
+
+	/**
+	 * Responses to queries start at this state. This state handles 
+	 * the decision tree associated with handling responses.
+	 */
+	QUERY_RESP_STATE,
+
+	/** Responses to priming queries finish at this state. */
+	PRIME_RESP_STATE,
+
+	/** Responses that are to be returned upstream end at this state. 
+	 * As well as responses to target queries. */
+	FINISHED_STATE
+};
+
+/**
+ * Per query state for the iterator module.
+ */
+struct iter_qstate {
+	/** 
+	 * State of the iterator module.
+	 * This is the state that event is in or should sent to -- all 
+	 * requests should start with the INIT_REQUEST_STATE. All 
+	 * responses should start with QUERY_RESP_STATE. Subsequent 
+	 * processing of the event will change this state.
+	 */
+	enum iter_state state;
+
+	/** 
+	 * Final state for the iterator module.
+	 * This is the state that responses should be routed to once the 
+	 * response is final. For externally initiated queries, this 
+	 * will be FINISHED_STATE, locally initiated queries will have 
+	 * different final states.
+	 */
+	enum iter_state final_state;
+
+	/** 
+	 * The depth of this query, this means the depth of recursion.
+	 * This address is needed for another query, which is an address
+	 * needed for another query, etc. Original client query has depth 0.
+	 */
+	int depth;
+
+	/**
+	 * The response
+	 */
+	struct dns_msg* response;
+
+	/** 
+	 * This is a list of RRsets that must be prepended to the 
+	 * ANSWER section of a response before being sent upstream.
+	 */
+	struct iter_prep_list* an_prepend_list;
+	/** Last element of the prepend list */
+	struct iter_prep_list* an_prepend_last;
+
+	/**
+	 * This is the list of RRsets that must be prepended to the
+	 * AUTHORITY section of the response before being sent upstream.
+	 */
+	struct iter_prep_list* ns_prepend_list;
+	/** Last element of the authority prepend list */
+	struct iter_prep_list* ns_prepend_last;
+
+	/** query name used for chasing the results. Initially the same as
+	 * the state qinfo, but after CNAMEs this will be different. 
+	 * The query info used to elicit the results needed. */
+	struct query_info qchase;
+	/** query flags to use when chasing the answer (i.e. RD flag) */
+	uint16_t chase_flags;
+
+	/** 
+	 * This is the current delegation point for an in-progress query. This
+	 * object retains state as to which delegation targets need to be
+	 * (sub)queried for vs which ones have already been visited.
+	 */
+	struct delegpt* dp;
+
+	/** Current delegation message - returned for non-RD queries */
+	struct dns_msg* deleg_msg;
+
+	/** number of outstanding target sub queries */
+	int num_target_queries;
+
+	/** outstanding direct queries */
+	int num_current_queries;
+
+	/** the number of times this query has been restarted. */
+	int query_restart_count;
+
+	/** the number of times this query as followed a referral. */
+	int referral_count;
+
+	/** 
+	 * expected dnssec information for this iteration step. 
+	 * If dnssec rrsigs are expected and not given, the server is marked
+	 * lame (dnssec-lame).
+	 */
+	int dnssec_expected;
+
+	/**
+	 * This is flag that, if true, means that this event is 
+	 * waiting for a stub priming query. 
+	 */
+	int wait_priming_stub;
+
+	/**
+	 * This is a flag that, if true, means that this query is
+	 * for (re)fetching glue from a zone. Since the address should
+	 * have been glue, query again to the servers that should have
+	 * been returning it as glue.
+	 * The delegation point must be set to the one that should *not*
+	 * be used when creating the state. A higher one will be attempted.
+	 */
+	int refetch_glue;
+
+	/** list of pending queries to authoritative servers. */
+	struct outbound_list outlist;
+};
+
+/**
+ * List of prepend items
+ */
+struct iter_prep_list {
+	/** next in list */
+	struct iter_prep_list* next;
+	/** rrset */
+	struct ub_packed_rrset_key* rrset;
+};
+
+/**
+ * Get the iterator function block.
+ * @return: function block with function pointers to iterator methods.
+ */
+struct module_func_block* iter_get_funcblock();
+
+/**
+ * Get iterator state as a string
+ * @param state: to convert
+ * @return constant string that is printable.
+ */
+const char* iter_state_to_string(enum iter_state state);
+
+/**
+ * See if iterator state is a response state
+ * @param s: to inspect
+ * @return true if response state.
+ */
+int iter_state_is_responsestate(enum iter_state s);
+
+/** iterator init */
+int iter_init(struct module_env* env, int id);
+
+/** iterator deinit */
+void iter_deinit(struct module_env* env, int id);
+
+/** iterator operate on a query */
+void iter_operate(struct module_qstate* qstate, enum module_ev event, int id,
+	struct outbound_entry* outbound);
+
+/**
+ * Return priming query results to interestes super querystates.
+ * 
+ * Sets the delegation point and delegation message (not nonRD queries).
+ * This is a callback from walk_supers.
+ *
+ * @param qstate: query state that finished.
+ * @param id: module id.
+ * @param super: the qstate to inform.
+ */
+void iter_inform_super(struct module_qstate* qstate, int id, 
+	struct module_qstate* super);
+
+/** iterator cleanup query state */
+void iter_clear(struct module_qstate* qstate, int id);
+
+/** iterator alloc size routine */
+size_t iter_get_mem(struct module_env* env, int id);
+
+#endif /* ITERATOR_ITERATOR_H */

makedist.sh

@@ -134,7 +134,7 @@ done
 # Check if SVNROOT is specified.
 if [ -z "$SVNROOT" ]; then
     if test -f .svn/entries; then
-	  eval `grep 'url=' .svn/entries | head -1`
+	  eval `svn info | grep 'URL:' | sed -e 's/URL: /url=/' | head -1`
 	  SVNROOT="$url"
     fi
     if test -z "$SVNROOT"; then
@@ -148,15 +148,11 @@ if test -z "$LDNSDIR"; then
 	  eval `grep 'ldnsdir=' Makefile`
 	  LDNSDIR="$ldnsdir"
     fi
-    if test -z "$LDNSDIR"; then
-	error "LDNSDIR not detected in Makefile, specify manually (using -l)"
-    fi
 fi
 
 # Start the packaging process.
 info "SVNROOT  is $SVNROOT"
 info "SNAPSHOT is $SNAPSHOT"
-info "LDNSDIR  is $LDNSDIR"
 
 #question "Do you wish to continue with these settings?" || error "User abort."
 
@@ -185,9 +181,15 @@ echo "#include \"util/configyyrename.h\"" > util/configlexer.c || error_cleanup
 flex -i -t util/configlexer.lex >> util/configlexer.c  || error_cleanup "Failed to create configlexer"
 bison -y -d -o util/configparser.c util/configparser.y || error_cleanup "Failed to create configparser"
 
-# copy ldns-testpkts from ldns examples
-#cp $LDNSDIR/examples/ldns-testpkts.c testcode/ldns-testpkts.c || error_cleanup "copy ldns/examples/.. failed"
-#cp $LDNSDIR/examples/ldns-testpkts.h testcode/ldns-testpkts.h || error_cleanup "copy ldns/examples/.. failed"
+# check shared code, ldns-testpkts from ldns examples, if possible.
+if test ! -z "$LDNSDIR"; then
+	if diff -q $LDNSDIR/examples/ldns-testpkts.c testcode/ldns-testpkts.c &&
+	   diff -q $LDNSDIR/examples/ldns-testpkts.h testcode/ldns-testpkts.h; then
+	   	info "ldns-testpkts.c and ldns-testpkts.h are OK"
+	else
+		error_cleanup "ldns-testpkts is different in ldns and unbound"
+	fi
+fi
 
 find . -name .c-mode-rc.el -exec rm {} \;
 find . -name .cvsignore -exec rm {} \;

services/cache/dns.c

@@ -0,0 +1,669 @@
+/*
+ * services/cache/dns.c - Cache services for DNS using msg and rrset caches.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the DNS cache.
+ */
+#include "config.h"
+#include "iterator/iter_delegpt.h"
+#include "services/cache/dns.h"
+#include "services/cache/rrset.h"
+#include "util/data/msgreply.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/dname.h"
+#include "util/module.h"
+#include "util/net_help.h"
+#include "util/regional.h"
+
+/** store rrsets in the rrset cache. 
+ * @param env: module environment with caches.
+ * @param rep: contains list of rrsets to store.
+ * @param now: current time(NULL).
+ */
+static void
+store_rrsets(struct module_env* env, struct reply_info* rep, uint32_t now)
+{
+        size_t i;
+        /* see if rrset already exists in cache, if not insert it. */
+        for(i=0; i<rep->rrset_count; i++) {
+                rep->ref[i].key = rep->rrsets[i];
+                rep->ref[i].id = rep->rrsets[i]->id;
+                if(rrset_cache_update(env->rrset_cache, &rep->ref[i],
+                        env->alloc, now)) /* it was in the cache */
+                        rep->rrsets[i] = rep->ref[i].key;
+        }
+}
+
+void 
+dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
+	hashvalue_t hash, struct reply_info* rep)
+{
+	struct msgreply_entry* e;
+	uint32_t now = time(NULL), ttl = rep->ttl;
+	size_t i;
+
+	/* store RRsets */
+        for(i=0; i<rep->rrset_count; i++) {
+		rep->ref[i].key = rep->rrsets[i];
+		rep->ref[i].id = rep->rrsets[i]->id;
+	}
+	reply_info_sortref(rep);
+	reply_info_set_ttls(rep, now);
+	store_rrsets(env, rep, now);
+	if(ttl == 0) {
+		/* we do not store the message, but we did store the RRs,
+		 * which could be useful for delegation information */
+		verbose(VERB_ALGO, "TTL 0: dropped msg from cache");
+		free(rep);
+		return;
+	}
+
+	/* store msg in the cache */
+	reply_info_sortref(rep);
+	if(!(e = query_info_entrysetup(qinfo, rep, hash))) {
+		log_err("store_msg: malloc failed");
+		return;
+	}
+	slabhash_insert(env->msg_cache, hash, &e->entry, rep, env->alloc);
+}
+
+/** allocate rrset in region - no more locks needed */
+static struct ub_packed_rrset_key*
+copy_rrset(struct ub_packed_rrset_key* key, struct regional* region, 
+	uint32_t now)
+{
+	struct ub_packed_rrset_key* ck = regional_alloc(region, 
+		sizeof(struct ub_packed_rrset_key));
+	struct packed_rrset_data* d;
+	struct packed_rrset_data* data = (struct packed_rrset_data*)
+		key->entry.data;
+	size_t dsize, i;
+	if(!ck)
+		return NULL;
+	ck->id = key->id;
+	memset(&ck->entry, 0, sizeof(ck->entry));
+	ck->entry.hash = key->entry.hash;
+	ck->entry.key = ck;
+	ck->rk = key->rk;
+	ck->rk.dname = regional_alloc_init(region, key->rk.dname, 
+		key->rk.dname_len);
+	if(!ck->rk.dname)
+		return NULL;
+	dsize = packed_rrset_sizeof(data);
+	d = (struct packed_rrset_data*)regional_alloc_init(region, data, dsize);
+	if(!d)
+		return NULL;
+	ck->entry.data = d;
+	packed_rrset_ptr_fixup(d);
+	/* make TTLs relative - once per rrset */
+	for(i=0; i<d->count + d->rrsig_count; i++)
+		d->rr_ttl[i] -= now;
+	d->ttl -= now;
+	return ck;
+}
+
+/** find closest NS or DNAME and returns the rrset (locked) */
+static struct ub_packed_rrset_key*
+find_closest_of_type(struct module_env* env, uint8_t* qname, size_t qnamelen, 
+	uint16_t qclass, uint32_t now, uint16_t searchtype, int stripfront)
+{
+	struct ub_packed_rrset_key *rrset;
+	uint8_t lablen;
+
+	if(stripfront) {
+		/* strip off so that DNAMEs have strict subdomain match */
+		lablen = *qname;
+		qname += lablen + 1;
+		qnamelen -= lablen + 1;
+	}
+
+	/* snip off front part of qname until the type is found */
+	while(qnamelen > 0) {
+		if((rrset = rrset_cache_lookup(env->rrset_cache, qname, 
+			qnamelen, searchtype, qclass, 0, now, 0)))
+			return rrset;
+
+		/* snip off front label */
+		lablen = *qname;
+		qname += lablen + 1;
+		qnamelen -= lablen + 1;
+	}
+	return NULL;
+}
+
+/** add addr to additional section */
+static void
+addr_to_additional(struct ub_packed_rrset_key* rrset, struct regional* region,
+	struct dns_msg* msg, uint32_t now)
+{
+	if((msg->rep->rrsets[msg->rep->rrset_count] = 
+		copy_rrset(rrset, region, now))) {
+		msg->rep->ar_numrrsets++;
+		msg->rep->rrset_count++;
+	}
+}
+
+/** find and add A and AAAA records for nameservers in delegpt */
+static int
+find_add_addrs(struct module_env* env, uint16_t qclass, 
+	struct regional* region, struct delegpt* dp, uint32_t now, 
+	struct dns_msg** msg)
+{
+	struct delegpt_ns* ns;
+	struct ub_packed_rrset_key* akey;
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_A, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_A(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			if(msg)
+				addr_to_additional(akey, region, *msg, now);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_AAAA(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			if(msg)
+				addr_to_additional(akey, region, *msg, now);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+	}
+	return 1;
+}
+
+/** find and add A and AAAA records for missing nameservers in delegpt */
+int
+cache_fill_missing(struct module_env* env, uint16_t qclass, 
+	struct regional* region, struct delegpt* dp)
+{
+	struct delegpt_ns* ns;
+	struct ub_packed_rrset_key* akey;
+	uint32_t now = time(NULL);
+	for(ns = dp->nslist; ns; ns = ns->next) {
+		if(ns->resolved)
+			continue;
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_A, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_A(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			log_nametypeclass(VERB_ALGO, "found in cache",
+				ns->name, LDNS_RR_TYPE_A, qclass);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+		akey = rrset_cache_lookup(env->rrset_cache, ns->name, 
+			ns->namelen, LDNS_RR_TYPE_AAAA, qclass, 0, now, 0);
+		if(akey) {
+			if(!delegpt_add_rrset_AAAA(dp, region, akey)) {
+				lock_rw_unlock(&akey->entry.lock);
+				return 0;
+			}
+			log_nametypeclass(VERB_ALGO, "found in cache",
+				ns->name, LDNS_RR_TYPE_AAAA, qclass);
+			lock_rw_unlock(&akey->entry.lock);
+		}
+	}
+	return 1;
+}
+
+/** find and add DS or NSEC to delegation msg */
+static void
+find_add_ds(struct module_env* env, struct regional* region, 
+	struct dns_msg* msg, struct delegpt* dp, uint32_t now)
+{
+	/* Lookup the DS or NSEC at the delegation point. */
+	struct ub_packed_rrset_key* rrset = rrset_cache_lookup(
+		env->rrset_cache, dp->name, dp->namelen, LDNS_RR_TYPE_DS, 
+		msg->qinfo.qclass, 0, now, 0);
+	if(!rrset) {
+		/* NOTE: this won't work for alternate NSEC schemes 
+		 *	(opt-in, NSEC3) */
+		rrset = rrset_cache_lookup(env->rrset_cache, dp->name, 
+			dp->namelen, LDNS_RR_TYPE_NSEC, msg->qinfo.qclass, 
+			0, now, 0);
+		/* Note: the PACKED_RRSET_NSEC_AT_APEX flag is not used.
+		 * since this is a referral, we need the NSEC at the parent
+		 * side of the zone cut, not the NSEC at apex side. */
+	}
+	if(rrset) {
+		/* add it to auth section. This is the second rrset. */
+		if((msg->rep->rrsets[msg->rep->rrset_count] = 
+			copy_rrset(rrset, region, now))) {
+			msg->rep->ns_numrrsets++;
+			msg->rep->rrset_count++;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+}
+
+/** create referral message with NS and query */
+static struct dns_msg*
+create_msg(uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct regional* region, struct ub_packed_rrset_key* nskey, 
+	struct packed_rrset_data* nsdata, uint32_t now)
+{
+	struct dns_msg* msg = (struct dns_msg*)regional_alloc(region,
+		sizeof(struct dns_msg));
+	if(!msg)
+		return NULL;
+	msg->qinfo.qname = regional_alloc_init(region, qname, qnamelen);
+	if(!msg->qinfo.qname)
+		return NULL;
+	msg->qinfo.qname_len = qnamelen;
+	msg->qinfo.qtype = qtype;
+	msg->qinfo.qclass = qclass;
+	/* non-packed reply_info, because it needs to grow the array */
+	msg->rep = (struct reply_info*)regional_alloc(region, 
+		sizeof(struct reply_info)-sizeof(struct rrset_ref));
+	if(!msg->rep)
+		return NULL;
+	memset(msg->rep, 0, 
+		sizeof(struct reply_info)-sizeof(struct rrset_ref));
+	msg->rep->flags = BIT_QR; /* with QR, no AA */
+	msg->rep->qdcount = 1;
+	/* allocate the array to as much as we could need:
+	 *	NS rrset + DS/NSEC rrset +
+	 *	A rrset for every NS RR
+	 *	AAAA rrset for every NS RR
+	 */
+	msg->rep->rrsets = (struct ub_packed_rrset_key**)
+		regional_alloc(region, 
+		(2 + nsdata->count*2)*sizeof(struct ub_packed_rrset_key*));
+	if(!msg->rep->rrsets)
+		return NULL;
+	msg->rep->rrsets[0] = copy_rrset(nskey, region, now);
+	if(!msg->rep->rrsets[0])
+		return NULL;
+	msg->rep->ns_numrrsets++;
+	msg->rep->rrset_count++;
+	return msg;
+}
+
+struct delegpt* 
+dns_cache_find_delegation(struct module_env* env, uint8_t* qname, 
+	size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct regional* region, struct dns_msg** msg, uint32_t now)
+{
+	/* try to find closest NS rrset */
+	struct ub_packed_rrset_key* nskey;
+	struct packed_rrset_data* nsdata;
+	struct delegpt* dp;
+
+	nskey = find_closest_of_type(env, qname, qnamelen, qclass, now,
+		LDNS_RR_TYPE_NS, 0);
+	if(!nskey) /* hope the caller has hints to prime or something */
+		return NULL;
+	nsdata = (struct packed_rrset_data*)nskey->entry.data;
+	/* got the NS key, create delegation point */
+	dp = delegpt_create(region);
+	if(!dp || !delegpt_set_name(dp, region, nskey->rk.dname)) {
+		lock_rw_unlock(&nskey->entry.lock);
+		log_err("find_delegation: out of memory");
+		return NULL;
+	}
+	/* create referral message */
+	if(msg) {
+		*msg = create_msg(qname, qnamelen, qtype, qclass, region, 
+			nskey, nsdata, now);
+		if(!*msg) {
+			lock_rw_unlock(&nskey->entry.lock);
+			log_err("find_delegation: out of memory");
+			return NULL;
+		}
+	}
+	if(!delegpt_rrset_add_ns(dp, region, nskey))
+		log_err("find_delegation: addns out of memory");
+	lock_rw_unlock(&nskey->entry.lock); /* first unlock before next lookup*/
+	/* find and add DS/NSEC (if any) */
+	if(msg)
+		find_add_ds(env, region, *msg, dp, now);
+	/* find and add A entries */
+	if(!find_add_addrs(env, qclass, region, dp, now, msg))
+		log_err("find_delegation: addrs out of memory");
+	return dp;
+}
+
+/** allocate dns_msg from query_info and reply_info */
+static struct dns_msg*
+gen_dns_msg(struct regional* region, struct query_info* q, size_t num)
+{
+	struct dns_msg* msg = (struct dns_msg*)regional_alloc(region, 
+		sizeof(struct dns_msg));
+	if(!msg)
+		return NULL;
+	memcpy(&msg->qinfo, q, sizeof(struct query_info));
+	msg->qinfo.qname = regional_alloc_init(region, q->qname, q->qname_len);
+	if(!msg->qinfo.qname)
+		return NULL;
+	/* allocate replyinfo struct and rrset key array separately */
+	msg->rep = (struct reply_info*)regional_alloc(region,
+		sizeof(struct reply_info) - sizeof(struct rrset_ref));
+	if(!msg->rep)
+		return NULL;
+	msg->rep->rrsets = (struct ub_packed_rrset_key**)
+		regional_alloc(region,
+		num * sizeof(struct ub_packed_rrset_key*));
+	if(!msg->rep->rrsets)
+		return NULL;
+	return msg;
+}
+
+/** generate dns_msg from cached message */
+static struct dns_msg*
+tomsg(struct module_env* env, struct msgreply_entry* e, struct reply_info* r, 
+	struct regional* region, uint32_t now, struct regional* scratch)
+{
+	struct dns_msg* msg;
+	size_t i;
+	if(now > r->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, &e->key, r->rrset_count);
+	if(!msg)
+		return NULL;
+	msg->rep->flags = r->flags;
+	msg->rep->qdcount = r->qdcount;
+	msg->rep->ttl = r->ttl;
+	msg->rep->security = r->security;
+	msg->rep->an_numrrsets = r->an_numrrsets;
+	msg->rep->ns_numrrsets = r->ns_numrrsets;
+	msg->rep->ar_numrrsets = r->ar_numrrsets;
+	msg->rep->rrset_count = r->rrset_count;
+	if(!rrset_array_lock(r->ref, r->rrset_count, now))
+		return NULL;
+	for(i=0; i<msg->rep->rrset_count; i++) {
+		msg->rep->rrsets[i] = copy_rrset(r->rrsets[i], region, now);
+		if(!msg->rep->rrsets[i]) {
+			rrset_array_unlock(r->ref, r->rrset_count);
+			return NULL;
+		}
+	}
+	rrset_array_unlock_touch(env->rrset_cache, scratch, r->ref, 
+		r->rrset_count);
+	return msg;
+}
+
+/** synthesize RRset-only response from cached RRset item */
+static struct dns_msg*
+rrset_msg(struct ub_packed_rrset_key* rrset, struct regional* region, 
+	uint32_t now, struct query_info* q)
+{
+	struct dns_msg* msg;
+	struct packed_rrset_data* d = (struct packed_rrset_data*)
+		rrset->entry.data;
+	if(now > d->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, q, 1); /* only the CNAME (or other) RRset */
+	if(!msg)
+		return NULL;
+	msg->rep->flags = BIT_QR; /* reply, no AA, no error */
+	msg->rep->qdcount = 1;
+	msg->rep->ttl = d->ttl - now;
+	msg->rep->security = sec_status_unchecked;
+	msg->rep->an_numrrsets = 1;
+	msg->rep->ns_numrrsets = 0;
+	msg->rep->ar_numrrsets = 0;
+	msg->rep->rrset_count = 1;
+	msg->rep->rrsets[0] = copy_rrset(rrset, region, now);
+	if(!msg->rep->rrsets[0]) /* copy CNAME */
+		return NULL;
+	return msg;
+}
+
+/** synthesize DNAME+CNAME response from cached DNAME item */
+static struct dns_msg*
+synth_dname_msg(struct ub_packed_rrset_key* rrset, struct regional* region, 
+	uint32_t now, struct query_info* q)
+{
+	struct dns_msg* msg;
+	struct ub_packed_rrset_key* ck;
+	struct packed_rrset_data* newd, *d = (struct packed_rrset_data*)
+		rrset->entry.data;
+	uint8_t* newname, *dtarg = NULL;
+	size_t newlen, dtarglen;
+	if(now > d->ttl)
+		return NULL;
+	msg = gen_dns_msg(region, q, 2); /* DNAME + CNAME RRset */
+	if(!msg)
+		return NULL;
+	msg->rep->flags = BIT_QR; /* reply, no AA, no error */
+	msg->rep->qdcount = 1;
+	msg->rep->ttl = d->ttl - now;
+	msg->rep->security = sec_status_unchecked;
+	msg->rep->an_numrrsets = 1;
+	msg->rep->ns_numrrsets = 0;
+	msg->rep->ar_numrrsets = 0;
+	msg->rep->rrset_count = 1;
+	msg->rep->rrsets[0] = copy_rrset(rrset, region, now);
+	if(!msg->rep->rrsets[0]) /* copy DNAME */
+		return NULL;
+	/* synth CNAME rrset */
+	get_cname_target(rrset, &dtarg, &dtarglen);
+	if(!dtarg)
+		return NULL;
+	newlen = q->qname_len + dtarglen - rrset->rk.dname_len;
+	if(newlen > LDNS_MAX_DOMAINLEN) {
+		msg->rep->flags |= LDNS_RCODE_YXDOMAIN;
+		return msg;
+	}
+	newname = (uint8_t*)regional_alloc(region, newlen);
+	if(!newname)
+		return NULL;
+	/* new name is concatenation of qname front (without DNAME owner)
+	 * and DNAME target name */
+	memcpy(newname, q->qname, q->qname_len-rrset->rk.dname_len);
+	memmove(newname+(q->qname_len-rrset->rk.dname_len), dtarg, dtarglen);
+	/* create rest of CNAME rrset */
+	ck = (struct ub_packed_rrset_key*)regional_alloc(region, 
+		sizeof(struct ub_packed_rrset_key));
+	if(!ck)
+		return NULL;
+	memset(&ck->entry, 0, sizeof(ck->entry));
+	msg->rep->rrsets[1] = ck;
+	ck->entry.key = ck;
+	ck->rk.type = htons(LDNS_RR_TYPE_CNAME);
+	ck->rk.rrset_class = rrset->rk.rrset_class;
+	ck->rk.flags = 0;
+	ck->rk.dname = regional_alloc_init(region, q->qname, q->qname_len);
+	if(!ck->rk.dname)
+		return NULL;
+	ck->rk.dname_len = q->qname_len;
+	ck->entry.hash = rrset_key_hash(&ck->rk);
+	newd = (struct packed_rrset_data*)regional_alloc(region,
+		sizeof(struct packed_rrset_data) + sizeof(size_t) + 
+		sizeof(uint8_t*) + sizeof(uint32_t) + sizeof(uint16_t) 
+		+ newlen);
+	if(!newd)
+		return NULL;
+	ck->entry.data = newd;
+	newd->ttl = 0; /* 0 for synthesized CNAME TTL */
+	newd->count = 1;
+	newd->rrsig_count = 0;
+	newd->trust = rrset_trust_ans_noAA;
+	newd->rr_len = (size_t*)((uint8_t*)newd + 
+		sizeof(struct packed_rrset_data));
+	newd->rr_len[0] = newlen + sizeof(uint16_t);
+	packed_rrset_ptr_fixup(newd);
+	newd->rr_ttl[0] = newd->ttl;
+	msg->rep->ttl = newd->ttl;
+	ldns_write_uint16(newd->rr_data[0], newlen);
+	memmove(newd->rr_data[0] + sizeof(uint16_t), newname, newlen);
+	msg->rep->an_numrrsets ++;
+	msg->rep->rrset_count ++;
+	return msg;
+}
+
+struct dns_msg* 
+dns_cache_lookup(struct module_env* env,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	struct regional* region, struct regional* scratch)
+{
+	struct lruhash_entry* e;
+	struct query_info k;
+	hashvalue_t h;
+	uint32_t now = (uint32_t)time(NULL);
+	struct ub_packed_rrset_key* rrset;
+
+	/* lookup first, this has both NXdomains and ANSWER responses */
+	k.qname = qname;
+	k.qname_len = qnamelen;
+	k.qtype = qtype;
+	k.qclass = qclass;
+	h = query_info_hash(&k);
+	e = slabhash_lookup(env->msg_cache, h, &k, 0);
+	if(e) {
+		struct msgreply_entry* key = (struct msgreply_entry*)e->key;
+		struct reply_info* data = (struct reply_info*)e->data;
+		struct dns_msg* msg = tomsg(env, key, data, region, now, 
+			scratch);
+		if(msg) {
+			lock_rw_unlock(&e->lock);
+			return msg;
+		}
+		/* could be msg==NULL; due to TTL or not all rrsets available */
+		lock_rw_unlock(&e->lock);
+	}
+
+	/* see if a DNAME exists. Checked for first, to enforce that DNAMEs
+	 * are more important, the CNAME is resynthesized and thus 
+	 * consistent with the DNAME */
+	if( (rrset=find_closest_of_type(env, qname, qnamelen, qclass, now,
+		LDNS_RR_TYPE_DNAME, 1))) {
+		/* synthesize a DNAME+CNAME message based on this */
+		struct dns_msg* msg = synth_dname_msg(rrset, region, now, &k);
+		if(msg) {
+			lock_rw_unlock(&rrset->entry.lock);
+			return msg;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+
+	/* see if we have CNAME for this domain */
+	if( (rrset=rrset_cache_lookup(env->rrset_cache, qname, qnamelen, 
+		LDNS_RR_TYPE_CNAME, qclass, 0, now, 0))) {
+		struct dns_msg* msg = rrset_msg(rrset, region, now, &k);
+		if(msg) {
+			lock_rw_unlock(&rrset->entry.lock);
+			return msg;
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+
+	/* construct DS, DNSKEY messages from rrset cache. */
+	if((qtype == LDNS_RR_TYPE_DS || qtype == LDNS_RR_TYPE_DNSKEY) &&
+		(rrset=rrset_cache_lookup(env->rrset_cache, qname, qnamelen, 
+		qtype, qclass, 0, now, 0))) {
+		/* if the rrset is from the additional section, and the
+		 * signatures have fallen off, then do not synthesize a msg
+		 * instead, allow a full query for signed results to happen.
+		 * Forego all rrset data from additional section, because
+		 * some signatures may not be present and cause validation
+		 * failure.
+		 */
+		struct packed_rrset_data *d = (struct packed_rrset_data*)
+			rrset->entry.data;
+		if(d->trust != rrset_trust_add_noAA && 
+			d->trust != rrset_trust_add_AA) {
+			struct dns_msg* msg = rrset_msg(rrset, region, now, &k);
+			if(msg) {
+				lock_rw_unlock(&rrset->entry.lock);
+				return msg;
+			}
+		}
+		lock_rw_unlock(&rrset->entry.lock);
+	}
+	return NULL;
+}
+
+int 
+dns_cache_store(struct module_env* env, struct query_info* msgqinf,
+        struct reply_info* msgrep, int is_referral)
+{
+	struct reply_info* rep = NULL;
+	/* alloc, malloc properly (not in region, like msg is) */
+	rep = reply_info_copy(msgrep, env->alloc, NULL);
+	if(!rep)
+		return 0;
+
+	if(is_referral) {
+		/* store rrsets */
+		struct rrset_ref ref;
+		uint32_t now = time(NULL);
+		size_t i;
+		for(i=0; i<rep->rrset_count; i++) {
+			packed_rrset_ttl_add((struct packed_rrset_data*)
+				rep->rrsets[i]->entry.data, now);
+			ref.key = rep->rrsets[i];
+			ref.id = rep->rrsets[i]->id;
+			/*ignore ret: it was in the cache, ref updated */
+			(void)rrset_cache_update(env->rrset_cache, &ref, 
+				env->alloc, now);
+		}
+		free(rep);
+		return 1;
+	} else {
+		/* store msg, and rrsets */
+		struct query_info qinf;
+		hashvalue_t h;
+
+		qinf = *msgqinf;
+		qinf.qname = memdup(msgqinf->qname, msgqinf->qname_len);
+		if(!qinf.qname) {
+			reply_info_parsedelete(rep, env->alloc);
+			return 0;
+		}
+		/* fixup flags to be sensible for a reply based on the cache */
+		/* this module means that RA is available. It is an answer QR. 
+		 * Not AA from cache. Not CD in cache (depends on client bit). */
+		rep->flags |= (BIT_RA | BIT_QR);
+		rep->flags &= ~(BIT_AA | BIT_CD);
+		h = query_info_hash(&qinf);
+		dns_cache_store_msg(env, &qinf, h, rep);
+		/* qname is used inside query_info_entrysetup, and set to 
+		 * NULL. If it has not been used, free it. free(0) is safe. */
+		free(qinf.qname);
+	}
+	return 1;
+}

services/cache/dns.h

@@ -0,0 +1,142 @@
+/*
+ * services/cache/dns.h - Cache services for DNS using msg and rrset caches.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the DNS cache.
+ */
+
+#ifndef SERVICES_CACHE_DNS_H
+#define SERVICES_CACHE_DNS_H
+#include "util/storage/lruhash.h"
+#include "util/data/msgreply.h"
+struct module_env;
+struct query_info;
+struct reply_info;
+struct regional;
+struct delegpt;
+
+/**
+ * Region allocated message reply
+ */
+struct dns_msg {
+	/** query info */
+	struct query_info qinfo;
+	/** reply info - ptr to packed repinfo structure */
+	struct reply_info *rep;
+};
+
+/**
+ * Allocate a dns_msg with malloc/alloc structure and store in dns cache.
+ *
+ * @param env: environment, with alloc structure and dns cache.
+ * @param qinf: query info, the query for which answer is stored.
+ * 	this is allocated in a region, and will be copied to malloc area
+ * 	before insertion.
+ * @param rep: reply in dns_msg from dns_alloc_msg for example.
+ * 	this is allocated in a region, and will be copied to malloc area
+ * 	before insertion.
+ * @param is_referral: If true, then the given message to be stored is a
+ *      referral. The cache implementation may use this as a hint.
+ *      It will store only the RRsets, not the message.
+ * @return 0 on alloc error (out of memory).
+ */
+int dns_cache_store(struct module_env* env, struct query_info* qinf,
+        struct reply_info* rep, int is_referral); 
+
+/**
+ * Store message in the cache. Stores in message cache and rrset cache.
+ * Both qinfo and rep should be malloced and are put in the cache.
+ * They should not be used after this call, as they are then in shared cache.
+ * Does not return errors, they are logged and only lead to less cache.
+ *
+ * @param env: module environment with the DNS cache.
+ * @param qinfo: query info
+ * @param hash: hash over qinfo.
+ * @param rep: reply info, together with qinfo makes up the message.
+ *	Adjusts the reply info TTLs to absolute time.
+ */
+void dns_cache_store_msg(struct module_env* env, struct query_info* qinfo,
+	hashvalue_t hash, struct reply_info* rep);
+
+/**
+ * Find a delegation from the cache.
+ * @param env: module environment with the DNS cache.
+ * @param qname: query name.
+ * @param qnamelen: length of qname.
+ * @param qtype: query type.
+ * @param qclass: query class.
+ * @param region: where to allocate result delegation.
+ * @param msg: if not NULL, delegation message is returned here, synthesized
+ *	from the cache.
+ * @param timenow: the time now, for checking if TTL on cache entries is OK.
+ * @return new delegation or NULL on error or if not found in cache.
+ */
+struct delegpt* dns_cache_find_delegation(struct module_env* env, 
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass, 
+	struct regional* region, struct dns_msg** msg, uint32_t timenow);
+
+/** 
+ * Find cached message 
+ * @param env: module environment with the DNS cache.
+ * @param qname: query name.
+ * @param qnamelen: length of qname.
+ * @param qtype: query type.
+ * @param qclass: query class.
+ * @param region: where to allocate result.
+ * @param scratch: where to allocate temporary data.
+ * @return new response message (alloced in region, rrsets do not have IDs).
+ * 	or NULL on error or if not found in cache.
+ *	TTLs are made relative to the current time.
+ */
+struct dns_msg* dns_cache_lookup(struct module_env* env,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	struct regional* region, struct regional* scratch);
+
+/** 
+ * find and add A and AAAA records for missing nameservers in delegpt 
+ * @param env: module environment with rrset cache
+ * @param qclass: which class to look in.
+ * @param region: where to store new dp info.
+ * @param dp: delegation point to fill missing entries.
+ * @return false on alloc failure.
+ */
+int cache_fill_missing(struct module_env* env, uint16_t qclass, 
+	struct regional* region, struct delegpt* dp);
+
+/** Find covering DNAME */
+
+#endif /* SERVICES_CACHE_DNS_H */

services/cache/infra.c

@@ -0,0 +1,550 @@
+/*
+ * services/cache/infra.c - infrastructure cache, server rtt and capabilities
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the infrastructure cache.
+ */
+#include "config.h"
+#include "services/cache/infra.h"
+#include "util/storage/slabhash.h"
+#include "util/storage/lookup3.h"
+#include "util/data/dname.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/config_file.h"
+
+size_t 
+infra_host_sizefunc(void* k, void* ATTR_UNUSED(d))
+{
+	struct infra_host_key* key = (struct infra_host_key*)k;
+	return sizeof(*key) + sizeof(struct infra_host_data) 
+		+ lock_get_mem(&key->entry.lock);
+}
+
+int 
+infra_host_compfunc(void* key1, void* key2)
+{
+	struct infra_host_key* k1 = (struct infra_host_key*)key1;
+	struct infra_host_key* k2 = (struct infra_host_key*)key2;
+	return sockaddr_cmp(&k1->addr, k1->addrlen, &k2->addr, k2->addrlen);
+}
+
+void 
+infra_host_delkeyfunc(void* k, void* ATTR_UNUSED(arg), int il)
+{
+	struct infra_host_key* key = (struct infra_host_key*)k;
+	if(!key)
+		return;
+	if(il) { lock_rw_unlock(&key->entry.lock); }
+	lock_rw_destroy(&key->entry.lock);
+	free(key);
+}
+
+void 
+infra_host_deldatafunc(void* d, void* ATTR_UNUSED(arg))
+{
+	struct infra_host_data* data = (struct infra_host_data*)d;
+	lruhash_delete(data->lameness);
+	free(data);
+}
+
+struct infra_cache* 
+infra_create(struct config_file* cfg)
+{
+	struct infra_cache* infra = (struct infra_cache*)calloc(1, 
+		sizeof(struct infra_cache));
+	/* the size of the lameness tables are not counted */
+	size_t maxmem = cfg->infra_cache_numhosts * 
+		(sizeof(struct infra_host_key)+sizeof(struct infra_host_data));
+	infra->hosts = slabhash_create(cfg->infra_cache_slabs,
+		INFRA_HOST_STARTSIZE, maxmem, &infra_host_sizefunc,
+		&infra_host_compfunc, &infra_host_delkeyfunc,
+		&infra_host_deldatafunc, NULL);
+	if(!infra->hosts) {
+		free(infra);
+		return NULL;
+	}
+	infra->host_ttl = cfg->host_ttl;
+	infra->lame_ttl = cfg->lame_ttl;
+	infra->max_lame_size = cfg->infra_cache_lame_size;
+	return infra;
+}
+
+void 
+infra_delete(struct infra_cache* infra)
+{
+	if(!infra)
+		return;
+	slabhash_delete(infra->hosts);
+	free(infra);
+}
+
+struct infra_cache* 
+infra_adjust(struct infra_cache* infra, struct config_file* cfg)
+{
+	size_t maxmem;
+	if(!infra)
+		return infra_create(cfg);
+	infra->host_ttl = cfg->host_ttl;
+	infra->lame_ttl = cfg->lame_ttl;
+	infra->max_lame_size = cfg->infra_cache_lame_size;
+	maxmem = cfg->infra_cache_numhosts * 
+		(sizeof(struct infra_host_key)+sizeof(struct infra_host_data));
+	if(maxmem != slabhash_get_size(infra->hosts) ||
+		cfg->infra_cache_slabs != infra->hosts->size) {
+		infra_delete(infra);
+		infra = infra_create(cfg);
+	}
+	return infra;
+}
+
+/** calculate the hash value for a host key */
+static hashvalue_t
+hash_addr(struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	hashvalue_t h = 0xab;
+	h = hashlittle(&addrlen, sizeof(addrlen), h);
+	h = hashlittle(addr, addrlen, h);
+	return h;
+}
+
+/** lookup version that does not check host ttl (you check it) */
+static struct lruhash_entry* 
+infra_lookup_host_nottl(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, int wr)
+{
+	struct infra_host_key k;
+	k.addrlen = addrlen;
+	memcpy(&k.addr, addr, addrlen);
+	k.entry.hash = hash_addr(addr, addrlen);
+	k.entry.key = (void*)&k;
+	k.entry.data = NULL;
+	return slabhash_lookup(infra->hosts, k.entry.hash, &k, wr);
+}
+
+struct infra_host_data* 
+infra_lookup_host(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, int wr,
+        time_t timenow, struct infra_host_key** key)
+{
+	struct infra_host_data* data;
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, wr);
+	*key = NULL;
+	if(!e)
+		return NULL;
+	/* check TTL */
+	data = (struct infra_host_data*)e->data;
+	if(data->ttl < timenow) {
+		lock_rw_unlock(&e->lock);
+		return NULL;
+	}
+	*key = (struct infra_host_key*)e->key;
+	return data;
+}
+
+/** 
+ * Create and init a new entry for a host 
+ * @param infra: infra structure with config parameters.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param tm: time now.
+ * @return: the new entry or NULL on malloc failure.
+ */
+static struct lruhash_entry*
+new_host_entry(struct infra_cache* infra, struct sockaddr_storage* addr, 
+	socklen_t addrlen, time_t tm)
+{
+	struct infra_host_data* data;
+	struct infra_host_key* key = (struct infra_host_key*)malloc(
+		sizeof(struct infra_host_key));
+	if(!key)
+		return NULL;
+	data = (struct infra_host_data*)malloc(
+		sizeof(struct infra_host_data));
+	if(!data) {
+		free(key);
+		return NULL;
+	}
+	lock_rw_init(&key->entry.lock);
+	key->entry.hash = hash_addr(addr, addrlen);
+	key->entry.key = (void*)key;
+	key->entry.data = (void*)data;
+	key->addrlen = addrlen;
+	memcpy(&key->addr, addr, addrlen);
+	data->ttl = tm + infra->host_ttl;
+	data->lameness = NULL;
+	data->edns_version = 0;
+	rtt_init(&data->rtt);
+	return &key->entry;
+}
+
+int 
+infra_host(struct infra_cache* infra, struct sockaddr_storage* addr,
+        socklen_t addrlen, time_t timenow, int* edns_vs, int* to)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 0);
+	struct infra_host_data* data;
+	if(e && ((struct infra_host_data*)e->data)->ttl < timenow) {
+		/* it expired, try to reuse existing entry */
+		lock_rw_unlock(&e->lock);
+		e = infra_lookup_host_nottl(infra, addr, addrlen, 1);
+		if(e) {
+			/* if its still there we have a writelock, init */
+			/* re-initialise */
+			data = (struct infra_host_data*)e->data;
+			data->ttl = timenow + infra->host_ttl;
+			rtt_init(&data->rtt);
+			/* do not touch lameness, it may be valid still */
+			data->edns_version = 0;
+		}
+	}
+	if(!e) {
+		/* insert new entry */
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		data = (struct infra_host_data*)e->data;
+		*to = rtt_timeout(&data->rtt);
+		*edns_vs = data->edns_version;
+		slabhash_insert(infra->hosts, e->hash, e, data, NULL);
+		return 1;
+	}
+	/* use existing entry */
+	data = (struct infra_host_data*)e->data;
+	*to = rtt_timeout(&data->rtt);
+	*edns_vs = data->edns_version;
+	lock_rw_unlock(&e->lock);
+	return 1;
+}
+
+/** hash lameness key */
+static hashvalue_t
+hash_lameness(uint8_t* name, size_t namelen)
+{
+	return hashlittle(name, namelen, 0xab);
+}
+
+int 
+infra_lookup_lame(struct infra_host_data* host,
+        uint8_t* name, size_t namelen, time_t timenow)
+{
+	struct lruhash_entry* e;
+	struct infra_lame_key k;
+	struct infra_lame_data *d;
+	int dl;
+	if(!host->lameness)
+		return 0;
+	k.entry.hash = hash_lameness(name, namelen);
+	k.zonename = name;
+	k.namelen = namelen;
+	k.entry.key = (void*)&k;
+	k.entry.data = NULL;
+	e = lruhash_lookup(host->lameness, k.entry.hash, &k, 0);
+	if(!e)
+		return 0;
+	d = (struct infra_lame_data*)e->data;
+	if(d->ttl < timenow) {
+		lock_rw_unlock(&e->lock);
+		return 0;
+	}
+	dl = d->isdnsseclame;
+	lock_rw_unlock(&e->lock);
+	return dl?2:1;
+}
+
+size_t 
+infra_lame_sizefunc(void* k, void* ATTR_UNUSED(d))
+{
+	struct infra_lame_key* key = (struct infra_lame_key*)k;
+	return sizeof(*key) + sizeof(struct infra_lame_data)
+		+ key->namelen + lock_get_mem(&key->entry.lock);
+}
+
+int 
+infra_lame_compfunc(void* key1, void* key2) 
+{
+	struct infra_lame_key* k1 = (struct infra_lame_key*)key1;
+	struct infra_lame_key* k2 = (struct infra_lame_key*)key2;
+	if(k1->namelen != k2->namelen) {
+		if(k1->namelen < k2->namelen)
+			return -1;
+		return 1;
+	}
+	return query_dname_compare(k1->zonename, k2->zonename);
+}
+
+void 
+infra_lame_delkeyfunc(void* k, void* ATTR_UNUSED(arg), int il)
+{
+	struct infra_lame_key* key = (struct infra_lame_key*)k;
+	if(!key) 
+		return;
+	if(il) { lock_rw_unlock(&key->entry.lock); }
+	lock_rw_destroy(&key->entry.lock);
+	free(key->zonename);
+	free(key);
+}
+
+void 
+infra_lame_deldatafunc(void* d, void* ATTR_UNUSED(arg))
+{
+	if(!d) 
+		return;
+	free(d);
+}
+
+int 
+infra_set_lame(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        uint8_t* name, size_t namelen, time_t timenow, int dnsseclame)
+{
+	struct infra_host_data* data;
+	struct lruhash_entry* e;
+	int needtoinsert = 0;
+	struct infra_lame_key* k;
+	struct infra_lame_data* d;
+	/* allocate at start, easier cleanup (no locks held) */
+	k = (struct infra_lame_key*)malloc(sizeof(*k));
+	if(!k) {
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	d = (struct infra_lame_data*)malloc(sizeof(*d));
+	if(!d) {
+		free(k);
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	k->zonename = memdup(name, namelen);
+	if(!k->zonename) {
+		free(d);
+		free(k);
+		log_err("set_lame: malloc failure");
+		return 0;
+	}
+	lock_rw_init(&k->entry.lock);
+	k->entry.hash = hash_lameness(name, namelen);
+	k->entry.key = (void*)k;
+	k->entry.data = (void*)d;
+	d->ttl = timenow + infra->lame_ttl;
+	d->isdnsseclame = dnsseclame;
+	k->namelen = namelen;
+	e = infra_lookup_host_nottl(infra, addr, addrlen, 1);
+	if(!e) {
+		/* insert it */
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow))) {
+			free(k->zonename);
+			free(k);
+			free(d);
+			log_err("set_lame: malloc failure");
+			return 0;
+		}
+		needtoinsert = 1;
+	}
+	/* got an entry, now set the zone lame */
+	data = (struct infra_host_data*)e->data;
+	if(!data->lameness) {
+		/* create hash table if not there already */
+		data->lameness = lruhash_create(INFRA_LAME_STARTSIZE,
+			infra->max_lame_size, infra_lame_sizefunc, 
+			infra_lame_compfunc, infra_lame_delkeyfunc,
+			infra_lame_deldatafunc, NULL);
+		if(!data->lameness) {
+			log_err("set_lame: malloc failure");
+			if(needtoinsert) slabhash_insert(infra->hosts, 
+				e->hash, e, e->data, NULL);
+			else 	{ lock_rw_unlock(&e->lock); }
+			free(k->zonename);
+			free(k);
+			free(d);
+			return 0;
+		}
+	}
+	/* inserts new entry, or updates TTL of older entry */
+	lruhash_insert(data->lameness, k->entry.hash, &k->entry, d, NULL);
+	
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return 1;
+}
+
+void 
+infra_update_tcp_works(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 1);
+	struct infra_host_data* data;
+	if(!e)
+		return; /* doesn't exist */
+	data = (struct infra_host_data*)e->data;
+	if(data->rtt.rto >= RTT_MAX_TIMEOUT)
+		/* do not disqualify this server altogether, it is better
+		 * than nothing */
+		data->rtt.rto = RTT_MAX_TIMEOUT-1;
+	lock_rw_unlock(&e->lock);
+}
+
+int 
+infra_rtt_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        int roundtrip, time_t timenow)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 1);
+	struct infra_host_data* data;
+	int needtoinsert = 0;
+	int rto = 1;
+	if(!e) {
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		needtoinsert = 1;
+	}
+	/* have an entry, update the rtt, and the ttl */
+	data = (struct infra_host_data*)e->data;
+	data->ttl = timenow + infra->host_ttl;
+	if(roundtrip == -1)
+		rtt_lost(&data->rtt);
+	else	rtt_update(&data->rtt, roundtrip);
+	if(data->rtt.rto > 0)
+		rto = data->rtt.rto;
+
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return rto;
+}
+
+int 
+infra_edns_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        int edns_version, time_t timenow)
+{
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 1);
+	struct infra_host_data* data;
+	int needtoinsert = 0;
+	if(!e) {
+		if(!(e = new_host_entry(infra, addr, addrlen, timenow)))
+			return 0;
+		needtoinsert = 1;
+	}
+	/* have an entry, update the rtt, and the ttl */
+	data = (struct infra_host_data*)e->data;
+	data->ttl = timenow + infra->host_ttl;
+	data->edns_version = edns_version;
+
+	if(needtoinsert)
+		slabhash_insert(infra->hosts, e->hash, e, e->data, NULL);
+	else 	{ lock_rw_unlock(&e->lock); }
+	return 1;
+}
+
+int 
+infra_get_lame_rtt(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+        uint8_t* name, size_t namelen, int* lame, int* dnsseclame,
+	int* rtt, time_t timenow)
+{
+	struct infra_host_data* host;
+	struct lruhash_entry* e = infra_lookup_host_nottl(infra, addr, 
+		addrlen, 0);
+	int lm;
+	if(!e) 
+		return 0;
+	host = (struct infra_host_data*)e->data;
+	*rtt = rtt_unclamped(&host->rtt);
+	/* check lameness first, if so, ttl on host does not matter anymore */
+	if((lm=infra_lookup_lame(host, name, namelen, timenow))) {
+		lock_rw_unlock(&e->lock);
+		if(lm == 1) {
+			*lame = 1;
+			*dnsseclame = 0;
+		} else {
+			*lame = 0;
+			*dnsseclame = 1;
+		}
+		return 1;
+	}
+	*lame = 0;
+	*dnsseclame = 0;
+	if(timenow > host->ttl) {
+		lock_rw_unlock(&e->lock);
+		return 0;
+	}
+	lock_rw_unlock(&e->lock);
+	return 1;
+}
+
+/** helper memory count for a host lame cache */
+static size_t
+count_host_lame(struct lruhash_entry* e)
+{
+	struct infra_host_data* host_data = (struct infra_host_data*)e->data;
+	if(!host_data->lameness)
+		return 0;
+	return lruhash_get_mem(host_data->lameness);
+}
+
+size_t 
+infra_get_mem(struct infra_cache* infra)
+{
+	size_t i, bin;
+	size_t s = sizeof(*infra) +
+		slabhash_get_mem(infra->hosts);
+	struct lruhash_entry* e;
+	for(i=0; i<infra->hosts->size; i++) {
+		lock_quick_lock(&infra->hosts->array[i]->lock);
+		for(bin=0; bin<infra->hosts->array[i]->size; bin++) {
+			lock_quick_lock(&infra->hosts->array[i]->
+				array[bin].lock);
+			/* count data size in bin items. */
+			for(e = infra->hosts->array[i]->array[bin].
+				overflow_list; e; e = e->overflow_next) {
+				lock_rw_rdlock(&e->lock);
+				s += count_host_lame(e);
+				lock_rw_unlock(&e->lock);
+			}
+			lock_quick_unlock(&infra->hosts->array[i]->
+				array[bin].lock);
+		}
+		lock_quick_unlock(&infra->hosts->array[i]->lock);
+	}
+	return s;
+}

services/cache/infra.h

@@ -0,0 +1,288 @@
+/*
+ * services/cache/infra.h - infrastructure cache, server rtt and capabilities
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the infrastructure cache.
+ */
+
+#ifndef SERVICES_CACHE_INFRA_H
+#define SERVICES_CACHE_INFRA_H
+#include "util/storage/lruhash.h"
+#include "util/rtt.h"
+struct slabhash;
+struct config_file;
+
+/**
+ * Host information kept for every server.
+ */
+struct infra_host_key {
+	/** the host address. */
+	struct sockaddr_storage addr;
+	/** length of addr. */
+	socklen_t addrlen;
+	/** hash table entry, data of type infra_host_data. */
+	struct lruhash_entry entry;
+};
+
+/**
+ * Host information encompasses host capabilities and retransmission timeouts.
+ */
+struct infra_host_data {
+	/** TTL value for this entry. absolute time. */
+	time_t ttl;
+	/** round trip times for timeout calculation */
+	struct rtt_info rtt;
+	/** Names of the zones that are lame. NULL=no lame zones. */
+	struct lruhash* lameness;
+	/** edns version that the host supports, -1 means no EDNS */
+	int edns_version;
+};
+
+/**
+ * Lameness information, per host, per zone. 
+ */
+struct infra_lame_key {
+	/** key is zone name in wireformat */
+	uint8_t* zonename;
+	/** length of zonename */
+	size_t namelen;
+	/** lruhash entry */
+	struct lruhash_entry entry;
+};
+
+/**
+ * Lameness information. Expires.
+ * This host is lame because it is in the cache.
+ */
+struct infra_lame_data {
+	/** TTL of this entry. absolute time. */
+	time_t ttl;
+	/** is the host lame (does not serve the zone authoritatively),
+	 * or is the host dnssec lame (does not serve DNSSEC data) */
+	int isdnsseclame;
+};
+
+/**
+ * Infra cache 
+ */
+struct infra_cache {
+	/** The hash table with hosts */
+	struct slabhash* hosts;
+	/** TTL value for host information, in seconds */
+	int host_ttl;
+	/** TTL for Lameness information, in seconds */
+	int lame_ttl;
+	/** infra lame cache max memory per host, in bytes */
+	size_t max_lame_size;
+};
+
+/** infra host cache default hash lookup size */
+#define INFRA_HOST_STARTSIZE 32
+/** infra lame cache default hash lookup size */
+#define INFRA_LAME_STARTSIZE 2
+
+/**
+ * Create infra cache.
+ * @param cfg: config parameters or NULL for defaults.
+ * @return: new infra cache, or NULL.
+ */
+struct infra_cache* infra_create(struct config_file* cfg);
+
+/**
+ * Delete infra cache.
+ * @param infra: infrastructure cache to delete.
+ */
+void infra_delete(struct infra_cache* infra);
+
+/**
+ * Adjust infra cache to use updated configuration settings.
+ * This may clean the cache. Operates a bit like realloc.
+ * There may be no threading or use by other threads.
+ * @param infra: existing cache. If NULL a new infra cache is returned.
+ * @param cfg: config options.
+ * @return the new infra cache pointer or NULL on error.
+ */
+struct infra_cache* infra_adjust(struct infra_cache* infra, 
+	struct config_file* cfg);
+
+/**
+ * Lookup host data
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param wr: set to true to get a writelock on the entry.
+ * @param timenow: what time it is now.
+ * @param key: the key for the host, returned so caller can unlock when done.
+ * @return: host data or NULL if not found or expired.
+ */
+struct infra_host_data* infra_lookup_host(struct infra_cache* infra, 
+	struct sockaddr_storage* addr, socklen_t addrlen, int wr, 
+	time_t timenow, struct infra_host_key** key);
+
+/**
+ * Find host information to send a packet. Creates new entry if not found.
+ * Lameness is empty. EDNS is 0 (try with first), and rtt is returned for 
+ * the first message to it.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param timenow: what time it is now.
+ * @param edns_vs: edns version it supports, is returned.
+ * @param to: timeout to use, is returned.
+ * @return: 0 on error.
+ */
+int infra_host(struct infra_cache* infra, struct sockaddr_storage* addr, 
+	socklen_t addrlen, time_t timenow, int* edns_vs, int* to);
+
+/**
+ * Check for lameness of this server for a particular zone.
+ * You must have a lock on the host structure.
+ * @param host: infrastructure cache data for the host. Caller holds lock.
+ * @param name: domain name of zone apex.
+ * @param namelen: length of domain name.
+ * @param timenow: what time it is now.
+ * @return: 0 if not lame or unknown or timed out, 1 if lame, 2 if dnsseclame.
+ */
+int infra_lookup_lame(struct infra_host_data* host,
+	uint8_t* name, size_t namelen, time_t timenow);
+
+/**
+ * Set a host to be lame for the given zone.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param name: domain name of zone apex.
+ * @param namelen: length of domain name.
+ * @param timenow: what time it is now.
+ * @param dnsseclame: if true the host is set dnssec lame.
+ *	if false, the host is marked lame (not serving the zone).
+ * @return: 0 on error.
+ */
+int infra_set_lame(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	uint8_t* name, size_t namelen, time_t timenow, int dnsseclame);
+
+/**
+ * Update rtt information for the host.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param roundtrip: estimate of roundtrip time in milliseconds or -1 for 
+ * 	timeout.
+ * @param timenow: what time it is now.
+ * @return: 0 on error. new rto otherwise.
+ */
+int infra_rtt_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	int roundtrip, time_t timenow);
+
+/**
+ * Update information for the host, store that a TCP transaction works.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ */
+void infra_update_tcp_works(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen);
+
+/**
+ * Update edns information for the host.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param edns_version: the version that it publishes.
+ * @param timenow: what time it is now.
+ * @return: 0 on error.
+ */
+int infra_edns_update(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen,
+	int edns_version, time_t timenow);
+
+/**
+ * Get Lameness information and average RTT if host is in the cache.
+ * @param infra: infrastructure cache.
+ * @param addr: host address.
+ * @param addrlen: length of addr.
+ * @param name: zone name.
+ * @param namelen: zone name length.
+ * @param lame: if function returns true, this returns lameness of the zone.
+ * @param dnsseclame: if function returns true, this returns if the zone
+ *	is dnssec-lame.
+ * @param rtt: if function returns true, this returns avg rtt of the server.
+ * 	The rtt value is unclamped and reflects recent timeouts.
+ * @param timenow: what time it is now.
+ * @return if found in cache, or false if not (or TTL bad).
+ */
+int infra_get_lame_rtt(struct infra_cache* infra,
+        struct sockaddr_storage* addr, socklen_t addrlen, 
+	uint8_t* name, size_t namelen, int* lame, int* dnsseclame,
+	int* rtt, time_t timenow);
+
+/**
+ * Get memory used by the infra cache.
+ * @param infra: infrastructure cache.
+ * @return memory in use in bytes.
+ */
+size_t infra_get_mem(struct infra_cache* infra);
+
+/** calculate size for the hashtable, does not count size of lameness,
+ * so the hashtable is a fixed number of items */
+size_t infra_host_sizefunc(void* k, void* d);
+
+/** compare two addresses, returns -1, 0, or +1 */
+int infra_host_compfunc(void* key1, void* key2);
+
+/** delete key, and destroy the lock */
+void infra_host_delkeyfunc(void* k, void* arg, int il);
+
+/** delete data and destroy the lameness hashtable */
+void infra_host_deldatafunc(void* d, void* arg);
+
+/** calculate size, which is fixed, zonename does not count so that
+ * a fixed number of items is stored */
+size_t infra_lame_sizefunc(void* k, void* d);
+
+/** compare zone names, returns -1, 0, +1 */
+int infra_lame_compfunc(void* key1, void* key2);
+
+/** free key, lock and zonename */
+void infra_lame_delkeyfunc(void* k, void* arg, int il);
+
+/** free the lameness data */
+void infra_lame_deldatafunc(void* d, void* arg);
+
+#endif /* SERVICES_CACHE_INFRA_H */

services/cache/rrset.c

@@ -0,0 +1,359 @@
+/*
+ * services/cache/rrset.c - Resource record set cache.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the rrset cache.
+ */
+#include "config.h"
+#include "services/cache/rrset.h"
+#include "util/storage/slabhash.h"
+#include "util/config_file.h"
+#include "util/data/packed_rrset.h"
+#include "util/data/msgreply.h"
+#include "util/regional.h"
+#include "util/alloc.h"
+
+struct rrset_cache* rrset_cache_create(struct config_file* cfg, 
+	struct alloc_cache* alloc)
+{
+	size_t slabs = (cfg?cfg->rrset_cache_slabs:HASH_DEFAULT_SLABS);
+	size_t startarray = HASH_DEFAULT_STARTARRAY;
+	size_t maxmem = (cfg?cfg->rrset_cache_size:HASH_DEFAULT_MAXMEM);
+
+	struct rrset_cache *r = (struct rrset_cache*)slabhash_create(slabs,
+		startarray, maxmem, ub_rrset_sizefunc, ub_rrset_compare,
+		ub_rrset_key_delete, rrset_data_delete, alloc);
+	return r;
+}
+
+void rrset_cache_delete(struct rrset_cache* r)
+{
+	if(!r) 
+		return;
+	slabhash_delete(&r->table);
+}
+
+struct rrset_cache* rrset_cache_adjust(struct rrset_cache *r, 
+	struct config_file* cfg, struct alloc_cache* alloc)
+{
+	if(!r || !cfg || cfg->rrset_cache_slabs != r->table.size ||
+		cfg->rrset_cache_size != slabhash_get_size(&r->table))
+	{
+		rrset_cache_delete(r);
+		r = rrset_cache_create(cfg, alloc);
+	}
+	return r;
+}
+
+void 
+rrset_cache_touch(struct rrset_cache* r, struct ub_packed_rrset_key* key,
+        hashvalue_t hash, rrset_id_t id)
+{
+	struct lruhash* table = slabhash_gettable(&r->table, hash);
+	/* 
+	 * This leads to locking problems, deadlocks, if the caller is 
+	 * holding any other rrset lock.
+	 * Because a lookup through the hashtable does:
+	 *	tablelock -> entrylock  (for that entry caller holds)
+	 * And this would do
+	 *	entrylock(already held) -> tablelock
+	 * And if two threads do this, it results in deadlock.
+	 * So, the caller must not hold entrylock.
+	 */
+	lock_quick_lock(&table->lock);
+	/* we have locked the hash table, the item can still be deleted.
+	 * because it could already have been reclaimed, but not yet set id=0.
+	 * This is because some lruhash routines have lazy deletion.
+	 * so, we must acquire a lock on the item to verify the id != 0.
+	 * also, with hash not changed, we are using the right slab.
+	 */
+	lock_rw_rdlock(&key->entry.lock);
+	if(key->id == id && key->entry.hash == hash) {
+		lru_touch(table, &key->entry);
+	}
+	lock_rw_unlock(&key->entry.lock);
+	lock_quick_unlock(&table->lock);
+}
+
+/** see if rrset needs to be updated in the cache */
+static int
+need_to_update_rrset(void* nd, void* cd, uint32_t timenow, int equal)
+{
+	struct packed_rrset_data* newd = (struct packed_rrset_data*)nd;
+	struct packed_rrset_data* cached = (struct packed_rrset_data*)cd;
+	/* 	o store if rrset has been validated */
+	if( newd->security > cached->security) {
+		return 1;
+	}
+        /*      o if current RRset is more trustworthy - insert it */
+        if( newd->trust > cached->trust ) {
+		/* if the cached rrset is bogus, and this one equal,
+		 * do not update the TTL - let it expire. */
+		if(equal && cached->ttl >= timenow && 
+			cached->security == sec_status_bogus)
+			return 0;
+                return 1;
+	}
+	/*	o item in cache has expired */
+	if( cached->ttl < timenow )
+		return 1;
+        /*      o same trust, but different in data - insert it */
+        if( newd->trust == cached->trust && !equal )
+                return 1;
+        return 0;
+}
+
+/** Update RRSet special key ID */
+static void
+rrset_update_id(struct rrset_ref* ref, struct alloc_cache* alloc)
+{
+	/* this may clear the cache and invalidate lock below */
+	uint64_t newid = alloc_get_id(alloc);
+	/* obtain writelock */
+	lock_rw_wrlock(&ref->key->entry.lock);
+	/* check if it was deleted in the meantime, if so, skip update */
+	if(ref->key->id == ref->id) {
+		ref->key->id = newid;
+		ref->id = newid;
+	}
+	lock_rw_unlock(&ref->key->entry.lock);
+}
+
+int 
+rrset_cache_update(struct rrset_cache* r, struct rrset_ref* ref,
+	struct alloc_cache* alloc, uint32_t timenow)
+{
+	struct lruhash_entry* e;
+	struct ub_packed_rrset_key* k = ref->key;
+	hashvalue_t h = k->entry.hash;
+	uint16_t rrset_type = ntohs(k->rk.type);
+	int equal = 0;
+	log_assert(ref->id != 0 && k->id != 0);
+	/* looks up item with a readlock - no editing! */
+	if((e=slabhash_lookup(&r->table, h, k, 0)) != 0) {
+		/* return id and key as they will be used in the cache
+		 * since the lruhash_insert, if item already exists, deallocs
+		 * the passed key in favor of the already stored key.
+		 * because of the small gap (see below) this key ptr and id
+		 * may prove later to be already deleted, which is no problem
+		 * as it only makes a cache miss. 
+		 */
+		ref->key = (struct ub_packed_rrset_key*)e->key;
+		ref->id = ref->key->id;
+		equal = rrsetdata_equal((struct packed_rrset_data*)k->entry.
+			data, (struct packed_rrset_data*)e->data);
+		if(!need_to_update_rrset(k->entry.data, e->data, timenow,
+			equal)) {
+			/* cache is superior, return that value */
+			lock_rw_unlock(&e->lock);
+			ub_packed_rrset_parsedelete(k, alloc);
+			return 1;
+		}
+		lock_rw_unlock(&e->lock);
+		/* Go on and insert the passed item.
+		 * small gap here, where entry is not locked.
+		 * possibly entry is updated with something else.
+		 * we then overwrite that with our data.
+		 * this is just too bad, its cache anyway. */
+		/* use insert to update entry to manage lruhash
+		 * cache size values nicely. */
+	}
+	log_assert(ref->key->id != 0);
+	slabhash_insert(&r->table, h, &k->entry, k->entry.data, alloc);
+	if(e) {
+		/* For NSEC, NSEC3, DNAME, when rdata is updated, update 
+		 * the ID number so that proofs in message cache are 
+		 * invalidated */
+		if((rrset_type == LDNS_RR_TYPE_NSEC 
+			|| rrset_type == LDNS_RR_TYPE_NSEC3
+			|| rrset_type == LDNS_RR_TYPE_DNAME) && !equal) {
+			rrset_update_id(ref, alloc);
+		}
+		return 1;
+	}
+	return 0;
+}
+
+struct ub_packed_rrset_key* 
+rrset_cache_lookup(struct rrset_cache* r, uint8_t* qname, size_t qnamelen, 
+	uint16_t qtype, uint16_t qclass, uint32_t flags, uint32_t timenow,
+	int wr)
+{
+	struct lruhash_entry* e;
+	struct ub_packed_rrset_key key;
+	
+	key.entry.key = &key;
+	key.entry.data = NULL;
+	key.rk.dname = qname;
+	key.rk.dname_len = qnamelen;
+	key.rk.type = htons(qtype);
+	key.rk.rrset_class = htons(qclass);
+	key.rk.flags = flags;
+
+	key.entry.hash = rrset_key_hash(&key.rk);
+
+	if((e = slabhash_lookup(&r->table, key.entry.hash, &key, wr))) {
+		/* check TTL */
+		struct packed_rrset_data* data = 
+			(struct packed_rrset_data*)e->data;
+		if(timenow > data->ttl) {
+			lock_rw_unlock(&e->lock);
+			return NULL;
+		}
+		/* we're done */
+		return (struct ub_packed_rrset_key*)e->key;
+	}
+	return NULL;
+}
+
+int 
+rrset_array_lock(struct rrset_ref* ref, size_t count, uint32_t timenow)
+{
+	size_t i;
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only lock items once */
+		lock_rw_rdlock(&ref[i].key->entry.lock);
+		if(ref[i].id != ref[i].key->id || timenow >
+			((struct packed_rrset_data*)(ref[i].key->entry.data))
+			->ttl) {
+			/* failure! rollback our readlocks */
+			rrset_array_unlock(ref, i+1);
+			return 0;
+		}
+	}
+	return 1;
+}
+
+void 
+rrset_array_unlock(struct rrset_ref* ref, size_t count)
+{
+	size_t i;
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only unlock items once */
+		lock_rw_unlock(&ref[i].key->entry.lock);
+	}
+}
+
+void 
+rrset_array_unlock_touch(struct rrset_cache* r, struct regional* scratch,
+	struct rrset_ref* ref, size_t count)
+{
+	hashvalue_t* h;
+	size_t i;
+	if(!(h = (hashvalue_t*)regional_alloc(scratch, 
+		sizeof(hashvalue_t)*count)))
+		log_warn("rrset LRU: memory allocation failed");
+	else 	/* store hash values */
+		for(i=0; i<count; i++)
+			h[i] = ref[i].key->entry.hash;
+	/* unlock */
+	for(i=0; i<count; i++) {
+		if(i>0 && ref[i].key == ref[i-1].key)
+			continue; /* only unlock items once */
+		lock_rw_unlock(&ref[i].key->entry.lock);
+	}
+	if(h) {
+		/* LRU touch, with no rrset locks held */
+		for(i=0; i<count; i++) {
+			if(i>0 && ref[i].key == ref[i-1].key)
+				continue; /* only touch items once */
+			rrset_cache_touch(r, ref[i].key, h[i], ref[i].id);
+		}
+	}
+}
+
+void 
+rrset_update_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset)
+{
+	uint32_t now = (uint32_t)time(0);
+	struct packed_rrset_data* updata = 
+		(struct packed_rrset_data*)rrset->entry.data;
+	struct lruhash_entry* e;
+	struct packed_rrset_data* cachedata;
+
+	/* hash it again to make sure it has a hash */
+	rrset->entry.hash = rrset_key_hash(&rrset->rk);
+
+	e = slabhash_lookup(&r->table, rrset->entry.hash, rrset, 1);
+	if(!e)
+		return; /* not in the cache anymore */
+	cachedata = (struct packed_rrset_data*)e->data;
+	if(!rrsetdata_equal(updata, cachedata)) {
+		lock_rw_unlock(&e->lock);
+		return; /* rrset has changed in the meantime */
+	}
+	/* update the cached rrset */
+	if(updata->security > cachedata->security) {
+		cachedata->trust = updata->trust;
+		cachedata->security = updata->security;
+		cachedata->ttl = updata->ttl + now;
+	}
+	lock_rw_unlock(&e->lock);
+}
+
+void 
+rrset_check_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset)
+{
+	uint32_t now = (uint32_t)time(0);
+	struct packed_rrset_data* updata = 
+		(struct packed_rrset_data*)rrset->entry.data;
+	struct lruhash_entry* e;
+	struct packed_rrset_data* cachedata;
+
+	/* hash it again to make sure it has a hash */
+	rrset->entry.hash = rrset_key_hash(&rrset->rk);
+
+	e = slabhash_lookup(&r->table, rrset->entry.hash, rrset, 0);
+	if(!e)
+		return; /* not in the cache anymore */
+	cachedata = (struct packed_rrset_data*)e->data;
+	if(now > cachedata->ttl || !rrsetdata_equal(updata, cachedata)) {
+		lock_rw_unlock(&e->lock);
+		return; /* expired, or rrset has changed in the meantime */
+	}
+	if(cachedata->security > updata->security) {
+		updata->security = cachedata->security;
+		if(cachedata->security == sec_status_bogus)
+			updata->ttl = cachedata->ttl - now;
+		updata->trust = cachedata->trust;
+	}
+	lock_rw_unlock(&e->lock);
+}

services/cache/rrset.h

@@ -0,0 +1,210 @@
+/*
+ * services/cache/rrset.h - Resource record set cache.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the rrset cache.
+ */
+
+#ifndef SERVICES_CACHE_RRSET_H
+#define SERVICES_CACHE_RRSET_H
+#include "util/storage/lruhash.h"
+#include "util/storage/slabhash.h"
+#include "util/data/packed_rrset.h"
+struct config_file;
+struct alloc_cache;
+struct rrset_ref;
+struct regional;
+
+/**
+ * The rrset cache
+ * Thin wrapper around hashtable, like a typedef.
+ */
+struct rrset_cache {
+	/** uses partitioned hash table */
+	struct slabhash table;
+};
+
+/**
+ * Create rrset cache
+ * @param cfg: config settings or NULL for defaults.
+ * @param alloc: initial default rrset key allocation.
+ * @return: NULL on error.
+ */
+struct rrset_cache* rrset_cache_create(struct config_file* cfg, 
+	struct alloc_cache* alloc);
+
+/**
+ * Delete rrset cache
+ * @param r: rrset cache to delete.
+ */
+void rrset_cache_delete(struct rrset_cache* r);
+
+/**
+ * Adjust settings of the cache to settings from the config file.
+ * May purge the cache. May recreate the cache.
+ * There may be no threading or use by other threads.
+ * @param r: rrset cache to adjust (like realloc).
+ * @param cfg: config settings or NULL for defaults.
+ * @param alloc: initial default rrset key allocation.
+ * @return 0 on error, or new rrset cache pointer on success.
+ */
+struct rrset_cache* rrset_cache_adjust(struct rrset_cache* r, 
+	struct config_file* cfg, struct alloc_cache* alloc);
+
+/**
+ * Touch rrset, with given pointer and id.
+ * Caller may not hold a lock on ANY rrset, this could give deadlock.
+ *
+ * This routine is faster than a hashtable lookup:
+ *	o no bin_lock is acquired.
+ *	o no walk through the bin-overflow-list. 
+ *	o no comparison of the entry key to find it.
+ *
+ * @param r: rrset cache.
+ * @param key: rrset key. Marked recently used (if it was not deleted
+ *	before the lock is acquired, in that case nothing happens).
+ * @param hash: hash value of the item. Please read it from the key when
+ *	you have it locked. Used to find slab from slabhash.
+ * @param id: used to check that the item is unchanged and not deleted.
+ */
+void rrset_cache_touch(struct rrset_cache* r, struct ub_packed_rrset_key* key,
+	hashvalue_t hash, rrset_id_t id);
+
+/**
+ * Update an rrset in the rrset cache. Stores the information for later use.
+ * Will lookup if the rrset is in the cache and perform an update if necessary.
+ * If the item was present, and superior, references are returned to that.
+ * The passed item is then deallocated with rrset_parsedelete.
+ *
+ * A superior rrset is:
+ *	o rrset with better trust value.
+ *	o same trust value, different rdata, newly passed rrset is inserted.
+ * If rdata is the same, TTL in the cache is updated.
+ *
+ * @param r: the rrset cache.
+ * @param ref: reference (ptr and id) to the rrset. Pass reference setup for
+ *	the new rrset. The reference may be changed if the cached rrset is
+ *	superior.
+ *	Before calling the rrset is presumed newly allocated and changeable.
+ *	Afer calling you do not hold a lock, and the rrset is inserted in
+ *	the hashtable so you need a lock to change it.
+ * @param alloc: how to allocate (and deallocate) the special rrset key.
+ * @param timenow: current time (to see if ttl in cache is expired).
+ * @return: true if the passed reference is updated, false if it is unchanged.
+ */
+int rrset_cache_update(struct rrset_cache* r, struct rrset_ref* ref, 
+	struct alloc_cache* alloc, uint32_t timenow);
+
+/**
+ * Lookup rrset. You obtain read/write lock. You must unlock before lookup
+ * anything of else.
+ * @param r: the rrset cache.
+ * @param qname: name of rrset to lookup.
+ * @param qnamelen: length of name of rrset to lookup.
+ * @param qtype: type of rrset to lookup (host order).
+ * @param qclass: class of rrset to lookup (host order).
+ * @param flags: rrset flags, or 0.
+ * @param timenow: used to compare with TTL.
+ * @param wr: set true to get writelock.
+ * @return packed rrset key pointer. Remember to unlock the key.entry.lock.
+ * 	or NULL if could not be found or it was timed out.
+ */
+struct ub_packed_rrset_key* rrset_cache_lookup(struct rrset_cache* r,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint32_t flags, uint32_t timenow, int wr);
+
+/**
+ * Obtain readlock on a (sorted) list of rrset references.
+ * Checks TTLs and IDs of the rrsets and rollbacks locking if not Ok.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ * @param timenow: used to compare with TTL.
+ * @return true on success, false on a failure, which can be that some
+ * 	RRsets have timed out, or that they do not exist any more, the
+ *	RRsets have been purged from the cache.
+ *	If true, you hold readlocks on all the ref items. 
+ */
+int rrset_array_lock(struct rrset_ref* ref, size_t count, uint32_t timenow);
+
+/**
+ * Unlock array (sorted) of rrset references.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ */
+void rrset_array_unlock(struct rrset_ref* ref, size_t count);
+
+/**
+ * Unlock array (sorted) of rrset references and at the same time
+ * touch LRU on the rrsets. It needs the scratch region for temporary
+ * storage as it uses the initial locks to obtain hash values.
+ * @param r: the rrset cache. In this cache LRU is updated.
+ * @param scratch: region for temporary storage of hash values.
+ *	if memory allocation fails, the lru touch fails silently,
+ *	but locks are released. memory errors are logged.
+ * @param ref: array of rrset references (key pointer and ID value).
+ *	duplicate references are allowed and handled.
+ * @param count: size of array.
+ */
+void rrset_array_unlock_touch(struct rrset_cache* r, struct regional* scratch,
+	struct rrset_ref* ref, size_t count);
+
+/**
+ * Update security status of an rrset. Looks up the rrset.
+ * If found, checks if rdata is equal.
+ * If so, it will update the security, trust and rrset-ttl values.
+ * The values are only updated if security is increased (towards secure).
+ * @param r: the rrset cache. 
+ * @param rrset: which rrset to attempt to update. This rrset is left 
+ * 	untouched. The rrset in the cache is updated in-place.
+ */
+void rrset_update_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset);
+
+/**
+ * Looks up security status of an rrset. Looks up the rrset.
+ * If found, checks if rdata is equal, and entry did not expire.
+ * If so, it will update the security, trust and rrset-ttl values.
+ * @param r: the rrset cache. 
+ * @param rrset: This rrset may change security status due to the cache.
+ * 	But its status will only improve, towards secure.
+ */
+void rrset_check_sec_status(struct rrset_cache* r, 
+	struct ub_packed_rrset_key* rrset);
+
+#endif /* SERVICES_CACHE_RRSET_H */

services/listen_dnsport.c

@@ -74,6 +74,7 @@ verbose_print_addr(struct addrinfo *addr)
 			(socklen_t)sizeof(buf)) == 0) {
 			strncpy(buf, "(null)", sizeof(buf));
 		}
+		buf[sizeof(buf)-1] = 0;
 		verbose(VERB_ALGO, "creating %s%s socket %s %d", 
 			addr->ai_socktype==SOCK_DGRAM?"udp":
 			addr->ai_socktype==SOCK_STREAM?"tcp":"otherproto",
@@ -288,7 +289,8 @@ listen_cp_insert(struct comm_point* c, struct listen_dnsport* front)
 
 struct listen_dnsport* 
 listen_create(struct comm_base* base, struct listen_port* ports,
-	size_t bufsize, comm_point_callback_t* cb, void *cb_arg)
+	size_t bufsize, int tcp_accept_count,
+	comm_point_callback_t* cb, void *cb_arg)
 {
 	struct listen_dnsport* front = (struct listen_dnsport*)
 		malloc(sizeof(struct listen_dnsport));
@@ -308,7 +310,7 @@ listen_create(struct comm_base* base, struct listen_port* ports,
 			cp = comm_point_create_udp(base, ports->fd, 
 				front->udp_buff, cb, cb_arg);
 		else 	cp = comm_point_create_tcp(base, ports->fd, 
-				TCP_ACCEPT_COUNT, bufsize, cb, cb_arg);
+				tcp_accept_count, bufsize, cb, cb_arg);
 		if(!cp) {
 			log_err("can't create commpoint");	
 			listen_delete(front);
@@ -381,10 +383,14 @@ listening_ports_open(struct config_file* cfg)
 	struct listen_port* list = NULL;
 	struct addrinfo hints;
 	int i, do_ip4, do_ip6;
+	int do_tcp;
 	char portbuf[32];
 	snprintf(portbuf, sizeof(portbuf), "%d", cfg->port);
 	do_ip4 = cfg->do_ip4;
 	do_ip6 = cfg->do_ip6;
+	do_tcp = cfg->do_tcp;
+	if(cfg->incoming_num_tcp == 0)
+		do_tcp = 0;
 
 	/* getaddrinfo */
 	memset(&hints, 0, sizeof(hints));
@@ -403,7 +409,7 @@ listening_ports_open(struct config_file* cfg)
 	if(cfg->num_ifs == 0) {
 		if(do_ip6) {
 			hints.ai_family = AF_INET6;
-			if(!ports_create_if(NULL, cfg->do_udp, cfg->do_tcp, 
+			if(!ports_create_if("::1", cfg->do_udp, do_tcp, 
 				&hints, portbuf, &list)) {
 				listening_ports_free(list);
 				return NULL;
@@ -411,7 +417,7 @@ listening_ports_open(struct config_file* cfg)
 		}
 		if(do_ip4) {
 			hints.ai_family = AF_INET;
-			if(!ports_create_if(NULL, cfg->do_udp, cfg->do_tcp, 
+			if(!ports_create_if("127.0.0.1", cfg->do_udp, do_tcp, 
 				&hints, portbuf, &list)) {
 				listening_ports_free(list);
 				return NULL;
@@ -423,7 +429,7 @@ listening_ports_open(struct config_file* cfg)
 				continue;
 			hints.ai_family = AF_INET6;
 			if(!ports_create_if(cfg->ifs[i], cfg->do_udp, 
-				cfg->do_tcp, &hints, portbuf, &list)) {
+				do_tcp, &hints, portbuf, &list)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -432,7 +438,7 @@ listening_ports_open(struct config_file* cfg)
 				continue;
 			hints.ai_family = AF_INET;
 			if(!ports_create_if(cfg->ifs[i], cfg->do_udp, 
-				cfg->do_tcp, &hints, portbuf, &list)) {
+				do_tcp, &hints, portbuf, &list)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -452,3 +458,16 @@ void listening_ports_free(struct listen_port* list)
 		list = nx;
 	}
 }
+
+size_t listen_get_mem(struct listen_dnsport* listen)
+{
+	size_t s = sizeof(*listen) + sizeof(*listen->base) + 
+		sizeof(*listen->udp_buff) + 
+		ldns_buffer_capacity(listen->udp_buff);
+	struct listen_list* p;
+	for(p = listen->cps; p; p = p->next) {
+		s += sizeof(*p);
+		s += comm_point_get_mem(p->com);
+	}
+	return s;
+}

services/listen_dnsport.h

@@ -48,9 +48,6 @@ struct listen_list;
 struct addrinfo;
 struct config_file;
 
-/** number of simultaneous open TCP connections for queries */
-#define TCP_ACCEPT_COUNT 10 
-
 /**
  * Listening for queries structure.
  * Contains list of query-listen sockets.
@@ -111,13 +108,15 @@ void listening_ports_free(struct listen_port* list);
  *	for default all ifs.
  * @param ports: the list of shared ports.
  * @param bufsize: size of datagram buffer.
+ * @param tcp_accept_count: max number of simultaneous TCP connections 
+ * 	from clients.
  * @param cb: callback function when a request arrives. It is passed
  *	  the packet and user argument. Return true to send a reply.
  * @param cb_arg: user data argument for callback function.
  * @return: the malloced listening structure, ready for use. NULL on error.
  */
 struct listen_dnsport* listen_create(struct comm_base* base,
-	struct listen_port* ports, size_t bufsize, 
+	struct listen_port* ports, size_t bufsize, int tcp_accept_count,
 	comm_point_callback_t* cb, void* cb_arg);
 
 /**
@@ -141,9 +140,17 @@ void listen_resume(struct listen_dnsport* listen);
 
 /**
  * delete the listening structure
+ * @param listen: listening structure.
  */
 void listen_delete(struct listen_dnsport* listen);
 
+/**
+ * get memory size used by the listening structs
+ * @param listen: listening structure.
+ * @return: size in bytes.
+ */
+size_t listen_get_mem(struct listen_dnsport* listen);
+
 /**
  * Create and bind nonblocking UDP socket
  * @param addr: address info ready to make socket.

services/mesh.c

@@ -0,0 +1,708 @@
+/*
+ * services/mesh.c - deal with mesh of query states and handle events for that.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist in dealing with a mesh of
+ * query states. This mesh is supposed to be thread-specific.
+ * It consists of query states (per qname, qtype, qclass) and connections
+ * between query states and the super and subquery states, and replies to
+ * send back to clients.
+ */
+#include "config.h"
+#include "services/mesh.h"
+#include "services/outbound_list.h"
+#include "services/cache/dns.h"
+#include "util/log.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/regional.h"
+#include "util/data/msgencode.h"
+#include "util/timehist.h"
+#include "util/fptr_wlist.h"
+#include "util/alloc.h"
+
+int
+mesh_state_compare(const void* ap, const void* bp)
+{
+	struct mesh_state* a = (struct mesh_state*)ap;
+	struct mesh_state* b = (struct mesh_state*)bp;
+
+	if(a->s.is_priming && !b->s.is_priming)
+		return -1;
+	if(!a->s.is_priming && b->s.is_priming)
+		return 1;
+
+	if((a->s.query_flags&BIT_RD) && !(b->s.query_flags&BIT_RD))
+		return -1;
+	if(!(a->s.query_flags&BIT_RD) && (b->s.query_flags&BIT_RD))
+		return 1;
+
+	if((a->s.query_flags&BIT_CD) && !(b->s.query_flags&BIT_CD))
+		return -1;
+	if(!(a->s.query_flags&BIT_CD) && (b->s.query_flags&BIT_CD))
+		return 1;
+
+	return query_info_compare(&a->s.qinfo, &b->s.qinfo);
+}
+
+int
+mesh_state_ref_compare(const void* ap, const void* bp)
+{
+	struct mesh_state_ref* a = (struct mesh_state_ref*)ap;
+	struct mesh_state_ref* b = (struct mesh_state_ref*)bp;
+	return mesh_state_compare(a->s, b->s);
+}
+
+struct mesh_area* 
+mesh_create(int num_modules, struct module_func_block** modfunc,
+	struct module_env* env)
+{
+	struct mesh_area* mesh = calloc(1, sizeof(struct mesh_area));
+	if(!mesh) {
+		log_err("mesh area alloc: out of memory");
+		return NULL;
+	}
+	mesh->histogram = timehist_setup();
+	if(!mesh->histogram) {
+		free(mesh);
+		log_err("mesh area alloc: out of memory");
+		return NULL;
+	}
+	mesh->num_modules = num_modules;
+	mesh->modfunc = modfunc;
+	mesh->env = env;
+	rbtree_init(&mesh->run, &mesh_state_compare);
+	rbtree_init(&mesh->all, &mesh_state_compare);
+	mesh->num_reply_addrs = 0;
+	mesh->num_reply_states = 0;
+	mesh->num_detached_states = 0;
+	return mesh;
+}
+
+/** help mesh delete delete mesh states */
+static void
+mesh_delete_helper(rbnode_t* n, void* ATTR_UNUSED(arg))
+{
+	struct mesh_state* mstate = (struct mesh_state*)n->key;
+	mesh_state_cleanup(mstate);
+}
+
+void 
+mesh_delete(struct mesh_area* mesh)
+{
+	if(!mesh)
+		return;
+	/* free all query states */
+	traverse_postorder(&mesh->all, &mesh_delete_helper, NULL);
+	timehist_delete(mesh->histogram);
+	free(mesh);
+}
+
+void mesh_new_client(struct mesh_area* mesh, struct query_info* qinfo,
+        uint16_t qflags, struct edns_data* edns, struct comm_reply* rep,
+        uint16_t qid)
+{
+	struct mesh_state* s = mesh_area_find(mesh, qinfo, qflags, 0);
+	int was_detached = 0;
+	int was_noreply = 0;
+	int added = 0;
+	/* see if it already exists, if not, create one */
+	if(!s) {
+		struct rbnode_t* n;
+		s = mesh_state_create(mesh->env,qinfo, qflags, 0);
+		if(!s) {
+			log_err("mesh_state_create: out of memory; SERVFAIL");
+			error_encode(rep->c->buffer, LDNS_RCODE_SERVFAIL,
+				qinfo, qid, qflags, edns);
+			comm_point_send_reply(rep);
+			return;
+		}
+		n = rbtree_insert(&mesh->all, &s->node);
+		log_assert(n != NULL);
+		/* set detached (it is now) */
+		mesh->num_detached_states++;
+		added = 1;
+	}
+	if(!s->reply_list && s->super_set.count == 0)
+		was_detached = 1;
+	if(!s->reply_list)
+		was_noreply = 1;
+	/* add reply to s */
+	if(!mesh_state_add_reply(s, edns, rep, qid, qflags)) {
+			log_err("mesh_new_client: out of memory; SERVFAIL");
+			error_encode(rep->c->buffer, LDNS_RCODE_SERVFAIL,
+				qinfo, qid, qflags, edns);
+			comm_point_send_reply(rep);
+			if(added)
+				mesh_state_delete(&s->s);
+			return;
+	}
+	/* update statistics */
+	if(was_detached) {
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	if(was_noreply) {
+		mesh->num_reply_states ++;
+	}
+	mesh->num_reply_addrs++;
+	if(added)
+		mesh_run(mesh, s, module_event_new, NULL);
+}
+
+void mesh_report_reply(struct mesh_area* mesh, struct outbound_entry* e,
+        int is_ok, struct comm_reply* reply)
+{
+	e->qstate->reply = reply;
+	mesh_run(mesh, e->qstate->mesh_info,
+		is_ok?module_event_reply:module_event_noreply, e);
+}
+
+struct mesh_state* 
+mesh_state_create(struct module_env* env, struct query_info* qinfo, 
+	uint16_t qflags, int prime)
+{
+	struct regional* region = alloc_reg_obtain(env->alloc);
+	struct mesh_state* mstate;
+	int i;
+	if(!region)
+		return NULL;
+	mstate = (struct mesh_state*)regional_alloc(region, 
+		sizeof(struct mesh_state));
+	if(!mstate) {
+		alloc_reg_release(env->alloc, region);
+		return NULL;
+	}
+	memset(mstate, 0, sizeof(*mstate));
+	mstate->node = *RBTREE_NULL;
+	mstate->run_node = *RBTREE_NULL;
+	mstate->node.key = mstate;
+	mstate->run_node.key = mstate;
+	mstate->reply_list = NULL;
+	rbtree_init(&mstate->super_set, &mesh_state_ref_compare);
+	rbtree_init(&mstate->sub_set, &mesh_state_ref_compare);
+	mstate->num_activated = 0;
+	/* init module qstate */
+	mstate->s.qinfo.qtype = qinfo->qtype;
+	mstate->s.qinfo.qclass = qinfo->qclass;
+	mstate->s.qinfo.qname_len = qinfo->qname_len;
+	mstate->s.qinfo.qname = regional_alloc_init(region, qinfo->qname,
+		qinfo->qname_len);
+	if(!mstate->s.qinfo.qname) {
+		alloc_reg_release(env->alloc, region);
+		return NULL;
+	}
+	/* remove all weird bits from qflags */
+	mstate->s.query_flags = (qflags & (BIT_RD|BIT_CD));
+	mstate->s.is_priming = prime;
+	mstate->s.reply = NULL;
+	mstate->s.region = region;
+	mstate->s.curmod = 0;
+	mstate->s.return_msg = 0;
+	mstate->s.return_rcode = LDNS_RCODE_NOERROR;
+	mstate->s.env = env;
+	mstate->s.mesh_info = mstate;
+	/* init modules */
+	for(i=0; i<env->mesh->num_modules; i++) {
+		mstate->s.minfo[i] = NULL;
+		mstate->s.ext_state[i] = module_state_initial;
+	}
+	return mstate;
+}
+
+void 
+mesh_state_cleanup(struct mesh_state* mstate)
+{
+	struct mesh_area* mesh;
+	int i;
+	if(!mstate)
+		return;
+	/* de-init modules */
+	mesh = mstate->s.env->mesh;
+	for(i=0; i<mesh->num_modules; i++) {
+		log_assert(fptr_whitelist_mod_clear(mesh->modfunc[i]->clear));
+		(*mesh->modfunc[i]->clear)(&mstate->s, i);
+		mstate->s.minfo[i] = NULL;
+		mstate->s.ext_state[i] = module_finished;
+	}
+	alloc_reg_release(mstate->s.env->alloc, mstate->s.region);
+}
+
+void 
+mesh_state_delete(struct module_qstate* qstate)
+{
+	struct mesh_area* mesh;
+	struct mesh_state_ref* super, ref;
+	struct mesh_state* mstate;
+	if(!qstate)
+		return;
+	mstate = qstate->mesh_info;
+	mesh = mstate->s.env->mesh;
+	mesh_detach_subs(&mstate->s);
+	if(!mstate->reply_list && mstate->super_set.count == 0) {
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	if(mstate->reply_list) {
+		log_assert(mesh->num_reply_states > 0);
+		mesh->num_reply_states--;
+	}
+	ref.node.key = &ref;
+	ref.s = mstate;
+	RBTREE_FOR(super, struct mesh_state_ref*, &mstate->super_set) {
+		(void)rbtree_delete(&super->s->sub_set, &ref);
+	}
+	(void)rbtree_delete(&mesh->run, mstate);
+	(void)rbtree_delete(&mesh->all, mstate);
+	mesh_state_cleanup(mstate);
+}
+
+void mesh_detach_subs(struct module_qstate* qstate)
+{
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state_ref* ref, lookup;
+	struct rbnode_t* n;
+	lookup.node.key = &lookup;
+	lookup.s = qstate->mesh_info;
+	RBTREE_FOR(ref, struct mesh_state_ref*, &qstate->mesh_info->sub_set) {
+		n = rbtree_delete(&ref->s->super_set, &lookup);
+		log_assert(n != NULL); /* must have been present */
+		if(!ref->s->reply_list && ref->s->super_set.count == 0) {
+			mesh->num_detached_states++;
+			log_assert(mesh->num_detached_states + 
+				mesh->num_reply_states <= mesh->all.count);
+		}
+	}
+	rbtree_init(&qstate->mesh_info->sub_set, &mesh_state_ref_compare);
+}
+
+int mesh_attach_sub(struct module_qstate* qstate, struct query_info* qinfo,
+        uint16_t qflags, int prime, struct module_qstate** newq)
+{
+	/* find it, if not, create it */
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state* sub = mesh_area_find(mesh, qinfo, qflags, prime);
+	if(!sub) {
+		struct rbnode_t* n;
+		/* create a new one */
+		sub = mesh_state_create(qstate->env, qinfo, qflags, prime);
+		if(!sub) {
+			log_err("mesh_attach_sub: out of memory");
+			return 0;
+		}
+		n = rbtree_insert(&mesh->all, &sub->node);
+		log_assert(n != NULL);
+		/* set detached (it is now) */
+		mesh->num_detached_states++;
+		/* set new query state to run */
+		n = rbtree_insert(&mesh->run, &sub->run_node);
+		log_assert(n != NULL);
+		*newq = &sub->s;
+	} else
+		*newq = NULL;
+	if(!mesh_state_attachment(qstate->mesh_info, sub))
+		return 0;
+	if(!sub->reply_list && sub->super_set.count == 1) {
+		/* it used to be detached, before this one got added */
+		log_assert(mesh->num_detached_states > 0);
+		mesh->num_detached_states--;
+	}
+	/* *newq will be run when inited after the current module stops */
+	return 1;
+}
+
+int mesh_state_attachment(struct mesh_state* super, struct mesh_state* sub)
+{
+	struct rbnode_t* n;
+	struct mesh_state_ref* subref; /* points to sub, inserted in super */
+	struct mesh_state_ref* superref; /* points to super, inserted in sub */
+	if( !(subref = regional_alloc(super->s.region,
+		sizeof(struct mesh_state_ref))) ||
+		!(superref = regional_alloc(sub->s.region,
+		sizeof(struct mesh_state_ref))) ) {
+		log_err("mesh_state_attachment: out of memory");
+		return 0;
+	}
+	superref->node.key = superref;
+	superref->s = super;
+	subref->node.key = subref;
+	subref->s = sub;
+	n = rbtree_insert(&sub->super_set, &superref->node);
+	log_assert(n != NULL);
+	n = rbtree_insert(&super->sub_set, &subref->node);
+	log_assert(n != NULL);
+	return 1;
+}
+
+/** subtract timers and the values do not overflow or become negative */
+static void
+timeval_subtract(struct timeval* d, struct timeval* end, struct timeval* start)
+{
+#ifndef S_SPLINT_S
+	d->tv_sec = end->tv_sec - start->tv_sec;
+	while(end->tv_usec < start->tv_usec) {
+		end->tv_usec += 1000000;
+		d->tv_sec--;
+	}
+	d->tv_usec = end->tv_usec - start->tv_usec;
+#endif
+}
+
+/** add timers and the values do not overflow or become negative */
+static void
+timeval_add(struct timeval* d, struct timeval* add)
+{
+#ifndef S_SPLINT_S
+	d->tv_sec += add->tv_sec;
+	d->tv_usec += add->tv_usec;
+	while(d->tv_usec > 1000000 ) {
+		d->tv_usec -= 1000000;
+		d->tv_sec++;
+	}
+#endif
+}
+
+/** divide sum of timers to get average */
+static void
+timeval_divide(struct timeval* avg, struct timeval* sum, size_t d)
+{
+#ifndef S_SPLINT_S
+	size_t leftover;
+	if(d == 0) {
+		avg->tv_sec = 0;
+		avg->tv_usec = 0;
+		return;
+	}
+	avg->tv_sec = sum->tv_sec / d;
+	avg->tv_usec = sum->tv_usec / d;
+	/* handle fraction from seconds divide */
+	leftover = sum->tv_sec - avg->tv_sec*d;
+	avg->tv_usec += (leftover*1000000)/d;
+#endif
+}
+
+/**
+ * Send reply to mesh reply entry
+ * @param m: mesh state to send it for.
+ * @param rcode: if not 0, error code.
+ * @param rep: reply to send (or NULL if rcode is set).
+ * @param r: reply entry
+ */
+static void
+mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
+	struct mesh_reply* r)
+{
+	struct timeval end_time;
+	int secure;
+	/* examine security status */
+	if(m->s.env->need_to_validate && !(r->qflags&BIT_CD) && rep && 
+		rep->security <= sec_status_bogus) {
+		rcode = LDNS_RCODE_SERVFAIL;
+	}
+	if(rep && rep->security == sec_status_secure)
+		secure = 1;
+	else	secure = 0;
+	/* send the reply */
+	if(rcode) {
+		error_encode(r->query_reply.c->buffer, rcode, &m->s.qinfo,
+			r->qid, r->qflags, &r->edns);
+		comm_point_send_reply(&r->query_reply);
+	} else {
+		size_t udp_size = r->edns.udp_size;
+		r->edns.edns_version = EDNS_ADVERTISED_VERSION;
+		r->edns.udp_size = EDNS_ADVERTISED_SIZE;
+		r->edns.ext_rcode = 0;
+		r->edns.bits &= EDNS_DO;
+		if(!reply_info_answer_encode(&m->s.qinfo, rep, r->qid, 
+			r->qflags, r->query_reply.c->buffer, 0, 1, 
+			m->s.env->scratch, udp_size, &r->edns, 
+			(int)(r->edns.bits & EDNS_DO), secure)) 
+		{
+			error_encode(r->query_reply.c->buffer, 
+				LDNS_RCODE_SERVFAIL, &m->s.qinfo, r->qid, 
+				r->qflags, &r->edns);
+		}
+		comm_point_send_reply(&r->query_reply);
+	}
+	/* account */
+	m->s.env->mesh->num_reply_addrs--;
+	if(gettimeofday(&end_time, NULL) < 0) {
+		log_err("gettimeofday: %s", strerror(errno));
+		return;
+	} else {
+		struct timeval duration;
+		timeval_subtract(&duration, &end_time, &r->start_time);
+		verbose(VERB_ALGO, "query took %d.%6.6d sec",
+			(int)duration.tv_sec, (int)duration.tv_usec);
+		m->s.env->mesh->replies_sent++;
+		timeval_add(&m->s.env->mesh->replies_sum_wait, &duration);
+		timehist_insert(m->s.env->mesh->histogram, &duration);
+	}
+}
+
+void mesh_query_done(struct mesh_state* mstate)
+{
+	struct mesh_reply* r;
+	struct reply_info* rep = (mstate->s.return_msg?
+		mstate->s.return_msg->rep:NULL);
+	for(r = mstate->reply_list; r; r = r->next) {
+		mesh_send_reply(mstate, mstate->s.return_rcode, rep, r);
+	}
+}
+
+void mesh_walk_supers(struct mesh_area* mesh, struct mesh_state* mstate)
+{
+	struct mesh_state_ref* ref;
+	RBTREE_FOR(ref, struct mesh_state_ref*, &mstate->super_set)
+	{
+		/* make super runnable */
+		(void)rbtree_insert(&mesh->run, &ref->s->run_node);
+		/* callback the function to inform super of result */
+		log_assert(fptr_whitelist_mod_inform_super(
+			mesh->modfunc[ref->s->s.curmod]->inform_super));
+		(*mesh->modfunc[ref->s->s.curmod]->inform_super)(&mstate->s, 
+			ref->s->s.curmod, &ref->s->s);
+	}
+}
+
+struct mesh_state* mesh_area_find(struct mesh_area* mesh,
+	struct query_info* qinfo, uint16_t qflags, int prime)
+{
+	struct mesh_state key;
+	struct mesh_state* result;
+
+	key.node.key = &key;
+	key.s.is_priming = prime;
+	key.s.qinfo = *qinfo;
+	key.s.query_flags = qflags;
+	
+	result = (struct mesh_state*)rbtree_search(&mesh->all, &key);
+	return result;
+}
+
+int mesh_state_add_reply(struct mesh_state* s, struct edns_data* edns,
+        struct comm_reply* rep, uint16_t qid, uint16_t qflags)
+{
+	struct mesh_reply* r = regional_alloc(s->s.region, 
+		sizeof(struct mesh_reply));
+	if(!r)
+		return 0;
+	r->query_reply = *rep;
+	r->edns = *edns;
+	r->qid = qid;
+	r->qflags = qflags;
+	if(gettimeofday(&r->start_time, NULL) < 0) {
+		log_err("addrep: gettimeofday: %s", strerror(errno));
+		memset(&r->start_time, 0, sizeof(r->start_time));
+	}
+	r->next = s->reply_list;
+	s->reply_list = r;
+	return 1;
+
+}
+
+/**
+ * Continue processing the mesh state at another module.
+ * Handles module to modules tranfer of control.
+ * Handles module finished.
+ * @param mesh: the mesh area.
+ * @param mstate: currently active mesh state.
+ * 	Deleted if finished, calls _done and _supers to 
+ * 	send replies to clients and inform other mesh states.
+ * 	This in turn may create additional runnable mesh states.
+ * @param s: state at which the current module exited.
+ * @param ev: the event sent to the module.
+ * 	returned is the event to send to the next module.
+ * @return true if continue processing at the new module.
+ * 	false if not continued processing is needed.
+ */
+static int
+mesh_continue(struct mesh_area* mesh, struct mesh_state* mstate,
+	enum module_ext_state s, enum module_ev* ev)
+{
+	mstate->num_activated++;
+	if(mstate->num_activated > MESH_MAX_ACTIVATION) {
+		/* module is looping. Stop it. */
+		log_err("internal error: looping module stopped");
+		log_query_info(VERB_DETAIL, "pass error for qstate",
+			&mstate->s.qinfo);
+		s = module_error;
+	}
+	if(s == module_wait_module) {
+		/* start next module */
+		mstate->s.curmod++;
+		if(mesh->num_modules == mstate->s.curmod) {
+			log_err("Cannot pass to next module; at last module");
+			log_query_info(VERB_DETAIL, "pass error for qstate",
+				&mstate->s.qinfo);
+			mstate->s.curmod--;
+			return mesh_continue(mesh, mstate, module_error, ev);
+		}
+		*ev = module_event_pass;
+		return 1;
+	}
+	if(s == module_error && mstate->s.return_rcode == LDNS_RCODE_NOERROR) {
+		/* error is bad, handle pass back up below */
+		mstate->s.return_rcode = LDNS_RCODE_SERVFAIL;
+	}
+	if(s == module_error || s == module_finished) {
+		if(mstate->s.curmod == 0) {
+			mesh_query_done(mstate);
+			mesh_walk_supers(mesh, mstate);
+			mesh_state_delete(&mstate->s);
+			return 0;
+		}
+		/* pass along the locus of control */
+		mstate->s.curmod --;
+		*ev = module_event_moddone;
+		return 1;
+	}
+	return 0;
+}
+
+void mesh_run(struct mesh_area* mesh, struct mesh_state* mstate,
+	enum module_ev ev, struct outbound_entry* e)
+{
+	enum module_ext_state s;
+	verbose(VERB_ALGO, "mesh_run: start");
+	while(mstate) {
+		/* run the module */
+		log_assert(fptr_whitelist_mod_operate(
+			mesh->modfunc[mstate->s.curmod]->operate));
+		(*mesh->modfunc[mstate->s.curmod]->operate)
+			(&mstate->s, ev, mstate->s.curmod, e);
+
+		/* examine results */
+		mstate->s.reply = NULL;
+		regional_free_all(mstate->s.env->scratch);
+		s = mstate->s.ext_state[mstate->s.curmod];
+		verbose(VERB_ALGO, "mesh_run: %s module exit state is %s", 
+			mesh->modfunc[mstate->s.curmod]->name, strextstate(s));
+		e = NULL;
+		if(mesh_continue(mesh, mstate, s, &ev))
+			continue;
+
+		/* run more modules */
+		ev = module_event_pass;
+		if(mesh->run.count > 0) {
+			/* pop random element off the runnable tree */
+			mstate = (struct mesh_state*)mesh->run.root->key;
+			(void)rbtree_delete(&mesh->run, mstate);
+		} else mstate = NULL;
+	}
+	if(verbosity >= VERB_ALGO) {
+		mesh_stats(mesh, "mesh_run: end");
+		mesh_log_list(mesh);
+	}
+}
+
+void 
+mesh_log_list(struct mesh_area* mesh)
+{
+	char buf[30];
+	struct mesh_state* m;
+	int num = 0;
+	RBTREE_FOR(m, struct mesh_state*, &mesh->all) {
+		snprintf(buf, sizeof(buf), "%d%s%s%s%s%s mod%d %s", 
+			num++, (m->s.is_priming)?"p":"",  /* prime */
+			(m->s.query_flags&BIT_RD)?"RD":"",
+			(m->s.query_flags&BIT_CD)?"CD":"",
+			(m->super_set.count==0)?"d":"", /* detached */
+			(m->sub_set.count!=0)?"c":"",  /* children */
+			m->s.curmod, (m->reply_list)?"hr":"nr"); /*hasreply*/
+		log_query_info(VERB_ALGO, buf, &m->s.qinfo);
+	}
+}
+
+void 
+mesh_stats(struct mesh_area* mesh, const char* str)
+{
+	log_info("%s %u states (%u with reply, %u detached), "
+		"%u waiting replies", str, (unsigned)mesh->all.count, 
+		(unsigned)mesh->num_reply_states,
+		(unsigned)mesh->num_detached_states,
+		(unsigned)mesh->num_reply_addrs);
+	if(mesh->replies_sent > 0) {
+		struct timeval avg;
+		timeval_divide(&avg, &mesh->replies_sum_wait, 
+			mesh->replies_sent);
+		log_info("sent %u replies, with average wait "
+			"of %d.%6.6d sec", (unsigned)mesh->replies_sent,
+			(int)avg.tv_sec, (int)avg.tv_usec);
+		log_info("histogram of reply wait times");
+		timehist_log(mesh->histogram);
+	}
+}
+
+size_t 
+mesh_get_mem(struct mesh_area* mesh)
+{
+	struct mesh_state* m;
+	size_t s = sizeof(*mesh) + sizeof(struct timehist) +
+		sizeof(struct th_buck)*mesh->histogram->num;
+	RBTREE_FOR(m, struct mesh_state*, &mesh->all) {
+		/* all, including m itself allocated in qstate region */
+		s += regional_get_mem(m->s.region);
+	}
+	return s;
+}
+
+/** helper recursive rbtree find routine */
+static int
+find_in_subsub(struct mesh_state* m, struct mesh_state* tofind)
+{
+	struct mesh_state_ref* r;
+	RBTREE_FOR(r, struct mesh_state_ref*, &m->sub_set) {
+		if(r->s == tofind || find_in_subsub(r->s, tofind))
+			return 1;
+	}
+	return 0;
+}
+
+int 
+mesh_detect_cycle(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t flags, int prime)
+{
+	struct mesh_area* mesh = qstate->env->mesh;
+	struct mesh_state* cyc_m = qstate->mesh_info;
+	struct mesh_state* dep_m = mesh_area_find(mesh, qinfo, flags, prime);
+	if(!dep_m)
+		return 0;
+	if(dep_m == cyc_m || find_in_subsub(dep_m, cyc_m))
+		return 1;
+	return 0;
+}

services/mesh.h

@@ -0,0 +1,387 @@
+/*
+ * services/mesh.h - deal with mesh of query states and handle events for that.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to assist in dealing with a mesh of
+ * query states. This mesh is supposed to be thread-specific.
+ * It consists of query states (per qname, qtype, qclass) and connections
+ * between query states and the super and subquery states, and replies to
+ * send back to clients.
+ */
+
+#ifndef SERVICES_MESH_H
+#define SERVICES_MESH_H
+
+#include "util/rbtree.h"
+#include "util/netevent.h"
+#include "util/data/msgparse.h"
+#include "util/module.h"
+struct mesh_state;
+struct mesh_reply;
+struct query_info;
+struct reply_info;
+struct outbound_entry;
+struct timehist;
+
+/**
+ * Maximum number of mesh state activations. Any more is likely an
+ * infinite loop in the module. It is then terminated.
+ */
+#define MESH_MAX_ACTIVATION 1000
+
+/** 
+ * Mesh of query states
+ */
+struct mesh_area {
+	/** the number of modules */
+	int num_modules;
+	/** the module callbacks, array of num_modules length (ref only) */
+	struct module_func_block** modfunc;
+	/** environment for new states */
+	struct module_env* env;
+
+	/** set of runnable queries (mesh_state.run_node) */
+	rbtree_t run;
+	/** rbtree of all current queries (mesh_state.node)*/
+	rbtree_t all;
+
+	/** count of the total number of mesh_reply entries */
+	size_t num_reply_addrs;
+	/** count of the number of mesh_states that have mesh_replies 
+	 * Because a state can send results to multiple reply addresses,
+	 * this number must be equal or lower than num_reply_addrs. */
+	size_t num_reply_states;
+	/** number of mesh_states that have no mesh_replies, and also
+	 * an empty set of super-states, thus are 'toplevel' or detached
+	 * internal opportunistic queries */
+	size_t num_detached_states;
+
+	/** number of replies sent */
+	size_t replies_sent;
+	/** sum of waiting times for the replies */
+	struct timeval replies_sum_wait;
+	/** histogram of time values */
+	struct timehist* histogram;
+};
+
+/**
+ * A mesh query state
+ * Unique per qname, qtype, qclass (from the qstate).
+ * And RD / CD flag; in case a client turns it off.
+ * And priming queries are different from ordinary queries (because of hints).
+ *
+ * The entire structure is allocated in a region, this region is the qstate
+ * region. All parts (rbtree nodes etc) are also allocated in the region.
+ */
+struct mesh_state {
+	/** node in mesh_area all tree, key is this struct. Must be first. */
+	rbnode_t node;
+	/** node in mesh_area runnable tree, key is this struct */
+	rbnode_t run_node;
+	/** the query state. Note that the qinfo and query_flags 
+	 * may not change. */
+	struct module_qstate s;
+	/** the list of replies to clients for the results */
+	struct mesh_reply* reply_list;
+	/** set of superstates (that want this state's result) 
+	 * contains struct mesh_state_ref* */
+	rbtree_t super_set;
+	/** set of substates (that this state needs to continue)
+	 * contains struct mesh_state_ref* */
+	rbtree_t sub_set;
+	/** number of activations for the mesh state */
+	size_t num_activated;
+};
+
+/**
+ * Rbtree reference to a mesh_state.
+ * Used in super_set and sub_set. 
+ */
+struct mesh_state_ref {
+	/** node in rbtree for set, key is this structure */
+	rbnode_t node;
+	/** the mesh state */
+	struct mesh_state* s;
+};
+
+/**
+ * Reply to a client
+ */
+struct mesh_reply {
+	/** next in reply list */
+	struct mesh_reply* next;
+	/** the query reply destination, packet buffer and where to send. */
+	struct comm_reply query_reply;
+	/** edns data from query */
+	struct edns_data edns;
+	/** the time when request was entered */
+	struct timeval start_time;
+	/** id of query, in network byteorder. */
+	uint16_t qid;
+	/** flags of query, for reply flags */
+	uint16_t qflags;
+};
+
+/* ------------------- Functions for worker -------------------- */
+
+/**
+ * Allocate mesh, to empty.
+ * @param num_modules: number of modules that are present.
+ * @param modfunc: array passed (alloced and deleted by caller), that has
+ * 	num_modules function callbacks for the modules.
+ * @param env: environment for new queries.
+ * @return mesh: the new mesh or NULL on error.
+ */
+struct mesh_area* mesh_create(int num_modules, 
+	struct module_func_block** modfunc, struct module_env* env);
+
+/**
+ * Delete mesh, and all query states and replies in it.
+ * @param mesh: the mesh to delete.
+ */
+void mesh_delete(struct mesh_area* mesh);
+
+/**
+ * New query incoming from clients. Create new query state if needed, and
+ * add mesh_reply to it. Returns error to client on malloc failures.
+ * Will run the mesh area queries to process if a new query state is created.
+ *
+ * @param mesh: the mesh.
+ * @param qinfo: query from client.
+ * @param qflags: flags from client query.
+ * @param edns: edns data from client query.
+ * @param rep: where to reply to.
+ * @param qid: query id to reply with.
+ */
+void mesh_new_client(struct mesh_area* mesh, struct query_info* qinfo,
+	uint16_t qflags, struct edns_data* edns, struct comm_reply* rep, 
+	uint16_t qid);
+
+/**
+ * Handle new event from the wire. A serviced query has returned.
+ * The query state will be made runnable, and the mesh_area will process
+ * query states until processing is complete.
+ *
+ * @param mesh: the query mesh.
+ * @param e: outbound entry, with query state to run and reply pointer.
+ * @param is_ok: if true, reply is OK, otherwise a timeout happened.
+ * @param reply: the comm point reply info.
+ */
+void mesh_report_reply(struct mesh_area* mesh, struct outbound_entry* e,
+	int is_ok, struct comm_reply* reply);
+
+/* ------------------- Functions for module environment --------------- */
+
+/**
+ * Detach-subqueries.
+ * Remove all sub-query references from this query state.
+ * Keeps super-references of those sub-queries correct.
+ * Updates stat items in mesh_area structure.
+ * @param qstate: used to find mesh state.
+ */
+void mesh_detach_subs(struct module_qstate* qstate);
+
+/**
+ * Attach subquery.
+ * Creates it if it does not exist already.
+ * Keeps sub and super references correct.
+ * Updates stat items in mesh_area structure.
+ * Pass if it is priming query or not.
+ * return:
+ * 	o if error (malloc) happened.
+ * 	o need to initialise the new state (module init; it is a new state).
+ * 	  so that the next run of the query with this module is successful.
+ * 	o no init needed, attachment successful.
+ *
+ * @param qstate: the state to find mesh state, and that wants to receive
+ * 	the results from the new subquery.
+ * @param qinfo: what to query for (copied).
+ * @param qflags: what flags to use (RD / CD flag or not).
+ * @param prime: if it is a (stub) priming query.
+ * @param newq: If the new subquery needs initialisation, it is returned,
+ * 	otherwise NULL is returned.
+ * @return: false on error, true if success (and init may be needed).
+ */
+int mesh_attach_sub(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t qflags, int prime, struct module_qstate** newq);
+
+/**
+ * Query state is done, send messages to reply entries.
+ * Encode messages using reply entry values and the querystate (with original
+ * qinfo), using given reply_info.
+ * Pass errcode != 0 if an error reply is needed.
+ * If no reply entries, nothing is done.
+ * Must be called before a module can module_finished or return module_error.
+ * The module must handle the super query states itself as well.
+ *
+ * @param mstate: mesh state that is done. return_rcode and return_msg
+ * 	are used for replies.
+ * 	return_rcode: if not 0 (NOERROR) an error is sent back (and 
+ * 		return_msg is ignored).
+ * 	return_msg: reply to encode and send back to clients.
+ */
+void mesh_query_done(struct mesh_state* mstate);
+
+/**
+ * Call inform_super for the super query states that are interested in the 
+ * results from this query state. These can then be changed for error 
+ * or results.
+ * Called when a module is module_finished or returns module_error.
+ * The super query states become runnable with event module_event_pass.
+ *
+ * @param mesh: mesh area to add newly runnable modules to.
+ * @param mstate: the state that has results, used to find mesh state.
+ */
+void mesh_walk_supers(struct mesh_area* mesh, struct mesh_state* mstate);
+
+/**
+ * Delete mesh state, cleanup and also rbtrees and so on.
+ * Will detach from all super/subnodes.
+ * @param qstate: to remove.
+ */
+void mesh_state_delete(struct module_qstate* qstate);
+
+/* ------------------- Functions for mesh -------------------- */
+
+/**
+ * Create and initialize a new mesh state and its query state
+ * Does not put the mesh state into rbtrees and so on.
+ * @param env: module environment to set.
+ * @param qinfo: query info that the mesh is for.
+ * @param qflags: flags for query (RD / CD flag).
+ * @param prime: if true, it is a priming query, set is_priming on mesh state.
+ * @return: new mesh state or NULL on allocation error.
+ */
+struct mesh_state* mesh_state_create(struct module_env* env, 
+	struct query_info* qinfo, uint16_t qflags, int prime);
+
+/**
+ * Cleanup a mesh state and its query state. Does not do rbtree or 
+ * reference cleanup.
+ * @param mstate: mesh state to cleanup. Its pointer may no longer be used
+ * 	afterwards. Cleanup rbtrees before calling this function.
+ */
+void mesh_state_cleanup(struct mesh_state* mstate);
+
+/**
+ * Find a mesh state in the mesh area. Pass relevant flags.
+ *
+ * @param mesh: the mesh area to look in.
+ * @param qinfo: what query
+ * @param qflags: if RD / CD bit is set or not.
+ * @param prime: if it is a priming query.
+ * @return: mesh state or NULL if not found.
+ */
+struct mesh_state* mesh_area_find(struct mesh_area* mesh, 
+	struct query_info* qinfo, uint16_t qflags, int prime);
+
+/**
+ * Setup attachment super/sub relation between super and sub mesh state.
+ * The relation must not be present when calling the function.
+ * Does not update stat items in mesh_area.
+ * @param super: super state.
+ * @param sub: sub state.
+ * @return: 0 on alloc error.
+ */
+int mesh_state_attachment(struct mesh_state* super, struct mesh_state* sub);
+
+/**
+ * Create new reply structure and attach it to a mesh state.
+ * Does not update stat items in mesh area.
+ * @param s: the mesh state.
+ * @param edns: edns data for reply (bufsize).
+ * @param rep: comm point reply info.
+ * @param qid: ID of reply.
+ * @param qflags: original query flags.
+ * @return: 0 on alloc error.
+ */
+int mesh_state_add_reply(struct mesh_state* s, struct edns_data* edns, 
+	struct comm_reply* rep, uint16_t qid, uint16_t qflags);
+
+/**
+ * Run the mesh. Run all runnable mesh states. Which can create new
+ * runnable mesh states. Until completion. Automatically called by
+ * mesh_report_reply and mesh_new_client as needed.
+ * @param mesh: mesh area.
+ * @param mstate: first mesh state to run.
+ * @param ev: event the mstate. Others get event_pass.
+ * @param e: if a reply, its outbound entry.
+ */
+void mesh_run(struct mesh_area* mesh, struct mesh_state* mstate, 
+	enum module_ev ev, struct outbound_entry* e);
+
+/**
+ * Print some stats about the mesh to the log.
+ * @param mesh: the mesh to print it for.
+ * @param str: descriptive string to go with it.
+ */
+void mesh_stats(struct mesh_area* mesh, const char* str);
+
+/**
+ * Print all the states in the mesh to the log.
+ * @param mesh: the mesh to print all states of.
+ */
+void mesh_log_list(struct mesh_area* mesh);
+
+/**
+ * Calculate memory size in use by mesh and all queries inside it.
+ * @param mesh: the mesh to examine.
+ * @return size in bytes.
+ */
+size_t mesh_get_mem(struct mesh_area* mesh);
+
+/**
+ * Find cycle; see if the given mesh is in the targets sub, or sub-sub, ...
+ * trees.
+ * @param qstate: given mesh querystate.
+ * @param qinfo: query info for dependency.
+ * @param flags: query flags of dependency.
+ * @param prime: if dependency is a priming query or not.
+ * @return true if the name,type,class exists and the given qstate mesh exists
+ * 	as a dependency of that name. Thus if qstate becomes dependent on
+ * 	name,type,class then a cycle is created.
+ */
+int mesh_detect_cycle(struct module_qstate* qstate, struct query_info* qinfo,
+	uint16_t flags, int prime);
+
+/** compare two mesh_states */
+int mesh_state_compare(const void* ap, const void* bp);
+
+/** compare two mesh references */
+int mesh_state_ref_compare(const void* ap, const void* bp);
+
+#endif /* SERVICES_MESH_H */

services/outbound_list.c

@@ -0,0 +1,88 @@
+/*
+ * services/outbound_list.c - keep list of outbound serviced queries.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to help a module keep track of the
+ * queries it has outstanding to authoritative servers.
+ */
+#include "config.h"
+#include "services/outbound_list.h"
+#include "services/outside_network.h"
+
+void 
+outbound_list_init(struct outbound_list* list)
+{
+	list->first = NULL;
+}
+
+void 
+outbound_list_clear(struct outbound_list* list)
+{
+	struct outbound_entry *p, *np;
+	p = list->first;
+	while(p) {
+		np = p->next;
+		outnet_serviced_query_stop(p->qsent, p);
+		/* in region, no free needed */
+		p = np;
+	}
+	outbound_list_init(list);
+}
+
+void 
+outbound_list_insert(struct outbound_list* list, struct outbound_entry* e)
+{
+	if(list->first)
+		list->first->prev = e;
+	e->next = list->first;
+	e->prev = NULL;
+	list->first = e;
+}
+
+void 
+outbound_list_remove(struct outbound_list* list, struct outbound_entry* e)
+{
+	if(!e)
+		return;
+	outnet_serviced_query_stop(e->qsent, e);
+	if(e->next)
+		e->next->prev = e->prev;
+	if(e->prev)
+		e->prev->next = e->next;
+	else	list->first = e->next;
+	/* in region, no free needed */
+}

services/outbound_list.h

@@ -0,0 +1,105 @@
+/*
+ * services/outbound_list.h - keep list of outbound serviced queries.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to help a module keep track of the
+ * queries it has outstanding to authoritative servers.
+ */
+#ifndef SERVICES_OUTBOUND_LIST_H
+#define SERVICES_OUTBOUND_LIST_H
+struct outbound_entry;
+struct serviced_query;
+struct module_qstate;
+
+/**
+ * The outbound list. This structure is part of the module specific query
+ * state.
+ */
+struct outbound_list {
+	/** The linked list of outbound query entries. */
+	struct outbound_entry* first;
+};
+
+/**
+ * Outbound list entry. A serviced query sent by a module processing the
+ * query from the qstate. Double linked list to aid removal.
+ */
+struct outbound_entry {
+	/** next in list */
+	struct outbound_entry* next;
+	/** prev in list */
+	struct outbound_entry* prev;
+	/** The query that was sent out */
+	struct serviced_query* qsent;
+	/** the module query state that sent it */
+	struct module_qstate* qstate;
+};
+
+/**
+ * Init the user allocated outbound list structure
+ * @param list: the list structure.
+ */
+void outbound_list_init(struct outbound_list* list);
+
+/**
+ * Clear the user owner outbound list structure.
+ * Deletes serviced queries.
+ * @param list: the list structure. It is cleared, but the list struct itself
+ * 	is callers responsability to delete.
+ */
+void outbound_list_clear(struct outbound_list* list);
+
+/**
+ * Insert new entry into the list. Caller must allocate the entry with malloc.
+ * qstate and qsent are set by caller.
+ * @param list: the list to add to.
+ * @param e: entry to add, it is only half initialised at call start, fully
+ *	initialised at call end.
+ */
+void outbound_list_insert(struct outbound_list* list, 
+	struct outbound_entry* e);
+
+/**
+ * Remove an entry from the list, and deletes it. 
+ * Deletes serviced query in the entry.
+ * @param list: the list to remove from.
+ * @param e: the entry to remove.
+ */
+void outbound_list_remove(struct outbound_list* list, 
+	struct outbound_entry* e);
+
+#endif /* SERVICES_OUTBOUND_LIST_H */

services/outside_network.h

@@ -51,6 +51,7 @@ struct pending_timeout;
 struct ub_randstate;
 struct pending_tcp;
 struct waiting_tcp;
+struct infra_cache;
 
 /**
  * Send queries to outside servers and wait for answers from servers.
@@ -63,30 +64,39 @@ struct outside_network {
 	/** buffer shared by UDP connections, since there is only one
 	    datagram at any time. */
 	ldns_buffer* udp_buff;
+	/** serviced_callbacks malloc overhead when processing multiple
+	 * identical serviced queries to the same server. */
+	size_t svcd_overhead;
 
 	/** 
 	 * Array of udp comm point* that are used to listen to pending events.
 	 * Each is on a different port. This is for ip4 ports.
 	 */
-	struct comm_point **udp4_ports;
+	struct comm_point** udp4_ports;
 	/** number of udp4 ports */
 	size_t num_udp4;
 
 	/**
 	 * The opened ip6 ports.
 	 */
-	struct comm_point **udp6_ports;
+	struct comm_point** udp6_ports;
 	/** number of udp6 ports */
 	size_t num_udp6;
 
 	/** pending udp answers. sorted by id, addr */
-	rbtree_t *pending;
+	rbtree_t* pending;
+	/** serviced queries, sorted by qbuf, addr, dnssec */
+	rbtree_t* serviced;
+	/** host cache, pointer but not owned by outnet. */
+	struct infra_cache* infra;
+	/** where to get random numbers */
+	struct ub_randstate* rnd;
 
 	/**
 	 * Array of tcp pending used for outgoing TCP connections.
 	 * Each can be used to establish a TCP connection with a server.
-	 * The file descriptors are -1 if its free, need to be opened for 
-	 * the tcp connection. Can be used for ip4 and ip6.
+	 * The file descriptors are -1 if they are free, and need to be 
+	 * opened for the tcp connection. Can be used for ip4 and ip6.
 	 */
 	struct pending_tcp **tcp_conns;
 	/** number of tcp communication points. */
@@ -170,6 +180,66 @@ struct waiting_tcp {
 	void* cb_arg;
 };
 
+/**
+ * Callback to party interested in serviced query results.
+ */
+struct service_callback {
+	/** next in callback list */
+	struct service_callback* next;
+	/** callback function */
+	comm_point_callback_t* cb;
+	/** user argument for callback function */
+	void* cb_arg;
+};
+
+/**
+ * Query service record.
+ * Contains query and destination. UDP, TCP, EDNS are all tried.
+ * complete with retries and timeouts. A number of interested parties can
+ * receive a callback.
+ */
+struct serviced_query {
+	/** The rbtree node, key is this record */
+	rbnode_t node;
+	/** The query that needs to be answered. Starts with flags u16,
+	 * then qdcount, ..., including qname, qtype, qclass. Does not include
+	 * EDNS record. */
+	uint8_t* qbuf;
+	/** length of qbuf. */
+	size_t qbuflen;
+	/** If an EDNS section is included, the DO/CD bit will be turned on. */
+	int dnssec;
+	/** where to send it */
+	struct sockaddr_storage addr;
+	/** length of addr field in use. */
+	socklen_t addrlen;
+	/** current status */
+	enum serviced_query_status {
+		/** initial status */
+		serviced_initial,
+		/** UDP with EDNS sent */
+		serviced_query_UDP_EDNS,
+		/** UDP without EDNS sent */
+		serviced_query_UDP,
+		/** TCP with EDNS sent */
+		serviced_query_TCP_EDNS,
+		/** TCP without EDNS sent */
+		serviced_query_TCP
+	} status;
+	/** true if serviced_query is scheduled for deletion already */
+	int to_be_deleted;
+	/** number of UDP retries */
+	int retry;
+	/** time last UDP was sent */
+	struct timeval last_sent_time;
+	/** outside network this is part of */
+	struct outside_network* outnet;
+	/** list of interested parties that need callback on results. */
+	struct service_callback* cblist;
+	/** the UDP or TCP query that is pending, see status which */
+	void* pending;
+};
+
 /**
  * Create outside_network structure with N udp ports.
  * @param base: the communication base to use for event handling.
@@ -183,11 +253,14 @@ struct waiting_tcp {
  * @param port_base: if -1 system assigns ports, otherwise try to get
  *    the ports numbered from this starting number.
  * @param num_tcp: number of outgoing tcp buffers to preallocate.
+ * @param infra: pointer to infra cached used for serviced queries.
+ * @param rnd: stored to create random numbers for serviced queries.
  * @return: the new structure (with no pending answers) or NULL on error.
  */
 struct outside_network* outside_network_create(struct comm_base* base,
 	size_t bufsize, size_t num_ports, char** ifs, int num_ifs,
-	int do_ip4, int do_ip6, int port_base, size_t num_tcp);
+	int do_ip4, int do_ip6, int port_base, size_t num_tcp, 
+	struct infra_cache* infra, struct ub_randstate* rnd);
 
 /**
  * Delete outside_network structure.
@@ -202,17 +275,16 @@ void outside_network_delete(struct outside_network* outnet);
  * @param packet: wireformat query to send to destination.
  * @param addr: address to send to.
  * @param addrlen: length of addr.
- * @param timeout: in seconds from now.
+ * @param timeout: in milliseconds from now.
  * @param callback: function to call on error, timeout or reply.
- *    The routine does not return an error, instead it calls the callback,
- *    with an error code if an error happens.
  * @param callback_arg: user argument for callback function.
  * @param rnd: random state for generating ID and port.
+ * @return: NULL on error for malloc or socket. Else the pending query object.
  */
-void pending_udp_query(struct outside_network* outnet, ldns_buffer* packet, 
-	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* callback, void* callback_arg,
-	struct ub_randstate* rnd);
+struct pending* pending_udp_query(struct outside_network* outnet, 
+	ldns_buffer* packet, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+	void* callback_arg, struct ub_randstate* rnd);
 
 /**
  * Send TCP query. May wait for TCP buffer. Selects ID to be random, and 
@@ -225,15 +297,14 @@ void pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
  *    Timer starts running now. Timer may expire if all buffers are used,
  *    without any query been sent to the server yet.
  * @param callback: function to call on error, timeout or reply.
- *    The routine does not return an error, instead it calls the callback,
- *    with an error code if an error happens.
  * @param callback_arg: user argument for callback function.
  * @param rnd: random state for generating ID.
+ * @return: false on error for malloc or socket. Else the pending TCP object.
  */
-void pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet, 
-	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* callback, void* callback_arg,
-	struct ub_randstate* rnd);
+struct waiting_tcp* pending_tcp_query(struct outside_network* outnet, 
+	ldns_buffer* packet, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+	void* callback_arg, struct ub_randstate* rnd);
 
 /**
  * Delete pending answer.
@@ -243,4 +314,87 @@ void pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
  */
 void pending_delete(struct outside_network* outnet, struct pending* p);
 
+/**
+ * Perform a serviced query to the authoritative servers.
+ * Duplicate efforts are detected, and EDNS, TCP and UDP retry is performed.
+ * @param outnet: outside network, with rbtree of serviced queries.
+ * @param qname: what qname to query.
+ * @param qnamelen: length of qname in octets including 0 root label.
+ * @param qtype: rrset type to query (host format)
+ * @param qclass: query class. (host format)
+ * @param flags: flags u16 (host format), includes opcode, CD bit.
+ * @param dnssec: if set, DO bit is set in EDNS queries.
+ *	If the value includes BIT_CD, CD bit is set when in EDNS queries.
+ *	If the value includes BIT_DO, DO bit is set when in EDNS queries.
+ * @param callback: callback function.
+ * @param callback_arg: user argument to callback function.
+ * @param addr: to which server to send the query.
+ * @param addrlen: length of addr.
+ * @param buff: scratch buffer to create query contents in. Empty on exit.
+ * @param arg_compare: function to compare callback args, return true if 
+ * 	identical. It is given the callback_arg and args that are listed.
+ * @return 0 on error, or pointer to serviced query that is used to answer
+ *	this serviced query may be shared with other callbacks as well.
+ */
+struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint16_t flags, int dnssec, struct sockaddr_storage* addr, 
+	socklen_t addrlen, comm_point_callback_t* callback, 
+	void* callback_arg, ldns_buffer* buff, 
+	int (*arg_compare)(void*,void*));
+
+/**
+ * Remove service query callback.
+ * If that leads to zero callbacks, the query is completely cancelled.
+ * @param sq: serviced query to adjust.
+ * @param cb_arg: callback argument of callback that needs removal.
+ *	same as the callback_arg to outnet_serviced_query().
+ */
+void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg);
+
+/**
+ * Get memory size in use by outside network.
+ * Counts buffers and outstanding query (serviced queries) malloced data.
+ * @param outnet: outside network structure.
+ * @return size in bytes.
+ */
+size_t outnet_get_mem(struct outside_network* outnet);
+
+/**
+ * Get memory size in use by serviced query while it is servicing callbacks.
+ * This takes into account the pre-deleted status of it; it will be deleted
+ * when the callbacks are done.
+ * @param sq: serviced query. 
+ * @return size in bytes.
+ */
+size_t serviced_get_mem(struct serviced_query* sq);
+
+/** callback for incoming udp answers from the network */
+int outnet_udp_cb(struct comm_point* c, void* arg, int error,
+	struct comm_reply *reply_info);
+
+/** callback for pending tcp connections */
+int outnet_tcp_cb(struct comm_point* c, void* arg, int error,
+	struct comm_reply *reply_info);
+
+/** callback for udp timeout */
+void pending_udp_timer_cb(void *arg);
+
+/** callback for outgoing TCP timer event */
+void outnet_tcptimer(void* arg);
+
+/** callback for serviced query UDP answers */
+int serviced_udp_callback(struct comm_point* c, void* arg, int error,
+        struct comm_reply* rep);
+
+/** TCP reply or error callback for serviced queries */
+int serviced_tcp_callback(struct comm_point* c, void* arg, int error,
+        struct comm_reply* rep);
+
+/** compare function of pending rbtree */
+int pending_cmp(const void* key1, const void* key2);
+
+/** compare function of serviced query rbtree */
+int serviced_cmp(const void* key1, const void* key2);
+
 #endif /* OUTSIDE_NETWORK_H */

testcode/checklocks.c

@@ -54,7 +54,7 @@
 /** How long to wait before lock attempt is a failure. */
 #define CHECK_LOCK_TIMEOUT 30 /* seconds */
 /** How long to wait before join attempt is a failure. */
-#define CHECK_JOIN_TIMEOUT 240 /* seconds */
+#define CHECK_JOIN_TIMEOUT 120 /* seconds */
 
 /** if key has been created */
 static int key_created = 0;
@@ -67,6 +67,9 @@ int check_locking_order = 1;
 /** the pid of this runset, reasonably unique. */
 static pid_t check_lock_pid;
 
+/** print all possible debug info on the state of the system */
+static void total_debug_info();
+
 /** print pretty lock error and exit */
 static void lock_error(struct checked_lock* lock, 
 	const char* func, const char* file, int line, const char* err)
@@ -82,6 +85,8 @@ static void lock_error(struct checked_lock* lock,
 		(lock->type==check_lock_mutex)?"mutex": (
 		(lock->type==check_lock_spinlock)?"spinlock": (
 		(lock->type==check_lock_rwlock)?"rwlock": "badtype")), err);
+	log_err("complete status display:");
+	total_debug_info();
 	fatal_exit("bailing out");
 }
 
@@ -189,7 +194,7 @@ prot_check(struct checked_lock* lock,
 	}
 }
 
-/** Copy protected memory region. */
+/** Copy protected memory region */
 static void 
 prot_store(struct checked_lock* lock)
 {
@@ -200,7 +205,24 @@ prot_store(struct checked_lock* lock)
 	}
 }
 
-/** write lock trace info to file, while you hold those locks. */
+/** get memory held by lock */
+size_t 
+lock_get_mem(void* pp)
+{
+	size_t s;
+	struct checked_lock* lock = *(struct checked_lock**)pp;
+	struct protected_area* p;
+	s = sizeof(struct checked_lock);
+	acquire_locklock(lock, __func__, __FILE__, __LINE__);
+	for(p = lock->prot; p; p = p->next) {
+		s += sizeof(struct protected_area);
+		s += p->size;
+	}
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+	return s;
+}
+
+/** write lock trace info to file, while you hold those locks */
 static void
 ordercheck_locklock(struct thr_check* thr, struct checked_lock* lock)
 {
@@ -220,7 +242,7 @@ ordercheck_locklock(struct thr_check* thr, struct checked_lock* lock)
 		log_err("fwrite: %s", strerror(errno));
 }
 
-/** write ordercheck lock creation details to file. */
+/** write ordercheck lock creation details to file */
 static void 
 ordercheck_lockcreate(struct thr_check* thr, struct checked_lock* lock)
 {
@@ -335,7 +357,7 @@ checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
 			e->create_func, e->create_file, e->create_line,
 			(unsigned int)e->contention_count, 
 			(unsigned int)e->history_count,
-			100*e->contention_count/e->history_count);
+			(int)(100*e->contention_count/e->history_count));
 	}
 
 	/* delete it */
@@ -360,7 +382,7 @@ checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
 	free(e);
 }
 
-/** finish acquiring lock, shared between _(rd|wr||)lock() routines. */
+/** finish acquiring lock, shared between _(rd|wr||)lock() routines */
 static void 
 finish_acquire_lock(struct thr_check* thr, struct checked_lock* lock,
         const char* func, const char* file, int line)
@@ -511,6 +533,7 @@ static int timed_spinlock(void* arg, struct timespec* to)
 #ifndef S_SPLINT_S
 		if(time(NULL) >= to->tv_sec)
 			return ETIMEDOUT;
+		usleep(1000); /* in 1/1000000s of a second */
 #endif
 	}
 	return err;
@@ -606,7 +629,7 @@ checklock_unlock(enum check_lock_type type, struct checked_lock* lock,
 	}
 }
 
-/** open order info debug file, thr->num must be valid. */
+/** open order info debug file, thr->num must be valid */
 static void 
 open_lockorder(struct thr_check* thr)
 {
@@ -626,7 +649,7 @@ open_lockorder(struct thr_check* thr)
 		log_err("fwrite: %s", strerror(errno));
 }
 
-/** checklock thread main, Inits thread structure. */
+/** checklock thread main, Inits thread structure */
 static void* checklock_main(void* arg)
 {
 	struct thr_check* thr = (struct thr_check*)arg; 
@@ -699,13 +722,102 @@ checklock_thrcreate(pthread_t* id, void* (*func)(void*), void* arg)
 	LOCKRET(pthread_create(id, NULL, checklock_main, thr));
 }
 
-/** signal handler for join timeout, Exits. */
+/** count number of thread infos */
+static int
+count_thread_infos()
+{
+	int cnt = 0;
+	int i;
+	for(i=0; i<THRDEBUG_MAX_THREADS; i++)
+		if(thread_infos[i])
+			cnt++;
+	return cnt;
+}
+
+/** print lots of info on a lock */
+static void
+lock_debug_info(struct checked_lock* lock)
+{
+	if(!lock) return;
+	log_info("+++ Lock %x, %d %d create %s %s %d", (int)lock, 
+		lock->create_thread, lock->create_instance, 
+		lock->create_func, lock->create_file, lock->create_line);
+	log_info("lock type: %s",
+		(lock->type==check_lock_mutex)?"mutex": (
+		(lock->type==check_lock_spinlock)?"spinlock": (
+		(lock->type==check_lock_rwlock)?"rwlock": "badtype")));
+	log_info("lock contention %u, history:%u, hold:%d, wait:%d", 
+		(unsigned)lock->contention_count, (unsigned)lock->history_count,
+		lock->hold_count, lock->wait_count);
+	log_info("last touch %s %s %d", lock->holder_func, lock->holder_file,
+		lock->holder_line);
+	log_info("holder thread %d, writeholder thread %d",
+		lock->holder?lock->holder->num:-1,
+		lock->writeholder?lock->writeholder->num:-1);
+}
+
+/** print debug locks held by a thread */
+static void
+held_debug_info(struct thr_check* thr, struct checked_lock* lock)
+{
+	if(!lock) return;
+	lock_debug_info(lock);
+	held_debug_info(thr, lock->next_held_lock[thr->num]);
+}
+
+/** print debug info for a thread */
+static void
+thread_debug_info(struct thr_check* thr)
+{
+	struct checked_lock* w = NULL;
+	struct checked_lock* f = NULL;
+	struct checked_lock* l = NULL;
+	if(!thr) return;
+	log_info("pthread id is %x", (int)thr->id);
+	log_info("thread func is %x", (int)thr->func);
+	log_info("thread arg is %x (%d)", (int)thr->arg, 
+		(thr->arg?*(int*)thr->arg:0));
+	log_info("thread num is %d", thr->num);
+	log_info("locks created %d", thr->locks_created);
+	log_info("open file for lockinfo: %s", 
+		thr->order_info?"yes, flushing":"no");
+	fflush(thr->order_info);
+	w = thr->waiting;
+	f = thr->holding_first;
+	l = thr->holding_last;
+	log_info("thread waiting for a lock: %s %x", w?"yes":"no", (int)w);
+	lock_debug_info(w);
+	log_info("thread holding first: %s, last: %s", f?"yes":"no", 
+		l?"yes":"no");
+	held_debug_info(thr, f);
+}
+
+static void
+total_debug_info()
+{
+	int i;
+	log_info("checklocks: supervising %d threads.",
+		count_thread_infos());
+	if(!key_created) {
+		log_info("No thread debug key created yet");
+	}
+	for(i=0; i<THRDEBUG_MAX_THREADS; i++) {
+		if(thread_infos[i]) {
+			log_info("*** Thread %d information: ***", i);
+			thread_debug_info(thread_infos[i]);
+		}
+	}
+}
+
+/** signal handler for join timeout, Exits */
 static RETSIGTYPE joinalarm(int ATTR_UNUSED(sig))
 {
+	log_err("join thread timeout. hangup or deadlock. Info follows.");
+	total_debug_info();
 	fatal_exit("join thread timeout. hangup or deadlock.");
 }
 
-/** wait for thread with a timeout. */
+/** wait for thread with a timeout */
 void 
 checklock_thrjoin(pthread_t thread)
 {
@@ -718,3 +830,20 @@ checklock_thrjoin(pthread_t thread)
 }
 
 #endif /* USE_THREAD_DEBUG */
+
+int order_lock_cmp(const void* e1, const void* e2)
+{
+	struct order_id* o1 = (struct order_id*)e1;
+	struct order_id* o2 = (struct order_id*)e2;
+	if(o1->thr < o2->thr) return -1;
+	if(o1->thr > o2->thr) return 1;
+	if(o1->instance < o2->instance) return -1;
+	if(o1->instance > o2->instance) return 1;
+	return 0;
+}
+
+int
+codeline_cmp(const void* a, const void* b)
+{
+	return strcmp((const char*)a, (const char*)b);
+}

testcode/checklocks.h

@@ -196,6 +196,13 @@ void lock_protect(void* lock, void* area, size_t size);
  */
 void lock_unprotect(void* lock, void* area);
 
+/**
+ * Get memory associated with a checked lock
+ * @param lock: the checked lock, any type. (struct checked_lock**).
+ * @return: in bytes, including protected areas.
+ */
+size_t lock_get_mem(void* lock);
+
 /**
  * Initialise checklock. Sets up internal debug structures.
  */
@@ -334,4 +341,16 @@ typedef pthread_key_t ub_thread_key_t;
 
 #endif /* USE_THREAD_DEBUG */
 
+/** keep track of lock id in lock-verify application */
+struct order_id {
+	/** the thread id that created it */
+	int thr;
+	/** the instance number of creation */
+	int instance;
+};
+/** compare two order_ids */
+int order_lock_cmp(const void* e1, const void* e2);
+/** compare two codeline structs for rbtree */
+int codeline_cmp(const void* a, const void* b);
+
 #endif /* TESTCODE_CHECK_LOCKS_H */

testcode/do-tests.sh

@@ -2,8 +2,12 @@
 
 NEED_SPLINT='00-lint.tpkg'
 NEED_DOXYGEN='01-doc.tpkg'
+NEED_LDNS_TESTNS='fwd_no_edns.tpkg fwd_tcp_tc.tpkg fwd_tcp.tpkg fwd_three_service.tpkg fwd_three.tpkg fwd_ttlexpire.tpkg fwd_udp.tpkg fwd_tcp_tc6.tpkg fwd_compress_c00c.tpkg'
+NEED_XXD='fwd_compress_c00c.tpkg'
+NEED_NC='fwd_compress_c00c.tpkg'
 
 cd testdata;
+sh ../testcode/mini_tpkg.sh clean
 for test in `ls *.tpkg`; do
 	SKIP=0
 	if echo $NEED_SPLINT | grep $test >/dev/null; then
@@ -16,10 +20,26 @@ for test in `ls *.tpkg`; do
 			SKIP=1;
 		fi
 	fi
+	if echo $NEED_LDNS_TESTNS | grep $test >/dev/null; then
+		if test ! -x "`which ldns-testns`"; then
+			SKIP=1;
+		fi
+	fi
+	if echo $NEED_XXD | grep $test >/dev/null; then
+		if test ! -x "`which xxd`"; then
+			SKIP=1;
+		fi
+	fi
+	if echo $NEED_NC | grep $test >/dev/null; then
+		if test ! -x "`which nc`"; then
+			SKIP=1;
+		fi
+	fi
 	if test $SKIP -eq 0; then
 		echo $test
-		tpkg -a ../.. exe $test
+		sh ../testcode/mini_tpkg.sh -a ../.. exe $test
 	else
 		echo "skip $test"
 	fi
 done
+sh ../testcode/mini_tpkg.sh report

testcode/fake_event.c

@@ -48,11 +48,15 @@
 #include "testcode/fake_event.h"
 #include "util/netevent.h"
 #include "util/net_help.h"
+#include "util/data/msgparse.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgencode.h"
 #include "services/listen_dnsport.h"
 #include "services/outside_network.h"
 #include "testcode/replay.h"
 #include "testcode/ldns-testpkts.h"
 #include "util/log.h"
+#include <signal.h>
 
 /** Global variable: the scenario. Saved here for when event_init is done. */
 static struct replay_scenario* saved_scenario = NULL;
@@ -70,7 +74,7 @@ fake_event_cleanup()
 	saved_scenario = NULL;
 }
 
-/** helper function that logs a ldns_pkt packet to logfile. */
+/** helper function that logs a ldns_pkt packet to logfile */
 static void
 log_pkt(const char* desc, ldns_pkt* pkt)
 {
@@ -140,6 +144,10 @@ pending_matches_current(struct replay_runtime* runtime,
 		return 0;
 	/* see if any of the pending queries matches */
 	for(p = runtime->pending_list; p; p = p->next) {
+		if(runtime->now->addrlen != 0 &&
+			sockaddr_cmp(&p->addr, p->addrlen, &runtime->now->addr,
+			runtime->now->addrlen) != 0)
+			continue;
 		if((e=find_match(runtime->now->match, p->pkt, p->transport))) {
 			*entry = e;
 			*pend = p;
@@ -164,10 +172,15 @@ pending_find_match(struct replay_runtime* runtime, struct entry** entry,
 	struct replay_range* p = runtime->scenario->range_list;
 	while(p) {
 		if(p->start_step <= timenow && timenow <= p->end_step &&
+		  (p->addrlen == 0 || sockaddr_cmp(&p->addr, p->addrlen,
+		  	&pend->addr, pend->addrlen) == 0) &&
 		  (*entry = find_match(p->match, pend->pkt, pend->transport))) {
 			log_info("matched query time %d in range [%d, %d] "
 				"with entry line %d", timenow, 
 				p->start_step, p->end_step, (*entry)->lineno);
+			if(p->addrlen != 0)
+				log_addr(0, "matched ip", &p->addr, p->addrlen);
+			log_pkt("matched pkt: ", (*entry)->reply_list->reply);
 			return 1;
 		}
 		p = p->next_range;
@@ -189,6 +202,7 @@ pending_matches_range(struct replay_runtime* runtime,
 	struct fake_pending* p = runtime->pending_list;
 	/* slow, O(N*N), but it works as advertised with weird matching */
 	while(p) {
+		log_info("check of pending");
 		if(pending_find_match(runtime, entry, p)) {
 			*pend = p;
 			return 1;
@@ -262,6 +276,10 @@ answer_callback_from_entry(struct replay_runtime* runtime,
         struct entry* entry, struct fake_pending* pend)
 {
 	struct comm_point c;
+	struct comm_reply repinfo;
+	void* cb_arg = pend->cb_arg;
+	comm_point_callback_t* cb = pend->callback;
+
 	memset(&c, 0, sizeof(c));
 	c.fd = -1;
 	c.buffer = ldns_buffer_new(runtime->bufsize);
@@ -269,11 +287,15 @@ answer_callback_from_entry(struct replay_runtime* runtime,
 	if(pend->transport == transport_tcp)
 		c.type = comm_tcp;
 	fill_buffer_with_reply(c.buffer, entry, pend->pkt);
-	if((*pend->callback)(&c, pend->cb_arg, NETEVENT_NOERROR, NULL)) {
+	repinfo.c = &c;
+	repinfo.addrlen = pend->addrlen;
+	memcpy(&repinfo.addr, &pend->addr, pend->addrlen);
+	if(!pend->serviced)
+		pending_list_delete(runtime, pend);
+	if((*cb)(&c, cb_arg, NETEVENT_NOERROR, &repinfo)) {
 		fatal_exit("testbound: unexpected: callback returned 1");
 	}
 	ldns_buffer_free(c.buffer);
-	pending_list_delete(runtime, pend);
 }
 
 /** Check the now moment answer check event */
@@ -288,7 +310,10 @@ answer_check_it(struct replay_runtime* runtime)
 		enum transport_type tr = transport_tcp;
 		if(ans->repinfo.c->type == comm_udp)
 			tr = transport_udp;
-		if(find_match(runtime->now->match, ans->pkt, tr)) {
+		if((runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&ans->repinfo.addr, ans->repinfo.addrlen) == 0) &&
+			find_match(runtime->now->match, ans->pkt, tr)) {
 			struct replay_answer *n = ans->next;
 			log_info("testbound matched event entry from line %d",
 				runtime->now->match->lineno);
@@ -323,12 +348,19 @@ fake_front_query(struct replay_runtime* runtime, struct replay_moment *todo)
 	memset(&repinfo, 0, sizeof(repinfo));
 	repinfo.c = (struct comm_point*)calloc(1, sizeof(struct comm_point));
 	repinfo.addrlen = (socklen_t)sizeof(struct sockaddr_in);
+	if(todo->addrlen != 0) {
+		repinfo.addrlen = todo->addrlen;
+		memcpy(&repinfo.addr, &todo->addr, todo->addrlen);
+	}
 	repinfo.c->fd = -1;
 	repinfo.c->ev = (struct internal_event*)runtime;
 	repinfo.c->buffer = ldns_buffer_new(runtime->bufsize);
-	repinfo.c->type = comm_udp;
+	if(todo->match->match_transport == transport_tcp)
+		repinfo.c->type = comm_tcp;
+	else	repinfo.c->type = comm_udp;
 	fill_buffer_with_reply(repinfo.c->buffer, todo->match, NULL);
 	log_info("testbound: incoming QUERY");
+	log_pkt("query pkt", todo->match->reply_list->reply);
 	/* call the callback for incoming queries */
 	if((*runtime->callback_query)(repinfo.c, runtime->cb_arg, 
 		NETEVENT_NOERROR, &repinfo)) {
@@ -347,7 +379,11 @@ fake_pending_callback(struct replay_runtime* runtime,
 	struct replay_moment* todo, int error)
 {
 	struct fake_pending* p = runtime->pending_list;
+	struct comm_reply repinfo;
 	struct comm_point c;
+	void* cb_arg = p->cb_arg;
+	comm_point_callback_t* cb = p->callback;
+
 	memset(&c, 0, sizeof(c));
 	if(!p) fatal_exit("No pending queries.");
 	log_assert(todo->qname == NULL); /* or find that one */
@@ -357,13 +393,17 @@ fake_pending_callback(struct replay_runtime* runtime,
 		c.type = comm_tcp;
 	if(todo->evt_type == repevt_back_reply && todo->match) {
 		fill_buffer_with_reply(c.buffer, todo->match, p->pkt);
-	}	
-	if((*p->callback)(&c, p->cb_arg, error, NULL)) {
+	}
+	repinfo.c = &c;
+	repinfo.addrlen = p->addrlen;
+	memcpy(&repinfo.addr, &p->addr, p->addrlen);
+	if(!p->serviced)
+		pending_list_delete(runtime, p);
+	if((*cb)(&c, cb_arg, error, &repinfo)) {
 		fatal_exit("unexpected: pending callback returned 1");
 	}
 	/* delete the pending item. */
 	ldns_buffer_free(c.buffer);
-	pending_list_delete(runtime, p);
 }
 
 /**
@@ -473,19 +513,25 @@ run_scenario(struct replay_runtime* runtime)
 	} while(runtime->now);
 
 	if(runtime->pending_list) {
+		struct fake_pending* p;
+		log_err("testbound: there are still messages pending.");
+		for(p = runtime->pending_list; p; p=p->next)
+			log_pkt("pending msg", p->pkt);
 		fatal_exit("testbound: there are still messages pending.");
 	}
 	if(runtime->answer_list) {
 		fatal_exit("testbound: there are unmatched answers.");
 	}
 	log_info("testbound: exiting fake runloop.");
+	runtime->exit_cleanly = 1;
 }
 
 /*********** Dummy routines ***********/
 
 struct listen_dnsport* 
 listen_create(struct comm_base* base, struct listen_port* ATTR_UNUSED(ports),
-	size_t bufsize, comm_point_callback_t* cb, void* cb_arg)
+	size_t bufsize, int ATTR_UNUSED(tcp_accept_count),
+	comm_point_callback_t* cb, void* cb_arg)
 {
 	struct replay_runtime* runtime = (struct replay_runtime*)base;
 	struct listen_dnsport* l= calloc(1, sizeof(struct listen_dnsport));
@@ -551,13 +597,19 @@ comm_base_dispatch(struct comm_base* b)
 {
 	struct replay_runtime* runtime = (struct replay_runtime*)b;
 	run_scenario(runtime);
+	if(runtime->sig_cb)
+		(*runtime->sig_cb)(SIGTERM, runtime->sig_cb_arg);
+	else	exit(0); /* OK exit when LIBEVENT_SIGNAL_PROBLEM exists */
 }
 
 void 
-comm_base_exit(struct comm_base* ATTR_UNUSED(b))
+comm_base_exit(struct comm_base* b)
 {
-	/* some sort of failure */
-	fatal_exit("testbound: comm_base_exit was called.");
+	struct replay_runtime* runtime = (struct replay_runtime*)b;
+	if(!runtime->exit_cleanly) {
+		/* some sort of failure */
+		fatal_exit("testbound: comm_base_exit was called.");
+	}
 }
 
 struct comm_signal* 
@@ -634,7 +686,8 @@ outside_network_create(struct comm_base* base, size_t bufsize,
 	size_t ATTR_UNUSED(num_ports), char** ATTR_UNUSED(ifs), 
 	int ATTR_UNUSED(num_ifs), int ATTR_UNUSED(do_ip4), 
 	int ATTR_UNUSED(do_ip6), int ATTR_UNUSED(port_base),
-	size_t ATTR_UNUSED(num_tcp))
+	size_t ATTR_UNUSED(num_tcp), struct infra_cache* ATTR_UNUSED(infra),
+	struct ub_randstate* ATTR_UNUSED(rnd))
 {
 	struct outside_network* outnet =  calloc(1, 
 		sizeof(struct outside_network));
@@ -656,7 +709,7 @@ outside_network_delete(struct outside_network* outnet)
 	free(outnet);
 }
 
-void 
+struct pending* 
 pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
 	comm_point_callback_t* callback, void* callback_arg,
@@ -676,9 +729,11 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	pend->addrlen = addrlen;
 	pend->callback = callback;
 	pend->cb_arg = callback_arg;
-	pend->timeout = timeout;
+	pend->timeout = timeout/1000;
 	pend->transport = transport_udp;
 	pend->pkt = NULL;
+	pend->serviced = 0;
+	pend->runtime = runtime;
 	status = ldns_buffer2pkt_wire(&pend->pkt, packet);
 	if(status != LDNS_STATUS_OK) {
 		log_err("ldns error parsing udp output packet: %s",
@@ -689,6 +744,9 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 
 	/* see if it matches the current moment */
 	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
 		find_match(runtime->now->match, pend->pkt, pend->transport)) {
 		log_info("testbound: matched pending to event. "
 			"advance time between events.");
@@ -701,9 +759,10 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	/* add to list */
 	pend->next = runtime->pending_list;
 	runtime->pending_list = pend;
+	return (struct pending*)pend;
 }
 
-void 
+struct waiting_tcp* 
 pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
 	comm_point_callback_t* callback, void* callback_arg,
@@ -726,6 +785,8 @@ pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 	pend->timeout = timeout;
 	pend->transport = transport_tcp;
 	pend->pkt = NULL;
+	pend->runtime = runtime;
+	pend->serviced = 0;
 	status = ldns_buffer2pkt_wire(&pend->pkt, packet);
 	if(status != LDNS_STATUS_OK) {
 		log_err("ldns error parsing tcp output packet: %s",
@@ -736,6 +797,9 @@ pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 
 	/* see if it matches the current moment */
 	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
 		find_match(runtime->now->match, pend->pkt, pend->transport)) {
 		log_info("testbound: matched pending to event. "
 			"advance time between events.");
@@ -748,6 +812,112 @@ pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 	/* add to list */
 	pend->next = runtime->pending_list;
 	runtime->pending_list = pend;
+	return (struct waiting_tcp*)pend;
+}
+
+struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
+        uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint16_t flags, int dnssec, struct sockaddr_storage* addr,
+	socklen_t addrlen, comm_point_callback_t* callback,
+	void* callback_arg, ldns_buffer* ATTR_UNUSED(buff),
+	int (*arg_compare)(void*,void*))
+{
+	struct replay_runtime* runtime = (struct replay_runtime*)outnet->base;
+	struct fake_pending* pend = (struct fake_pending*)calloc(1,
+		sizeof(struct fake_pending));
+	ldns_status status;
+	(void)arg_compare;
+	log_assert(pend);
+	log_nametypeclass(VERB_OPS, "pending serviced query", 
+		qname, qtype, qclass);
+	verbose(VERB_OPS, "pending serviced query flags%s%s%s%s", 
+		(flags&BIT_RD)?" RD":"", (flags&BIT_CD)?" CD":"",
+		(flags&~(BIT_RD|BIT_CD))?" MORE":"", (dnssec)?" DO":"");
+
+	/* create packet with EDNS */
+	pend->buffer = ldns_buffer_new(512);
+	log_assert(pend->buffer);
+	ldns_buffer_write_u16(pend->buffer, 0); /* id */
+	ldns_buffer_write_u16(pend->buffer, flags);
+	ldns_buffer_write_u16(pend->buffer, 1); /* qdcount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* ancount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* nscount */
+	ldns_buffer_write_u16(pend->buffer, 0); /* arcount */
+	ldns_buffer_write(pend->buffer, qname, qnamelen);
+	ldns_buffer_write_u16(pend->buffer, qtype);
+	ldns_buffer_write_u16(pend->buffer, qclass);
+	ldns_buffer_flip(pend->buffer);
+	if(1) {
+		/* add edns */
+		struct edns_data edns;
+		edns.edns_present = 1;
+		edns.ext_rcode = 0;
+		edns.edns_version = EDNS_ADVERTISED_VERSION;
+		edns.udp_size = EDNS_ADVERTISED_SIZE;
+		edns.bits = 0;
+		if(dnssec)
+			edns.bits = EDNS_DO;
+		attach_edns_record(pend->buffer, &edns);
+	}
+	memcpy(&pend->addr, addr, addrlen);
+	pend->addrlen = addrlen;
+	pend->callback = callback;
+	pend->cb_arg = callback_arg;
+	pend->timeout = UDP_QUERY_TIMEOUT;
+	pend->transport = transport_udp; /* pretend UDP */
+	pend->pkt = NULL;
+	pend->runtime = runtime;
+	pend->serviced = 1;
+	status = ldns_buffer2pkt_wire(&pend->pkt, pend->buffer);
+	if(status != LDNS_STATUS_OK) {
+		log_err("ldns error parsing serviced output packet: %s",
+			ldns_get_errorstr_by_id(status));
+		fatal_exit("internal error");
+	}
+	/*log_pkt("pending serviced query: ", pend->pkt);*/
+
+	/* see if it matches the current moment */
+	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		(runtime->now->addrlen == 0 || sockaddr_cmp(
+			&runtime->now->addr, runtime->now->addrlen,
+			&pend->addr, pend->addrlen) == 0) &&
+		find_match(runtime->now->match, pend->pkt, pend->transport)) {
+		log_info("testbound: matched pending to event. "
+			"advance time between events.");
+		log_info("testbound: do STEP %d %s", runtime->now->time_step,
+			repevt_string(runtime->now->evt_type));
+		advance_moment(runtime);
+		/* still create the pending, because we need it to callback */
+	} 
+	log_info("testbound: created fake pending");
+	/* add to list */
+	pend->next = runtime->pending_list;
+	runtime->pending_list = pend;
+	return (struct serviced_query*)pend;
+}
+
+void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
+{
+	struct fake_pending* pend = (struct fake_pending*)sq;
+	struct replay_runtime* runtime = pend->runtime;
+	/* delete from the list */
+	struct fake_pending* p = runtime->pending_list, *prev=NULL;
+	while(p) {
+		if(p == pend) {
+			log_assert(p->cb_arg == cb_arg);
+			log_info("serviced pending delete");
+			if(prev)
+				prev->next = p->next;
+			else 	runtime->pending_list = p->next;
+			ldns_buffer_free(p->buffer);
+			ldns_pkt_free(p->pkt);
+			free(p);
+			return;
+		}
+		prev = p;
+		p = p->next;
+	}
+	log_info("double delete of pending serviced query");
 }
 
 struct listen_port* listening_ports_open(struct config_file* ATTR_UNUSED(cfg))
@@ -782,4 +952,115 @@ void comm_point_delete(struct comm_point* c)
 	free(c);
 }
 
+size_t listen_get_mem(struct listen_dnsport* ATTR_UNUSED(listen))
+{
+	return 0;
+}
+
+size_t outnet_get_mem(struct outside_network* ATTR_UNUSED(outnet))
+{
+	return 0;
+}
+
+size_t comm_point_get_mem(struct comm_point* ATTR_UNUSED(c))
+{
+	return 0;
+}
+
+size_t serviced_get_mem(struct serviced_query* ATTR_UNUSED(c))
+{
+	return 0;
+}
+
+/* fake for fptr wlist */
+int outnet_udp_cb(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply *ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int outnet_tcp_cb(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply *ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+void pending_udp_timer_cb(void *ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void outnet_tcptimer(void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_point_udp_callback(int ATTR_UNUSED(fd), short ATTR_UNUSED(event), 
+	void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_point_tcp_accept_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_point_tcp_handle_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_timer_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_signal_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+void comm_point_local_handle_callback(int ATTR_UNUSED(fd), 
+	short ATTR_UNUSED(event), void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
+int serviced_udp_callback(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int serviced_tcp_callback(struct comm_point* ATTR_UNUSED(c), 
+	void* ATTR_UNUSED(arg), int ATTR_UNUSED(error),
+        struct comm_reply* ATTR_UNUSED(reply_info))
+{
+	log_assert(0);
+	return 0;
+}
+
+int pending_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
+{
+	log_assert(0);
+	return 0;
+}
+
+int serviced_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
+{
+	log_assert(0);
+	return 0;
+}
+
 /*********** End of Dummy routines ***********/

testcode/ldns-testpkts.c

@@ -31,7 +31,7 @@ struct sockaddr_storage;
 /** string to show in warnings and errors */
 static const char* prog_name = "ldns-testpkts";
 
-/** logging routine, provided by caller. */
+/** logging routine, provided by caller */
 void verbose(int lvl, const char* msg, ...);
 
 /** print error and exit */
@@ -68,7 +68,7 @@ static bool str_keyword(const char** str, const char* keyword)
 	if(strncmp(*str, keyword, len) != 0)
 		return false;
 	*str += len;
-	while(isspace(**str))
+	while(isspace((int)**str))
 		(*str)++;
 	return true;
 }
@@ -108,6 +108,8 @@ static void matchline(const char* line, struct entry* e)
 			e->match_all = true;
 		} else if(str_keyword(&parse, "ttl")) {
 			e->match_ttl = true;
+		} else if(str_keyword(&parse, "DO")) {
+			e->match_do = true;
 		} else if(str_keyword(&parse, "UDP")) {
 			e->match_transport = transport_udp;
 		} else if(str_keyword(&parse, "TCP")) {
@@ -118,7 +120,7 @@ static void matchline(const char* line, struct entry* e)
 				error("expected = or : in MATCH: %s", line);
 			parse++;
 			e->ixfr_soa_serial = (uint32_t)strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else {
 			error("could not parse MATCH: '%s'", parse);
@@ -155,6 +157,8 @@ static void replyline(const char* line, ldns_pkt *reply)
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_NXDOMAIN);
 		} else if(str_keyword(&parse, "NOTIMPL")) {
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_NOTIMPL);
+		} else if(str_keyword(&parse, "REFUSED")) {
+			ldns_pkt_set_rcode(reply, LDNS_RCODE_REFUSED);
 		} else if(str_keyword(&parse, "YXDOMAIN")) {
 			ldns_pkt_set_rcode(reply, LDNS_RCODE_YXDOMAIN);
 		} else if(str_keyword(&parse, "YXRRSET")) {
@@ -180,6 +184,9 @@ static void replyline(const char* line, ldns_pkt *reply)
 			ldns_pkt_set_ra(reply, true);
 		} else if(str_keyword(&parse, "AD")) {
 			ldns_pkt_set_ad(reply, true);
+		} else if(str_keyword(&parse, "DO")) {
+			ldns_pkt_set_edns_udp_size(reply, 4096);
+			ldns_pkt_set_edns_do(reply, true);
 		} else {
 			error("could not parse REPLY: '%s'", parse);
 		}
@@ -196,13 +203,15 @@ static void adjustline(const char* line, struct entry* e,
 			return;
 		if(str_keyword(&parse, "copy_id")) {
 			e->copy_id = true;
+		} else if(str_keyword(&parse, "copy_query")) {
+			e->copy_query = true;
 		} else if(str_keyword(&parse, "sleep=")) {
 			e->sleeptime = (unsigned int) strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else if(str_keyword(&parse, "packet_sleep=")) {
 			pkt->packet_sleep = (unsigned int) strtol(parse, (char**)&parse, 10);
-			while(isspace(*parse)) 
+			while(isspace((int)*parse)) 
 				parse++;
 		} else {
 			error("could not parse ADJUST: '%s'", parse);
@@ -220,11 +229,13 @@ static struct entry* new_entry()
 	e->match_qname = false;
 	e->match_all = false;
 	e->match_ttl = false;
+	e->match_do = false;
 	e->match_serial = false;
 	e->ixfr_soa_serial = 0;
 	e->match_transport = transport_any;
 	e->reply_list = NULL;
 	e->copy_id = false;
+	e->copy_query = false;
 	e->sleeptime = 0;
 	e->next = NULL;
 	return e;
@@ -380,7 +391,7 @@ get_origin(const char* name, int lineno, ldns_rdf** origin, char* parse)
 	*origin = NULL;
 
 	end=parse;
-	while(!isspace(*end) && !isendline(*end))
+	while(!isspace((int)*end) && !isendline(*end))
 		end++;
 	store = *end;
 	*end = 0;
@@ -394,7 +405,7 @@ get_origin(const char* name, int lineno, ldns_rdf** origin, char* parse)
 
 /* Reads one entry from file. Returns entry or NULL on error. */
 struct entry*
-read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl, 
+read_entry(FILE* in, const char* name, int *lineno, uint32_t* default_ttl, 
 	ldns_rdf** origin, ldns_rdf** prev_rr)
 {
 	struct entry* current = NULL;
@@ -410,7 +421,7 @@ read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl,
 		parse = line;
 		(*lineno) ++;
 		
-		while(isspace(*parse))
+		while(isspace((int)*parse))
 			parse++;
 		/* test for keywords */
 		if(isendline(*parse))
@@ -428,7 +439,7 @@ read_entry(FILE* in, const char* name, int *lineno, uint16_t* default_ttl,
 			get_origin(name, *lineno, origin, (char*)parse);
 			continue;
 		} else if(str_keyword(&parse, "$TTL")) {
-			*default_ttl = (uint16_t)atoi(parse);
+			*default_ttl = (uint32_t)atoi(parse);
 			continue;
 		}
 
@@ -502,7 +513,7 @@ read_datafile(const char* name)
 	struct entry* current = NULL;
 	FILE *in;
 	int lineno = 0;
-	uint16_t default_ttl = 0;
+	uint32_t default_ttl = 0;
 	ldns_rdf* origin = NULL;
 	ldns_rdf* prev_rr = NULL;
 	int entry_num = 0;
@@ -523,6 +534,8 @@ read_datafile(const char* name)
 	verbose(1, "%s: Read %d entries\n", prog_name, entry_num);
 
 	fclose(in);
+	ldns_rdf_deep_free(origin);
+	ldns_rdf_deep_free(prev_rr);
 	return list;
 }
 
@@ -663,6 +676,10 @@ find_match(struct entry* entries, ldns_pkt* query_pkt,
 				verbose(3, "bad serial\n");
 				continue;
 		}
+		if(p->match_do && !ldns_pkt_edns_do(query_pkt)) {
+			verbose(3, "no DO bit set\n");
+			continue;
+		}
 		if(p->match_transport != transport_any && p->match_transport != transport) {
 			verbose(3, "bad transport\n");
 			continue;
@@ -683,6 +700,12 @@ adjust_packet(struct entry* match, ldns_pkt* answer_pkt, ldns_pkt* query_pkt)
 	/* copy & adjust packet */
 	if(match->copy_id)
 		ldns_pkt_set_id(answer_pkt, ldns_pkt_id(query_pkt));
+	if(match->copy_query) {
+		ldns_rr_list* list = ldns_pkt_get_section_clone(query_pkt,
+			LDNS_SECTION_QUESTION);
+		ldns_rr_list_deep_free(ldns_pkt_question(answer_pkt));
+		ldns_pkt_set_question(answer_pkt, list);
+	}
 	if(match->sleeptime > 0) {
 		verbose(3, "sleeping for %d seconds\n", match->sleeptime);
 		sleep(match->sleeptime);

testcode/ldns-testpkts.h

@@ -45,19 +45,22 @@
 	; 'serial=1023' makes the query match if ixfr serial is 1023. 
 	; 'all' has to match header byte for byte and all rrs in packet.
 	; 'ttl' used with all, rrs in packet must also have matching TTLs.
+	; 'DO' will match only queries with DO bit set.
 	MATCH [opcode] [qtype] [qname] [serial=<value>] [all] [ttl]
-	MATCH [UDP|TCP]
+	MATCH [UDP|TCP] DO
 	MATCH ...
 	; Then the REPLY header is specified.
 	REPLY opcode, rcode or flags.
 		(opcode)  QUERY IQUERY STATUS NOTIFY UPDATE
 		(rcode)   NOERROR FORMERR SERVFAIL NXDOMAIN NOTIMPL YXDOMAIN
 		 		YXRRSET NXRRSET NOTAUTH NOTZONE
-		(flags)   QR AA TC RD CD RA AD
+		(flags)   QR AA TC RD CD RA AD DO
 	REPLY ...
 	; any additional actions to do.
 	; 'copy_id' copies the ID from the query to the answer.
 	ADJUST copy_id
+	; 'copy_query' copies the query name, type and class to the answer.
+	ADJUST copy_query
 	; 'sleep=10' sleeps for 10 seconds before giving the answer (TCP is open)
 	ADJUST [sleep=<num>]    ; sleep before giving any reply
 	ADJUST [packet_sleep=<num>]  ; sleep before this packet in sequence
@@ -160,6 +163,8 @@ struct entry {
 	bool match_all;
 	/** match ttls in the packet */
 	bool match_ttl;
+	/** match DO bit */
+	bool match_do;
 	/** match query serial with this value. */
 	uint32_t ixfr_soa_serial; 
 	/** match on UDP/TCP */
@@ -171,6 +176,8 @@ struct entry {
 	/** how to adjust the reply packet */
 	/** copy over the ID from the query into the answer */
 	bool copy_id; 
+	/** copy the query nametypeclass from query into the answer */
+	bool copy_query;
 	/** in seconds */
 	unsigned int sleeptime; 
 
@@ -207,7 +214,7 @@ void delete_entry(struct entry* list);
  * @return: The entry read (malloced) or NULL if no entry could be read.
  */
 struct entry* read_entry(FILE* in, const char* name, int *lineno, 
-	uint16_t* default_ttl, ldns_rdf** origin, ldns_rdf** prev_rr);
+	uint32_t* default_ttl, ldns_rdf** origin, ldns_rdf** prev_rr);
 
 /**
  * finds entry in list, or returns NULL.

testcode/lock_verify.c

@@ -46,18 +46,12 @@
 #include "config.h"
 #include "util/log.h"
 #include "util/rbtree.h"
+#include "util/locks.h"
+#include "testcode/checklocks.h"
 
 /* --- data structures --- */
 struct lock_ref;
 
-/** key for lock lookup */
-struct order_id {
-	/** the thread id that created it */
-	int thr;
-	/** the instance number of creation */
-	int instance;
-};
-
 /** a lock */
 struct order_lock {
 	/** rbnode in all tree */
@@ -103,18 +97,6 @@ usage()
 	printf("lock_verify <trace files>\n");
 }
 
-/** compare two order_ids */
-int order_lock_cmp(const void* e1, const void* e2)
-{
-	struct order_id* o1 = (struct order_id*)e1;
-	struct order_id* o2 = (struct order_id*)e2;
-	if(o1->thr < o2->thr) return -1;
-	if(o1->thr > o2->thr) return 1;
-	if(o1->instance < o2->instance) return -1;
-	if(o1->instance > o2->instance) return 1;
-	return 0;
-}
-
 /** read header entry. 
  * @param in: file to read header of.
  * @return: False if it does not belong to the rest. */
@@ -139,6 +121,9 @@ read_header(FILE* in)
 		the_time = t;
 		the_pid = p;
 		memset(threads, 0, 256*sizeof(int));
+		if(thrno >= 256) {
+			fatal_exit("Thread number too big. %d", thrno);
+		}
 		threads[thrno] = 1;
 		have_values = 1;
 		printf(" trace %d from pid %u on %s", thrno, 
@@ -306,7 +291,7 @@ static void found_cycle(struct lock_ref* visit, int level)
 	}
 }
 
-/** Detect cycle by comparing visited now with all (unvisited) bigger nodes. */
+/** Detect cycle by comparing visited now with all (unvisited) bigger nodes */
 static int detect_cycle(struct lock_ref* visit, struct lock_ref* from)
 {
 	struct lock_ref* p = from;
@@ -332,7 +317,7 @@ static void search_cycle(struct lock_ref* visit, int level,
 	/* check for cycle */
 	if(detect_cycle(visit, from) && level != 0) {
 		found_cycle(visit, level);
-		return;
+		fatal_exit("found lock order cycle");
 	}
 	/* recurse */
 	if(!visit->lock->visited)
@@ -400,7 +385,7 @@ main(int argc, char* argv[])
 		usage();
 		return 1;
 	}
-	log_init(NULL);
+	log_init(NULL, 0, NULL);
 	log_ident_set("lock-verify");
 	/* init */
 	all_locks = rbtree_create(order_lock_cmp);

testcode/memstats.c

@@ -0,0 +1,265 @@
+/*
+ * testcode/memstats.c - debug tool to show memory allocation statistics.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program reads a log file and prints the memory allocation summed
+ * up.
+ */
+#include "config.h"
+#include "util/log.h"
+#include "util/rbtree.h"
+#include "util/locks.h"
+#include "testcode/checklocks.h"
+#include <sys/stat.h>
+
+/**
+ * The allocation statistics block
+ */
+struct codeline {
+	/** rbtree node */
+	rbnode_t node;
+	/** the name of the file:linenumber */
+	char* codeline;
+	/** the name of the function */
+	char* func;
+	/** number of bytes allocated */
+	uint64_t alloc;
+	/** number of bytes freed */
+	uint64_t free;
+};
+
+/**
+ * Other allocation stats
+ */
+struct alloc_misc {
+	/** number of region allocs */
+	uint64_t region_alloc;
+};
+
+/** print usage and exit */
+static void
+usage()
+{
+	printf("usage:	memstats <logfile>\n");
+	printf("statistics are printed on stdout.\n");
+	exit(1);
+}
+
+/** match logfile line to see if it needs accounting processing */
+static int
+match(char* line)
+{
+	/* f.e.:
+	 * [1187340064] unbound[24604:0] info: ul/rb.c:81 r_create malloc(12)
+	 * 0123456789 123456789 123456789 123456789
+	 */
+	if(strlen(line) < 36) /* up to 'info: ' */
+		return 0;
+	if(strncmp(line+30, "info: ", 6) != 0)
+		return 0;
+	if(strncmp(line+36, "stat ", 5) == 0)
+		return 0; /* skip the hex dumps */
+	if(strstr(line+36, "malloc("))
+		return 1;
+	else if(strstr(line+36, "calloc("))
+		return 1;
+	/* skip reallocs */
+	return 0;
+}
+
+/** read up the region stats */
+static void
+read_region_stat(char* line, struct alloc_misc* misc)
+{
+	long num = 0;
+	if(sscanf(line+50, "%ld", &num) != 1) {
+		printf("%s\n%s\n", line, line+50);
+		fatal_exit("unhandled region");
+	}
+	misc->region_alloc += num;
+}
+
+/** find or alloc codeline in tree */
+static struct codeline*
+get_codeline(rbtree_t* tree, char* key, char* func)
+{
+	struct codeline* cl = (struct codeline*)rbtree_search(tree, key);
+	if(!cl) {
+		cl = calloc(1, sizeof(*cl));
+		if(!cl) return 0;
+		cl->codeline = strdup(key);
+		if(!cl->codeline) return 0;
+		cl->func = strdup(func);
+		if(!cl->func) return 0;
+		cl->alloc = 0;
+		cl->node.key = cl->codeline;
+		(void)rbtree_insert(tree, &cl->node);
+	}
+	return cl;
+}
+
+/** read up the malloc stats */
+static void
+read_malloc_stat(char* line, rbtree_t* tree)
+{
+	char codeline[10240];
+	char name[10240];
+	int skip = 0;
+	long num = 0;
+	struct codeline* cl = 0;
+	if(sscanf(line+36, "%s %s %n", codeline, name, &skip) != 2) {
+		printf("%s\n%s\n", line, line+36);
+		fatal_exit("unhandled malloc");
+	}
+	if(sscanf(line+36+skip+7, "%ld", &num) != 1) {
+		printf("%s\n%s\n", line, line+36+skip+7);
+		fatal_exit("unhandled malloc");
+	}
+	cl = get_codeline(tree, codeline, name);
+	if(!cl)
+		fatal_exit("alloc failure");
+	cl->alloc += num;
+}
+
+/** read up the calloc stats */
+static void
+read_calloc_stat(char* line, rbtree_t* tree)
+{
+	char codeline[10240];
+	char name[10240];
+	int skip = 0;
+	long num = 0, sz = 0;
+	struct codeline* cl = 0;
+	if(sscanf(line+36, "%s %s %n", codeline, name, &skip) != 2) {
+		printf("%s\n%s\n", line, line+36);
+		fatal_exit("unhandled calloc");
+	}
+	if(sscanf(line+36+skip+7, "%ld, %ld", &num, &sz) != 2) {
+		printf("%s\n%s\n", line, line+36+skip+7);
+		fatal_exit("unhandled calloc");
+	}
+
+	cl = get_codeline(tree, codeline, name);
+	if(!cl)
+		fatal_exit("alloc failure");
+	cl->alloc += num*sz;
+}
+
+/** get size of file */
+static off_t
+get_file_size(const char* fname)
+{
+	struct stat s;
+	if(stat(fname, &s) < 0) {
+		fatal_exit("could not stat %s: %s", fname, strerror(errno));
+	}
+	return s.st_size;
+}
+
+/** read the logfile */
+static void
+readfile(rbtree_t* tree, const char* fname, struct alloc_misc* misc)
+{
+	off_t total = get_file_size(fname);
+	off_t done = (off_t)0;
+	int report = 0;
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	if(!in)
+		fatal_exit("could not open %s: %s", fname, strerror(errno));
+	printf("Reading %s of size %lld\n", fname, (long long)total);
+	while(fgets(buf, 102400, in)) {
+		buf[102400-1] = 0;
+		done += (off_t)strlen(buf);
+		/* progress count */
+		if((int)(((double)done / (double)total)*100.) > report) {
+			report = (int)(((double)done / (double)total)*100.);
+			fprintf(stderr, " %d%%", report);
+		}
+
+		if(!match(buf))
+			continue;
+		if(strncmp(buf+36, "region ", 7) == 0)
+			read_region_stat(buf, misc);
+		else if(strstr(buf+36, "malloc("))
+			read_malloc_stat(buf, tree);
+		else if(strstr(buf+36, "calloc("))
+			read_calloc_stat(buf, tree);
+		else {
+			printf("%s\n", buf);
+			fatal_exit("unhandled input");
+		}
+	}
+	fprintf(stderr, " done\n");
+	fclose(in);
+}
+
+/** print memory stats */
+static void
+printstats(rbtree_t* tree, struct alloc_misc* misc)
+{
+	struct codeline* cl;
+	uint64_t total = 0;
+	printf("%12lld in region alloc\n", (long long)misc->region_alloc);
+	total += misc->region_alloc;
+	RBTREE_FOR(cl, struct codeline*, tree) {
+		printf("%12lld in %s %s\n", (long long)cl->alloc, 
+			cl->codeline, cl->func);
+		total += cl->alloc;
+	}
+	printf("------------\n");
+	printf("%12lld total in %ld code lines\n", (long long)total, 
+		(long)tree->count);
+	printf("\n");
+}
+
+/** main program */
+int main(int argc, const char* argv[])
+{
+	rbtree_t* tree = 0;
+	struct alloc_misc misc;
+	if(argc != 2) {
+		usage();
+	}
+	tree = rbtree_create(codeline_cmp);
+	if(!tree)
+		fatal_exit("alloc failure");
+	memset(&misc, 0, sizeof(misc));
+	readfile(tree, argv[1], &misc);
+	printstats(tree, &misc);
+	return 0;
+}

testcode/mini_tpkg.sh

@@ -0,0 +1,111 @@
+# tpkg that only exes the files.
+args="../.."
+if test "$1" = "-a"; then
+	args=$2
+	shift
+	shift
+fi
+	
+if test "$1" = "clean"; then
+	echo "rm -f result.* .done* .tpkg.var.master .tpkg.var.test"
+	rm -f result.* .done* .tpkg.var.master .tpkg.var.test
+	exit 0
+fi
+if test "$1" = "fake"; then
+	echo "minitpkg fake $2"
+	echo "fake" > .done-`basename $2 .tpkg`
+	exit 0
+fi
+if test "$1" = "report" || test "$2" = "report"; then
+	echo "Minitpkg Report"
+	for result in result.*; do
+		name=`echo $result | sed -e 's/result\.//'`
+		if test -f ".done-$name"; then
+			if test "$1" != "-q"; then
+				echo "** PASSED ** : $name"
+			fi
+		else
+			echo "!! FAILED !! : $name"
+		fi
+	done
+	exit 0
+fi
+
+if test "$1" != 'exe'; then
+	# usage
+	echo "mini tpkg. Reduced functionality for old shells."
+	echo "	tpkg exe <file>"
+	echo "	tpkg fake <file>"
+	echo "	tpkg clean"
+	echo "	tpkg [-q] report"
+	exit 1
+fi
+shift
+
+# do not execute if the disk is too full
+#DISKLIMIT=100000
+# This check is not portable (to Solaris 10).
+#avail=`df . | tail -1 | awk '{print $4}'`
+#if test "$avail" -lt "$DISKLIMIT"; then
+	#echo "minitpkg: The disk is too full! Only $avail."
+	#exit 1
+#fi
+
+name=`basename $1 .tpkg`
+dir=$name.$$
+result=result.$name
+done=.done-$name
+success="no"
+shell="bash"
+
+# check already done
+if test -f .done-$name; then
+	echo "minitpkg .done-$name exists. skip test."
+	exit 0
+fi
+
+# Extract
+echo "minitpkg extract $1 to $dir"
+mkdir $dir
+gzip -cd $name.tpkg | (cd $dir; tar xf -)
+cd $dir
+mv $name.dir/* .
+
+# EXE
+echo "minitpkg exe $name"
+echo "minitpkg exe $name" > $result
+if test -f $name.pre; then
+	echo "minitpkg exe $name.pre"
+	echo "minitpkg exe $name.pre" >> $result
+	$shell $name.pre $args >> $result
+	if test $? -ne 0; then
+		echo "Warning: $name.pre did not exit successfully"
+	fi
+fi
+if test -f $name.test; then
+	echo "minitpkg exe $name.test"
+	echo "minitpkg exe $name.test" >> $result
+	$shell $name.test $args >>$result 2>&1
+	if test $? -ne 0; then
+		echo "$name: FAILED" >> $result
+		echo "$name: FAILED"
+		success="no"
+	else
+		echo "$name: PASSED" >> $result
+		echo "$name: PASSED" > ../.done-$name
+		echo "$name: PASSED"
+		success="yes"
+	fi
+fi
+if test -f $name.post; then
+	echo "minitpkg exe $name.post"
+	echo "minitpkg exe $name.post" >> $result
+	$shell $name.post $args >> $result
+	if test $? -ne 0; then
+		echo "Warning: $name.post did not exit successfully"
+	fi
+fi
+
+mv $result ..
+cd ..
+rm -rf $dir

testcode/pktview.c

@@ -67,7 +67,7 @@ void read_input(ldns_buffer* pkt, FILE* in)
 	hex_to_buf(pkt, buf);
 }
 
-/** analyze domain name in packet, possibly compressed. */
+/** analyze domain name in packet, possibly compressed */
 void analyze_dname(ldns_buffer* pkt)
 {
 	size_t oldpos = ldns_buffer_position(pkt);
@@ -111,13 +111,18 @@ void analyze_rdata(ldns_buffer*pkt, const ldns_rr_descriptor* desc,
 		}
 		rdf++;
 	}
-	if(rdlen)
+	if(rdlen) {
+		size_t i;
 		printf(" remain[%d]\n", (int)rdlen);
+		for(i=0; i<rdlen; i++)
+			printf(" %2.2X", (unsigned)ldns_buffer_current(pkt)[i]);
+		printf("\n");
+	}
 	else	printf("\n");
 	ldns_buffer_skip(pkt, (ssize_t)rdlen);
 }
 
-/** analyze rr in packet. */
+/** analyze rr in packet */
 void analyze_rr(ldns_buffer* pkt, int q)
 {
 	uint16_t type, dclass, len;
@@ -134,9 +139,9 @@ void analyze_rr(ldns_buffer* pkt, int q)
 		printf("\n");
 	} else {
 		ttl = ldns_buffer_read_u32(pkt);
-		printf(" ttl %d (0x%x)", ttl, ttl);
+		printf(" ttl %d (0x%x)", (int)ttl, (unsigned)ttl);
 		len = ldns_buffer_read_u16(pkt);
-		printf(" rdata len %d:\n", len);
+		printf(" rdata len %d:\n", (int)len);
 		if(ldns_rr_descript(type))
 			analyze_rdata(pkt, ldns_rr_descript(type), len);
 		else ldns_buffer_skip(pkt, (ssize_t)len);
@@ -152,25 +157,28 @@ void analyze(ldns_buffer* pkt)
 	if(ldns_buffer_limit(pkt) < 12) return;
 
 	i = ldns_buffer_read_u16(pkt);
-	printf("id (hostorder): %d (0x%x)\n", i, i);
+	printf("id (hostorder): %d (0x%x)\n", (int)i, (unsigned)i);
 	f = ldns_buffer_read_u16(pkt);
-	printf("flags: 0x%x\n", f);
+	printf("flags: 0x%x\n", (unsigned)f);
 	qd = ldns_buffer_read_u16(pkt);
-	printf("qdcount: %d\n", qd);
+	printf("qdcount: %d\n", (int)qd);
 	an = ldns_buffer_read_u16(pkt);
-	printf("ancount: %d\n", an);
+	printf("ancount: %d\n", (int)an);
 	ns = ldns_buffer_read_u16(pkt);
-	printf("nscount: %d\n", ns);
+	printf("nscount: %d\n", (int)ns);
 	ar = ldns_buffer_read_u16(pkt);
-	printf("arcount: %d\n", ar);
+	printf("arcount: %d\n", (int)ar);
 	
 	printf(";-- query section\n");
 	while(ldns_buffer_remaining(pkt) > 0) {
-		if(rrnum == qd) printf(";-- answer section\n");
-		if(rrnum == qd+an) printf(";-- authority section\n");
-		if(rrnum == qd+an+ns) printf(";-- additional section\n");
+		if(rrnum == (int)qd) 
+			printf(";-- answer section\n");
+		if(rrnum == (int)qd+(int)an) 
+			printf(";-- authority section\n");
+		if(rrnum == (int)qd+(int)an+(int)ns) 
+			printf(";-- additional section\n");
 		printf("rr %d ", rrnum);
-		analyze_rr(pkt, rrnum < qd);
+		analyze_rr(pkt, rrnum < (int)qd);
 		rrnum++;
 	}
 }

testcode/readhex.c

@@ -46,7 +46,7 @@ static void
 skip_whites(const char** p)
 {
 	while(1) {
-		while(isspace(**p))
+		while(isspace((int)**p))
 			(*p)++;
 		if(**p == ';') {
 			/* comment, skip until newline */
@@ -68,11 +68,11 @@ void hex_to_buf(ldns_buffer* pkt, const char* hex)
 		skip_whites(&p);
 		if(ldns_buffer_position(pkt) == ldns_buffer_limit(pkt))
 			fatal_exit("hex_to_buf: buffer too small");
-		if(!isalnum(*p))
+		if(!isalnum((int)*p))
 			break;
 		val = ldns_hexdigit_to_int(*p++) << 4;
 		skip_whites(&p);
-		log_assert(*p && isalnum(*p));
+		log_assert(*p && isalnum((int)*p));
 		val |= ldns_hexdigit_to_int(*p++);
 		ldns_buffer_write_u8(pkt, (uint8_t)val);
 		skip_whites(&p);

testcode/replay.c

@@ -41,6 +41,7 @@
 
 #include "config.h"
 #include "util/log.h"
+#include "util/net_help.h"
 #include "testcode/replay.h"
 #include "testcode/ldns-testpkts.h"
 
@@ -85,6 +86,18 @@ replay_range_delete(struct replay_range* rng)
 	free(rng);
 }
 
+/** strip whitespace from end of string */
+static void
+strip_end_white(char* p)
+{
+	size_t i;
+	for(i = strlen(p); i > 0; i--) {
+		if(isspace((int)p[i-1]))
+			p[i-1] = 0;
+		else return;
+	}
+}
+
 /** 
  * Read a range from file. 
  * @param remain: Rest of line (after RANGE keyword).
@@ -99,7 +112,7 @@ replay_range_delete(struct replay_range* rng)
  */
 static struct replay_range*
 replay_range_read(char* remain, FILE* in, const char* name, int* lineno, 
-	char* line, uint16_t* ttl, ldns_rdf** or, ldns_rdf** prev)
+	char* line, uint32_t* ttl, ldns_rdf** or, ldns_rdf** prev)
 {
 	struct replay_range* rng = (struct replay_range*)malloc(
 		sizeof(struct replay_range));
@@ -120,10 +133,25 @@ replay_range_read(char* remain, FILE* in, const char* name, int* lineno,
 	while(fgets(line, MAX_LINE_LEN-1, in)) {
 		(*lineno)++;
 		parse = line;
-		while(isspace(*parse))
+		while(isspace((int)*parse))
 			parse++;
-		if(!*parse || *parse == ';')
+		if(!*parse || *parse == ';') {
+			pos = ftello(in);
 			continue;
+		}
+		if(parse_keyword(&parse, "ADDRESS")) {
+			while(isspace((int)*parse))
+				parse++;
+			strip_end_white(parse);
+			if(!extstrtoaddr(parse, &rng->addr, &rng->addrlen)) {
+				log_err("Line %d: could not read ADDRESS: %s", 
+					*lineno, parse);
+				free(rng);
+				return NULL;
+			}
+			pos = ftello(in);
+			continue;
+		}
 		if(parse_keyword(&parse, "RANGE_END")) {
 			return rng;
 		}
@@ -158,7 +186,7 @@ replay_range_read(char* remain, FILE* in, const char* name, int* lineno,
  */
 static struct replay_moment*
 replay_moment_read(char* remain, FILE* in, const char* name, int* lineno, 
-	uint16_t* ttl, ldns_rdf** or, ldns_rdf** prev)
+	uint32_t* ttl, ldns_rdf** or, ldns_rdf** prev)
 {
 	struct replay_moment* mom = (struct replay_moment*)malloc(
 		sizeof(struct replay_moment));
@@ -173,7 +201,7 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 		return NULL;
 	}
 	remain += skip;
-	while(isspace(*remain))
+	while(isspace((int)*remain))
 		remain++;
 	if(parse_keyword(&remain, "NOTHING")) {
 		mom->evt_type = repevt_nothing;
@@ -198,6 +226,16 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 		free(mom);
 		return NULL;
 	}
+	while(isspace((int)*remain))
+		remain++;
+	if(parse_keyword(&remain, "ADDRESS")) {
+		if(!extstrtoaddr(remain, &mom->addr, &mom->addrlen)) {
+			log_err("line %d: could not parse ADDRESS: %s", 
+				*lineno, remain);
+			free(mom);
+			return NULL;
+		}
+	}
 
 	if(readentry) {
 		mom->match = read_entry(in, name, lineno, ttl, or, prev);
@@ -210,12 +248,12 @@ replay_moment_read(char* remain, FILE* in, const char* name, int* lineno,
 	return mom;
 }
 
-/** makes scenario with title on rest of line. */
+/** makes scenario with title on rest of line */
 static struct replay_scenario*
 make_scenario(char* line)
 {
 	struct replay_scenario* scen;
-	while(isspace(*line))
+	while(isspace((int)*line))
 		line++;
 	if(!*line) {
 		log_err("scenario: no title given");
@@ -239,7 +277,7 @@ replay_scenario_read(FILE* in, const char* name, int* lineno)
 	char line[MAX_LINE_LEN];
 	char *parse;
 	struct replay_scenario* scen = NULL;
-	uint16_t ttl = 3600;
+	uint32_t ttl = 3600;
 	ldns_rdf* or = NULL;
 	ldns_rdf* prev = NULL;
 	line[MAX_LINE_LEN-1]=0;
@@ -247,7 +285,7 @@ replay_scenario_read(FILE* in, const char* name, int* lineno)
 	while(fgets(line, MAX_LINE_LEN-1, in)) {
 		parse=line;
 		(*lineno)++;
-		while(isspace(*parse))
+		while(isspace((int)*parse))
 			parse++;
 		if(!*parse) 
 			continue; /* empty line */

testcode/replay.h

@@ -44,11 +44,13 @@
  * ; comment line.
  * SCENARIO_BEGIN name_of_scenario
  * RANGE_BEGIN start_time end_time
+ *    ; give ip of the virtual server, it matches any ip if not present.
+ *    ADDRESS ip_address 
  *    match_entries
  * RANGE_END
  * ; more RANGE items.
  * ; go to the next moment
- * STEP time_step event_type
+ * STEP time_step event_type [ADDRESS ip_address]
  * ; event_type can be:
  *	o NOTHING - nothing
  *	o QUERY - followed by entry
@@ -158,6 +160,11 @@ struct replay_moment {
 	/** The sent packet must match this. Incoming events, the data. */
 	struct entry* match;
 
+	/** address that must be matched, or packet remote host address. */
+	struct sockaddr_storage addr;
+	/** length of addr, if 0, then any address will do */
+	socklen_t addrlen;
+
 	/** what pending query should timeout or is answered. or 
 	 * NULL for last sent query. 
 	 * Unused at this time.
@@ -173,6 +180,10 @@ struct replay_range {
 	int start_step;
 	/** end step of time range. */
 	int end_step;
+	/** address of where this range is served. */
+	struct sockaddr_storage addr;
+	/** length of addr, if 0, then any address will do */
+	socklen_t addrlen;
 
 	/** Matching list */
 	struct entry* match;
@@ -217,6 +228,8 @@ struct replay_runtime {
 	void (*sig_cb)(int, void*);
 	/** signal handler user arg */
 	void *sig_cb_arg;
+	/** time to exit cleanly */
+	int exit_cleanly;
 
 	/** size of buffers */
 	size_t bufsize;
@@ -245,6 +258,10 @@ struct fake_pending {
 	ldns_pkt* pkt;
 	/** by what transport was the query sent out */
 	enum transport_type transport;
+	/** if this is a serviced query */
+	int serviced;
+	/** the runtime structure this is part of */
+	struct replay_runtime* runtime;
 };
 
 /**

testcode/signit.c

@@ -0,0 +1,249 @@
+/*
+ * testcode/signit.c - debug tool to sign rrsets with given keys.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program signs rrsets with the given keys. It can be used to 
+ * construct input to test the validator with.
+ */
+#include "config.h"
+#include "util/log.h"
+#include "util/config_file.h"
+#include "util/net_help.h"
+
+/**
+ * Key settings
+ */
+struct keysets {
+	/** signature inception */
+	uint32_t incep;
+	/** signature expiration */
+	uint32_t expi;
+	/** owner name */
+	char* owner;
+	/** keytag */
+	uint16_t keytag;
+	/** DNSKEY flags */
+	uint16_t flags;
+};
+
+/** print usage and exit */
+static void
+usage()
+{
+	printf("usage:	signit expi ince keytag owner keyfile\n");
+	printf("present rrset data on stdin.\n");
+	printf("signed data is printed to stdout.\n");
+	printf("\n");
+	printf("Or use:	signit NSEC3PARAM hash flags iter salt\n");
+	printf("present names on stdin, hashed names are printed to stdout.\n");
+	exit(1);
+}
+
+/** read expi ince keytag owner from cmdline */
+static void
+parse_cmdline(char *argv[], struct keysets* s)
+{
+	s->expi = cfg_convert_timeval(argv[1]);
+	s->incep = cfg_convert_timeval(argv[2]);
+	s->keytag = (uint16_t)atoi(argv[3]);
+	s->owner = argv[4];
+	s->flags = DNSKEY_BIT_ZSK; /* to enforce signing */
+}
+
+/** read all key files, exit on error */
+static ldns_key_list*
+read_keys(int num, char* names[], struct keysets* set)
+{
+	int i;
+	ldns_key_list* keys = ldns_key_list_new();
+	ldns_key* k;
+	ldns_rdf* rdf;
+	ldns_status s;
+	int b;
+	FILE* in;
+
+	if(!keys) fatal_exit("alloc failure");
+	for(i=0; i<num; i++) {
+		printf("read keyfile %s\n", names[i]);
+		in = fopen(names[i], "r");
+		if(!in) fatal_exit("could not open %s: %s", names[i],
+				strerror(errno));
+		s = ldns_key_new_frm_fp(&k, in);
+		fclose(in);
+		if(s != LDNS_STATUS_OK)
+			fatal_exit("bad keyfile %s: %s", names[i],
+				ldns_get_errorstr_by_id(s));
+		ldns_key_set_expiration(k, set->expi);
+		ldns_key_set_inception(k, set->incep);
+		s = ldns_str2rdf_dname(&rdf, set->owner);
+		if(s != LDNS_STATUS_OK)
+			fatal_exit("bad owner name %s: %s", set->owner,
+				ldns_get_errorstr_by_id(s));
+		ldns_key_set_pubkey_owner(k, rdf);
+		ldns_key_set_flags(k, set->flags);
+		ldns_key_set_keytag(k, set->keytag);
+		b = ldns_key_list_push_key(keys, k);
+		log_assert(b);
+	}
+	return keys;
+}
+
+/** read list of rrs from the file */
+static ldns_rr_list*
+read_rrs(FILE* in)
+{
+	uint32_t my_ttl = 3600;
+	ldns_rdf *my_origin = NULL;
+	ldns_rdf *my_prev = NULL;
+	ldns_status s;
+	int line_nr = 1;
+	int b;
+
+	ldns_rr_list* list;
+	ldns_rr *rr;
+
+	list = ldns_rr_list_new();
+	if(!list) fatal_exit("alloc error");
+
+	while(!feof(in)) {
+		s = ldns_rr_new_frm_fp_l(&rr, in, &my_ttl, &my_origin,
+			&my_prev, &line_nr);
+		if(s == LDNS_STATUS_SYNTAX_TTL || 
+			s == LDNS_STATUS_SYNTAX_ORIGIN ||
+			s == LDNS_STATUS_SYNTAX_EMPTY)
+			continue;
+		else if(s != LDNS_STATUS_OK)
+			fatal_exit("parse error in line %d: %s", line_nr,
+				ldns_get_errorstr_by_id(s));
+		b = ldns_rr_list_push_rr(list, rr);
+		log_assert(b);
+	}
+	printf("read %d lines\n", line_nr);
+
+	return list;
+}
+
+/** sign the rrs with the keys */
+static void
+signit(ldns_rr_list* rrs, ldns_key_list* keys)
+{
+	ldns_rr_list* rrset;
+	ldns_rr_list* sigs;
+	
+	while(ldns_rr_list_rr_count(rrs) > 0) {
+		rrset = ldns_rr_list_pop_rrset(rrs);
+		if(!rrset) fatal_exit("copy alloc failure");
+		sigs = ldns_sign_public(rrset, keys);
+		if(!sigs) fatal_exit("failed to sign");
+		ldns_rr_list_print(stdout, rrset);
+		ldns_rr_list_print(stdout, sigs);
+		printf("\n");
+		ldns_rr_list_free(rrset);
+		ldns_rr_list_free(sigs);
+	}
+}
+
+/** process keys and signit */
+static void
+process_keys(int argc, char* argv[])
+{
+	ldns_rr_list* rrs;
+	ldns_key_list* keys;
+	struct keysets settings;
+	log_assert(argc == 6);
+
+	parse_cmdline(argv, &settings);
+	keys = read_keys(1, argv+5, &settings);
+	rrs = read_rrs(stdin);
+	signit(rrs, keys);
+
+	ldns_rr_list_deep_free(rrs);
+	ldns_key_list_free(keys);
+}
+
+/** process nsec3 params and perform hashing */
+static void
+process_nsec3(int argc, char* argv[])
+{
+	char line[10240];
+	ldns_rdf* salt;
+	ldns_rdf* in, *out;
+	ldns_status status;
+	status = ldns_str2rdf_nsec3_salt(&salt, argv[5]);
+	if(status != LDNS_STATUS_OK)
+		fatal_exit("Could not parse salt %s: %s", argv[5],
+			ldns_get_errorstr_by_id(status));
+	log_assert(argc == 6);
+	while(fgets(line, (int)sizeof(line), stdin)) {
+		if(strlen(line) > 0)
+			line[strlen(line)-1] = 0; /* remove trailing newline */
+		if(line[0]==0)
+			continue;
+		status = ldns_str2rdf_dname(&in, line);
+		if(status != LDNS_STATUS_OK)
+			fatal_exit("Could not parse name %s: %s", line,
+				ldns_get_errorstr_by_id(status));
+		ldns_rdf_print(stdout, in);
+		printf(" -> ");
+		/* arg 3 is flags, unused */
+		out = ldns_nsec3_hash_name(in, (uint8_t)atoi(argv[2]), 
+			(uint16_t)atoi(argv[4]),
+			ldns_rdf_data(salt)[0], ldns_rdf_data(salt)+1);
+		if(!out)
+			fatal_exit("Could not hash %s", line);
+		ldns_rdf_print(stdout, out);
+		printf("\n");
+		ldns_rdf_deep_free(in);
+		ldns_rdf_deep_free(out);
+	}
+	ldns_rdf_deep_free(salt);
+}
+
+/** main program */
+int main(int argc, char* argv[])
+{
+	log_init(NULL, 0, NULL);
+	if(argc != 6) {
+		usage();
+	}
+	if(strcmp(argv[1], "NSEC3PARAM") == 0) {
+		process_nsec3(argc, argv);
+		return 0;
+	}
+	process_keys(argc, argv);
+	return 0;
+}

testcode/testbed.sh

@@ -43,10 +43,10 @@ function dotest()
 	fi
 
 	cat >makeconf.mak.$$ << EOF 
-configure:	configure.ac
-	$AC_CMD
-	touch configure
-Makefile:	configure Makefile.in
+#configure:	configure.ac
+#	$AC_CMD
+#	touch configure
+Makefile:	Makefile.in #configure
 	./configure $CONFIGURE_FLAGS $DISABLE
 	touch Makefile 
 EOF
@@ -77,12 +77,11 @@ EOF
 		# echossh $1 "gtar xzf unbound_ttt.tar.gz && rm unbound_ttt.tar.gz"
 	fi
 	echossh $1 "cd $2; $MAKE_CMD -f makeconf.mak.$$ Makefile"
-	echossh $1 "cd $2; $MAKE_CMD"
+	echossh $1 "cd $2; $MAKE_CMD all tests"
 	echossh $1 "cd $2; $MAKE_CMD doc"
 	if test $RUN_TEST = yes; then
-	echossh $1 "cd $2/testdata; tpkg clean"
 	echossh $1 "cd $2; bash testcode/do-tests.sh"
-	echossh $1 "cd $2/testdata; tpkg -q report" | tee -a $REPORT_FILE
+	echossh $1 "cd $2/testdata; sh ../testcode/mini_tpkg.sh -q report" | tee -a $REPORT_FILE
 	fi
 	echossh $1 "cd $2; rm -f makeconf.mak.$$"
 	rm -f makeconf.mak.$$

testcode/testbound.c

@@ -53,6 +53,8 @@
 
 /** maximum line length for lines in the replay file. */
 #define MAX_LINE_LEN 1024
+/** the config file (removed at exit) */
+static char cfgfile[MAX_LINE_LEN];
 
 /** give commandline usage for testbound. */
 static void
@@ -82,7 +84,7 @@ add_opts(char* optarg, int* pass_argc, char* pass_argv[])
 {
 	char *p = optarg, *np;
 	size_t len;
-	while(p && isspace(*p)) 
+	while(p && isspace((int)*p)) 
 		p++;
 	while(p && *p) {
 		/* find location of next string and length of this one */
@@ -100,12 +102,12 @@ add_opts(char* optarg, int* pass_argc, char* pass_argv[])
 		(*pass_argc)++;
 		/* go to next option */
 	        p = np;
-		while(p && isspace(*p)) 
+		while(p && isspace((int)*p)) 
 			p++;
 	}
 }
 
-/** pretty print commandline for unbound in this test. */
+/** pretty print commandline for unbound in this test */
 static void
 echo_cmdline(int argc, char* argv[])
 {
@@ -133,10 +135,16 @@ setup_config(FILE* in, char* configfile, int* lineno,
 	if(!cfg) fatal_exit("could not open %s: %s", 
 			configfile, strerror(errno));
 	line[MAX_LINE_LEN-1] = 0;
+	/* some basic settings to not pollute the host system */
+	fprintf(cfg, "server:	use-syslog: no\n");
+	fprintf(cfg, "		directory: \"\"\n");
+	fprintf(cfg, "		chroot: \"\"\n");
+	fprintf(cfg, "		username: \"\"\n");
+	fprintf(cfg, "		pidfile: \"\"\n");
 	while(fgets(line, MAX_LINE_LEN-1, in)) {
 		parse = line;
 		(*lineno)++;
-		while(isspace(*parse))
+		while(isspace((int)*parse))
 			parse++;
 		if(!*parse || parse[0] == ';')
 			continue;
@@ -179,6 +187,12 @@ setup_playback(const char* filename, char* configfile,
 	log_info("Scenario: %s", scen->title);
 	return scen;
 }
+
+/** remove config file at exit */
+void remove_configfile(void)
+{
+	unlink(cfgfile);
+}
 	
 /**
  * Main fake event test program. Setup, teardown and report errors.
@@ -195,11 +209,11 @@ main(int argc, char* argv[])
 	int init_optind = optind;
 	char* init_optarg = optarg;
 	struct replay_scenario* scen = NULL;
-	char cfgfile[MAX_LINE_LEN];
 
-	log_init(NULL);
+	log_init(NULL, 0, NULL);
 	log_info("Start of %s testbound program.", PACKAGE_STRING);
 	/* determine commandline options for the daemon */
+	cfgfile[0] = 0;
 	pass_argc = 1;
 	pass_argv[0] = "unbound";
 	add_opts("-d", &pass_argc, pass_argv);
@@ -224,6 +238,8 @@ main(int argc, char* argv[])
 		testbound_usage();
 		return 1;
 	}
+	if(atexit(&remove_configfile) != 0)
+		fatal_exit("atexit() failed: %s", strerror(errno));
 
 	/* setup test environment */
 	scen = setup_playback(playback_file, cfgfile, &pass_argc, pass_argv);
@@ -240,9 +256,10 @@ main(int argc, char* argv[])
 	/* run the normal daemon */
 	res = daemon_main(pass_argc, pass_argv);
 
-	unlink(cfgfile);
 	fake_event_cleanup();
 	for(c=1; c<pass_argc; c++)
 		free(pass_argv[c]);
+	if(res == 0)
+		log_info("Testbound Exit Success");
 	return res;
 }

testcode/unitanchor.c

@@ -0,0 +1,118 @@
+/*
+ * testcode/unitanchor.c - unit test for trust anchor storage.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls trust anchor unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/data/dname.h"
+#include "testcode/unitmain.h"
+#include "validator/val_anchor.h"
+
+/** test empty set */
+static void
+test_anchor_empty(struct val_anchors* a)
+{
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c) == NULL);
+}
+
+/** test set of one anchor */
+static void
+test_anchor_one(ldns_buffer* buff, struct val_anchors* a)
+{
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchor_store_str(a, buff, 
+		"nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"));
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c) != NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c) != NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c) != NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002oo\000", 4, c) == NULL);
+}
+
+/** test with several anchors */
+static void
+test_anchors(ldns_buffer* buff, struct val_anchors* a)
+{
+	struct trust_anchor* ta;
+	uint16_t c = LDNS_RR_CLASS_IN;
+	unit_assert(anchor_store_str(a, buff, 
+	"labs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A"));
+	unit_assert(anchors_lookup(a, (uint8_t*)"\000", 1, c) == NULL);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\003com\000", 5, c) == NULL);
+	unit_assert(anchors_lookup(a, 
+		(uint8_t*)"\007example\003com\000", 11, c) == NULL);
+	unit_assert(ta = anchors_lookup(a, (uint8_t*)"\002nl\000", 4, c));
+	unit_assert(query_dname_compare(ta->name, (uint8_t*)"\002nl\000")==0);
+	unit_assert(ta = anchors_lookup(a, 
+		(uint8_t*)"\004labs\002nl\000", 9, c));
+	unit_assert(query_dname_compare(ta->name, 
+		(uint8_t*)"\004labs\002nl\000") == 0);
+	unit_assert(ta = anchors_lookup(a, 
+		(uint8_t*)"\004fabs\002nl\000", 9, c));
+	unit_assert(query_dname_compare(ta->name, 
+		(uint8_t*)"\002nl\000") == 0);
+	unit_assert(anchors_lookup(a, (uint8_t*)"\002oo\000", 4, c) == NULL);
+}
+
+void anchors_test()
+{
+	ldns_buffer* buff = ldns_buffer_new(65800);
+	struct val_anchors* a;
+	unit_assert(a = anchors_create());
+	ldns_buffer_flip(buff);
+	test_anchor_empty(a);
+	test_anchor_one(buff, a);
+	test_anchors(buff, a);
+	anchors_delete(a);
+	ldns_buffer_free(buff);
+}

testcode/unitdname.c

@@ -76,21 +76,21 @@ static void
 dname_test_qdtl(ldns_buffer* buff)
 {
 	ldns_buffer_write_at(buff, 0, "\012abCDeaBCde\003cOm\000", 16);
-	query_dname_tolower(ldns_buffer_begin(buff), 16);
+	query_dname_tolower(ldns_buffer_begin(buff));
 	unit_assert( memcmp(ldns_buffer_begin(buff), 
 		"\012abcdeabcde\003com\000", 16) == 0);
 
 	ldns_buffer_write_at(buff, 0, "\001+\012abC{e-ZYXe\003NET\000", 18);
-	query_dname_tolower(ldns_buffer_begin(buff), 18);
+	query_dname_tolower(ldns_buffer_begin(buff));
 	unit_assert( memcmp(ldns_buffer_begin(buff), 
 		"\001+\012abc{e-zyxe\003net\000", 18) == 0);
 
 	ldns_buffer_write_at(buff, 0, "\000", 1);
-	query_dname_tolower(ldns_buffer_begin(buff), 1);
+	query_dname_tolower(ldns_buffer_begin(buff));
 	unit_assert( memcmp(ldns_buffer_begin(buff), "\000", 1) == 0);
 
 	ldns_buffer_write_at(buff, 0, "\002NL\000", 4);
-	query_dname_tolower(ldns_buffer_begin(buff), 4);
+	query_dname_tolower(ldns_buffer_begin(buff));
 	unit_assert( memcmp(ldns_buffer_begin(buff), "\002nl\000", 4) == 0);
 }
 
@@ -361,6 +361,377 @@ dname_test_dname_lab_cmp()
 	unit_assert(ml == 4);
 }
 
+/** test dname_subdomain_c */
+static void
+dname_test_subdomain()
+{
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"",
+		(uint8_t*)""));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\003com",
+		(uint8_t*)""));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"",
+		(uint8_t*)"\003com"));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\003com"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\007example\003com"));
+	unit_assert(dname_subdomain_c(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)""));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003com"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003org"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\007example\003net",
+		(uint8_t*)"\003org"));
+	unit_assert(!dname_subdomain_c(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\007example\003org"));
+}
+
+/** test dname_strict_subdomain */
+static void
+dname_test_strict_subdomain()
+{
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"", 1,
+		(uint8_t*)"", 1));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\003com", 2,
+		(uint8_t*)"", 1));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"", 1,
+		(uint8_t*)"\003com", 2));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\007example\003com", 3,
+		(uint8_t*)"\003com", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003com", 2,
+		(uint8_t*)"\007example\003com", 3));
+	unit_assert(dname_strict_subdomain(
+		(uint8_t*)"\007example\003com", 3,
+		(uint8_t*)"", 1));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\003com", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\003org", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\007example\003net", 3,
+		(uint8_t*)"\003org", 2));
+	unit_assert(!dname_strict_subdomain(
+		(uint8_t*)"\003net", 2,
+		(uint8_t*)"\007example\003org", 3));
+}
+
+/** test dname_is_root */
+static void
+dname_test_isroot()
+{
+	unit_assert(dname_is_root((uint8_t*)"\000"));
+	unit_assert(!dname_is_root((uint8_t*)"\001a\000"));
+	unit_assert(!dname_is_root((uint8_t*)"\005abvcd\003com\000"));
+	/* malformed dname in this test, but should work */
+	unit_assert(!dname_is_root((uint8_t*)"\077a\000"));
+	unit_assert(dname_is_root((uint8_t*)"\000"));
+}
+
+/** test dname_remove_label */
+static void
+dname_test_removelabel()
+{
+	uint8_t* orig = (uint8_t*)"\007example\003com\000";
+	uint8_t* n = orig;
+	size_t l = 13;
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+8 );
+	unit_assert( l == 5 );
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+12 );
+	unit_assert( l == 1 );
+	dname_remove_label(&n, &l);
+	unit_assert( n == orig+12 );
+	unit_assert( l == 1 );
+}
+
+/** test dname_signame_label_count */
+static void
+dname_test_sigcount()
+{
+	unit_assert(dname_signame_label_count((uint8_t*)"\000") == 0);
+	unit_assert(dname_signame_label_count((uint8_t*)"\001*\000") == 0);
+	unit_assert(dname_signame_label_count((uint8_t*)"\003xom\000") == 1);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\003xom\000") == 1);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\007example\003xom\000") == 2);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\007example\003xom\000") == 2);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\003www\007example\003xom\000") == 3);
+	unit_assert(dname_signame_label_count(
+		(uint8_t*)"\001*\003www\007example\003xom\000") == 3);
+}
+
+/** test dname_is_wild routine */
+static void
+dname_test_iswild()
+{
+	unit_assert( !dname_is_wild((uint8_t*)"\000") );
+	unit_assert( dname_is_wild((uint8_t*)"\001*\000") );
+	unit_assert( !dname_is_wild((uint8_t*)"\003net\000") );
+	unit_assert( dname_is_wild((uint8_t*)"\001*\003net\000") );
+}
+
+/** test dname_canonical_compare */
+static void
+dname_test_canoncmp()
+{
+	/* equality */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net\000",
+		(uint8_t*)"\003net\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003net\000",
+		(uint8_t*)"\007example\003net\000"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004test\007example\003net\000",
+		(uint8_t*)"\004test\007example\003net\000"
+		) == 0);
+
+	/* subdomains */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\000"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\003com"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\003com"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\007example\003com"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003com",
+		(uint8_t*)"\000"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\000",
+		(uint8_t*)"\007example\003com"
+		) == -1);
+
+	/* compare rightmost label */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003com",
+		(uint8_t*)"\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003com"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003net",
+		(uint8_t*)"\003org"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example\003net",
+		(uint8_t*)"\003org"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003org",
+		(uint8_t*)"\007example\003net"
+		) == 1);
+
+	/* label length makes a difference; but only if rest is equal */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004neta",
+		(uint8_t*)"\003net"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\002ne",
+		(uint8_t*)"\004neta"
+		) == -1);
+
+	/* label content */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003aag\007example\003net",
+		(uint8_t*)"\003bla\007example\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bla\007example\003net",
+		(uint8_t*)"\003aag\007example\003net"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bla\003aag\007example\003net",
+		(uint8_t*)"\003aag\003bla\007example\003net"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\02sn\003opt\003aag\007example\003net",
+		(uint8_t*)"\02sn\003opt\003bla\007example\003net"
+		) == -1);
+
+	/* lowercase during compare */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\003bLa\007examPLe\003net",
+		(uint8_t*)"\003bla\007eXAmple\003nET"
+		) == 0);
+
+	/* example from 4034 */
+	/* example a.example yljkjljk.a.example Z.a.example zABC.a.EXAMPLE
+	 z.example \001.z.example *.z.example \200.z.example */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"",
+		(uint8_t*)"\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)"\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\001Z\001a\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == -1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001\200\001z\007example"
+		) == -1);
+	/* same example in reverse */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)""
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\001Z\001a\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001z\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == 1);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\200\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == 1);
+	/* same example for equality */
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\007example",
+		(uint8_t*)"\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001a\007example",
+		(uint8_t*)"\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\010yljkjljk\001a\007example",
+		(uint8_t*)"\010yljkjljk\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001Z\001a\007example",
+		(uint8_t*)"\001Z\001a\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\004zABC\001a\007EXAMPLE",
+		(uint8_t*)"\004zABC\001a\007EXAMPLE"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001z\007example",
+		(uint8_t*)"\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\001\001z\007example",
+		(uint8_t*)"\001\001\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001*\001z\007example",
+		(uint8_t*)"\001*\001z\007example"
+		) == 0);
+	unit_assert( dname_canonical_compare(
+		(uint8_t*)"\001\200\001z\007example",
+		(uint8_t*)"\001\200\001z\007example"
+		) == 0);
+}
+
+/** Test dname_get_shared_topdomain */
+static void
+dname_test_topdomain()
+{
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"",
+			(uint8_t*)""), 
+		(uint8_t*)"") == 0);
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"\003www\007example\003com",
+			(uint8_t*)"\003www\007example\003com"), 
+		(uint8_t*)"\003www\007example\003com") == 0);
+	unit_assert( query_dname_compare(
+		dname_get_shared_topdomain(
+			(uint8_t*)"\003www\007example\003com",
+			(uint8_t*)"\003bla\007example\003com"), 
+		(uint8_t*)"\007example\003com") == 0);
+}
+
 void dname_test()
 {
 	ldns_buffer* buff = ldns_buffer_new(65800);
@@ -372,5 +743,13 @@ void dname_test()
 	dname_test_count_size_labels();
 	dname_test_dname_lab_cmp();
 	dname_test_pkt_dname_len(buff);
+	dname_test_strict_subdomain();
+	dname_test_subdomain();
+	dname_test_isroot();
+	dname_test_removelabel();
+	dname_test_sigcount();
+	dname_test_iswild();
+	dname_test_canoncmp();
+	dname_test_topdomain();
 	ldns_buffer_free(buff);
 }

testcode/unitlruhash.c

@@ -42,36 +42,24 @@
 #include "testcode/unitmain.h"
 #include "util/log.h"
 #include "util/storage/lruhash.h"
+#include "util/storage/slabhash.h" /* for the test structures */
 
-/* --- test representation --- */
-/** structure contains test key */
-struct testkey {
-	/** the key id */
-	int id;
-	/** the entry */
-	struct lruhash_entry entry;
-};
-/** structure contains test data */
-struct testdata {
-	/** data value */
-	int data;
-};
+/** use this type for the lruhash test key */
+typedef struct slabhash_testkey testkey_t;
+/** use this type for the lruhash test data */
+typedef struct slabhash_testdata testdata_t;
 
-/** sizefunc for lruhash */
-static size_t test_sizefunc(void*, void*);
-/** comparefunc for lruhash */
-static int test_compfunc(void*, void*);
-/** delkey for lruhash */
-static void test_delkey(void*, void*);
-/** deldata for lruhash */
-static void test_deldata(void*, void*);
-/* --- end test representation --- */
+/** delete key */
+static void delkey(struct slabhash_testkey* k) {
+	lock_rw_destroy(&k->entry.lock); free(k);}
+/** delete data */
+static void deldata(struct slabhash_testdata* d) {free(d);}
 
-/** hash func, very bad to improve collisions. */
+/** hash func, very bad to improve collisions */
 static hashvalue_t myhash(int id) {return (hashvalue_t)id & 0x0f;}
-/** allocate new key, fill in hash. */
-static struct testkey* newkey(int id) {
-	struct testkey* k = (struct testkey*)calloc(1, sizeof(struct testkey));
+/** allocate new key, fill in hash */
+static testkey_t* newkey(int id) {
+	testkey_t* k = (testkey_t*)calloc(1, sizeof(testkey_t));
 	if(!k) fatal_exit("out of memory");
 	k->id = id;
 	k->entry.hash = myhash(id);
@@ -80,28 +68,23 @@ static struct testkey* newkey(int id) {
 	return k;
 }
 /** new data el */
-static struct testdata* newdata(int val) {
-	struct testdata* d = (struct testdata*)calloc(1, 
-		sizeof(struct testdata));
+static testdata_t* newdata(int val) {
+	testdata_t* d = (testdata_t*)calloc(1, 
+		sizeof(testdata_t));
 	if(!d) fatal_exit("out of memory");
 	d->data = val;
 	return d;
 }
-/** delete key */
-static void delkey(struct testkey* k) {	
-	lock_rw_destroy(&k->entry.lock); free(k);}
-/** delete data */
-static void deldata(struct testdata* d) {free(d);}
 
 /** test bin_find_entry function and bin_overflow_remove */
 static void
 test_bin_find_entry(struct lruhash* table)
 {
-	struct testkey* k = newkey(12);
-	struct testdata* d = newdata(128);
-	struct testkey* k2 = newkey(12 + 1024);
-	struct testkey* k3 = newkey(14);
-	struct testkey* k4 = newkey(12 + 1024*2);
+	testkey_t* k = newkey(12);
+	testdata_t* d = newdata(128);
+	testkey_t* k2 = newkey(12 + 1024);
+	testkey_t* k3 = newkey(14);
+	testkey_t* k4 = newkey(12 + 1024*2);
 	hashvalue_t h = myhash(12);
 	struct lruhash_bin bin;
 	memset(&bin, 0, sizeof(bin));
@@ -178,8 +161,8 @@ test_bin_find_entry(struct lruhash* table)
 /** test lru_front lru_remove */
 static void test_lru(struct lruhash* table)
 {
-	struct testkey* k = newkey(12);
-	struct testkey* k2 = newkey(14);
+	testkey_t* k = newkey(12);
+	testkey_t* k2 = newkey(14);
 	lock_quick_lock(&table->lock);
 
 	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
@@ -225,10 +208,10 @@ static void test_lru(struct lruhash* table)
 static void
 test_short_table(struct lruhash* table) 
 {
-	struct testkey* k = newkey(12);
-	struct testkey* k2 = newkey(14);
-	struct testdata* d = newdata(128);
-	struct testdata* d2 = newdata(129);
+	testkey_t* k = newkey(12);
+	testkey_t* k2 = newkey(14);
+	testdata_t* d = newdata(128);
+	testdata_t* d2 = newdata(129);
 	
 	k->entry.data = d;
 	k2->entry.data = d2;
@@ -249,11 +232,11 @@ test_short_table(struct lruhash* table)
 
 /** test adding a random element */
 static void
-testadd(struct lruhash* table, struct testdata* ref[])
+testadd(struct lruhash* table, testdata_t* ref[])
 {
 	int numtoadd = random() % HASHTESTMAX;
-	struct testdata* data = newdata(numtoadd);
-	struct testkey* key = newkey(numtoadd);
+	testdata_t* data = newdata(numtoadd);
+	testkey_t* key = newkey(numtoadd);
 	key->entry.data = data;
 	lruhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
 	ref[numtoadd] = data;
@@ -261,10 +244,10 @@ testadd(struct lruhash* table, struct testdata* ref[])
 
 /** test adding a random element */
 static void
-testremove(struct lruhash* table, struct testdata* ref[])
+testremove(struct lruhash* table, testdata_t* ref[])
 {
 	int num = random() % HASHTESTMAX;
-	struct testkey* key = newkey(num);
+	testkey_t* key = newkey(num);
 	lruhash_remove(table, myhash(num), key);
 	ref[num] = NULL;
 	delkey(key);
@@ -272,12 +255,12 @@ testremove(struct lruhash* table, struct testdata* ref[])
 
 /** test adding a random element */
 static void
-testlookup(struct lruhash* table, struct testdata* ref[])
+testlookup(struct lruhash* table, testdata_t* ref[])
 {
 	int num = random() % HASHTESTMAX;
-	struct testkey* key = newkey(num);
+	testkey_t* key = newkey(num);
 	struct lruhash_entry* en = lruhash_lookup(table, myhash(num), key, 0);
-	struct testdata* data = en? (struct testdata*)en->data : NULL;
+	testdata_t* data = en? (testdata_t*)en->data : NULL;
 	if(en) {
 		unit_assert(en->key);
 		unit_assert(en->data);
@@ -321,17 +304,17 @@ check_table(struct lruhash* table)
 
 	/* this assertion is specific to the unit test */
 	unit_assert( table->space_used == 
-		table->num * test_sizefunc(NULL, NULL) );
+		table->num * test_slabhash_sizefunc(NULL, NULL) );
 	lock_quick_unlock(&table->lock);
 }
 
 /** test adding a random element (unlimited range) */
 static void
-testadd_unlim(struct lruhash* table, struct testdata** ref)
+testadd_unlim(struct lruhash* table, testdata_t** ref)
 {
 	int numtoadd = random() % (HASHTESTMAX * 10);
-	struct testdata* data = newdata(numtoadd);
-	struct testkey* key = newkey(numtoadd);
+	testdata_t* data = newdata(numtoadd);
+	testkey_t* key = newkey(numtoadd);
 	key->entry.data = data;
 	lruhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
 	if(ref)
@@ -340,10 +323,10 @@ testadd_unlim(struct lruhash* table, struct testdata** ref)
 
 /** test adding a random element (unlimited range) */
 static void
-testremove_unlim(struct lruhash* table, struct testdata** ref)
+testremove_unlim(struct lruhash* table, testdata_t** ref)
 {
 	int num = random() % (HASHTESTMAX*10);
-	struct testkey* key = newkey(num);
+	testkey_t* key = newkey(num);
 	lruhash_remove(table, myhash(num), key);
 	if(ref)
 		ref[num] = NULL;
@@ -352,12 +335,12 @@ testremove_unlim(struct lruhash* table, struct testdata** ref)
 
 /** test adding a random element (unlimited range) */
 static void
-testlookup_unlim(struct lruhash* table, struct testdata** ref)
+testlookup_unlim(struct lruhash* table, testdata_t** ref)
 {
 	int num = random() % (HASHTESTMAX*10);
-	struct testkey* key = newkey(num);
+	testkey_t* key = newkey(num);
 	struct lruhash_entry* en = lruhash_lookup(table, myhash(num), key, 0);
-	struct testdata* data = en? (struct testdata*)en->data : NULL;
+	testdata_t* data = en? (testdata_t*)en->data : NULL;
 	if(en) {
 		unit_assert(en->key);
 		unit_assert(en->data);
@@ -377,17 +360,22 @@ static void
 test_long_table(struct lruhash* table) 
 {
 	/* assuming it all fits in the hastable, this check will work */
-	struct testdata* ref[HASHTESTMAX * 100];
+	testdata_t* ref[HASHTESTMAX * 100];
 	size_t i;
 	memset(ref, 0, sizeof(ref));
 	/* test assumption */
-	if(0) log_info(" size %d x %d < %d", (int)test_sizefunc(NULL, NULL), 
+	if(0) log_info(" size %d x %d < %d", (int)test_slabhash_sizefunc(NULL, NULL), 
 		(int)HASHTESTMAX, (int)table->space_max);
-	unit_assert( test_sizefunc(NULL, NULL)*HASHTESTMAX < table->space_max);
+	unit_assert( test_slabhash_sizefunc(NULL, NULL)*HASHTESTMAX < table->space_max);
 	if(0) lruhash_status(table, "unit test", 1);
 	srandom(48);
 	for(i=0; i<1000; i++) {
 		/* what to do? */
+		if(i == 500) {
+			lruhash_clear(table);
+			memset(ref, 0, sizeof(ref));
+			continue;
+		}
 		switch(random() % 4) {
 			case 0:
 			case 3:
@@ -469,7 +457,7 @@ test_thr_main(void* arg)
 	return NULL;
 }
 
-/** test hash table access by multiple threads. */
+/** test hash table access by multiple threads */
 static void
 test_threaded_table(struct lruhash* table)
 {
@@ -496,40 +484,16 @@ void lruhash_test()
 	struct lruhash* table ;
 	printf("lruhash test\n");
 	table = lruhash_create(2, 4096, 
-		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+		test_slabhash_sizefunc, test_slabhash_compfunc, 
+		test_slabhash_delkey, test_slabhash_deldata, NULL);
 	test_bin_find_entry(table);
 	test_lru(table);
 	test_short_table(table);
 	test_long_table(table);
 	lruhash_delete(table);
 	table = lruhash_create(2, 4096, 
-		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+		test_slabhash_sizefunc, test_slabhash_compfunc, 
+		test_slabhash_delkey, test_slabhash_deldata, NULL);
 	test_threaded_table(table);
 	lruhash_delete(table);
 }
-
-static size_t test_sizefunc(void* ATTR_UNUSED(key), void* ATTR_UNUSED(data))
-{
-	return sizeof(struct testkey) + sizeof(struct testdata);
-}
-
-static int test_compfunc(void* key1, void* key2)
-{
-	struct testkey* k1 = (struct testkey*)key1;
-	struct testkey* k2 = (struct testkey*)key2;
-	if(k1->id == k2->id)
-		return 0;
-	if(k1->id > k2->id)
-		return 1;
-	return -1;
-}
-
-static void test_delkey(void* key, void* ATTR_UNUSED(arg))
-{
-	delkey((struct testkey*)key);
-}
-
-static void test_deldata(void* data, void* ATTR_UNUSED(arg))
-{
-	deldata((struct testdata*)data);
-}

testcode/unitmain.c

@@ -119,6 +119,107 @@ net_test()
 	unit_assert( !is_pow2(259) );
 }
 
+#include "util/rtt.h"
+/** test RTT code */
+static void
+rtt_test()
+{
+	int init = 376;
+	int i;
+	struct rtt_info r;
+	rtt_init(&r);
+	/* initial value sensible */
+	unit_assert( rtt_timeout(&r) == init );
+	rtt_lost(&r);
+	unit_assert( rtt_timeout(&r) == init*2 );
+	rtt_lost(&r);
+	unit_assert( rtt_timeout(&r) == init*4 );
+	rtt_update(&r, 4000);
+	unit_assert( rtt_timeout(&r) >= 2000 );
+	rtt_lost(&r);
+	for(i=0; i<100; i++) {
+		rtt_lost(&r); 
+		unit_assert( rtt_timeout(&r) > RTT_MIN_TIMEOUT-1);
+		unit_assert( rtt_timeout(&r) < RTT_MAX_TIMEOUT+1);
+	}
+}
+
+#include "services/cache/infra.h"
+#include "util/config_file.h"
+/** test host cache */
+static void
+infra_test()
+{
+	int one = 1;
+	uint8_t* zone = (uint8_t*)"\007example\003com\000";
+	size_t zonelen = 13;
+	struct infra_cache* slab;
+	struct config_file* cfg = config_create();
+	time_t now = 0;
+	int vs, to;
+	struct infra_host_key* k;
+	struct infra_host_data* d;
+	int init = 376;
+
+	slab = infra_create(cfg);
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == init );
+
+	unit_assert( infra_rtt_update(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), -1, now) );
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == init*2 );
+
+	unit_assert( infra_edns_update(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), -1, now) );
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == -1 && to == init*2 );
+
+	now += cfg->host_ttl + 10;
+	unit_assert( infra_host(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), now, &vs, &to) );
+	unit_assert( vs == 0 && to == init );
+	
+	unit_assert( infra_set_lame(slab, (struct sockaddr_storage*)&one, 
+		(socklen_t)sizeof(int), zone, zonelen, now, 0) );
+	unit_assert( (d=infra_lookup_host(slab, (struct sockaddr_storage*)&one,
+		(socklen_t)sizeof(int), 0, now, &k)) );
+	unit_assert( d->ttl == now+cfg->host_ttl );
+	unit_assert( d->edns_version == 0 );
+	unit_assert( infra_lookup_lame(d, zone, zonelen, now)==1 );
+	unit_assert( !infra_lookup_lame(d, zone, zonelen, 
+		now+cfg->lame_ttl+10) );
+	unit_assert( !infra_lookup_lame(d, (uint8_t*)"\000", 1, now) );
+	lock_rw_unlock(&k->entry.lock);
+
+	infra_delete(slab);
+	config_delete(cfg);
+}
+
+#include "util/random.h"
+/** test randomness */
+static void
+rnd_test()
+{
+	struct ub_randstate r;
+	int num = 100, i;
+	long int a[100];
+	unit_assert( ub_initstate((unsigned)time(NULL), &r, 256) );
+	for(i=0; i<num; i++) {
+		a[i] = ub_random(&r);
+		unit_assert(a[i] >= 0);
+		unit_assert((size_t)a[i] <= (size_t)RAND_MAX);
+		if(i > 5)
+			unit_assert(a[i] != a[i-1] || a[i] != a[i-2] ||
+				a[i] != a[i-3] || a[i] != a[i-4] ||
+				a[i] != a[i-5] || a[i] != a[i-6]);
+	}
+	ub_randfree(&r);
+}
+
 /**
  * Main unit test program. Setup, teardown and report errors.
  * @param argc: arg count.
@@ -127,7 +228,7 @@ net_test()
 int 
 main(int argc, char* argv[])
 {
-	log_init(NULL);
+	log_init(NULL, 0, NULL);
 	if(argc != 1) {
 		printf("usage: %s\n", argv[0]);
 		printf("\tperforms unit tests.\n");
@@ -135,11 +236,16 @@ main(int argc, char* argv[])
 	}
 	printf("Start of %s unit test.\n", PACKAGE_STRING);
 	checklock_start();
+	rnd_test();
+	verify_test();
 	net_test();
 	dname_test();
+	anchors_test();
+	rtt_test();
 	alloc_test();
 	lruhash_test();
 	slabhash_test();
+	infra_test();
 	msgparse_test();
 	checklock_stop();
 	printf("%d checks ok.\n", testcount);

testcode/unitmain.h

@@ -55,5 +55,9 @@ void slabhash_test();
 void msgparse_test();
 /** unit test dname handling functions */
 void dname_test();
+/** unit test trust anchor storage functions */
+void anchors_test();
+/** unit test for verification functions */
+void verify_test();
 
 #endif /* TESTCODE_UNITMAIN_H */

testcode/unitmsgparse.c

@@ -43,13 +43,16 @@
 #include "testcode/unitmain.h"
 #include "util/data/msgparse.h"
 #include "util/data/msgreply.h"
+#include "util/data/msgencode.h"
 #include "util/alloc.h"
-#include "util/region-allocator.h"
+#include "util/regional.h"
 #include "util/net_help.h"
 #include "testcode/readhex.h"
 
 /** verbose message parse unit test */
 static int vbmp = 0;
+/** if matching within a section should disregard the order of RRs. */
+static int matches_nolocation = 0;
 
 /** match two rr lists */
 static int
@@ -64,18 +67,25 @@ match_list(ldns_rr_list* q, ldns_rr_list *p)
 	}
 	for(i=0; i<ldns_rr_list_rr_count(q); i++)
 	{
-		if(ldns_rr_compare(ldns_rr_list_rr(q, i),
-			ldns_rr_list_rr(p, i)) != 0) {
-			verbose(3, "rr %d different", i);
-			return 0;
+		if(matches_nolocation) {
+			if(!ldns_rr_list_contains_rr(p, ldns_rr_list_rr(q, i)))
+			{
+				verbose(3, "rr %u not found", (unsigned)i);
+				return 0;
+			}
+		} else {
+			if(ldns_rr_compare(ldns_rr_list_rr(q, i),
+				ldns_rr_list_rr(p, i)) != 0) {
+				verbose(3, "rr %u different", (unsigned)i);
+				return 0;
+			}
+			/* and check the ttl */
+			if(ldns_rr_ttl(ldns_rr_list_rr(q, i)) !=
+				ldns_rr_ttl(ldns_rr_list_rr(p, i))) {
+				verbose(3, "rr %u ttl different", (unsigned)i);
+				return 0;
+			}
 		}
-		/* and check the ttl */
-		if(ldns_rr_ttl(ldns_rr_list_rr(q, i)) !=
-			ldns_rr_ttl(ldns_rr_list_rr(p, i))) {
-			verbose(3, "rr %d ttl different", i);
-			return 0;
-		}
-
 	}
 	return 1;
 }
@@ -180,11 +190,11 @@ test_buffers(ldns_buffer* pkt, ldns_buffer* out)
 			if(lim-count < sz) rem = lim-count;
 			if(memcmp(ldns_buffer_at(pkt, count), 
 				ldns_buffer_at(out, count), rem) == 0) {
-				log_info("same %d %d", count, rem);
+				log_info("same %d %d", (int)count, (int)rem);
 				log_hex("same: ", ldns_buffer_at(pkt, count),
 					rem);
 			} else {
-				log_info("diff %d %d", count, rem);
+				log_info("diff %d %d", (int)count, (int)rem);
 				log_hex("difp: ", ldns_buffer_at(pkt, count),
 					rem);
 				log_hex("difo: ", ldns_buffer_at(out, count),
@@ -196,10 +206,8 @@ test_buffers(ldns_buffer* pkt, ldns_buffer* out)
 	s1 = ldns_buffer2pkt_wire(&p1, pkt);
 	s2 = ldns_buffer2pkt_wire(&p2, out);
 	if(vbmp) {
-		log_hex("orig in hex", ldns_buffer_begin(pkt),
-			ldns_buffer_limit(pkt));
-		log_hex("unbound out in hex", ldns_buffer_begin(out),
-			ldns_buffer_limit(out));
+		log_buf(0, "orig in hex", pkt);
+		log_buf(0, "unbound out in hex", out);
 		printf("\npacket from unbound (%d):\n", 
 			(int)ldns_buffer_limit(out));
 		ldns_pkt_print(stdout, p2);
@@ -223,7 +231,7 @@ test_buffers(ldns_buffer* pkt, ldns_buffer* out)
 	return 0;
 }
 
-/** check if unbound formerr equals ldns formerr. */
+/** check if unbound formerr equals ldns formerr */
 static void
 checkformerr(ldns_buffer* pkt)
 {
@@ -250,7 +258,7 @@ testpkt(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
 	uint16_t id;
 	uint16_t flags;
 	uint32_t timenow = 0;
-	region_type *region = region_create(malloc, free);
+	struct regional* region = regional_create();
 	struct edns_data edns;
 
 	hex_to_buf(pkt, hex);
@@ -269,7 +277,7 @@ testpkt(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
 	} else {
 		const size_t lim = 512;
 		ret = reply_info_encode(&qi, rep, id, flags, out, timenow,
-			region, 65535);
+			region, 65535, (int)(edns.bits & EDNS_DO) );
 		unit_assert(ret != 0); /* udp packets should fit */
 		attach_edns_record(out, &edns);
 		if(vbmp) printf("inlen %u outlen %u\n", 
@@ -280,7 +288,8 @@ testpkt(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
 		if(ldns_buffer_limit(out) > lim) {
 			ret = reply_info_encode(&qi, rep, id, flags, out, 
 				timenow, region, 
-				lim - calc_edns_field_size(&edns));
+				lim - calc_edns_field_size(&edns),
+				(int)(edns.bits & EDNS_DO));
 			unit_assert(ret != 0); /* should fit, but with TC */
 			attach_edns_record(out, &edns);
 			if( LDNS_QDCOUNT(ldns_buffer_begin(out)) !=
@@ -298,10 +307,10 @@ testpkt(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
 
 	query_info_clear(&qi);
 	reply_info_parsedelete(rep, alloc);
-	region_destroy(region);
+	regional_destroy(region);
 }
 
-/** simple test of parsing. */
+/** simple test of parsing */
 static void
 simpletest(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out)
 {
@@ -352,7 +361,7 @@ simpletest(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out)
 	" 64 b9 00 04 c0 3a 80 1e  ");
 }
 
-/** simple test of parsing, pcat file. */
+/** simple test of parsing, pcat file */
 static void
 testfromfile(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
 	const char* fname)
@@ -379,7 +388,7 @@ testfromfile(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
 	fclose(in);
 }
 
-/** simple test of parsing, drill file. */
+/** simple test of parsing, drill file */
 static void
 testfromdrillfile(ldns_buffer* pkt, struct alloc_cache* alloc, 
 	ldns_buffer* out, const char* fname)
@@ -431,6 +440,10 @@ void msgparse_test()
 	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.4");
 	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.5");
 
+	matches_nolocation = 1; /* RR order not important for the next test */
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.6");
+	matches_nolocation = 0; 
+
 	/* cleanup */
 	alloc_clear(&alloc);
 	alloc_clear(&super_a);

testcode/unitslabhash.c

@@ -43,40 +43,25 @@
 #include "util/log.h"
 #include "util/storage/slabhash.h"
 
-/* --- test representation --- */
-/** structure contains test key */
-struct slabtestkey {
-	/** the key id */
-	int id;
-	/** the entry */
-	struct lruhash_entry entry;
-};
-/** structure contains test data */
-struct slabtestdata {
-	/** data value */
-	int data;
-};
+/** use this type for the slabhash test key */
+typedef struct slabhash_testkey testkey_t;
+/** use this type for the slabhash test data */
+typedef struct slabhash_testdata testdata_t;
 
-/** sizefunc for lruhash */
-static size_t test_sizefunc(void*, void*);
-/** comparefunc for lruhash */
-static int test_compfunc(void*, void*);
-/** delkey for lruhash */
-static void test_delkey(void*, void*);
-/** deldata for lruhash */
-static void test_deldata(void*, void*);
-/* --- end test representation --- */
+/** delete key */
+static void delkey(struct slabhash_testkey* k) {
+	lock_rw_destroy(&k->entry.lock); free(k);}
 
-/** hash func, very bad to improve collisions, both high and low bits. */
+/** hash func, very bad to improve collisions, both high and low bits */
 static hashvalue_t myhash(int id) {
 	hashvalue_t h = (hashvalue_t)id & 0x0f;
 	h |= (h << 28);
 	return h;
 }
 
-/** allocate new key, fill in hash. */
-static struct slabtestkey* newkey(int id) {
-	struct slabtestkey* k = (struct slabtestkey*)calloc(1, sizeof(struct slabtestkey));
+/** allocate new key, fill in hash */
+static testkey_t* newkey(int id) {
+	testkey_t* k = (testkey_t*)calloc(1, sizeof(testkey_t));
 	if(!k) fatal_exit("out of memory");
 	k->id = id;
 	k->entry.hash = myhash(id);
@@ -85,27 +70,22 @@ static struct slabtestkey* newkey(int id) {
 	return k;
 }
 /** new data el */
-static struct slabtestdata* newdata(int val) {
-	struct slabtestdata* d = (struct slabtestdata*)calloc(1, 
-		sizeof(struct slabtestdata));
+static testdata_t* newdata(int val) {
+	testdata_t* d = (testdata_t*)calloc(1, 
+		sizeof(testdata_t));
 	if(!d) fatal_exit("out of memory");
 	d->data = val;
 	return d;
 }
-/** delete key */
-static void delkey(struct slabtestkey* k) {	
-	lock_rw_destroy(&k->entry.lock); free(k);}
-/** delete data */
-static void deldata(struct slabtestdata* d) {free(d);}
 
 /** test hashtable using short sequence */
 static void
 test_short_table(struct slabhash* table) 
 {
-	struct slabtestkey* k = newkey(12);
-	struct slabtestkey* k2 = newkey(14);
-	struct slabtestdata* d = newdata(128);
-	struct slabtestdata* d2 = newdata(129);
+	testkey_t* k = newkey(12);
+	testkey_t* k2 = newkey(14);
+	testdata_t* d = newdata(128);
+	testdata_t* d2 = newdata(129);
 	
 	k->entry.data = d;
 	k2->entry.data = d2;
@@ -126,11 +106,11 @@ test_short_table(struct slabhash* table)
 
 /** test adding a random element */
 static void
-testadd(struct slabhash* table, struct slabtestdata* ref[])
+testadd(struct slabhash* table, testdata_t* ref[])
 {
 	int numtoadd = random() % HASHTESTMAX;
-	struct slabtestdata* data = newdata(numtoadd);
-	struct slabtestkey* key = newkey(numtoadd);
+	testdata_t* data = newdata(numtoadd);
+	testkey_t* key = newkey(numtoadd);
 	key->entry.data = data;
 	slabhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
 	ref[numtoadd] = data;
@@ -138,10 +118,10 @@ testadd(struct slabhash* table, struct slabtestdata* ref[])
 
 /** test adding a random element */
 static void
-testremove(struct slabhash* table, struct slabtestdata* ref[])
+testremove(struct slabhash* table, testdata_t* ref[])
 {
 	int num = random() % HASHTESTMAX;
-	struct slabtestkey* key = newkey(num);
+	testkey_t* key = newkey(num);
 	slabhash_remove(table, myhash(num), key);
 	ref[num] = NULL;
 	delkey(key);
@@ -149,12 +129,12 @@ testremove(struct slabhash* table, struct slabtestdata* ref[])
 
 /** test adding a random element */
 static void
-testlookup(struct slabhash* table, struct slabtestdata* ref[])
+testlookup(struct slabhash* table, testdata_t* ref[])
 {
 	int num = random() % HASHTESTMAX;
-	struct slabtestkey* key = newkey(num);
+	testkey_t* key = newkey(num);
 	struct lruhash_entry* en = slabhash_lookup(table, myhash(num), key, 0);
-	struct slabtestdata* data = en? (struct slabtestdata*)en->data : NULL;
+	testdata_t* data = en? (testdata_t*)en->data : NULL;
 	if(en) {
 		unit_assert(en->key);
 		unit_assert(en->data);
@@ -198,7 +178,7 @@ check_lru_table(struct lruhash* table)
 
 	/* this assertion is specific to the unit test */
 	unit_assert( table->space_used == 
-		table->num * test_sizefunc(NULL, NULL) );
+		table->num * test_slabhash_sizefunc(NULL, NULL) );
 	lock_quick_unlock(&table->lock);
 }
 
@@ -213,11 +193,11 @@ check_table(struct slabhash* table)
 
 /** test adding a random element (unlimited range) */
 static void
-testadd_unlim(struct slabhash* table, struct slabtestdata** ref)
+testadd_unlim(struct slabhash* table, testdata_t** ref)
 {
 	int numtoadd = random() % (HASHTESTMAX * 10);
-	struct slabtestdata* data = newdata(numtoadd);
-	struct slabtestkey* key = newkey(numtoadd);
+	testdata_t* data = newdata(numtoadd);
+	testkey_t* key = newkey(numtoadd);
 	key->entry.data = data;
 	slabhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
 	if(ref)
@@ -226,10 +206,10 @@ testadd_unlim(struct slabhash* table, struct slabtestdata** ref)
 
 /** test adding a random element (unlimited range) */
 static void
-testremove_unlim(struct slabhash* table, struct slabtestdata** ref)
+testremove_unlim(struct slabhash* table, testdata_t** ref)
 {
 	int num = random() % (HASHTESTMAX*10);
-	struct slabtestkey* key = newkey(num);
+	testkey_t* key = newkey(num);
 	slabhash_remove(table, myhash(num), key);
 	if(ref)
 		ref[num] = NULL;
@@ -238,12 +218,12 @@ testremove_unlim(struct slabhash* table, struct slabtestdata** ref)
 
 /** test adding a random element (unlimited range) */
 static void
-testlookup_unlim(struct slabhash* table, struct slabtestdata** ref)
+testlookup_unlim(struct slabhash* table, testdata_t** ref)
 {
 	int num = random() % (HASHTESTMAX*10);
-	struct slabtestkey* key = newkey(num);
+	testkey_t* key = newkey(num);
 	struct lruhash_entry* en = slabhash_lookup(table, myhash(num), key, 0);
-	struct slabtestdata* data = en? (struct slabtestdata*)en->data : NULL;
+	testdata_t* data = en? (testdata_t*)en->data : NULL;
 	if(en) {
 		unit_assert(en->key);
 		unit_assert(en->data);
@@ -263,7 +243,7 @@ static void
 test_long_table(struct slabhash* table) 
 {
 	/* assuming it all fits in the hastable, this check will work */
-	struct slabtestdata* ref[HASHTESTMAX * 100];
+	testdata_t* ref[HASHTESTMAX * 100];
 	size_t i;
 	memset(ref, 0, sizeof(ref));
 	/* test assumption */
@@ -271,6 +251,11 @@ test_long_table(struct slabhash* table)
 	srandom(48);
 	for(i=0; i<1000; i++) {
 		/* what to do? */
+		if(i == 500) {
+			slabhash_clear(table);
+			memset(ref, 0, sizeof(ref));
+			continue;
+		}
 		switch(random() % 4) {
 			case 0:
 			case 3:
@@ -351,7 +336,7 @@ test_thr_main(void* arg)
 	return NULL;
 }
 
-/** test hash table access by multiple threads. */
+/** test hash table access by multiple threads */
 static void
 test_threaded_table(struct slabhash* table)
 {
@@ -378,38 +363,14 @@ void slabhash_test()
 	struct slabhash* table;
 	printf("slabhash test\n");
 	table = slabhash_create(4, 2, 5200, 
-		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+		test_slabhash_sizefunc, test_slabhash_compfunc, 
+		test_slabhash_delkey, test_slabhash_deldata, NULL);
 	test_short_table(table);
 	test_long_table(table);
 	slabhash_delete(table);
 	table = slabhash_create(4, 2, 5200, 
-		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+		test_slabhash_sizefunc, test_slabhash_compfunc, 
+		test_slabhash_delkey, test_slabhash_deldata, NULL);
 	test_threaded_table(table);
 	slabhash_delete(table);
 }
-
-static size_t test_sizefunc(void* ATTR_UNUSED(key), void* ATTR_UNUSED(data))
-{
-	return sizeof(struct slabtestkey) + sizeof(struct slabtestdata);
-}
-
-static int test_compfunc(void* key1, void* key2)
-{
-	struct slabtestkey* k1 = (struct slabtestkey*)key1;
-	struct slabtestkey* k2 = (struct slabtestkey*)key2;
-	if(k1->id == k2->id)
-		return 0;
-	if(k1->id > k2->id)
-		return 1;
-	return -1;
-}
-
-static void test_delkey(void* key, void* ATTR_UNUSED(arg))
-{
-	delkey((struct slabtestkey*)key);
-}
-
-static void test_deldata(void* data, void* ATTR_UNUSED(arg))
-{
-	deldata((struct slabtestdata*)data);
-}

testcode/unitverify.c

@@ -0,0 +1,465 @@
+/*
+ * testcode/unitverify.c - unit test for signature verification routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls verification unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "validator/val_sigcrypt.h"
+#include "validator/val_nsec.h"
+#include "validator/val_nsec3.h"
+#include "validator/validator.h"
+#include "testcode/ldns-testpkts.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgparse.h"
+#include "util/data/dname.h"
+#include "util/regional.h"
+#include "util/alloc.h"
+#include "util/rbtree.h"
+#include "util/net_help.h"
+#include "util/module.h"
+#include "util/config_file.h"
+
+/** verbose signature test */
+static int vsig = 0;
+
+/** entry to packet buffer with wireformat */
+static void
+entry_to_buf(struct entry* e, ldns_buffer* pkt)
+{
+	unit_assert(e->reply_list);
+	if(e->reply_list->reply_from_hex) {
+		ldns_buffer_copy(pkt, e->reply_list->reply_from_hex);
+	} else {
+		ldns_status status;
+		size_t answer_size;
+		uint8_t* ans = NULL;
+		status = ldns_pkt2wire(&ans, e->reply_list->reply, 
+			&answer_size);
+		if(status != LDNS_STATUS_OK) {
+			log_err("could not create reply: %s",
+				ldns_get_errorstr_by_id(status));
+			fatal_exit("error in test");
+		}
+		ldns_buffer_clear(pkt);
+		ldns_buffer_write(pkt, ans, answer_size);
+		ldns_buffer_flip(pkt);
+		free(ans);
+	}
+}
+
+/** entry to reply info conversion */
+static void
+entry_to_repinfo(struct entry* e, struct alloc_cache* alloc, 
+	struct regional* region, ldns_buffer* pkt, struct query_info* qi, 
+	struct reply_info** rep)
+{
+	int ret;
+	struct edns_data edns;
+	entry_to_buf(e, pkt);
+	/* lock alloc lock to please lock checking software. 
+	 * alloc_special_obtain assumes it is talking to a ub-alloc,
+	 * and does not need to perform locking. Here the alloc is
+	 * the only one, so we lock it here */
+	lock_quick_lock(&alloc->lock);
+	ret = reply_info_parse(pkt, alloc, qi, rep, region, &edns);
+	lock_quick_unlock(&alloc->lock);
+	if(ret != 0) {
+		printf("parse code %d: %s\n", ret,
+			ldns_lookup_by_id(ldns_rcodes, ret)->name);
+		unit_assert(ret != 0);
+	}
+}
+
+/** extract DNSKEY rrset from answer and convert it */
+static struct ub_packed_rrset_key* 
+extract_keys(struct entry* e, struct alloc_cache* alloc, 
+	struct regional* region, ldns_buffer* pkt)
+{
+	struct ub_packed_rrset_key* dnskey = NULL;
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	size_t i;
+
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+	for(i=0; i<rep->an_numrrsets; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == LDNS_RR_TYPE_DNSKEY) {
+			dnskey = rep->rrsets[i];
+			rep->rrsets[i] = NULL;
+			break;
+		}
+	}
+	unit_assert(dnskey);
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+	return dnskey;
+}
+
+/** return true if answer should be bogus */
+static int
+should_be_bogus(struct ub_packed_rrset_key* rrset)
+{
+	struct packed_rrset_data* d = (struct packed_rrset_data*)rrset->
+		entry.data;
+	if(d->rrsig_count == 0)
+		return 1;
+	return 0;
+}
+
+/** verify and test one rrset against the key rrset */
+static void
+verifytest_rrset(struct module_env* env, struct val_env* ve, 
+	struct ub_packed_rrset_key* rrset, struct ub_packed_rrset_key* dnskey)
+{
+	enum sec_status sec;
+	if(vsig) {
+		log_nametypeclass(VERB_DETAIL, "verify of rrset",
+			rrset->rk.dname, ntohs(rrset->rk.type),
+			ntohs(rrset->rk.rrset_class));
+	}
+	sec = dnskeyset_verify_rrset(env, ve, rrset, dnskey);
+	if(vsig) {
+		printf("verify outcome is: %s\n", sec_status_to_string(sec));
+	}
+	if(should_be_bogus(rrset)) {
+		unit_assert(sec == sec_status_bogus);
+	} else {
+		unit_assert(sec == sec_status_secure);
+	}
+}
+
+/** verify and test an entry - every rr in the message */
+static void
+verifytest_entry(struct entry* e, struct alloc_cache* alloc, 
+	struct regional* region, ldns_buffer* pkt, 
+	struct ub_packed_rrset_key* dnskey, struct module_env* env, 
+	struct val_env* ve)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	size_t i;
+
+	regional_free_all(region);
+	if(vsig) {
+		printf("verifying pkt:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+
+	for(i=0; i<rep->rrset_count; i++) {
+		verifytest_rrset(env, ve, rep->rrsets[i], dnskey);
+	}
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+/** find RRset in reply by type */
+static struct ub_packed_rrset_key*
+find_rrset_type(struct reply_info* rep, uint16_t type)
+{
+	size_t i;
+	for(i=0; i<rep->rrset_count; i++) {
+		if(ntohs(rep->rrsets[i]->rk.type) == type)
+			return rep->rrsets[i];
+	}
+	return NULL;
+}
+
+/** DS sig test an entry - get DNSKEY and DS in entry and verify */
+static void
+dstest_entry(struct entry* e, struct alloc_cache* alloc, 
+	struct regional* region, ldns_buffer* pkt, struct module_env* env)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	struct ub_packed_rrset_key* ds, *dnskey;
+	int ret;
+
+	regional_free_all(region);
+	if(vsig) {
+		printf("verifying DS-DNSKEY match:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, pkt, &qinfo, &rep);
+	ds = find_rrset_type(rep, LDNS_RR_TYPE_DS);
+	dnskey = find_rrset_type(rep, LDNS_RR_TYPE_DNSKEY);
+	/* check test is OK */
+	unit_assert(ds && dnskey);
+
+	ret = ds_digest_match_dnskey(env, dnskey, 0, ds, 0);
+	if(strncmp((char*)qinfo.qname, "\003yes", 4) == 0) {
+		if(vsig) {
+			printf("result(yes)= %s\n", ret?"yes":"no");
+		}
+		unit_assert(ret);
+	} else if (strncmp((char*)qinfo.qname, "\002no", 3) == 0) {
+		if(vsig) {
+			printf("result(no)= %s\n", ret?"yes":"no");
+		}
+		unit_assert(!ret);
+		verbose(VERB_DETAIL, "DS fail: OK; matched unit test");
+	} else {
+		fatal_exit("Bad qname in DS unit test, yes or no");
+	}
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+/** verify from a file */
+static void
+verifytest_file(const char* fname, const char* at_date)
+{
+	/* 
+	 * The file contains a list of ldns-testpkts entries.
+	 * The first entry must be a query for DNSKEY.
+	 * The answer rrset is the keyset that will be used for verification
+	 */
+	struct ub_packed_rrset_key* dnskey;
+	struct regional* region = regional_create();
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+	struct module_env env;
+	struct val_env ve;
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	alloc_init(&alloc, NULL, 1);
+	memset(&env, 0, sizeof(env));
+	memset(&ve, 0, sizeof(ve));
+	env.scratch = region;
+	env.scratch_buffer = buf;
+	ve.date_override = cfg_convert_timeval(at_date);
+	unit_assert(region && buf);
+	dnskey = extract_keys(list, &alloc, region, buf);
+	if(vsig) log_nametypeclass(VERB_DETAIL, "test dnskey",
+			dnskey->rk.dname, ntohs(dnskey->rk.type), 
+			ntohs(dnskey->rk.rrset_class));
+	/* ready to go! */
+	for(e = list->next; e; e = e->next) {
+		verifytest_entry(e, &alloc, region, buf, dnskey, &env, &ve);
+	}
+
+	ub_packed_rrset_parsedelete(dnskey, &alloc);
+	delete_entry(list);
+	regional_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+/** verify DS matches DNSKEY from a file */
+static void
+dstest_file(const char* fname)
+{
+	/* 
+	 * The file contains a list of ldns-testpkts entries.
+	 * The first entry must be a query for DNSKEY.
+	 * The answer rrset is the keyset that will be used for verification
+	 */
+	struct regional* region = regional_create();
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+	struct module_env env;
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	alloc_init(&alloc, NULL, 1);
+	memset(&env, 0, sizeof(env));
+	env.scratch = region;
+	env.scratch_buffer = buf;
+	unit_assert(region && buf);
+
+	/* ready to go! */
+	for(e = list; e; e = e->next) {
+		dstest_entry(e, &alloc, region, buf, &env);
+	}
+
+	delete_entry(list);
+	regional_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+/** helper for unittest of NSEC routines */
+static int
+unitest_nsec_has_type_rdata(char* bitmap, size_t len, uint16_t type)
+{
+	return nsecbitmap_has_type_rdata((uint8_t*)bitmap, len, type);
+}
+
+/** Test NSEC type bitmap routine */
+static void
+nsectest()
+{
+	/* bitmap starts at type bitmap rdata field */
+	/* from rfc 4034 example */
+	char* bitmap = "\000\006\100\001\000\000\000\003"
+		"\004\033\000\000\000\000\000\000"
+		"\000\000\000\000\000\000\000\000"
+		"\000\000\000\000\000\000\000\000"
+		"\000\000\000\000\040";
+	size_t len = 37;
+
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 0));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_A));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 2));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 3));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 4));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 5));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 6));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 7));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 8));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 9));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 10));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 11));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 12));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 13));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 14));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_MX));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_RRSIG));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, LDNS_RR_TYPE_NSEC));
+	unit_assert(unitest_nsec_has_type_rdata(bitmap, len, 1234));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1233));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1235));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1236));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1237));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1238));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1239));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 1240));
+	unit_assert(!unitest_nsec_has_type_rdata(bitmap, len, 2230));
+}
+
+/** Test hash algo - NSEC3 hash it and compare result */
+static void
+nsec3_hash_test_entry(struct entry* e, rbtree_t* ct,
+	struct alloc_cache* alloc, struct regional* region, 
+	ldns_buffer* buf)
+{
+	struct query_info qinfo;
+	struct reply_info* rep = NULL;
+	struct ub_packed_rrset_key* answer, *nsec3;
+	struct nsec3_cached_hash* hash;
+	int ret;
+	uint8_t* qname;
+
+	if(vsig) {
+		printf("verifying NSEC3 hash:\n");
+		ldns_pkt_print(stdout, e->reply_list->reply);
+		printf("\n");
+	}
+	entry_to_repinfo(e, alloc, region, buf, &qinfo, &rep);
+	nsec3 = find_rrset_type(rep, LDNS_RR_TYPE_NSEC3);
+	answer = find_rrset_type(rep, LDNS_RR_TYPE_AAAA);
+	qname = regional_alloc_init(region, qinfo.qname, qinfo.qname_len);
+	/* check test is OK */
+	unit_assert(nsec3 && answer && qname);
+
+	ret = nsec3_hash_name(ct, region, buf, nsec3, 0, qname,
+		qinfo.qname_len, &hash);
+	if(ret != 1) {
+		printf("Bad nsec3_hash_name retcode %d\n", ret);
+		unit_assert(ret == 1);
+	}
+	unit_assert(hash->dname && hash->hash && hash->hash_len &&
+		hash->b32 && hash->b32_len);
+	unit_assert(hash->b32_len == (size_t)answer->rk.dname[0]);
+	/* does not do lowercasing. */
+	unit_assert(memcmp(hash->b32, answer->rk.dname+1, hash->b32_len) 
+		== 0);
+
+	reply_info_parsedelete(rep, alloc);
+	query_info_clear(&qinfo);
+}
+
+
+/** Read file to test NSEC3 hash algo */
+static void
+nsec3_hash_test(const char* fname)
+{
+	/* 
+	 * The list contains a list of ldns-testpkts entries.
+	 * Every entry is a test.
+	 * 	The qname is hashed.
+	 * 	The answer section AAAA RR name is the required result.
+	 * 	The auth section NSEC3 is used to get hash parameters.
+	 * The hash cache is maintained per file.
+	 *
+	 * The test does not perform canonicalization during the compare.
+	 */
+	rbtree_t ct;
+	struct regional* region = regional_create();
+	struct alloc_cache alloc;
+	ldns_buffer* buf = ldns_buffer_new(65535);
+	struct entry* e;
+	struct entry* list = read_datafile(fname);
+
+	if(!list)
+		fatal_exit("could not read %s: %s", fname, strerror(errno));
+	rbtree_init(&ct, &nsec3_hash_cmp);
+	alloc_init(&alloc, NULL, 1);
+	unit_assert(region && buf);
+
+	/* ready to go! */
+	for(e = list; e; e = e->next) {
+		nsec3_hash_test_entry(e, &ct, &alloc, region, buf);
+	}
+
+	delete_entry(list);
+	regional_destroy(region);
+	alloc_clear(&alloc);
+	ldns_buffer_free(buf);
+}
+
+void 
+verify_test()
+{
+	printf("verify test\n");
+	verifytest_file("testdata/test_signatures.1", "20070818005004");
+	dstest_file("testdata/test_ds_sig.1");
+	nsectest();
+	nsec3_hash_test("testdata/test_nsec3_hash.1");
+}

testdata/Kexample.com.+003+02854.ds

@@ -0,0 +1 @@
+example.com.	3600	IN	DS	2854 3 1 46e4ffc6e9a4793b488954bd3f0cc6af0dfb201b

testdata/Kexample.com.+003+02854.key

@@ -0,0 +1 @@
+example.com.	3600	IN	DNSKEY	256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}

testdata/Kexample.com.+003+02854.private

@@ -0,0 +1,7 @@
+Private-key-format: v1.2
+Algorithm: 3 (DSA)
+Prime(p): +sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKQ==
+Subprime(q): tctSxapStjclgRHdPhMEkgizvSM=
+Base(g): LGE7N8tUTWCoDQ/B5lHx21jdo1BJGfew+nAmvoyL8+pAoAwytn1yQml/X5tAw46/GDPPZTUZLXxvfJkpyBMwRg==
+Private_value(x): XMs4XYi1oNckzTPvGMkgG5IiuzY=
+Public_value(y): ic3fxmWM4vwTdg80NDLK0sGF37DSxTgW7PDyTFuC2CMXnjnuq1IJFjhewgvQ4f3XbkNUK7CYZmQBBX3egcmFfg==

testdata/Kexample.com.+005+30899.ds

@@ -0,0 +1 @@
+example.com.	3600	IN	DS	30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512

testdata/Kexample.com.+005+30899.key

@@ -0,0 +1 @@
+example.com.	3600	IN	DNSKEY	256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b}

testdata/Kexample.com.+005+30899.private

@@ -0,0 +1,10 @@
+Private-key-format: v1.2
+Algorithm: 5 (RSASHA1)
+Modulus: 0ONXIUfQxB7f2iMyBQKmp2w5UX5SaEbOcs9YxbKESE+3Vn9K/j3g7nsHcZLMzR+sJ1OEC/KXyhUYeTyGFQSytw==
+PublicExponent: Aw==
+PrivateExponent: i0I6Fi/ggr8/5sIhWKxvGkgmNlQ28C80TIo7LncC2t6ar2Q5rpyiDxEHvFLfphRh108ZOqf2tQdHx7tXTx5Gqw==
+Prime1: 9WS85Q92ilTAuGiVi+KesKzrFqF98l2Gpu4003hfmbc=
+Prime2: 2eqsD2jcY4Mgw26A8XFiaLdxx5J4s10Dhd9ur6X3rwE=
+Exponent1: o5h97gpPBuMrJZsOXUG/IHNHZGupTD5ZxJ7N4lA/u88=
+Exponent2: kUcdX5s9l6zAgkmrS6DsRc+hL7b7Ij4CWT+fH8P6dKs=
+Coefficient: PVZrElFmz9tWa4kwu9jArjcocycYu0eBycgguQ03J7w=

testdata/fetch_glue.rpl

@@ -0,0 +1,149 @@
+; config options
+stub-zone:
+	name: "."
+	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
+CONFIG_END
+
+SCENARIO_BEGIN Fetch glue when missing or timed out.
+
+; K.ROOT-SERVERS.NET.
+RANGE_BEGIN 0 100
+	ADDRESS 193.0.14.129 
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+. IN NS
+SECTION ANSWER
+. IN NS	K.ROOT-SERVERS.NET.
+SECTION ADDITIONAL
+K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION AUTHORITY
+com.	IN NS	a.gtld-servers.net.
+SECTION ADDITIONAL
+a.gtld-servers.net.	IN 	A	192.5.6.30
+ENTRY_END
+RANGE_END
+
+; a.gtld-servers.net.
+RANGE_BEGIN 0 100
+	ADDRESS 192.5.6.30
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+; is missing the glue!
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. 	IN 	A
+SECTION ANSWER
+ns.example.com.		IN 	A	1.2.3.4
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. 	IN 	AAAA
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+RANGE_END
+
+; ns.example.com.
+RANGE_BEGIN 0 100
+	ADDRESS 1.2.3.4
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com.		IN 	A	1.2.3.4
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. IN A
+SECTION ANSWER
+ns.example.com. IN A	1.2.3.4
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. IN AAAA
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com. IN A	1.2.3.4
+ENTRY_END
+
+RANGE_END
+
+STEP 1 QUERY
+ENTRY_BEGIN
+REPLY RD
+SECTION QUESTION
+www.example.com. IN A
+ENTRY_END
+
+; recursion happens here.
+STEP 10 CHECK_ANSWER
+ENTRY_BEGIN
+MATCH all
+REPLY QR RD RA NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com.		IN 	A	1.2.3.4
+ENTRY_END
+
+; due to ordering of answer packets, this is still outstanding, remove it
+STEP 21 CHECK_OUT_QUERY
+ENTRY_BEGIN
+ADJUST copy_id
+MATCH qname qtype
+REPLY QR
+SECTION QUESTION
+ns.example.com IN AAAA
+ENTRY_END
+
+SCENARIO_END

testdata/fetch_glue_cname.rpl

@@ -0,0 +1,161 @@
+; config options
+stub-zone:
+	name: "."
+	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
+CONFIG_END
+
+SCENARIO_BEGIN Fetch cnamed glue when missing or timed out.
+
+; K.ROOT-SERVERS.NET.
+RANGE_BEGIN 0 100
+	ADDRESS 193.0.14.129 
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+. IN NS
+SECTION ANSWER
+. IN NS	K.ROOT-SERVERS.NET.
+SECTION ADDITIONAL
+K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION AUTHORITY
+com.	IN NS	a.gtld-servers.net.
+SECTION ADDITIONAL
+a.gtld-servers.net.	IN 	A	192.5.6.30
+ENTRY_END
+RANGE_END
+
+; a.gtld-servers.net.
+RANGE_BEGIN 0 100
+	ADDRESS 192.5.6.30
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+; is missing the glue!
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY AA QR NOERROR
+SECTION QUESTION
+ns.example.com. 	IN 	A
+SECTION ANSWER
+ns.example.com.		IN	CNAME	ns2.example.com.
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY AA QR NOERROR
+SECTION QUESTION
+ns2.example.com. 	IN 	A
+SECTION ANSWER
+ns2.example.com.	IN 	A	1.2.3.4
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. 	IN 	AAAA
+SECTION AUTHORITY
+example.com.		IN 	NS	ns.example.com.
+ENTRY_END
+RANGE_END
+
+; ns.example.com.
+RANGE_BEGIN 0 100
+	ADDRESS 1.2.3.4
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com.		IN 	A	1.2.3.4
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. IN A
+SECTION ANSWER
+ns.example.com. IN A	1.2.3.4
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+ENTRY_END
+
+ENTRY_BEGIN
+MATCH opcode qtype qname
+ADJUST copy_id
+REPLY QR NOERROR
+SECTION QUESTION
+ns.example.com. IN AAAA
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com. IN A	1.2.3.4
+ENTRY_END
+
+RANGE_END
+
+STEP 1 QUERY
+ENTRY_BEGIN
+REPLY RD
+SECTION QUESTION
+www.example.com. IN A
+ENTRY_END
+
+; recursion happens here.
+STEP 10 CHECK_ANSWER
+ENTRY_BEGIN
+MATCH all
+REPLY QR RD RA NOERROR
+SECTION QUESTION
+www.example.com. IN A
+SECTION ANSWER
+www.example.com. IN A	10.20.30.40
+SECTION AUTHORITY
+example.com.	IN NS	ns.example.com.
+SECTION ADDITIONAL
+ns.example.com.		IN 	A	1.2.3.4
+ENTRY_END
+
+; due to ordering of answer packets, this is still outstanding, remove it
+STEP 21 CHECK_OUT_QUERY
+ENTRY_BEGIN
+ADJUST copy_id
+MATCH qname qtype
+REPLY QR
+SECTION QUESTION
+ns.example.com IN AAAA
+ENTRY_END
+
+SCENARIO_END

testdata/fwd.rpl

@@ -1,5 +1,6 @@
 ; This is a comment.
 ; config options go here.
+forward-zone: name: "." forward-addr: 216.0.0.1
 CONFIG_END
 
 SCENARIO_BEGIN Sample of a valid query

testdata/fwd_cached.rpl

@@ -1,5 +1,6 @@
 ; This is a comment.
 ; config options go here.
+forward-zone: name: "." forward-addr: 216.0.0.1
 CONFIG_END
 
 SCENARIO_BEGIN Query receives answer from the cache
@@ -35,8 +36,7 @@ ENTRY_END
 STEP 4 CHECK_ANSWER
 ENTRY_BEGIN
 	MATCH all 
-	; first reply, have AA set.
-	REPLY QR AA RD RA
+	REPLY QR RD RA
 	SECTION QUESTION
 	www.example.com. IN A
 	SECTION ANSWER