Test uses less file descriptors.

git-svn-id: file:///svn/unbound/trunk@303 be551aaa-1e26-0410-a405-d3ace91eadb9

Makefile.in

@@ -44,22 +44,26 @@ endif
 BUILD=build/
 
 LINT=splint
-LINTFLAGS=+quiet -weak -warnposix -unrecog -Din_addr_t=uint32_t -Du_int=unsigned -Du_char=uint8_t -preproc
+LINTFLAGS=+quiet -weak -warnposix -unrecog -Din_addr_t=uint32_t -Du_int=unsigned -Du_char=uint8_t -preproc -Drlimit=rlimit64
 # compat with openssl linux edition.
 LINTFLAGS+="-DBN_ULONG=unsigned long" -Dkrb5_int32=int "-Dkrb5_ui_4=unsigned int" -DPQ_64BIT=uint64_t
 
 INSTALL=$(srcdir)/install-sh 
 
-COMMON_SRC=$(wildcard services/*.c util/*.c) util/configparser.c util/configlexer.c testcode/checklocks.c
+COMMON_SRC=$(wildcard services/*.c util/*.c util/data/*.c util/storage/*.c) util/configparser.c util/configlexer.c testcode/checklocks.c
 COMMON_OBJ=$(addprefix $(BUILD),$(COMMON_SRC:.c=.o))
 COMPAT_OBJ=$(addprefix $(BUILD)compat/,$(LIBOBJS))
-UNITTEST_SRC=testcode/unitmain.c $(COMMON_SRC)
+UNITTEST_SRC=$(wildcard testcode/unit*.c) testcode/readhex.c $(COMMON_SRC)
 UNITTEST_OBJ=$(addprefix $(BUILD),$(UNITTEST_SRC:.c=.o)) $(COMPAT_OBJ)
 DAEMON_SRC=$(wildcard daemon/*.c) $(COMMON_SRC)
 DAEMON_OBJ=$(addprefix $(BUILD),$(DAEMON_SRC:.c=.o)) $(COMPAT_OBJ)
-TESTBOUND_SRC=testcode/testbound.c testcode/ldns-testpkts.c daemon/worker.c daemon/daemon.c testcode/replay.c testcode/fake_event.c $(filter-out util/netevent.c services/listen_dnsport.c services/outside_network.c, $(COMMON_SRC))
+TESTBOUND_SRC=testcode/testbound.c testcode/ldns-testpkts.c daemon/worker.c daemon/daemon.c daemon/stats.c testcode/replay.c testcode/fake_event.c $(filter-out util/netevent.c services/listen_dnsport.c services/outside_network.c, $(COMMON_SRC))
 TESTBOUND_OBJ=$(addprefix $(BUILD),$(TESTBOUND_SRC:.c=.o)) $(COMPAT_OBJ)
-ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) $(TESTBOUND_SRC)
+LOCKVERIFY_SRC=testcode/lock_verify.c $(COMMON_SRC)
+LOCKVERIFY_OBJ=$(addprefix $(BUILD),$(LOCKVERIFY_SRC:.c=.o)) $(COMPAT_OBJ)
+PKTVIEW_SRC=testcode/pktview.c testcode/readhex.c $(COMMON_SRC)
+PKTVIEW_OBJ=$(addprefix $(BUILD),$(PKTVIEW_SRC:.c=.o)) $(COMPAT_OBJ)
+ALL_SRC=$(COMMON_SRC) $(UNITTEST_SRC) $(DAEMON_SRC) $(TESTBOUND_SRC) $(LOCKVERIFY_SRC)
 ALL_OBJ=$(addprefix $(BUILD),$(ALL_SRC:.c=.o) $(addprefix compat/,$(LIBOBJS))) $(COMPAT_OBJ)
 
 COMPILE=$(LIBTOOL) --tag=CC --mode=compile $(CC) $(CPPFLAGS) $(CFLAGS)
@@ -73,7 +77,7 @@ $(BUILD)%.o:    $(srcdir)/%.c
 
 .PHONY:	clean realclean doc lint all 
 
-all:	$(COMMON_OBJ) unbound unittest testbound
+all:	$(COMMON_OBJ) unbound unittest testbound lock-verify pktview
 
 unbound:	$(DAEMON_OBJ)
 	$(INFO) Link $@
@@ -87,10 +91,18 @@ testbound:	$(TESTBOUND_OBJ)
 	$(INFO) Link $@
 	$Q$(LINK) -o $@ $^ $(LIBS)
 
-testcode/ldns-testpkts.c:	$(ldnsdir)/examples/ldns-testpkts.c \
-			$(ldnsdir)/examples/ldns-testpkts.h
-	cp $(ldnsdir)/examples/ldns-testpkts.c testcode/ldns-testpkts.c
-	cp $(ldnsdir)/examples/ldns-testpkts.h testcode/ldns-testpkts.h
+lock-verify:	$(LOCKVERIFY_OBJ)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $^ $(LIBS)
+
+pktview:	$(PKTVIEW_OBJ)
+	$(INFO) Link $@
+	$Q$(LINK) -o $@ $^ $(LIBS)
+
+#testcode/ldns-testpkts.c:	$(ldnsdir)/examples/ldns-testpkts.c \
+#			$(ldnsdir)/examples/ldns-testpkts.h
+#	cp $(ldnsdir)/examples/ldns-testpkts.c testcode/ldns-testpkts.c
+#	cp $(ldnsdir)/examples/ldns-testpkts.h testcode/ldns-testpkts.h
 
 util/config_file.c:	util/configparser.h
 util/configlexer.c:  $(srcdir)/util/configlexer.lex util/configparser.h
@@ -114,12 +126,13 @@ realclean: clean
 	rm -f util/configlexer.c util/configparser.c util/configparser.h
 	rm -f Makefile 
 
-lint:
-	$Qfor i in $(filter-out util/configparser.c,$(filter-out util/configlexer.c,$(sort $(ALL_SRC)))); do \
-		echo lint $$i; \
-		$(LINT) $(LINTFLAGS) -I. -I$(srcdir) -I$(ldnsdir)/include $(srcdir)/$$i ; \
-		if [ $$? -ne 0 ] ; then exit 1 ; fi ; \
-	done
+$(BUILD)%.lint:    $(srcdir)/%.c 
+	$(INFO) Lint $<
+	@if test ! -d $(dir $@); then $(INSTALL) -d $(patsubst %/,%,$(dir $@)); fi
+	$Q$(LINT) $(LINTFLAGS) -I. -I$(srcdir) -I$(ldnsdir)/include $<
+	$Qtouch $@
+
+lint:	$(addprefix $(BUILD),$(filter-out util/configparser.lint,$(filter-out util/configlexer.lint,$(sort $(ALL_SRC:.c=.lint)))))
 
 tags:	$(srcdir)/*.[ch] $(srcdir)/*/*.[ch]
 	ctags  -f $(srcdir)/tags $(srcdir)/*.[ch] $(srcdir)/*/*.[ch]

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 AC_PREREQ(2.57)
 
-AC_INIT(unbound, 0.1, wouter@nlnetlabs.nl, unbound)
+AC_INIT(unbound, 0.3, wouter@nlnetlabs.nl, unbound)
 
 CFLAGS=
 AC_AIX
@@ -90,7 +90,7 @@ CHECK_COMPILER_FLAG(O2, [CFLAGS="$CFLAGS -O2"])
 
 AC_CHECK_HEADERS([getopt.h time.h],,, [AC_INCLUDES_DEFAULT])
 
-CHECK_COMPILER_FLAG_NEEDED(-std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600,
+CHECK_COMPILER_FLAG_NEEDED(-std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600 -D_ALL_SOURCE,
 [
 #include "confdefs.h"
 #include <stdlib.h>
@@ -121,7 +121,7 @@ int test() {
 	str = gai_strerror(0);
 	return a;
 }
-], [CFLAGS="$CFLAGS -std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600"])
+], [CFLAGS="$CFLAGS -std=c99 -D__EXTENSIONS__ -D_BSD_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600 -D_ALL_SOURCE"])
 
 CHECK_COMPILER_FLAG_NEEDED(-std=c99,
 [
@@ -269,7 +269,7 @@ AC_PROG_LIBTOOL
 
 # Checks for header files.
 AC_HEADER_STDC
-AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h arpa/inet.h],,, [AC_INCLUDES_DEFAULT])
+AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/uio.h arpa/inet.h],,, [AC_INCLUDES_DEFAULT])
 
 # check for types
 AC_CHECK_TYPE(int8_t, char)
@@ -313,6 +313,29 @@ AC_INCLUDES_DEFAULT
 #endif
 ])
 
+# Check the type of struct msghdr msg_iovlen.
+AC_MSG_CHECKING([sign of struct msghdr.msg_iovlen])
+AC_RUN_IFELSE([
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/socket.h>
+int main()
+{
+	struct msghdr h;
+	h.msg_iovlen = -1;
+	if(10/h.msg_iovlen == 0) return 0; /* unsigned */
+	return 1; /* signed */
+}
+], [ 
+# if compiled and exit status 0
+cv_msgiovlen_type="unsigned" ], [ 
+# not compiled or bad exit status
+cv_msgiovlen_type="int" ], 
+# when cross compiling
+[ cv_msgiovlen_type="int" ])
+AC_MSG_RESULT($cv_msgiovlen_type)
+AC_DEFINE_UNQUOTED([TYPE_MSGIOVLEN], $cv_msgiovlen_type, [Define to 'int' or type of struct msghdr.msg_iovlen.])
+
 # Checks for libraries.
 AC_ARG_WITH(ssl, AC_HELP_STRING([--with-ssl=pathname],
                                     [enable SSL (will check /usr/local/ssl
@@ -489,6 +512,10 @@ AH_BOTTOM([
 #include <sys/socket.h>
 #endif
 
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+
 #ifdef HAVE_NETINET_IN_H
 #include <netinet/in.h>
 #endif

daemon/daemon.c

@@ -47,6 +47,8 @@
 #include "daemon/worker.h"
 #include "util/log.h"
 #include "util/config_file.h"
+#include "util/data/msgreply.h"
+#include "util/storage/slabhash.h"
 #include "services/listen_dnsport.h"
 #include <signal.h>
 
@@ -116,7 +118,24 @@ daemon_init()
 	signal_handling_record();
 	checklock_start();
 	daemon->need_to_exit = 0;
-	alloc_init(&daemon->superalloc, NULL);
+	daemon->msg_cache = slabhash_create(HASH_DEFAULT_SLABS, 
+		HASH_DEFAULT_STARTARRAY, HASH_DEFAULT_MAXMEM, 
+		msgreply_sizefunc, query_info_compare,
+		query_entry_delete, reply_info_delete, NULL);
+	if(!daemon->msg_cache) {
+		free(daemon);
+		return NULL;
+	}
+	daemon->rrset_cache = slabhash_create(HASH_DEFAULT_SLABS, 
+		HASH_DEFAULT_STARTARRAY, HASH_DEFAULT_MAXMEM, 
+		ub_rrset_sizefunc, ub_rrset_compare,
+		ub_rrset_key_delete, rrset_data_delete, &daemon->superalloc);
+	if(!daemon->rrset_cache) {
+		slabhash_delete(daemon->msg_cache);
+		free(daemon);
+		return NULL;
+	}
+	alloc_init(&daemon->superalloc, NULL, 0);
 	return daemon;	
 }
 
@@ -148,7 +167,7 @@ daemon_create_workers(struct daemon* daemon)
 		sizeof(struct worker*));
 	for(i=0; i<daemon->num; i++) {
 		if(!(daemon->workers[i] = worker_create(daemon, i)))
-			fatal_exit("malloc failure");
+			fatal_exit("could not create worker");
 	}
 }
 
@@ -182,7 +201,6 @@ static void*
 thread_start(void* arg)
 {
 	struct worker* worker = (struct worker*)arg;
-	int num = worker->thread_num;
 	log_thread_set(&worker->thread_num);
 	ub_thread_blocksigs();
 #if !defined(HAVE_PTHREAD) && !defined(HAVE_SOLARIS_THREADS)
@@ -190,10 +208,10 @@ thread_start(void* arg)
 	close(worker->cmd_send_fd);
 	worker->cmd_send_fd = -1;
 	close_other_pipes(worker->daemon, worker->thread_num);
-#endif /* no threads */
+#endif
 	if(!worker_init(worker, worker->daemon->cfg, worker->daemon->ports,
 		BUFSZ, 0))
-		fatal_exit("Could not initialize thread #%d", num);
+		fatal_exit("Could not initialize thread");
 
 	worker_work(worker);
 	return NULL;
@@ -303,6 +321,8 @@ daemon_delete(struct daemon* daemon)
 	if(!daemon)
 		return;
 	listening_ports_free(daemon->ports);
+	slabhash_delete(daemon->msg_cache);
+	slabhash_delete(daemon->rrset_cache);
 	alloc_clear(&daemon->superalloc);
 	free(daemon->cwd);
 	free(daemon->pidfile);

daemon/daemon.h

@@ -47,6 +47,7 @@
 struct config_file;
 struct worker;
 struct listen_port;
+struct slabhash;
 
 /**
  * Structure holding worker list.
@@ -71,6 +72,10 @@ struct daemon {
 	int need_to_exit;
 	/** master allocation cache */
 	struct alloc_cache superalloc;
+	/** the message cache, content is struct msgreply_entry* */
+	struct slabhash* msg_cache;
+	/** the rrset cache, content is struct ub_packed_rrset_key* */
+	struct slabhash* rrset_cache;
 };
 
 /**

daemon/stats.c

@@ -0,0 +1,71 @@
+/*
+ * daemon/stats.c - collect runtime performance indicators.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file describes the data structure used to collect runtime performance
+ * numbers. These 'statistics' may be of interest to the operator.
+ */
+#include "config.h"
+#include "daemon/stats.h"
+#include "daemon/worker.h"
+
+void server_stats_init(struct server_stats* stats)
+{
+	memset(stats, 0, sizeof(stats));
+}
+
+void server_stats_querymiss(struct server_stats* stats, struct worker* worker)
+{
+	stats->num_queries_missed_cache++;
+	stats->sum_query_list_size += worker->num_requests;
+	if(worker->num_requests > stats->max_query_list_size)
+		stats->max_query_list_size = worker->num_requests;
+}
+
+void server_stats_log(struct server_stats* stats, int threadnum)
+{
+	log_info("server stats for thread %d: %u queries, %u from cache", 
+		threadnum, (unsigned)stats->num_queries, 
+		(unsigned)(stats->num_queries - 
+			stats->num_queries_missed_cache));
+	log_info("server stats for thread %d: requestlist max %u avg %g "
+		"exceeded %u", threadnum, (unsigned)stats->max_query_list_size,
+		stats->num_queries_missed_cache?
+		(double)stats->sum_query_list_size/
+		stats->num_queries_missed_cache:0.0,
+		(unsigned)stats->num_query_list_exceeded);
+}

daemon/stats.h

@@ -0,0 +1,77 @@
+/*
+ * daemon/stats.h - collect runtime performance indicators.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file describes the data structure used to collect runtime performance
+ * numbers. These 'statistics' may be of interest to the operator.
+ */
+
+#ifndef DAEMON_STATS_H
+#define DAEMON_STATS_H
+struct worker;
+
+/** per worker statistics */
+struct server_stats {
+	/** number of queries from clients received. */
+	size_t num_queries;
+	/** number of queries that had a cache-miss. */
+	size_t num_queries_missed_cache;
+
+	/**
+	 * Sum of the querylistsize of the worker for 
+	 * every query that missed cache. To calculate average.
+	 */
+	size_t sum_query_list_size;
+	/** max value of query list size reached. */
+	size_t max_query_list_size;
+	/** number of times that the query_list_size was insufficient */
+	size_t num_query_list_exceeded;
+};
+
+/** 
+ * Initialize server stats to 0.
+ * @param stats: what to init (this is alloced by the caller).
+ */
+void server_stats_init(struct server_stats* stats);
+
+/** add query if it missed the cache */
+void server_stats_querymiss(struct server_stats* stats, struct worker* worker);
+
+/** display the stats to the log */
+void server_stats_log(struct server_stats* stats, int threadnum);
+
+#endif /* DAEMON_STATS_H */

daemon/unbound.c

@@ -44,9 +44,13 @@
 #include "util/log.h"
 #include "daemon/daemon.h"
 #include "util/config_file.h"
+#include "util/storage/slabhash.h"
+#include "services/listen_dnsport.h"
+#include "util/data/msgreply.h"
 #include <signal.h>
 #include <fcntl.h>
 #include <pwd.h>
+#include <sys/resource.h>
 
 /** print usage. */
 static void usage()
@@ -62,11 +66,45 @@ static void usage()
 	printf("Report bugs to %s\n", PACKAGE_BUGREPORT);
 }
 
+/** check file descriptor count */
+static void
+checkrlimits(struct config_file* cfg)
+{
+	int list = ((cfg->do_ip4?1:0) + (cfg->do_ip6?1:0)) * 
+		((cfg->do_udp?1:0) + (cfg->do_tcp?1 + TCP_ACCEPT_COUNT:0));
+	size_t ifs = (size_t)(cfg->num_ifs==0?1:cfg->num_ifs);
+	size_t listen_num = list*ifs;
+	size_t outnum = cfg->outgoing_num_ports*ifs + cfg->outgoing_num_tcp;
+	size_t misc = 4; /* logfile, pidfile, stdout... */
+	size_t perthread = listen_num + outnum + 2/*cmdpipe*/ + 2/*libevent*/
+		+ misc; 
+#if !defined(HAVE_PTHREAD) && !defined(HAVE_SOLARIS_THREADS)
+	int numthread = 1; /* it forks */
+#else
+	int numthread = cfg->num_threads;
+#endif
+	size_t total = numthread * perthread + misc;
+	struct rlimit rlim;
+	if(getrlimit(RLIMIT_NOFILE, &rlim) < 0) {
+		log_warn("getrlimit: %s", strerror(errno));
+		return;
+	}
+	if((size_t)rlim.rlim_cur < total) {
+		log_err("Not enough sockets available. Increase "
+			"ulimit(open files).");
+		log_err("or decrease number of threads, outgoing num ports, "
+			"outgoing num tcp or number of interfaces");
+		log_err("estimate %u fds high mark, %u available", 
+			(unsigned)total, (unsigned)rlim.rlim_cur);
+		fatal_exit("Not enough file descriptors available");
+	}
+}
+
 /** to changedir, logfile */
 static void
 apply_dir(struct daemon* daemon, struct config_file* cfg, int cmdline_verbose)
 {
-	/* apply changes if they have changed */
+	/* apply if they have changed */
 	daemon->cfg = cfg;
 	verbosity = cmdline_verbose + cfg->verbosity;
 	if(cfg->directory && cfg->directory[0]) {
@@ -77,9 +115,33 @@ apply_dir(struct daemon* daemon, struct config_file* cfg, int cmdline_verbose)
 			}
 			free(daemon->cwd);
 			if(!(daemon->cwd = strdup(cfg->directory)))
-				fatal_exit("malloc failed");
+				log_err("cwd: malloc failed");
 		}
 	}
+	if(cfg->msg_cache_size != slabhash_get_size(daemon->msg_cache) ||
+		cfg->msg_cache_slabs != daemon->msg_cache->size) {
+		slabhash_delete(daemon->msg_cache);
+		daemon->msg_cache = slabhash_create(cfg->msg_cache_slabs, 
+			HASH_DEFAULT_STARTARRAY, cfg->msg_cache_size, 
+			msgreply_sizefunc, query_info_compare,
+			query_entry_delete, reply_info_delete, NULL);
+		if(!daemon->msg_cache) {
+			fatal_exit("malloc failure updating config settings");
+		}
+	}
+	if(cfg->rrset_cache_size != slabhash_get_size(daemon->rrset_cache) ||
+		cfg->rrset_cache_slabs != daemon->rrset_cache->size) {
+		slabhash_delete(daemon->rrset_cache);
+		daemon->rrset_cache = slabhash_create(cfg->rrset_cache_slabs, 
+			HASH_DEFAULT_STARTARRAY, cfg->rrset_cache_size, 
+			ub_rrset_sizefunc, ub_rrset_compare,
+			ub_rrset_key_delete, rrset_data_delete, 
+			&daemon->superalloc);
+		if(!daemon->rrset_cache) {
+			fatal_exit("malloc failure updating config settings");
+		}
+	}
+	checkrlimits(cfg);
 }
 
 /** Read existing pid from pidfile. */
@@ -93,10 +155,16 @@ readpid (const char* file)
 	ssize_t l;
 
 	if ((fd = open(file, O_RDONLY)) == -1) {
+		if(errno != ENOENT)
+			log_err("Could not read pidfile %s: %s",
+				file, strerror(errno));
 		return -1;
 	}
 
 	if (((l = read(fd, pidbuf, sizeof(pidbuf)))) == -1) {
+		if(errno != ENOENT)
+			log_err("Could not read pidfile %s: %s",
+				file, strerror(errno));
 		close(fd);
 		return -1;
 	}
@@ -105,7 +173,6 @@ readpid (const char* file)
 
 	/* Empty pidfile means no pidfile... */
 	if (l == 0) {
-		errno = ENOENT;
 		return -1;
 	}
 
@@ -143,13 +210,8 @@ static void
 checkoldpid(struct config_file* cfg)
 {
 	pid_t old;
-	if((old = readpid(cfg->pidfile)) == -1) {
-		if(errno != ENOENT) {
-			log_err("Could not read pidfile %s: %s",
-				cfg->pidfile, strerror(errno));
-		}
-	} else {
-		/** see if it is still alive */
+	if((old = readpid(cfg->pidfile)) != -1) {
+		/* see if it is still alive */
 		if(kill(old, 0) == 0 || errno == EPERM)
 			log_warn("unbound is already running as pid %u.", 
 				(unsigned)old);
@@ -158,6 +220,34 @@ checkoldpid(struct config_file* cfg)
 	}
 }
 
+/** detach from command line */
+static void
+detach(struct config_file* cfg)
+{
+	int fd;
+	/* Take off... */
+	switch (fork()) {
+		case 0:
+			break;
+		case -1:
+			unlink(cfg->pidfile);
+			fatal_exit("fork failed: %s", strerror(errno));
+		default:
+			/* exit interactive session */
+			exit(0);
+	}
+	/* detach */
+	if(setsid() == -1)
+		fatal_exit("setsid() failed: %s", strerror(errno));
+	if ((fd = open("/dev/null", O_RDWR, 0)) != -1) {
+		(void)dup2(fd, STDIN_FILENO);
+		(void)dup2(fd, STDOUT_FILENO);
+		(void)dup2(fd, STDERR_FILENO);
+		if (fd > 2)
+			(void)close(fd);
+	}
+}
+
 /** daemonize, drop user priviliges and chroot if needed */
 static void
 do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
@@ -186,32 +276,12 @@ do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
 	/* init logfile just before fork */
 	log_init(cfg->logfile);
 	if(!debug_mode && cfg->do_daemonize) {
-		int fd;
-		/* Take off... */
-		switch (fork()) {
-			case 0:
-				break;
-			case -1:
-				unlink(cfg->pidfile);
-				fatal_exit("fork failed: %s", strerror(errno));
-			default:
-				/* exit interactive session */
-				exit(0);
-		}
-		/* detach */
-		if(setsid() == -1)
-			fatal_exit("setsid() failed: %s", strerror(errno));
-		if ((fd = open("/dev/null", O_RDWR, 0)) != -1) {
-			(void)dup2(fd, STDIN_FILENO);
-			(void)dup2(fd, STDOUT_FILENO);
-			(void)dup2(fd, STDERR_FILENO);
-			if (fd > 2)
-				(void)close(fd);
-		}
+		detach(cfg);
 	}
 	if(cfg->pidfile && cfg->pidfile[0]) {
 		writepid(cfg->pidfile, getpid());
-		daemon->pidfile = strdup(cfg->pidfile);
+		if(!(daemon->pidfile = strdup(cfg->pidfile)))
+			log_err("pidf: malloc failed");
 	}
 }
 
@@ -222,7 +292,8 @@ do_chroot(struct daemon* daemon, struct config_file* cfg, int debug_mode)
  *    These increase verbosity as specified in the config file.
  * @param debug_mode: if set, do not daemonize.
  */
-static void run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode)
+static void 
+run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode)
 {
 	struct config_file* cfg = NULL;
 	struct daemon* daemon = NULL;

daemon/worker.c

@@ -47,8 +47,11 @@
 #include "daemon/daemon.h"
 #include "util/netevent.h"
 #include "util/config_file.h"
+#include "util/region-allocator.h"
+#include "util/storage/slabhash.h"
 #include "services/listen_dnsport.h"
 #include "services/outside_network.h"
+#include "util/data/msgparse.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #  include <sys/types.h>
@@ -56,10 +59,17 @@
 #include <netdb.h>
 #include <signal.h>
 
+/** size of ID+FLAGS in a DNS message */
+#define DNS_ID_AND_FLAGS 4
 /** timeout in seconds for UDP queries to auth servers. TODO: proper rtt */
-#define UDP_QUERY_TIMEOUT 5
-/** the size of ID and flags, opcode, rcode in dns packet */
-#define ID_AND_FLAGS 4
+#define UDP_QUERY_TIMEOUT 4
+/** timeout in seconds for TCP queries to auth servers. TODO: proper rtt */
+#define TCP_QUERY_TIMEOUT 30 
+/** Advertised version of EDNS capabilities */
+#define EDNS_ADVERTISED_VERSION 	0
+/** Advertised size of EDNS capabilities */
+#define EDNS_ADVERTISED_SIZE 	4096
+
 
 void 
 worker_send_cmd(struct worker* worker, ldns_buffer* buffer,
@@ -75,18 +85,129 @@ worker_send_cmd(struct worker* worker, ldns_buffer* buffer,
 		log_err("write socket: %s", strerror(errno));
 }
 
+/** release workrequest back to the freelist,
+ * note that the w->qinfo still needs to be cleared after this. 
+ */
+static void
+req_release(struct work_query* w)
+{
+	if(w->worker->num_requests == w->worker->request_size)  {
+		/* no longer at max, start accepting again. */
+		listen_resume(w->worker->front);
+	}
+	log_assert(w->worker->num_requests >= 1);
+	w->worker->num_requests --;
+	w->next = w->worker->free_queries;
+	w->worker->free_queries = w;
+}
+
+/** create error and fill into buffer */
+static void
+replyerror_fillbuf(int r, struct comm_reply* repinfo, uint16_t id,
+	uint16_t qflags, struct query_info* qinfo)
+{
+	ldns_buffer* buf = repinfo->c->buffer;
+	uint16_t flags;
+	verbose(VERB_DETAIL, "reply with error");
+	
+	ldns_buffer_clear(buf);
+	ldns_buffer_write(buf, &id, sizeof(uint16_t));
+	flags = (uint16_t)(BIT_QR | r); /* QR and retcode*/
+	flags |= (qflags & (BIT_RD|BIT_CD)); /* copy RD and CD bit */
+	ldns_buffer_write_u16(buf, flags);
+	flags = 1;
+	ldns_buffer_write_u16(buf, flags);
+	flags = 0;
+	ldns_buffer_write(buf, &flags, sizeof(uint16_t));
+	ldns_buffer_write(buf, &flags, sizeof(uint16_t));
+	ldns_buffer_write(buf, &flags, sizeof(uint16_t));
+	ldns_buffer_write(buf, qinfo->qname, qinfo->qnamesize);
+	ldns_buffer_write_u16(buf, qinfo->qtype);
+	ldns_buffer_write_u16(buf, qinfo->qclass);
+	ldns_buffer_flip(buf);
+}
+
 /** reply to query with given error code */
 static void 
-replyerror(int r, struct worker* worker)
+replyerror(int r, struct work_query* w)
 {
-	LDNS_QR_SET(ldns_buffer_begin(worker->query_reply.c->buffer));
-	LDNS_RCODE_SET(ldns_buffer_begin(worker->query_reply.c->buffer), r);
-	comm_point_send_reply(&worker->query_reply);
-	if(worker->num_requests == 1)  {
-		/* no longer at max, start accepting again. */
-		listen_resume(worker->front);
+	w->edns.edns_version = EDNS_ADVERTISED_VERSION;
+	w->edns.udp_size = EDNS_ADVERTISED_SIZE;
+	w->edns.ext_rcode = 0;
+	w->edns.bits &= EDNS_DO;
+	replyerror_fillbuf(r, &w->query_reply, w->query_id, w->query_flags,
+		&w->qinfo);
+	attach_edns_record(w->query_reply.c->buffer, &w->edns);
+	comm_point_send_reply(&w->query_reply);
+	req_release(w);
+	query_info_clear(&w->qinfo);
+}
+
+/** see if rrset needs to be updated in the cache */
+static int
+need_to_update_rrset(struct packed_rrset_data* newd, 
+	struct packed_rrset_data* cached)
+{
+	/*	o if current RRset is more trustworthy - insert it */
+	if( newd->trust > cached->trust )
+		return 1;
+	/*	o same trust, but different in data - insert it */
+	if( newd->trust == cached->trust &&
+		!rrsetdata_equal(newd, cached))
+		return 1;
+	/*	o see if TTL is better than TTL in cache. */
+	/*	  if so, see if rrset+rdata is the same */
+	/*	  if so, update TTL in cache, even if trust is worse. */
+	if( newd->ttl > cached->ttl &&
+		rrsetdata_equal(newd, cached))
+		return 1;
+	return 0;
+}
+
+/** store rrsets in the rrset cache. */
+static void
+worker_store_rrsets(struct worker* worker, struct reply_info* rep)
+{
+	struct lruhash_entry* e;
+	size_t i;
+	/* see if rrset already exists in cache, if not insert it. */
+	/* if it does exist: check to insert it */
+	for(i=0; i<rep->rrset_count; i++) {
+		rep->ref[i].key = rep->rrsets[i];
+		rep->ref[i].id = rep->rrsets[i]->id;
+		/* looks up item with a readlock - no editing! */
+		if((e=slabhash_lookup(worker->daemon->rrset_cache,
+			rep->rrsets[i]->entry.hash, rep->rrsets[i]->entry.key,
+			0)) != 0) {
+			struct packed_rrset_data* data = 
+				(struct packed_rrset_data*)e->data;
+			struct packed_rrset_data* rd = 
+				(struct packed_rrset_data*)
+				rep->rrsets[i]->entry.data;
+			rep->ref[i].key = (struct ub_packed_rrset_key*)e->key;
+			rep->ref[i].id = rep->rrsets[i]->id;
+			/* found in cache, do checks above */
+			if(!need_to_update_rrset(rd, data)) {
+				lock_rw_unlock(&e->lock);
+				ub_packed_rrset_parsedelete(rep->rrsets[i],
+					&worker->alloc);
+				rep->rrsets[i] = rep->ref[i].key;
+				continue; /* use cached item instead */
+			}
+			if(rd->trust < data->trust)
+				rd->trust = data->trust;
+			lock_rw_unlock(&e->lock);
+			/* small gap here, where entry is not locked.
+			 * possibly entry is updated with something else.
+			 * this is just too bad, its cache anyway. */
+			/* use insert to update entry to manage lruhash
+			 * cache size values nicely. */
+		}
+		slabhash_insert(worker->daemon->rrset_cache, 
+			rep->rrsets[i]->entry.hash, &rep->rrsets[i]->entry,
+			rep->rrsets[i]->entry.data, &worker->alloc);
+		if(e) rep->rrsets[i] = rep->ref[i].key;
 	}
-	worker->num_requests --;
 }
 
 /** process incoming replies from the network */
@@ -94,43 +215,94 @@ static int
 worker_handle_reply(struct comm_point* c, void* arg, int error, 
 	struct comm_reply* ATTR_UNUSED(reply_info))
 {
-	struct worker* worker = (struct worker*)arg;
+	struct work_query* w = (struct work_query*)arg;
+	struct query_info qinf;
+	struct reply_info* rep;
+	struct msgreply_entry* e;
+	struct edns_data svr_edns; /* unused server edns advertisement */
+	uint16_t us;
+	int r;
+
 	verbose(VERB_DETAIL, "reply to query with stored ID %d", 
-		worker->query_id);
-	LDNS_ID_SET(ldns_buffer_begin(worker->query_reply.c->buffer),
-		worker->query_id);
+		ntohs(w->query_id)); /* byteswapped so same as dig prints */
 	if(error != 0) {
-		replyerror(LDNS_RCODE_SERVFAIL, worker);
+		replyerror(LDNS_RCODE_SERVFAIL, w);
+		return 0;
+	}
+	/* sanity check. */
+	if(!LDNS_QR_WIRE(ldns_buffer_begin(c->buffer)))
+		return 0; /* not a reply. */
+	if(LDNS_OPCODE_WIRE(ldns_buffer_begin(c->buffer)) != LDNS_PACKET_QUERY)
+		return 0; /* not a reply to a query. */
+	if(LDNS_QDCOUNT(ldns_buffer_begin(c->buffer)) > 1)
+		return 0; /* too much in the query section */
+	/* see if it is truncated */
+	if(LDNS_TC_WIRE(ldns_buffer_begin(c->buffer)) && c->type == comm_udp) {
+		log_info("TC: truncated. retry in TCP mode.");
+		qinfo_query_encode(w->worker->back->udp_buff, &w->qinfo);
+		pending_tcp_query(w->worker->back, w->worker->back->udp_buff, 
+			&w->worker->fwd_addr, w->worker->fwd_addrlen, 
+			TCP_QUERY_TIMEOUT, worker_handle_reply, w, 
+			w->worker->rndstate);
 		return 0;
 	}
 	/* woohoo a reply! */
-	ldns_buffer_clear(worker->query_reply.c->buffer);
-	ldns_buffer_skip(worker->query_reply.c->buffer, ID_AND_FLAGS);
-	ldns_buffer_write(worker->query_reply.c->buffer, 
-		ldns_buffer_at(c->buffer, ID_AND_FLAGS), 
-		ldns_buffer_limit(c->buffer) - ID_AND_FLAGS);
-	LDNS_QR_SET(ldns_buffer_begin(worker->query_reply.c->buffer));
-	ldns_buffer_flip(worker->query_reply.c->buffer);
-	comm_point_send_reply(&worker->query_reply);
-	if(worker->num_requests == 1)  {
-		/* no longer at max, start accepting again. */
-		listen_resume(worker->front);
+	if((r=reply_info_parse(c->buffer, &w->worker->alloc, &qinf, &rep,
+		w->worker->scratchpad, &svr_edns))!=0) {
+		if(r == LDNS_RCODE_SERVFAIL)
+			log_err("reply_info_parse: out of memory");
+		/* formerr on my parse gives servfail to my client */
+		replyerror(LDNS_RCODE_SERVFAIL, w);
+		region_free_all(w->worker->scratchpad);
+		return 0;
 	}
-	worker->num_requests --;
+	us = w->edns.udp_size;
+	w->edns.edns_version = EDNS_ADVERTISED_VERSION;
+	w->edns.udp_size = EDNS_ADVERTISED_SIZE;
+	w->edns.ext_rcode = 0;
+	w->edns.bits &= EDNS_DO;
+	if(!reply_info_answer_encode(&qinf, rep, w->query_id, w->query_flags,
+		w->query_reply.c->buffer, 0, 0, w->worker->scratchpad, us, 
+		&w->edns)) {
+		replyerror(LDNS_RCODE_SERVFAIL, w);
+		query_info_clear(&qinf);
+		reply_info_parsedelete(rep, &w->worker->alloc);
+		region_free_all(w->worker->scratchpad);
+		return 0;
+	}
+	comm_point_send_reply(&w->query_reply);
+	region_free_all(w->worker->scratchpad);
+	req_release(w);
+	query_info_clear(&w->qinfo);
+	if(rep->ttl == 0) {
+		log_info("TTL 0: dropped");
+		query_info_clear(&qinf);
+		reply_info_parsedelete(rep, &w->worker->alloc);
+		return 0;
+	}
+	reply_info_set_ttls(rep, time(0));
+	worker_store_rrsets(w->worker, rep);
+	reply_info_sortref(rep);
+	/* store msg in the cache */
+	if(!(e = query_info_entrysetup(&qinf, rep, w->query_hash))) {
+		query_info_clear(&qinf);
+		reply_info_parsedelete(rep, &w->worker->alloc);
+		return 0;
+	}
+	slabhash_insert(w->worker->daemon->msg_cache, w->query_hash, 
+		&e->entry, rep, &w->worker->alloc);
 	return 0;
 }
 
 /** process incoming request */
 static void 
-worker_process_query(struct worker* worker) 
+worker_process_query(struct worker* worker, struct work_query* w) 
 {
 	/* query the forwarding address */
-	worker->query_id = LDNS_ID_WIRE(ldns_buffer_begin(
-		worker->query_reply.c->buffer));
-	verbose(VERB_DETAIL, "process_query ID %d", worker->query_id);
-	pending_udp_query(worker->back, worker->query_reply.c->buffer, 
+	verbose(VERB_DETAIL, "process_query ID %d", ntohs(w->query_id));
+	pending_udp_query(worker->back, w->query_reply.c->buffer, 
 		&worker->fwd_addr, worker->fwd_addrlen, UDP_QUERY_TIMEOUT,
-		worker_handle_reply, worker, worker->rndstate);
+		worker_handle_reply, w, worker->rndstate);
 }
 
 /** check request sanity. Returns error code, 0 OK, or -1 discard. 
@@ -167,7 +339,7 @@ worker_check_request(ldns_buffer* pkt)
 			LDNS_NSCOUNT(ldns_buffer_begin(pkt)));
 		return LDNS_RCODE_FORMERR;
 	}
-	if(LDNS_ARCOUNT(ldns_buffer_begin(pkt)) != 0) {
+	if(LDNS_ARCOUNT(ldns_buffer_begin(pkt)) > 1) {
 		verbose(VERB_DETAIL, "request wrong nr ar=%d", 
 			LDNS_ARCOUNT(ldns_buffer_begin(pkt)));
 		return LDNS_RCODE_FORMERR;
@@ -205,6 +377,58 @@ worker_handle_control_cmd(struct comm_point* c, void* arg, int error,
 	return 0;
 }
 
+/** answer query from the cache */
+static int
+answer_from_cache(struct worker* worker, struct lruhash_entry* e, uint16_t id,
+	uint16_t flags, struct comm_reply* repinfo, struct edns_data* edns)
+{
+	struct msgreply_entry* mrentry = (struct msgreply_entry*)e->key;
+	struct reply_info* rep = (struct reply_info*)e->data;
+	uint32_t timenow = time(0);
+	uint16_t udpsize = edns->udp_size;
+	size_t i;
+	/* see if it is possible */
+	if(rep->ttl <= timenow) {
+		/* the rrsets may have been updated in the meantime */
+		/* but this ignores it */
+		return 0;
+	}
+	edns->edns_version = EDNS_ADVERTISED_VERSION;
+	edns->udp_size = EDNS_ADVERTISED_SIZE;
+	edns->ext_rcode = 0;
+	edns->bits &= EDNS_DO;
+	/* check rrsets */
+	for(i=0; i<rep->rrset_count; i++) {
+		if(i>0 && rep->ref[i].key == rep->ref[i-1].key)
+			continue; /* only lock items once */
+		lock_rw_rdlock(&rep->ref[i].key->entry.lock);
+		if(rep->ref[i].id != rep->ref[i].key->id ||
+			rep->ttl <= timenow) {
+			/* failure! rollback our readlocks */
+			size_t j;
+			for(j=0; j<=i; j++)
+				lock_rw_unlock(&rep->ref[j].key->entry.lock);
+			return 0;
+		}
+	}
+	/* locked and ids and ttls are OK. */
+	if(!reply_info_answer_encode(&mrentry->key, rep, id, flags, 
+		repinfo->c->buffer, timenow, 1, worker->scratchpad,
+		udpsize, edns)) {
+		replyerror_fillbuf(LDNS_RCODE_SERVFAIL, repinfo, id,
+			flags, &mrentry->key);
+	}
+	/* unlock */
+	for(i=0; i<rep->rrset_count; i++) {
+		if(i>0 && rep->ref[i].key == rep->ref[i-1].key)
+			continue; /* only unlock items once */
+		lock_rw_unlock(&rep->ref[i].key->entry.lock);
+	}
+	region_free_all(worker->scratchpad);
+	/* go and return this buffer to the client */
+	return 1;
+}
+
 /** handles callbacks from listening event interface */
 static int 
 worker_handle_request(struct comm_point* c, void* arg, int error,
@@ -212,6 +436,12 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 {
 	struct worker* worker = (struct worker*)arg;
 	int ret;
+	hashvalue_t h;
+	struct lruhash_entry* e;
+	struct query_info qinfo;
+	struct work_query* w;
+	struct edns_data edns;
+
 	verbose(VERB_DETAIL, "worker handle request");
 	if(error != NETEVENT_NOERROR) {
 		log_err("called with err=%d", error);
@@ -226,22 +456,84 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		comm_point_drop_reply(repinfo);
 		return 0;
 	}
-	if(worker->num_requests > 0) {
+	worker->stats.num_queries++;
+	/* see if query is in the cache */
+	if(!query_info_parse(&qinfo, c->buffer)) {
+		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
+		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), 
+			LDNS_RCODE_FORMERR);
+		return 1;
+	}
+	h = query_info_hash(&qinfo);
+	if((ret=parse_edns_from_pkt(c->buffer, &edns)) != 0) {
+		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
+		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), ret);
+		return 1;
+	}
+	if(edns.edns_present && edns.edns_version != 0) {
+		edns.ext_rcode = (uint8_t)(EDNS_RCODE_BADVERS>>4);
+		edns.edns_version = EDNS_ADVERTISED_VERSION;
+		edns.udp_size = EDNS_ADVERTISED_SIZE;
+		edns.bits &= EDNS_DO;
+		replyerror_fillbuf(EDNS_RCODE_BADVERS&0xf, repinfo, 
+			*(uint16_t*)ldns_buffer_begin(c->buffer),
+			ldns_buffer_read_u16_at(c->buffer, 2), &qinfo);
+		attach_edns_record(c->buffer, &edns);
+		return 1;
+	}
+	if(c->type != comm_udp)
+		edns.udp_size = 65535; /* max size for TCP replies */
+	if((e=slabhash_lookup(worker->daemon->msg_cache, h, &qinfo, 0))) {
+		/* answer from cache - we have acquired a readlock on it */
+		log_info("answer from the cache");
+		if(answer_from_cache(worker, e, 
+			*(uint16_t*)ldns_buffer_begin(c->buffer), 
+			ldns_buffer_read_u16_at(c->buffer, 2), repinfo, 
+			&edns)) {
+			lock_rw_unlock(&e->lock);
+			return 1;
+		}
+		log_info("answer from the cache -- data has timed out");
+		lock_rw_unlock(&e->lock);
+	}
+	ldns_buffer_rewind(c->buffer);
+	server_stats_querymiss(&worker->stats, worker);
+	/* perform memory allocation(s) */
+	if(!query_info_allocqname(&qinfo)) {
+		comm_point_drop_reply(repinfo);
+		return 0;
+	}
+
+	/* grab a work request structure for this new request */
+	if(!(w = worker->free_queries)) {
 		/* we could get this due to a slow tcp incoming query, 
 		   that started before we performed listen_pushback */
 		verbose(VERB_DETAIL, "worker: too many incoming requests "
 			"active. dropping incoming query.");
+		worker->stats.num_query_list_exceeded++;
 		comm_point_drop_reply(repinfo);
+		query_info_clear(&qinfo);
 		return 0;
 	}
-	/* answer it */
+	w->edns = edns;
+	worker->free_queries = w->next;
 	worker->num_requests ++;
-	if(worker->num_requests >= 1)  {
+	log_assert(worker->num_requests <= worker->request_size);
+	if(worker->num_requests == worker->request_size)  {
 		/* the max request number has been reached, stop accepting */
 		listen_pushback(worker->front);
 	}
-	memcpy(&worker->query_reply, repinfo, sizeof(struct comm_reply));
-	worker_process_query(worker);
+
+	/* init request */
+	w->next = NULL;
+	w->query_hash = h;
+	memcpy(&w->query_reply, repinfo, sizeof(struct comm_reply));
+	memcpy(&w->qinfo, &qinfo, sizeof(struct query_info));
+	memcpy(&w->query_id, ldns_buffer_begin(c->buffer), sizeof(uint16_t));
+	w->query_flags = ldns_buffer_read_u16_at(c->buffer, 2);
+
+	/* answer it */
+	worker_process_query(worker, w);
 	return 0;
 }
 
@@ -259,14 +551,17 @@ worker_sighandler(int sig, void* arg)
 			break;
 		case SIGINT:
 			log_info("caught signal SIGINT");
+			worker->need_to_restart = 0;
 			comm_base_exit(worker->base);
 			break;
 		case SIGQUIT:
 			log_info("caught signal SIGQUIT");
+			worker->need_to_restart = 0;
 			comm_base_exit(worker->base);
 			break;
 		case SIGTERM:
 			log_info("caught signal SIGTERM");
+			worker->need_to_restart = 0;
 			comm_base_exit(worker->base);
 			break;
 		default:
@@ -306,6 +601,40 @@ worker_create(struct daemon* daemon, int id)
 	return worker;
 }
 
+/** create request handling structures */
+static int
+reqs_init(struct worker* worker)
+{
+	size_t i;
+	for(i=0; i<worker->request_size; i++) {
+		struct work_query* q = (struct work_query*)calloc(1,
+			sizeof(struct work_query));
+		if(!q) return 0;
+		q->worker = worker;
+		q->next = worker->free_queries;
+		worker->free_queries = q;
+		q->all_next = worker->all_queries;
+		worker->all_queries = q;
+	}
+	return 1;
+}
+
+/** delete request list */
+static void
+reqs_delete(struct worker* worker)
+{
+	struct work_query* q = worker->all_queries;
+	struct work_query* n;
+	while(q) {
+		n = q->all_next;
+		log_assert(q->worker == worker);
+		/* comm_reply closed in outside_network_delete */
+		query_info_clear(&q->qinfo);
+		free(q);
+		q = n;
+	}
+}
+
 int
 worker_init(struct worker* worker, struct config_file *cfg, 
 	struct listen_port* ports, size_t buffer_size, int do_sigs)
@@ -350,7 +679,8 @@ worker_init(struct worker* worker, struct config_file *cfg,
 		cfg->outgoing_num_ports * worker->thread_num;
 	worker->back = outside_network_create(worker->base,
 		buffer_size, (size_t)cfg->outgoing_num_ports, cfg->ifs, 
-		cfg->num_ifs, cfg->do_ip4, cfg->do_ip6, startport);
+		cfg->num_ifs, cfg->do_ip4, cfg->do_ip6, startport, 
+		cfg->do_tcp?cfg->outgoing_num_tcp:0);
 	if(!worker->back) {
 		log_err("could not create outgoing sockets");
 		worker_delete(worker);
@@ -380,6 +710,11 @@ worker_init(struct worker* worker, struct config_file *cfg,
 			return 0;
 		}
 	}
+	worker->request_size = cfg->num_queries_per_thread;
+	if(!reqs_init(worker)) {
+		worker_delete(worker);
+		return 0;
+	}
 
 	/* set forwarder address */
 	if(cfg->fwd_address && cfg->fwd_address[0]) {
@@ -388,7 +723,11 @@ worker_init(struct worker* worker, struct config_file *cfg,
 			fatal_exit("could not set forwarder address");
 		}
 	}
-	alloc_init(&worker->alloc, &worker->daemon->superalloc);
+	server_stats_init(&worker->stats);
+	alloc_init(&worker->alloc, &worker->daemon->superalloc, 
+		worker->thread_num);
+	worker->scratchpad = region_create_custom(malloc, free, 
+		65536, 8192, 32, 1);
 	return 1;
 }
 
@@ -403,6 +742,8 @@ worker_delete(struct worker* worker)
 {
 	if(!worker) 
 		return;
+	server_stats_log(&worker->stats, worker->thread_num);
+	reqs_delete(worker);
 	listen_delete(worker->front);
 	outside_network_delete(worker->back);
 	comm_signal_delete(worker->comsig);
@@ -419,6 +760,7 @@ worker_delete(struct worker* worker)
 		close(worker->cmd_recv_fd);
 	worker->cmd_recv_fd = -1;
 	alloc_clear(&worker->alloc);
+	region_destroy(worker->scratchpad);
 	free(worker);
 }
 

daemon/worker.h

@@ -47,12 +47,16 @@
 #include "util/netevent.h"
 #include "util/locks.h"
 #include "util/alloc.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgparse.h"
+#include "daemon/stats.h"
 struct listen_dnsport;
 struct outside_network;
 struct config_file;
 struct daemon;
 struct listen_port;
 struct ub_randstate;
+struct region;
 
 /** size of table used for random numbers. large to be more secure. */
 #define RND_STATE_SIZE 256
@@ -63,6 +67,27 @@ enum worker_commands {
 	worker_cmd_quit
 };
 
+/** information per query that is in processing */
+struct work_query {
+	/** next query in freelist */
+	struct work_query* next;
+	/** the worker for this query */
+	struct worker* worker;
+	/** the query reply destination, packet buffer and where to send. */
+	struct comm_reply query_reply;
+	/** the query_info structure from the query */
+	struct query_info qinfo;
+	/** hash value of the query qinfo */
+	hashvalue_t query_hash;
+	/** next query in all-list */
+	struct work_query* all_next;
+	/** id of query, in network byteorder. */
+	uint16_t query_id;
+	/** flags uint16 from query */
+	uint16_t query_flags;
+	/** edns data from the query */
+	struct edns_data edns;
+};
 
 /**
  * Structure holding working information for unbound.
@@ -91,11 +116,13 @@ struct worker {
 	struct comm_point* cmd_com;
 
 	/** number of requests currently active */
-	int num_requests;
-	/** our one and only query, packet buffer and where to send. */
-	struct comm_reply query_reply;
-	/** id of query */
-	uint16_t query_id;
+	size_t num_requests;
+	/** number of requests that can be handled by this worker */
+	size_t request_size;
+	/** the free working queries */
+	struct work_query* free_queries;
+	/** list of all working queries */
+	struct work_query* all_queries;
 
 	/** address to forward to */
 	struct sockaddr_storage fwd_addr;
@@ -108,6 +135,10 @@ struct worker {
 	int need_to_restart;
 	/** allocation cache for this thread */
 	struct alloc_cache alloc;
+	/** per thread statistics */
+	struct server_stats stats;
+	/** thread scratch region */
+	struct region* scratchpad;
 };
 
 /**

doc/Changelog

@@ -1,3 +1,240 @@
+9 May 2007: Wouter
+	- outside network cleans up waiting tcp queries on exit.
+	- fallback to TCP.
+	- testbound replay with retry in TCP mode.
+	- tpkg test for retry in TCP mode, against ldns-testns server.
+	- daemon checks max number of open files and complains if not enough.
+	- test where data expires in the cache.
+	- compiletests: fixed empty body ifstatements in alloc.c, in case
+	  locks are disabled.
+
+8 May 2007: Wouter
+	- outgoing network keeps list of available tcp buffers for outgoing 
+	  tcp queries.
+	- outgoing-num-tcp config option.
+	- outgoing network keeps waiting list of queries waiting for buffer.
+	- netevent supports outgoing tcp commpoints, nonblocking connects.
+
+7 May 2007: Wouter
+	- EDNS read from query, used to make reply smaller.
+	- advertised edns value constants.
+	- EDNS BADVERS response, if asked for too high edns version.
+	- EDNS extended error reponses once the EDNS record from the query
+	  has successfully been parsed.
+
+4 May 2007: Wouter
+	- msgreply sizefunc is more accurate.
+	- config settings for rrset cache size and slabs.
+	- hashtable insert takes argument so that a thread can use its own
+	  alloc cache to store released keys.
+	- alloc cache special_release() locks if necessary.
+	- rrset trustworthiness type added.
+	- thread keeps a scratchpad region for handling messages.
+	- writev used in netevent to write tcp length and data after another.
+	  This saves a roundtrip on tcp replies.
+	- test for one rrset updated in the cache.
+	- test for one rrset which is not updated, as it is not deemed
+	  trustworthy enough.
+	- test for TTL refreshed in rrset.
+
+3 May 2007: Wouter
+	- fill refs. Use new parse and encode to answer queries.
+	- stores rrsets in cache.
+	- uses new msgreply format in cache.
+
+2 May 2007: Wouter
+	- dname unit tests in own file and spread out neatly in functions.
+	- more dname unit tests.
+	- message encoding creates truncated TC flagged messages if they do 
+	  not fit, and will leave out (whole)rrsets from additional if needed.
+
+1 May 2007: Wouter
+	- decompress query section, extremely lenient acceptance.
+	  But only for answers from other servers, not for plain queries.
+	- compression and decompression test cases.
+	- some stats added.
+	- example.conf interface: line is changed from 127.0.0.1 which leads
+	  to problems if used (restricting communication to the localhost),
+	  to a documentation and test address.
+
+27 April 2007: Wouter
+	- removed iov usage, it is not good for dns message encoding.
+	- owner name compression more optimal.
+	- rrsig owner name compression.
+	- rdata domain name compression.
+
+26 April 2007: Wouter
+	- floating point exception fix in lock-verify.
+	- lint uses make dependency
+	- fixup lint in dname owner domain name compression code.
+	- define for offset range that can be compressed to.
+
+25 April 2007: Wouter
+	- prettier code; parse_rrset->type kept in host byte order.
+	- datatype used for hashvalue of converted rrsig structure.
+	- unit test compares edns section data too.
+
+24 April 2007: Wouter
+	- ttl per RR, for RRSIG rrsets and others.
+	- dname_print debug function.
+	- if type is not known, size calc will skip DNAME decompression.
+	- RRSIG parsing and storing and putting in messages.
+	- dnssec enabled unit tests (from nlnetlabs.nl and se queries).
+	- EDNS extraction routine.
+
+20 April 2007: Wouter
+	- code comes through all of the unit tests now.
+	- disabled warning about spurious extra data.
+	- documented the RRSIG parse plan in msgparse.h.
+	- rrsig reading and outputting.
+
+19 April 2007: Wouter
+	- fix unit test to actually to tests.
+	- fix write iov helper, and fakevent code.
+	- extra builtin testcase (small packet).
+	- ttl converted to network format in packets.
+	- flags converted correctly
+	- rdatalen off by 2 error fixup.
+	- uses less iov space for header.
+
+18 April 2007: Wouter
+	- review of msgparse code.
+	- smaller test cases.
+
+17 April 2007: Wouter
+	- copy and decompress dnames.
+	- store calculated hash value too.
+	- routine to create message out of stored information.
+	- util/data/msgparse.c for message parsing code.
+	- unit test, and first fixes because of test.
+		* forgot rrset_count addition.
+		* did & of ptr on stack for memory position calculation.
+		* dname_pkt_copy forgot to read next label length.
+	- test from file and fixes
+		* double frees fixed in error conditions.
+		* types with less than full rdata allowed by parser.
+		  Some dynamic update packets seem to use it.
+
+16 April 2007: Wouter
+	- following a small change in LDNS, parsing code calculates the
+	  memory size to allocate for rrs.
+	- code to handle ID creation.
+
+13 April 2007: Wouter
+	- parse routines. Code that parses rrsets, rrs.
+
+12 April 2007: Wouter
+	- dname compare routine that preserves case, with unit tests.
+	
+11 April 2007: Wouter
+	- parse work - dname packet parse, msgparse, querysection parse,
+	  start of sectionparse.
+
+10 April 2007: Wouter
+	- Improved alignment of reply_info packet, nice for 32 and 64 bit.
+	- Put RRset counts in reply_info, because the number of RRs can change
+	  due to RRset updates.
+	- import of region-allocator code from nsd.
+	- set alloc special type to ub_packed_rrset_key.
+	  Uses lruhash entry overflow chain next pointer in alloc cache.
+	- doxygen documentation for region-allocator.
+	- setup for parse scratch data.
+
+5 April 2007: Wouter
+	- discussed packed rrset with Jelte.
+
+4 April 2007: Wouter
+	- moved to version 0.3.
+	- added util/data/dname.c
+	- layout of memory for rrsets.
+
+3 April 2007: Wouter
+	- detect sign of msghdr.msg_iovlen so that the cast to that type
+	  in netevent (which is there to please lint) can be correct.
+	  The type on several OSes ranges from int, int32, uint32, size_t.
+	  Detects unsigned or signed using math trick.
+	- constants for DNS flags. 
+	- compilation without locks fixup.
+	- removed include of unportable header from lookup3.c.
+	- more portable use of struct msghdr.
+	- casts for printf warning portability.
+	- tweaks to tests to port them to the testbed.
+	- 0.2 tag created.
+
+2 April 2007: Wouter
+	- check sizes of udp received messages, not too short.
+	- review changes. Some memmoves can be memcpys: 4byte aligned.
+	  set id correctly on cached answers. 
+	- review changes msgreply.c, memleak on error condition. AA flag
+	  clear on cached reply. Lowercase queries on hashing.
+	  unit test on lowercasing. Test AA bit not set on cached reply.
+	  Note that no TTLs are managed.
+
+29 March 2007: Wouter
+	- writev or sendmsg used when answering from cache.
+	  This avoids a copy of the data.
+	- do not do useless byteswap on query id. Store reply flags in uint16
+	  for easier access (and no repeated byteswapping).
+	- reviewed code.
+	- configure detects and config.h includes sys/uio.h for writev decl.
+
+28 March 2007: Wouter
+	- new config option: num-queries-per-thread.
+	- added tpkg test for answering three queries at the same time
+	  using one thread (from the query service list).
+
+27 March 2007: Wouter
+	- added test for cache and not cached answers, in testbound replays.
+	- testbound can give config file and commandline options from the
+	  replay file to unbound.
+	- created test that checks if items drop out of the cache.
+	- added word 'partitioned hash table' to documentation on slab hash.
+	  A slab hash is a partitioned hash table.
+	- worker can handle multiple queries at a time.
+
+26 March 2007: Wouter
+	- config settings for slab hash message cache.
+	- test for cached answer.
+	- Fixup deleting fake answer from testbound list.
+
+23 March 2007: Wouter
+	- review of yesterday's commits.
+	- covered up memory leak of the entry locks.
+	- answers from the cache correctly. Copies flags correctly.
+	- sanity check for incoming query replies.
+	- slabbed hash table. Much nicer contention, need dual cpu to see.
+
+22 March 2007: Wouter
+	- AIX configure check.
+	- lock-verify can handle references to locks that are created
+	  in files it has not yet read in.
+	- threaded hash table test. 
+	- unit test runs lock-verify afterwards and checks result.
+	- need writelock to update data on hash_insert.
+	- message cache code, msgreply code.
+
+21 March 2007: Wouter
+	- unit test of hash table, fixup locking problem in table_grow().
+	- fixup accounting of sizes for removing items from hashtable.
+	- unit test for hash table, single threaded test of integrity.
+	- lock-verify reports errors nicely. More quiet in operation.
+
+16 March 2007: Wouter
+	- lock-verifier, checks consistent order of locking.
+
+14 March 2007: Wouter
+	- hash table insert (and subroutines) and lookup implemented.
+	- hash table remove.
+	- unit tests for hash internal bin, lru functions.
+
+13 March 2007: Wouter
+	- lock_unprotect in checklocks.
+	- util/storage/lruhash.h for LRU hash table structure.
+
+12 March 2007: Wouter
+	- configure.ac moved to 0.2.
+	- query_info and replymsg util/data structure.
+
 9 March 2007: Wouter
 	- added rwlock writelock checking.
 	  So it will keep track of the writelock, and readlocks are enforced
@@ -6,6 +243,7 @@
 	- checklocks zeroes its destroyed lock after checking memory areas.
 	- unit test for alloc.
 	- identifier for union in checklocks to please older compilers.
+	- created 0.1 tag.
 
 8 March 2007: Wouter
 	- Reviewed checklock code.

doc/TODO

@@ -3,3 +3,18 @@ o use real entropy to make random (ID, port) numbers more random.
 o in production mode, do not free memory on exit. In debug mode, test leaks.
 o profile memory allocation, and if performance issues, use special memory
   allocator. For example, with caches per thread.
+o #define BIT_... different on bigendian and smallendian systems so that
+  the htons on flags is not needed to send a message from the cache.
+o possible optimization with delayed malloc of msgreply after parse
+  (reuse parse structures and ptrs to packet in meantime).
+o possible optimization, so that precious id number resource is not depleted
+  by parsing of messages. Delay malloc, as above, or try to reverse release
+  special id numbers, and if you release the next_id number for the thread
+  it reuses that id number.
+o use rbtree to compress domain names in messages, sorted AXFRs of 16Kb
+  and so on. For current packets, even worstcase, list, is ok. Only if we
+  start to send out compressed AXFRs then it becomes a problem.
+  We can cap AXFR packets on 256 or 1024 domain names, for example.
+o speed up pkt domain name decompression loop detection using counter perhaps.
+o detect OS/400 pthreads implementation that allows upgrading to writelock
+  on pthreads rwlocks and use it to examine-rd before storing-wr rrset cache.

doc/example.conf

@@ -20,7 +20,10 @@ server:
 
 	# specify the interfaces to answer queries from by ip-address.
 	# If you give none the default (all) interface is used.
-	# interface: 127.0.0.1
+	# specify every interface on a new 'interface:' labelled line.
+	# interface: 192.0.2.153
+	# interface: 192.0.2.154
+	# interface: 2001:DB8::5
 
 	# port to answer queries from
 	# port: 53
@@ -36,6 +39,30 @@ server:
 	# But also takes more system resources (for open sockets).
 	# outgoing-range: 16
 
+	# number of outgoing simultaneous tcp buffers to hold per thread.
+	# outgoing-num-tcp: 10
+
+	# the amount of memory to use for the message cache.
+	# in bytes. default is 4 Mb
+	# msg-cache-size: 4194304
+
+	# the number of slabs to use for the message cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# msg-cache-slabs: 4
+
+	# the number of queries that a thread gets to service.
+	# num-queries-per-thread: 1024
+
+	# the amount of memory to use for the RRset cache.
+	# in bytes. default is 4 Mb
+	# rrset-cache-size: 4194304
+
+	# the number of slabs to use for the RRset cache.
+	# the number of slabs must be a power of 2.
+	# more slabs reduce lock contention, but fragment memory usage.
+	# rrset-cache-slabs: 4
+
 	# Enable IPv4, "yes" or "no".
 	# do-ip4: yes
 

doc/unbound.conf.5

@@ -63,6 +63,22 @@ Number of ports to open. This number is opened per thread for every outgoing
 query interface. Must be at least 1. Default is 16.
 Larger numbers give more protection against spoofing attempts, but need
 extra resources from the operating system.
+.It \fBoutgoing-num-tcp:\fR <number>
+Number of outgoing TCP buffers to allocate per thread. Default is 10. If set
+to 0, or if do_tcp is "no", no TCP queries to authoritative servers are done.
+.It \fBmsg-cache-size:\fR <number>
+Number of bytes size of the message cache. Default is 4 megabytes.
+.It \fBmsg-cache-slabs:\fR <number>
+Number of slabs in the message cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. Setting (close) to the number of cpus is a 
+reasonable guess.
+.It \fBnum-queries-per-thread:\fR <number>
+The number of queries that every thread will service simultaneously.
+.It \fBrrset-cache-size:\fR <number>
+Number of bytes size of the RRset cache. Default is 4 megabytes.
+.It \fBrrset-cache-slabs:\fR <number>
+Number of slabs in the RRset cache. Slabs reduce lock contention by threads.
+Must be set to a power of 2. 
 .It \fBdo-ip4:\fR <yes or no>
 Enable or disable whether ip4 queries are answered. Default is yes.
 .It \fBdo-ip6:\fR <yes or no>

services/listen_dnsport.c

@@ -54,8 +54,6 @@
 
 /** number of queued TCP connections for listen() */
 #define TCP_BACKLOG 5 
-/** number of simultaneous open TCP connections for queries */
-#define TCP_COUNT 10 
 
 /**
  * Debug print of the getaddrinfo returned address.
@@ -310,7 +308,7 @@ listen_create(struct comm_base* base, struct listen_port* ports,
 			cp = comm_point_create_udp(base, ports->fd, 
 				front->udp_buff, cb, cb_arg);
 		else 	cp = comm_point_create_tcp(base, ports->fd, 
-				TCP_COUNT, bufsize, cb, cb_arg);
+				TCP_ACCEPT_COUNT, bufsize, cb, cb_arg);
 		if(!cp) {
 			log_err("can't create commpoint");	
 			listen_delete(front);

services/listen_dnsport.h

@@ -48,6 +48,9 @@ struct listen_list;
 struct addrinfo;
 struct config_file;
 
+/** number of simultaneous open TCP connections for queries */
+#define TCP_ACCEPT_COUNT 10 
+
 /**
  * Listening for queries structure.
  * Contains list of query-listen sockets.

services/outside_network.c

@@ -109,6 +109,110 @@ pending_cmp(const void* key1, const void* key2)
 	}
 }
 
+/** delete waiting_tcp entry. Does not unlink from waiting list. 
+ * @param w: to delete.
+ */
+static void
+waiting_tcp_delete(struct waiting_tcp* w)
+{
+	if(!w) return;
+	if(w->timer)
+		comm_timer_delete(w->timer);
+	free(w);
+}
+
+/** use next free buffer to service a tcp query */
+static void
+outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt)
+{
+	struct pending_tcp* pend = w->outnet->tcp_free;
+	int s;
+	log_assert(pend);
+	log_assert(pkt);
+	/* open socket */
+#ifndef INET6
+	if(addr_is_ip6(addr))
+		s = socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP);
+	else
+#endif
+		s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
+	if(s == -1) {
+		log_err("outgoing tcp: socket: %s", strerror(errno));
+		log_addr(&w->addr, w->addrlen);
+		(void)(*w->cb)(NULL, w->cb_arg, NETEVENT_CLOSED, NULL);
+		waiting_tcp_delete(w);
+		return;
+	}
+	fd_set_nonblock(s);
+	if(connect(s, (struct sockaddr*)&w->addr, w->addrlen) == -1) {
+		if(errno != EINPROGRESS) {
+			log_err("outgoing tcp: connect: %s", strerror(errno));
+			log_addr(&w->addr, w->addrlen);
+			close(s);
+			(void)(*w->cb)(NULL, w->cb_arg, NETEVENT_CLOSED, NULL);
+			waiting_tcp_delete(w);
+			return;
+		}
+	}
+	w->pkt = NULL;
+	w->next_waiting = (void*)pend;
+	pend->id = LDNS_ID_WIRE(pkt);
+	w->outnet->tcp_free = pend->next_free;
+	pend->next_free = NULL;
+	pend->query = w;
+	ldns_buffer_clear(pend->c->buffer);
+	ldns_buffer_write(pend->c->buffer, pkt, w->pkt_len);
+	ldns_buffer_flip(pend->c->buffer);
+	pend->c->tcp_is_reading = 0;
+	pend->c->tcp_byte_count = 0;
+	comm_point_start_listening(pend->c, s, -1);
+	return;
+}
+
+/** see if buffers can be used to service TCP queries. */
+static void
+use_free_buffer(struct outside_network* outnet)
+{
+	struct waiting_tcp* w;
+	while(outnet->tcp_free && outnet->tcp_wait_first) {
+		w = outnet->tcp_wait_first;
+		outnet->tcp_wait_first = w->next_waiting;
+		if(outnet->tcp_wait_last == w)
+			outnet->tcp_wait_last = NULL;
+		outnet_tcp_take_into_use(w, w->pkt);
+	}
+}
+
+/** callback for pending tcp connections */
+static int 
+outnet_tcp_cb(struct comm_point* c, void* arg, int error,
+	struct comm_reply *reply_info)
+{
+	struct pending_tcp* pend = (struct pending_tcp*)arg;
+	struct outside_network* outnet = pend->query->outnet;
+	verbose(VERB_ALGO, "outnettcp cb");
+	if(error != NETEVENT_NOERROR) {
+		log_info("outnettcp got tcp error %d", error);
+		/* pass error below and exit */
+	} else {
+		/* check ID */
+		if(ldns_buffer_limit(c->buffer) < sizeof(uint16_t) ||
+			LDNS_ID_WIRE(ldns_buffer_begin(c->buffer))!=pend->id) {
+			log_info("outnettcp: bad ID in reply, from:");
+			log_addr(&pend->query->addr, pend->query->addrlen);
+			error = NETEVENT_CLOSED;
+		}
+	}
+	(void)(*pend->query->cb)(c, pend->query->cb_arg, error, reply_info);
+	comm_point_close(c);
+	pend->next_free = outnet->tcp_free;
+	outnet->tcp_free = pend;
+	waiting_tcp_delete(pend->query);
+	pend->query = NULL;
+	use_free_buffer(outnet);
+	return 0;
+}
+
 /** callback for incoming udp answers from the network. */
 static int 
 outnet_udp_cb(struct comm_point* c, void* arg, int error,
@@ -123,6 +227,10 @@ outnet_udp_cb(struct comm_point* c, void* arg, int error,
 		log_info("outnetudp got udp error %d", error);
 		return 0;
 	}
+	if(ldns_buffer_limit(c->buffer) < LDNS_HEADER_SIZE) {
+		log_info("outnetudp udp too short");
+		return 0;
+	}
 	log_assert(reply_info);
 
 	/* setup lookup key */
@@ -267,10 +375,35 @@ pending_udp_timer_cb(void *arg)
 	pending_delete(p->outnet, p);
 }
 
+/** create pending_tcp buffers */
+static int
+create_pending_tcp(struct outside_network* outnet, size_t bufsize)
+{
+	size_t i;
+	if(outnet->num_tcp == 0)
+		return 1; /* no tcp needed, nothing to do */
+	if(!(outnet->tcp_conns = (struct pending_tcp **)calloc(
+			outnet->num_tcp, sizeof(struct pending_tcp*))))
+		return 0;
+	for(i=0; i<outnet->num_tcp; i++) {
+		if(!(outnet->tcp_conns[i] = (struct pending_tcp*)calloc(1, 
+			sizeof(struct pending_tcp))))
+			return 0;
+		outnet->tcp_conns[i]->next_free = outnet->tcp_free;
+		outnet->tcp_free = outnet->tcp_conns[i];
+		outnet->tcp_conns[i]->c = comm_point_create_tcp_out(
+			outnet->base, bufsize, outnet_tcp_cb, 
+			outnet->tcp_conns[i]);
+		if(!outnet->tcp_conns[i]->c)
+			return 0;
+	}
+	return 1;
+}
+
 struct outside_network* 
 outside_network_create(struct comm_base *base, size_t bufsize, 
 	size_t num_ports, char** ifs, int num_ifs, int do_ip4, 
-	int do_ip6, int port_base)
+	int do_ip6, int port_base, size_t num_tcp)
 {
 	struct outside_network* outnet = (struct outside_network*)
 		calloc(1, sizeof(struct outside_network));
@@ -280,6 +413,7 @@ outside_network_create(struct comm_base *base, size_t bufsize,
 		return NULL;
 	}
 	outnet->base = base;
+	outnet->num_tcp = num_tcp;
 #ifndef INET6
 	do_ip6 = 0;
 #endif
@@ -291,7 +425,8 @@ outside_network_create(struct comm_base *base, size_t bufsize,
 			outnet->num_udp4+1, sizeof(struct comm_point*))) ||
 		!(outnet->udp6_ports = (struct comm_point **)calloc(
 			outnet->num_udp6+1, sizeof(struct comm_point*))) ||
-		!(outnet->pending = rbtree_create(pending_cmp)) ) {
+		!(outnet->pending = rbtree_create(pending_cmp)) ||
+		!create_pending_tcp(outnet, bufsize)) {
 		log_err("malloc failed");
 		outside_network_delete(outnet);
 		return NULL;
@@ -372,6 +507,25 @@ outside_network_delete(struct outside_network* outnet)
 			comm_point_delete(outnet->udp6_ports[i]);
 		free(outnet->udp6_ports);
 	}
+	if(outnet->tcp_conns) {
+		size_t i;
+		for(i=0; i<outnet->num_tcp; i++)
+			if(outnet->tcp_conns[i]) {
+				comm_point_delete(outnet->tcp_conns[i]->c);
+				waiting_tcp_delete(outnet->tcp_conns[i]->query);
+				free(outnet->tcp_conns[i]);
+			}
+		free(outnet->tcp_conns);
+	}
+	if(outnet->tcp_wait_first) {
+		struct waiting_tcp* p = outnet->tcp_wait_first, *np;
+		while(p) {
+			np = p->next_waiting;
+			waiting_tcp_delete(p);
+			p = np;
+		}
+	}
+
 	free(outnet);
 }
 
@@ -527,3 +681,86 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	tv.tv_usec = 0;
 	comm_timer_set(pend->timer, &tv);
 }
+
+/** callback for outgoing TCP timer event */
+static void
+outnet_tcptimer(void* arg)
+{
+	struct waiting_tcp* w = (struct waiting_tcp*)arg;
+	struct outside_network* outnet = w->outnet;
+	if(w->pkt) {
+		/* it is on the waiting list */
+		struct waiting_tcp* p=outnet->tcp_wait_first, *prev=NULL;
+		while(p) {
+			if(p == w) {
+				if(prev) prev->next_waiting = w->next_waiting;
+				else	outnet->tcp_wait_first=w->next_waiting;
+				outnet->tcp_wait_last = prev;
+				break;
+			}
+			prev = p;
+			p=p->next_waiting;
+		}
+	} else {
+		/* it was in use */
+		struct pending_tcp* pend=(struct pending_tcp*)w->next_waiting;
+		comm_point_close(pend->c);
+		pend->query = NULL;
+		pend->next_free = outnet->tcp_free;
+		outnet->tcp_free = pend;
+	}
+	(void)(*w->cb)(NULL, w->cb_arg, NETEVENT_TIMEOUT, NULL);
+	waiting_tcp_delete(w);
+	use_free_buffer(outnet);
+}
+
+void 
+pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet, 
+	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
+	comm_point_callback_t* callback, void* callback_arg,
+	struct ub_randstate* rnd)
+{
+	struct pending_tcp* pend = outnet->tcp_free;
+	struct waiting_tcp* w;
+	struct timeval tv;
+	uint16_t id;
+	/* if no buffer is free allocate space to store query */
+	w = (struct waiting_tcp*)malloc(sizeof(struct waiting_tcp) 
+		+ (pend?0:ldns_buffer_limit(packet)));
+	if(!w) {
+		/* callback user for the error */
+		(void)(*callback)(NULL, callback_arg, NETEVENT_CLOSED, NULL);
+		return;
+	}
+	if(!(w->timer = comm_timer_create(outnet->base, outnet_tcptimer, w))) {
+		free(w);
+		(void)(*callback)(NULL, callback_arg, NETEVENT_CLOSED, NULL);
+		return;
+	}
+	w->pkt = NULL;
+	w->pkt_len = ldns_buffer_limit(packet);
+	/* id uses lousy random() TODO use better and entropy */
+	id = ((unsigned)ub_random(rnd)>>8) & 0xffff;
+	LDNS_ID_SET(ldns_buffer_begin(packet), id);
+	memcpy(&w->addr, addr, addrlen);
+	w->addrlen = addrlen;
+	w->outnet = outnet;
+	w->cb = callback;
+	w->cb_arg = callback_arg;
+	tv.tv_sec = timeout;
+	tv.tv_usec = 0;
+	comm_timer_set(w->timer, &tv);
+	if(pend) {
+		/* we have a buffer available right now */
+		outnet_tcp_take_into_use(w, ldns_buffer_begin(packet));
+	} else {
+		/* queue up */
+		w->pkt = (uint8_t*)w + sizeof(struct waiting_tcp);
+		memmove(w->pkt, ldns_buffer_begin(packet), w->pkt_len);
+		w->next_waiting = NULL;
+		if(outnet->tcp_wait_last)
+			outnet->tcp_wait_last->next_waiting = w;
+		else	outnet->tcp_wait_first = w;
+		outnet->tcp_wait_last = w;
+	}
+}

services/outside_network.h

@@ -49,6 +49,8 @@
 struct pending;
 struct pending_timeout;
 struct ub_randstate;
+struct pending_tcp;
+struct waiting_tcp;
 
 /**
  * Send queries to outside servers and wait for answers from servers.
@@ -77,8 +79,24 @@ struct outside_network {
 	/** number of udp6 ports */
 	size_t num_udp6;
 
-	/** pending answers. sorted by id, addr */
+	/** pending udp answers. sorted by id, addr */
 	rbtree_t *pending;
+
+	/**
+	 * Array of tcp pending used for outgoing TCP connections.
+	 * Each can be used to establish a TCP connection with a server.
+	 * The file descriptors are -1 if its free, need to be opened for 
+	 * the tcp connection. Can be used for ip4 and ip6.
+	 */
+	struct pending_tcp **tcp_conns;
+	/** number of tcp communication points. */
+	size_t num_tcp;
+	/** list of tcp comm points that are free for use */
+	struct pending_tcp* tcp_free;
+	/** list of tcp queries waiting for a buffer */
+	struct waiting_tcp* tcp_wait_first;
+	/** last of waiting query list */
+	struct waiting_tcp* tcp_wait_last;
 };
 
 /**
@@ -105,6 +123,53 @@ struct pending {
 	struct outside_network* outnet;
 };
 
+/**
+ * Pending TCP query to server.
+ */
+struct pending_tcp {
+	/** next in list of free tcp comm points, or NULL. */
+	struct pending_tcp* next_free;
+	/** the ID for the query; checked in reply */
+	uint16_t id;
+	/** tcp comm point it was sent on (and reply must come back on). */
+	struct comm_point* c;
+	/** the query being serviced, NULL if the pending_tcp is unused. */
+	struct waiting_tcp* query;
+};
+
+/**
+ * Query waiting for TCP buffer.
+ */
+struct waiting_tcp {
+	/** 
+	 * next in waiting list.
+	 * if pkt==0, this points to the pending_tcp structure.
+	 */
+	struct waiting_tcp* next_waiting;
+	/** timeout event; timer keeps running whether the query is
+	 * waiting for a buffer or the tcp reply is pending */
+	struct comm_timer* timer;
+	/** the outside network it is part of */
+	struct outside_network* outnet;
+	/** remote address. */
+	struct sockaddr_storage addr;
+	/** length of addr field in use. */
+	socklen_t addrlen;
+	/** 
+	 * The query itself, the query packet to send.
+	 * allocated after the waiting_tcp structure.
+	 * set to NULL when the query is serviced and it part of pending_tcp.
+	 * if this is NULL, the next_waiting points to the pending_tcp.
+	 */
+	uint8_t* pkt;
+	/** length of query packet. */
+	size_t pkt_len;
+	/** callback for the timeout, error or reply to the message */
+	comm_point_callback_t* cb;
+	/** callback user argument */
+	void* cb_arg;
+};
+
 /**
  * Create outside_network structure with N udp ports.
  * @param base: the communication base to use for event handling.
@@ -117,11 +182,12 @@ struct pending {
  * @param do_ip6: service IP6.
  * @param port_base: if -1 system assigns ports, otherwise try to get
  *    the ports numbered from this starting number.
+ * @param num_tcp: number of outgoing tcp buffers to preallocate.
  * @return: the new structure (with no pending answers) or NULL on error.
  */
 struct outside_network* outside_network_create(struct comm_base* base,
 	size_t bufsize, size_t num_ports, char** ifs, int num_ifs,
-	int do_ip4, int do_ip6, int port_base);
+	int do_ip4, int do_ip6, int port_base, size_t num_tcp);
 
 /**
  * Delete outside_network structure.
@@ -148,6 +214,27 @@ void pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	comm_point_callback_t* callback, void* callback_arg,
 	struct ub_randstate* rnd);
 
+/**
+ * Send TCP query. May wait for TCP buffer. Selects ID to be random, and 
+ * checks id.
+ * @param outnet: provides the event handling.
+ * @param packet: wireformat query to send to destination. copied from.
+ * @param addr: address to send to.
+ * @param addrlen: length of addr.
+ * @param timeout: in seconds from now.
+ *    Timer starts running now. Timer may expire if all buffers are used,
+ *    without any query been sent to the server yet.
+ * @param callback: function to call on error, timeout or reply.
+ *    The routine does not return an error, instead it calls the callback,
+ *    with an error code if an error happens.
+ * @param callback_arg: user argument for callback function.
+ * @param rnd: random state for generating ID.
+ */
+void pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet, 
+	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
+	comm_point_callback_t* callback, void* callback_arg,
+	struct ub_randstate* rnd);
+
 /**
  * Delete pending answer.
  * @param outnet: outside network the pending query is part of.

testcode/checklocks.c

@@ -46,18 +46,26 @@
  * are passed to thread_create to make the thread numbers here the same as 
  * those used for logging which is nice.
  *
- * Todo: - check global ordering of instances of locks.
- *	 - refcount statistics.
+ * Todo: 
  *	 - debug status print, of thread lock stacks, and current waiting.
  */
 #ifdef USE_THREAD_DEBUG
 
+/** How long to wait before lock attempt is a failure. */
+#define CHECK_LOCK_TIMEOUT 30 /* seconds */
+/** How long to wait before join attempt is a failure. */
+#define CHECK_JOIN_TIMEOUT 240 /* seconds */
+
 /** if key has been created */
 static int key_created = 0;
 /** we hide the thread debug info with this key. */
 static ub_thread_key_t thr_debug_key;
 /** the list of threads, so all threads can be examined. NULL if unused. */
 static struct thr_check* thread_infos[THRDEBUG_MAX_THREADS];
+/** do we check locking order */
+int check_locking_order = 1;
+/** the pid of this runset, reasonably unique. */
+static pid_t check_lock_pid;
 
 /** print pretty lock error and exit */
 static void lock_error(struct checked_lock* lock, 
@@ -66,8 +74,9 @@ static void lock_error(struct checked_lock* lock,
 	log_err("lock error (description follows)");
 	log_err("Created at %s %s:%d", lock->create_func, 
 		lock->create_file, lock->create_line);
-	log_err("Previously %s %s:%d", lock->holder_func, 
-		lock->holder_file, lock->holder_line);
+	if(lock->holder_func && lock->holder_file)
+		log_err("Previously %s %s:%d", lock->holder_func, 
+			lock->holder_file, lock->holder_line);
 	log_err("At %s %s:%d", func, file, line);
 	log_err("Error for %s lock: %s",
 		(lock->type==check_lock_mutex)?"mutex": (
@@ -132,6 +141,31 @@ lock_protect(void *p, void* area, size_t size)
 	LOCKRET(pthread_mutex_unlock(&lock->lock));
 }
 
+/** remove protected region */
+void
+lock_unprotect(void* mangled, void* area)
+{
+	struct checked_lock* lock = *(struct checked_lock**)mangled;
+	struct protected_area* p, **prevp;
+	if(!lock) 
+		return;
+	acquire_locklock(lock, __func__, __FILE__, __LINE__);
+	p = lock->prot;
+	prevp = &lock->prot;
+	while(p) {
+		if(p->region == area) {
+			*prevp = p->next;
+			free(p->hold);
+			free(p);
+			LOCKRET(pthread_mutex_unlock(&lock->lock));
+			return;
+		}
+		prevp = &p->next;
+		p = p->next;
+	}
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+}
+
 /** 
  * Check protected memory region. Memory compare. Exit on error. 
  * @param lock: which lock to check.
@@ -166,6 +200,45 @@ prot_store(struct checked_lock* lock)
 	}
 }
 
+/** write lock trace info to file, while you hold those locks. */
+static void
+ordercheck_locklock(struct thr_check* thr, struct checked_lock* lock)
+{
+	int info[4];
+	if(!check_locking_order) return;
+	if(!thr->holding_first) return; /* no older lock, no info */
+	/* write: <lock id held> <lock id new> <file> <line> */
+	info[0] = thr->holding_first->create_thread;
+	info[1] = thr->holding_first->create_instance;
+	info[2] = lock->create_thread;
+	info[3] = lock->create_instance;
+	if(fwrite(info, 4*sizeof(int), 1, thr->order_info) != 1 ||
+		fwrite(lock->holder_file, strlen(lock->holder_file)+1, 1, 
+		thr->order_info) != 1 ||
+		fwrite(&lock->holder_line, sizeof(int), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
+
+/** write ordercheck lock creation details to file. */
+static void 
+ordercheck_lockcreate(struct thr_check* thr, struct checked_lock* lock)
+{
+	/* write: <ffff = create> <lock id> <file> <line> */
+	int cmd = -1;
+	if(!check_locking_order) return;
+
+	if( fwrite(&cmd, sizeof(int), 1, thr->order_info) != 1 ||
+		fwrite(&lock->create_thread, sizeof(int), 1, 
+			thr->order_info) != 1 ||
+		fwrite(&lock->create_instance, sizeof(int), 1, 
+			thr->order_info) != 1 ||
+		fwrite(lock->create_file, strlen(lock->create_file)+1, 1, 
+			thr->order_info) != 1 ||
+		fwrite(&lock->create_line, sizeof(int), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
 
 /** alloc struct, init lock empty */
 void 
@@ -174,13 +247,21 @@ checklock_init(enum check_lock_type type, struct checked_lock** lock,
 {
 	struct checked_lock* e = (struct checked_lock*)calloc(1, 
 		sizeof(struct checked_lock));
+	struct thr_check *thr = (struct thr_check*)pthread_getspecific(
+		thr_debug_key);
 	if(!e)
 		fatal_exit("%s %s %d: out of memory", func, file, line);
+	if(!thr)
+		fatal_exit("%s %s %d: lock_init no thread info", func, file,
+			line);
 	*lock = e;
 	e->type = type;
 	e->create_func = func;
 	e->create_file = file;
 	e->create_line = line;
+	e->create_thread = thr->num;
+	e->create_instance = thr->locks_created++;
+	ordercheck_lockcreate(thr, e);
 	LOCKRET(pthread_mutex_init(&e->lock, NULL));
 	switch(e->type) {
 		case check_lock_mutex:
@@ -228,7 +309,7 @@ void
 checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
         const char* func, const char* file, int line)
 {
-	const size_t contention_interest = 10;
+	const size_t contention_interest = 1; /* promille contented locks */
 	struct checked_lock* e;
 	if(!lock) 
 		return;
@@ -247,11 +328,14 @@ checklock_destroy(enum check_lock_type type, struct checked_lock** lock,
 	*lock = NULL; /* use after free will fail */
 	LOCKRET(pthread_mutex_unlock(&e->lock));
 
-	/* contention */
-	if(e->contention_count > contention_interest) {
-		log_info("lock created %s %s %d has contention %u",
+	/* contention, look at fraction in trouble. */
+	if(e->history_count > 1 &&
+	   1000*e->contention_count/e->history_count > contention_interest) {
+		log_info("lock created %s %s %d has contention %u of %u (%d%%)",
 			e->create_func, e->create_file, e->create_line,
-			(unsigned int)e->contention_count);
+			(unsigned int)e->contention_count, 
+			(unsigned int)e->history_count,
+			100*e->contention_count/e->history_count);
 	}
 
 	/* delete it */
@@ -288,6 +372,7 @@ finish_acquire_lock(struct thr_check* thr, struct checked_lock* lock,
 	lock->holder_func = func;
 	lock->holder_file = file;
 	lock->holder_line = line;
+	ordercheck_locklock(thr, lock);
 	
 	/* insert in thread lock list, as first */
 	lock->prev_held_lock[thr->num] = NULL;
@@ -355,6 +440,7 @@ checklock_lockit(enum check_lock_type type, struct checked_lock* lock,
 
 	acquire_locklock(lock, func, file, line);
 	lock->contention_count += contend;
+	lock->history_count++;
 	if(exclusive && lock->hold_count > 0)
 		lock_error(lock, func, file, line, "got nonexclusive lock");
 	if(type==check_lock_rwlock && getwr && lock->writeholder)
@@ -520,6 +606,26 @@ checklock_unlock(enum check_lock_type type, struct checked_lock* lock,
 	}
 }
 
+/** open order info debug file, thr->num must be valid. */
+static void 
+open_lockorder(struct thr_check* thr)
+{
+	char buf[24];
+	time_t t;
+	snprintf(buf, sizeof(buf), "ublocktrace.%d", thr->num);
+	thr->order_info = fopen(buf, "w");
+	if(!thr->order_info)
+		fatal_exit("could not open %s: %s", buf, strerror(errno));
+	thr->locks_created = 0;
+	t = time(NULL);
+	/* write: <time_stamp> <runpid> <thread_num> */
+	if(fwrite(&t, sizeof(t), 1, thr->order_info) != 1 ||
+		fwrite(&thr->num, sizeof(thr->num), 1, thr->order_info) != 1 || 
+		fwrite(&check_lock_pid, sizeof(check_lock_pid), 1, 
+		thr->order_info) != 1)
+		log_err("fwrite: %s", strerror(errno));
+}
+
 /** checklock thread main, Inits thread structure. */
 static void* checklock_main(void* arg)
 {
@@ -532,8 +638,12 @@ static void* checklock_main(void* arg)
 	log_assert(thread_infos[thr->num] == NULL);
 	thread_infos[thr->num] = thr;
 	LOCKRET(pthread_setspecific(thr_debug_key, thr));
+	if(check_locking_order)
+		open_lockorder(thr);
 	ret = thr->func(thr->arg);
 	thread_infos[thr->num] = NULL;
+	if(check_locking_order)
+		fclose(thr->order_info);
 	free(thr);
 	return ret;
 }
@@ -547,9 +657,12 @@ void checklock_start()
 		if(!thisthr)
 			fatal_exit("thrcreate: out of memory");
 		key_created = 1;
+		check_lock_pid = getpid();
 		LOCKRET(pthread_key_create(&thr_debug_key, NULL));
 		LOCKRET(pthread_setspecific(thr_debug_key, thisthr));
 		thread_infos[0] = thisthr;
+		if(check_locking_order)
+			open_lockorder(thisthr);
 	}
 }
 
@@ -558,6 +671,8 @@ void checklock_stop()
 {
 	if(key_created) {
 		int i;
+		if(check_locking_order)
+			fclose(thread_infos[0]->order_info);
 		free(thread_infos[0]);
 		thread_infos[0] = NULL;
 		for(i = 0; i < THRDEBUG_MAX_THREADS; i++)

testcode/checklocks.h

@@ -71,12 +71,10 @@
 /******************* THREAD DEBUG ************************/
 #include <pthread.h>
 
-/** How long to wait before lock attempt is a failure. */
-#define CHECK_LOCK_TIMEOUT 5 /* seconds */
-/** How long to wait before join attempt is a failure. */
-#define CHECK_JOIN_TIMEOUT 120 /* seconds */
 /** How many threads to allocate for */
 #define THRDEBUG_MAX_THREADS 32 /* threads */
+/** do we check locking order */
+extern int check_locking_order;
 
 /**
  * Protection memory area.
@@ -106,6 +104,10 @@ struct thr_check {
 	void* arg;
 	/** number of thread in list structure */
 	int num;
+	/** instance number - how many locks have been created by thread */
+	int locks_created;
+	/** file to write locking order information to */
+	FILE* order_info;
 	/** 
 	 * List of locks that this thread is holding, double
 	 * linked list. The first element is the most recent lock acquired.
@@ -128,8 +130,12 @@ struct checked_lock {
 	const char* create_func, *create_file;
 	/** where was this lock created */
 	int create_line;
+	/** unique instance identifier */
+	int create_thread, create_instance;
 	/** contention count */
 	size_t contention_count;
+	/** number of times locked, ever */
+	size_t history_count;
 	/** hold count (how many threads are holding this lock) */
 	int hold_count;
 	/** how many threads are waiting for this lock */
@@ -182,6 +188,14 @@ struct checked_lock {
  */
 void lock_protect(void* lock, void* area, size_t size);
 
+/**
+ * Remove protected area from lock.
+ * No need to call this when deleting the lock.
+ * @param lock: the lock, any type, (struct checked_lock**).
+ * @param area: pointer to memory.
+ */
+void lock_unprotect(void* lock, void* area);
+
 /**
  * Initialise checklock. Sets up internal debug structures.
  */

testcode/fake_event.c

@@ -47,6 +47,7 @@
 #include "config.h"
 #include "testcode/fake_event.h"
 #include "util/netevent.h"
+#include "util/net_help.h"
 #include "services/listen_dnsport.h"
 #include "services/outside_network.h"
 #include "testcode/replay.h"
@@ -280,7 +281,7 @@ static void
 answer_check_it(struct replay_runtime* runtime)
 {
 	struct replay_answer* ans = runtime->answer_list, 
-		**prev = &runtime->answer_list;
+		*prev = NULL;
 	log_assert(runtime && runtime->now && 
 		runtime->now->evt_type == repevt_front_reply);
 	while(ans) {
@@ -294,12 +295,16 @@ answer_check_it(struct replay_runtime* runtime)
 			log_info("testbound: do STEP %d %s", 
 				runtime->now->time_step,
 				repevt_string(runtime->now->evt_type));
-			*prev = ans->next;
+			if(prev)
+				prev->next = ans->next;
+			else 	runtime->answer_list = ans->next;
+			if(!ans->next)
+				runtime->answer_last = prev;
 			delete_replay_answer(ans);
 			ans = n;
 			return;
 		} else {
-			prev = &ans->next;
+			prev = ans;
 			ans = ans->next;
 		}
 	}
@@ -605,6 +610,15 @@ comm_point_send_reply(struct comm_reply* repinfo)
 	log_pkt("reply pkt: ", ans->pkt);
 }
 
+void 
+comm_point_send_reply_iov(struct comm_reply* repinfo, struct iovec* iov,
+        size_t iovlen)
+{
+	/* skip tcp len at [0]. */
+	write_iov_buffer(repinfo->c->buffer, iov+1, iovlen-1);
+	comm_point_send_reply(repinfo);
+}
+
 void 
 comm_point_drop_reply(struct comm_reply* repinfo)
 {
@@ -619,7 +633,8 @@ struct outside_network*
 outside_network_create(struct comm_base* base, size_t bufsize, 
 	size_t ATTR_UNUSED(num_ports), char** ATTR_UNUSED(ifs), 
 	int ATTR_UNUSED(num_ifs), int ATTR_UNUSED(do_ip4), 
-	int ATTR_UNUSED(do_ip6), int ATTR_UNUSED(port_base))
+	int ATTR_UNUSED(do_ip6), int ATTR_UNUSED(port_base),
+	size_t ATTR_UNUSED(num_tcp))
 {
 	struct outside_network* outnet =  calloc(1, 
 		sizeof(struct outside_network));
@@ -688,6 +703,53 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	runtime->pending_list = pend;
 }
 
+void 
+pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
+	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
+	comm_point_callback_t* callback, void* callback_arg,
+	struct ub_randstate* ATTR_UNUSED(rnd))
+{
+	struct replay_runtime* runtime = (struct replay_runtime*)outnet->base;
+	struct fake_pending* pend = (struct fake_pending*)calloc(1,
+		sizeof(struct fake_pending));
+	ldns_status status;
+	log_assert(pend);
+	pend->buffer = ldns_buffer_new(ldns_buffer_capacity(packet));
+	log_assert(pend->buffer);
+	ldns_buffer_write(pend->buffer, ldns_buffer_begin(packet),
+		ldns_buffer_limit(packet));
+	ldns_buffer_flip(pend->buffer);
+	memcpy(&pend->addr, addr, addrlen);
+	pend->addrlen = addrlen;
+	pend->callback = callback;
+	pend->cb_arg = callback_arg;
+	pend->timeout = timeout;
+	pend->transport = transport_tcp;
+	pend->pkt = NULL;
+	status = ldns_buffer2pkt_wire(&pend->pkt, packet);
+	if(status != LDNS_STATUS_OK) {
+		log_err("ldns error parsing tcp output packet: %s",
+			ldns_get_errorstr_by_id(status));
+		fatal_exit("Sending unparseable DNS packets to servers!");
+	}
+	log_pkt("pending tcp pkt: ", pend->pkt);
+
+	/* see if it matches the current moment */
+	if(runtime->now && runtime->now->evt_type == repevt_back_query &&
+		find_match(runtime->now->match, pend->pkt, pend->transport)) {
+		log_info("testbound: matched pending to event. "
+			"advance time between events.");
+		log_info("testbound: do STEP %d %s", runtime->now->time_step,
+			repevt_string(runtime->now->evt_type));
+		advance_moment(runtime);
+		/* still create the pending, because we need it to callback */
+	} 
+	log_info("testbound: created fake pending");
+	/* add to list */
+	pend->next = runtime->pending_list;
+	runtime->pending_list = pend;
+}
+
 struct listen_port* listening_ports_open(struct config_file* ATTR_UNUSED(cfg))
 {
 	return calloc(1, 1);

testcode/ldns-testpkts.c

@@ -106,6 +106,8 @@ static void matchline(const char* line, struct entry* e)
 			e->match_qname = true;
 		} else if(str_keyword(&parse, "all")) {
 			e->match_all = true;
+		} else if(str_keyword(&parse, "ttl")) {
+			e->match_ttl = true;
 		} else if(str_keyword(&parse, "UDP")) {
 			e->match_transport = transport_udp;
 		} else if(str_keyword(&parse, "TCP")) {
@@ -217,6 +219,7 @@ static struct entry* new_entry()
 	e->match_qtype = false;
 	e->match_qname = false;
 	e->match_all = false;
+	e->match_ttl = false;
 	e->match_serial = false;
 	e->ixfr_soa_serial = 0;
 	e->match_transport = transport_any;
@@ -555,7 +558,7 @@ static uint32_t get_serial(ldns_pkt* p)
 
 /** match two rr lists */
 static int
-match_list(ldns_rr_list* q, ldns_rr_list *p)
+match_list(ldns_rr_list* q, ldns_rr_list *p, bool mttl)
 {
 	size_t i;
 	if(ldns_rr_list_rr_count(q) != ldns_rr_list_rr_count(p))
@@ -567,6 +570,11 @@ match_list(ldns_rr_list* q, ldns_rr_list *p)
 			verbose(3, "rr %d different", i);
 			return 0;
 		}
+		if(mttl && ldns_rr_ttl(ldns_rr_list_rr(q, i)) !=
+			ldns_rr_ttl(ldns_rr_list_rr(p, i))) {
+			verbose(3, "rr %d ttl different", i);
+			return 0;
+		}
 	}
 	return 1;
 }
@@ -583,7 +591,7 @@ cmp_bool(int x, int y)
 
 /** match all of the packet */
 static int
-match_all(ldns_pkt* q, ldns_pkt* p)
+match_all(ldns_pkt* q, ldns_pkt* p, bool mttl)
 {
 	if(ldns_pkt_get_opcode(q) != ldns_pkt_get_opcode(p)) 
 	{ verbose(3, "allmatch: opcode different"); return 0;}
@@ -613,13 +621,13 @@ match_all(ldns_pkt* q, ldns_pkt* p)
 	{ verbose(3, "allmatch: nscount different"); return 0;}
 	if(ldns_pkt_arcount(q) != ldns_pkt_arcount(p))
 	{ verbose(3, "allmatch: arcount different"); return 0;}
-	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p)))
+	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p), mttl))
 	{ verbose(3, "allmatch: qd section different"); return 0;}
-	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p)))
+	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p), mttl))
 	{ verbose(3, "allmatch: an section different"); return 0;}
-	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p)))
+	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p), mttl))
 	{ verbose(3, "allmatch: ar section different"); return 0;}
-	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p))) 
+	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p), mttl)) 
 	{ verbose(3, "allmatch: ns section different"); return 0;}
 	return 1;
 }
@@ -659,7 +667,7 @@ find_match(struct entry* entries, ldns_pkt* query_pkt,
 			verbose(3, "bad transport\n");
 			continue;
 		}
-		if(p->match_all && !match_all(query_pkt, reply)) {
+		if(p->match_all && !match_all(query_pkt, reply, p->match_ttl)) {
 			verbose(3, "bad allmatch\n");
 			continue;
 		}

testcode/ldns-testpkts.h

@@ -44,7 +44,8 @@
 	; 'qname' makes the query match the qname from the reply
 	; 'serial=1023' makes the query match if ixfr serial is 1023. 
 	; 'all' has to match header byte for byte and all rrs in packet.
-	MATCH [opcode] [qtype] [qname] [serial=<value>] [all]
+	; 'ttl' used with all, rrs in packet must also have matching TTLs.
+	MATCH [opcode] [qtype] [qname] [serial=<value>] [all] [ttl]
 	MATCH [UDP|TCP]
 	MATCH ...
 	; Then the REPLY header is specified.
@@ -157,6 +158,8 @@ struct entry {
 	bool match_serial; 
 	/** match all of the packet */
 	bool match_all;
+	/** match ttls in the packet */
+	bool match_ttl;
 	/** match query serial with this value. */
 	uint32_t ixfr_soa_serial; 
 	/** match on UDP/TCP */

testcode/lock_verify.c

@@ -0,0 +1,423 @@
+/*
+ * testcode/lock_verify.c - verifier program for lock traces, checks order.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file checks the lock traces generated by checklock.c.
+ * Checks if locks are consistently locked in the same order.
+ * If not, this can lead to deadlock if threads execute the different
+ * ordering at the same time.
+ * 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/rbtree.h"
+
+/* --- data structures --- */
+struct lock_ref;
+
+/** key for lock lookup */
+struct order_id {
+	/** the thread id that created it */
+	int thr;
+	/** the instance number of creation */
+	int instance;
+};
+
+/** a lock */
+struct order_lock {
+	/** rbnode in all tree */
+	rbnode_t node;
+	/** lock id */
+	struct order_id id;
+	/** the creation file */
+	char* create_file;
+	/** creation line */
+	int create_line;
+	/** set of all locks that are smaller than this one (locked earlier) */
+	rbtree_t* smaller;
+	/** during depthfirstsearch, this is a linked list of the stack 
+	 * of locks. points to the next lock bigger than this one. */
+	struct lock_ref* dfs_next;
+	/** if lock has been visited (all smaller locks have been compared to
+	 * this lock), only need to compare this with all unvisited(bigger) 
+	 * locks */
+	int visited;
+};
+
+/** reference to a lock in a rbtree set */
+struct lock_ref {
+	/** rbnode, key is an order_id ptr */
+	rbnode_t node;
+	/** the lock referenced */
+	struct order_lock* lock;
+	/** why is this ref */
+	char* file;
+	/** line number */
+	int line;
+};
+
+/** count of errors detected */
+static int errors_detected = 0;
+/** verbose? */
+static int verb = 0;
+
+/** print program usage help */
+static void
+usage()
+{
+	printf("lock_verify <trace files>\n");
+}
+
+/** compare two order_ids */
+int order_lock_cmp(const void* e1, const void* e2)
+{
+	struct order_id* o1 = (struct order_id*)e1;
+	struct order_id* o2 = (struct order_id*)e2;
+	if(o1->thr < o2->thr) return -1;
+	if(o1->thr > o2->thr) return 1;
+	if(o1->instance < o2->instance) return -1;
+	if(o1->instance > o2->instance) return 1;
+	return 0;
+}
+
+/** read header entry. 
+ * @param in: file to read header of.
+ * @return: False if it does not belong to the rest. */
+static int 
+read_header(FILE* in)
+{
+	time_t t;
+	pid_t p;
+	int thrno;
+	static int have_values = 0;
+	static time_t the_time;
+	static pid_t the_pid;
+	static int threads[256];
+
+	if(fread(&t, sizeof(t), 1, in) != 1 ||	
+		fread(&thrno, sizeof(thrno), 1, in) != 1 ||
+		fread(&p, sizeof(p), 1, in) != 1) {
+		fatal_exit("fread failed");
+	}
+	/* check these values are sorta OK */
+	if(!have_values) {
+		the_time = t;
+		the_pid = p;
+		memset(threads, 0, 256*sizeof(int));
+		threads[thrno] = 1;
+		have_values = 1;
+		printf(" trace %d from pid %u on %s", thrno, 
+			(unsigned)p, ctime(&t));
+	} else {
+		if(the_pid != p) {
+			printf(" has pid %u, not %u. Skipped.\n",
+				(unsigned)p, (unsigned)the_pid);
+			return 0;
+		}
+		if(threads[thrno])
+			fatal_exit("same threadno in two files");
+		threads[thrno] = 1;
+		if( abs((int)(the_time - t)) > 3600)
+			fatal_exit("input files from different times: %u %u",
+				(unsigned)the_time, (unsigned)t);
+		printf(" trace of thread %u:%d\n", (unsigned)p, thrno);
+	}
+	return 1;
+}
+
+/** max length of strings: filenames and function names. */
+#define STRMAX 1024
+/** read a string from file, false on error */
+static int readup_str(char** str, FILE* in)
+{
+	char buf[STRMAX];
+	int len = 0;
+	int c;
+	/* ends in zero */
+	while( (c = fgetc(in)) != 0) {
+		if(c == EOF)
+			fatal_exit("eof in readstr, file too short");
+		buf[len++] = c;
+		if(len == STRMAX) {
+			fatal_exit("string too long, bad file format");
+		}
+	}
+	buf[len] = 0;
+	*str = strdup(buf);
+	return 1;
+}
+
+/** read creation entry */
+static void read_create(rbtree_t* all, FILE* in)
+{
+	struct order_lock* o = calloc(1, sizeof(struct order_lock));
+	if(!o) fatal_exit("malloc failure");
+	if(fread(&o->id.thr, sizeof(int), 1, in) != 1 ||	
+	   fread(&o->id.instance, sizeof(int), 1, in) != 1 ||	
+	   !readup_str(&o->create_file, in) ||
+	   fread(&o->create_line, sizeof(int), 1, in) != 1)
+		fatal_exit("fread failed");
+	o->smaller = rbtree_create(order_lock_cmp);
+	o->node.key = &o->id;
+	if(!rbtree_insert(all, &o->node)) {
+		/* already inserted */
+		struct order_lock* a = (struct order_lock*)rbtree_search(all, 
+			&o->id);
+		log_assert(a);
+		a->create_file = o->create_file;
+		a->create_line = o->create_line;
+		free(o->smaller);
+		free(o);
+		o = a;
+	}
+	if(verb) printf("read create %u %u %s %d\n", 
+		(unsigned)o->id.thr, (unsigned)o->id.instance,
+		o->create_file, o->create_line);
+}
+
+/** insert lock entry (empty) into list */
+static struct order_lock* 
+insert_lock(rbtree_t* all, struct order_id* id)
+{
+	struct order_lock* o = calloc(1, sizeof(struct order_lock));
+	if(!o) fatal_exit("malloc failure");
+	o->smaller = rbtree_create(order_lock_cmp);
+	o->id = *id;
+	o->node.key = &o->id;
+	if(!rbtree_insert(all, &o->node))
+		fatal_exit("insert fail should not happen");
+	return o;
+}
+
+/** read lock entry */
+static void read_lock(rbtree_t* all, FILE* in, int val)
+{
+	struct order_id prev_id, now_id;
+	struct lock_ref* ref;
+	struct order_lock* prev, *now;
+	ref = (struct lock_ref*)calloc(1, sizeof(struct lock_ref));
+	if(!ref) fatal_exit("malloc failure");
+	prev_id.thr = val;
+	if(fread(&prev_id.instance, sizeof(int), 1, in) != 1 ||	
+	   fread(&now_id.thr, sizeof(int), 1, in) != 1 ||	
+	   fread(&now_id.instance, sizeof(int), 1, in) != 1 ||	
+	   !readup_str(&ref->file, in) ||
+	   fread(&ref->line, sizeof(int), 1, in) != 1)
+		fatal_exit("fread failed");
+	if(verb) printf("read lock %u %u %u %u %s %d\n", 
+		(unsigned)prev_id.thr, (unsigned)prev_id.instance,
+		(unsigned)now_id.thr, (unsigned)now_id.instance,
+		ref->file, ref->line);
+	/* find the two locks involved */
+	prev = (struct order_lock*)rbtree_search(all, &prev_id);
+	now = (struct order_lock*)rbtree_search(all, &now_id);
+	/* if not there - insert 'em */
+	if(!prev) prev = insert_lock(all, &prev_id);
+	if(!now) now = insert_lock(all, &now_id);
+	ref->lock = prev;
+	ref->node.key = &prev->id;
+	if(!rbtree_insert(now->smaller, &ref->node)) {
+		free(ref->file);
+		free(ref);
+	}
+}
+
+/** read input file */
+static void readinput(rbtree_t* all, char* file)
+{
+	FILE *in = fopen(file, "r");
+	int fst;
+	if(!in) {
+		perror(file);
+		exit(1);
+	}
+	printf("file %s", file);
+	if(!read_header(in)) {
+		fclose(in);
+		return;
+	}
+	while(fread(&fst, sizeof(fst), 1, in) == 1) {
+		if(fst == -1)
+			read_create(all, in);
+		else	read_lock(all, in, fst);
+	}
+	fclose(in);
+}
+
+/** print cycle message */
+static void found_cycle(struct lock_ref* visit, int level)
+{
+	struct lock_ref* p;
+	int i = 0;
+	errors_detected++;
+	printf("Found inconsistent locking order of length %d\n", level);
+	printf("for lock %d %d created %s %d\n", 
+		visit->lock->id.thr, visit->lock->id.instance,
+		visit->lock->create_file, visit->lock->create_line);
+	printf("sequence is:\n");
+	p = visit;
+	while(p) {
+		struct order_lock* next = 
+			p->lock->dfs_next?p->lock->dfs_next->lock:visit->lock;
+		printf("[%d] is locked at line %s %d before lock %d %d\n",
+			i, p->file, p->line, next->id.thr, next->id.instance);
+		printf("[%d] lock %d %d is created at %s %d\n",
+			i, next->id.thr, next->id.instance,
+			next->create_file, next->create_line); 
+		i++;
+		p = p->lock->dfs_next;
+		if(p && p->lock == visit->lock)
+			break;
+	}
+}
+
+/** Detect cycle by comparing visited now with all (unvisited) bigger nodes. */
+static int detect_cycle(struct lock_ref* visit, struct lock_ref* from)
+{
+	struct lock_ref* p = from;
+	while(p) {
+		if(p->lock == visit->lock)
+			return 1;
+		p = p->lock->dfs_next;
+	}
+	return 0;
+}
+
+/** recursive function to depth first search for cycles.
+ * @param visit: the lock visited at this step.
+ *	its dfs_next pointer gives the visited lock up in recursion.
+ * 	same as lookfor at level 0.
+ * @param level: depth of recursion. 0 is start.
+ * @param from: search for matches from unvisited node upwards.
+ */
+static void search_cycle(struct lock_ref* visit, int level, 
+	struct lock_ref* from)
+{
+	struct lock_ref* ref;
+	/* check for cycle */
+	if(detect_cycle(visit, from) && level != 0) {
+		found_cycle(visit, level);
+		return;
+	}
+	/* recurse */
+	if(!visit->lock->visited)
+		from = visit;
+	if(verb > 1) fprintf(stderr, "[%d] visit lock %u %u %s %d\n", level,
+			(unsigned)visit->lock->id.thr, 
+			(unsigned)visit->lock->id.instance,
+			visit->lock->create_file, visit->lock->create_line);
+	RBTREE_FOR(ref, struct lock_ref*, visit->lock->smaller) {
+		ref->lock->dfs_next = visit;
+		search_cycle(ref, level+1, from);
+	}
+	visit->lock->visited = 1;
+}
+
+/** Check ordering of one lock */
+static void check_order_lock(struct order_lock* lock)
+{
+	struct lock_ref start;
+	if(lock->visited) return;
+
+	start.node.key = &lock->id;
+	start.lock = lock;
+	start.file = lock->create_file;
+	start.line = lock->create_line;
+
+	if(!lock->create_file)
+		log_err("lock %u %u does not have create info",
+			(unsigned)lock->id.thr, (unsigned)lock->id.instance);
+
+	/* depth first search to find cycle with this lock at head */
+	lock->dfs_next = NULL;
+	search_cycle(&start, 0, &start);
+}
+
+/** Check ordering of locks */
+static void check_order(rbtree_t* all_locks)
+{
+	/* check each lock */
+	struct order_lock* lock;
+	int i=0;
+	RBTREE_FOR(lock, struct order_lock*, all_locks) {
+		if(verb)
+		    printf("[%d/%d] Checking lock %d %d %s %d\n",
+			i, (int)all_locks->count,
+			lock->id.thr, lock->id.instance, 
+			lock->create_file, lock->create_line);
+		else if (i % ((all_locks->count/75)<1?1:all_locks->count/75) 
+			== 0) 
+		    fprintf(stderr, ".");
+		i++;
+		check_order_lock(lock);
+	}
+	fprintf(stderr, "\n");
+}
+
+/** main program to verify all traces passed */
+int
+main(int argc, char* argv[])
+{
+	rbtree_t* all_locks;
+	int i;
+	time_t starttime = time(NULL);
+	if(argc <= 1) {
+		usage();
+		return 1;
+	}
+	log_init(NULL);
+	log_ident_set("lock-verify");
+	/* init */
+	all_locks = rbtree_create(order_lock_cmp);
+	errors_detected = 0;
+
+	/* read the input files */
+	for(i=1; i<argc; i++) {
+		readinput(all_locks, argv[i]);
+	}
+
+	/* check ordering */
+	check_order(all_locks);
+
+	/* do not free a thing, OS will do it */
+	printf("checked %d locks in %d seconds with %d errors.\n", 
+		(int)all_locks->count, (int)(time(NULL)-starttime),
+		errors_detected);
+	if(errors_detected) return 1;
+	return 0;
+}

testcode/pktview.c

@@ -0,0 +1,192 @@
+/*
+ * testcode/pktview.c - debug program to disassemble a DNS packet.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This program shows a dns packet wire format.
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/data/dname.h"
+#include "util/data/msgparse.h"
+#include "testcode/unitmain.h"
+#include "testcode/readhex.h"
+
+/** usage information for pktview */
+void usage(char* argv[])
+{
+	printf("usage: %s\n", argv[0]);
+	printf("present hex packet on stdin.\n");
+	exit(1);
+}
+
+/** read hex input */
+void read_input(ldns_buffer* pkt, FILE* in)
+{
+	char buf[102400];
+	char* np = buf;
+	while(fgets(np, (int)sizeof(buf) - (np-buf), in)) {
+		if(buf[0] == ';') /* comment */
+			continue;
+		np = &np[strlen(np)];
+	}
+	hex_to_buf(pkt, buf);
+}
+
+/** analyze domain name in packet, possibly compressed. */
+void analyze_dname(ldns_buffer* pkt)
+{
+	size_t oldpos = ldns_buffer_position(pkt);
+	size_t len;
+	printf("[pos %d] dname: ", (int)oldpos);
+	dname_print(stdout, pkt, ldns_buffer_current(pkt));
+	len = pkt_dname_len(pkt);
+	printf(" len=%d", (int)len);
+	if(ldns_buffer_position(pkt)-oldpos != len)
+		printf(" comprlen=%d\n", 
+			(int)(ldns_buffer_position(pkt)-oldpos));
+	else	printf("\n");
+}
+
+/** analyze rdata in packet */
+void analyze_rdata(ldns_buffer*pkt, const ldns_rr_descriptor* desc, 
+	uint16_t rdlen)
+{
+	int rdf = 0;
+	int count = (int)desc->_dname_count;
+	size_t len, oldpos;
+	while(rdlen > 0 && count) {
+		switch(desc->_wireformat[rdf]) {
+		case LDNS_RDF_TYPE_DNAME:
+			oldpos = ldns_buffer_position(pkt);
+			analyze_dname(pkt);
+			rdlen -= ldns_buffer_position(pkt)-oldpos;
+			count --;
+			len = 0;
+			break;
+		case LDNS_RDF_TYPE_STR:
+			len = ldns_buffer_current(pkt)[0] + 1;
+			break;
+		default:
+			len = get_rdf_size(desc->_wireformat[rdf]);
+		}
+		if(len) {
+			printf(" wf[%d]", (int)len);
+			ldns_buffer_skip(pkt, (ssize_t)len);
+			rdlen -= len;
+		}
+		rdf++;
+	}
+	if(rdlen)
+		printf(" remain[%d]\n", (int)rdlen);
+	else	printf("\n");
+	ldns_buffer_skip(pkt, (ssize_t)rdlen);
+}
+
+/** analyze rr in packet. */
+void analyze_rr(ldns_buffer* pkt, int q)
+{
+	uint16_t type, dclass, len;
+	uint32_t ttl;
+	analyze_dname(pkt);
+	type = ldns_buffer_read_u16(pkt);
+	dclass = ldns_buffer_read_u16(pkt);
+	printf("type %s(%d)", ldns_rr_descript(type)?  
+		ldns_rr_descript(type)->_name: "??" , (int)type);
+	printf(" class %s(%d) ", ldns_lookup_by_id(ldns_rr_classes, 
+		(int)dclass)?ldns_lookup_by_id( ldns_rr_classes, 
+		(int)dclass)->name:"??", (int)dclass);
+	if(q) {
+		printf("\n");
+	} else {
+		ttl = ldns_buffer_read_u32(pkt);
+		printf(" ttl %d (0x%x)", ttl, ttl);
+		len = ldns_buffer_read_u16(pkt);
+		printf(" rdata len %d:\n", len);
+		if(ldns_rr_descript(type))
+			analyze_rdata(pkt, ldns_rr_descript(type), len);
+		else ldns_buffer_skip(pkt, (ssize_t)len);
+	}
+}
+
+/** analyse pkt */
+void analyze(ldns_buffer* pkt)
+{
+	uint16_t i, f, qd, an, ns, ar;
+	int rrnum = 0;
+	printf("packet length %d\n", (int)ldns_buffer_limit(pkt));
+	if(ldns_buffer_limit(pkt) < 12) return;
+
+	i = ldns_buffer_read_u16(pkt);
+	printf("id (hostorder): %d (0x%x)\n", i, i);
+	f = ldns_buffer_read_u16(pkt);
+	printf("flags: 0x%x\n", f);
+	qd = ldns_buffer_read_u16(pkt);
+	printf("qdcount: %d\n", qd);
+	an = ldns_buffer_read_u16(pkt);
+	printf("ancount: %d\n", an);
+	ns = ldns_buffer_read_u16(pkt);
+	printf("nscount: %d\n", ns);
+	ar = ldns_buffer_read_u16(pkt);
+	printf("arcount: %d\n", ar);
+	
+	printf(";-- query section\n");
+	while(ldns_buffer_remaining(pkt) > 0) {
+		if(rrnum == qd) printf(";-- answer section\n");
+		if(rrnum == qd+an) printf(";-- authority section\n");
+		if(rrnum == qd+an+ns) printf(";-- additional section\n");
+		printf("rr %d ", rrnum);
+		analyze_rr(pkt, rrnum < qd);
+		rrnum++;
+	}
+}
+
+/** main program for pktview */
+int main(int argc, char* argv[]) 
+{
+	ldns_buffer* pkt = ldns_buffer_new(65553);
+	if(argc != 1) {
+		usage(argv);
+	}
+	if(!pkt) fatal_exit("out of memory");
+
+	read_input(pkt, stdin);
+	analyze(pkt);
+
+	ldns_buffer_free(pkt);
+	return 0;
+}

testcode/readhex.c

@@ -0,0 +1,82 @@
+/*
+ * testcode/readhex.c - read hex data.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+#include "config.h"
+#include "testcode/readhex.h"
+#include "util/log.h"
+
+/** skip whitespace */
+static void
+skip_whites(const char** p)
+{
+	while(1) {
+		while(isspace(**p))
+			(*p)++;
+		if(**p == ';') {
+			/* comment, skip until newline */
+			while(**p && **p != '\n')
+				(*p)++;
+			if(**p == '\n')
+				(*p)++;
+		} else return;
+	}
+}
+
+/** takes a hex string and puts into buffer */
+void hex_to_buf(ldns_buffer* pkt, const char* hex)
+{
+	const char* p = hex;
+	int val;
+	ldns_buffer_clear(pkt);
+	while(*p) {
+		skip_whites(&p);
+		if(ldns_buffer_position(pkt) == ldns_buffer_limit(pkt))
+			fatal_exit("hex_to_buf: buffer too small");
+		if(!isalnum(*p))
+			break;
+		val = ldns_hexdigit_to_int(*p++) << 4;
+		skip_whites(&p);
+		log_assert(*p && isalnum(*p));
+		val |= ldns_hexdigit_to_int(*p++);
+		ldns_buffer_write_u8(pkt, (uint8_t)val);
+		skip_whites(&p);
+	}
+	ldns_buffer_flip(pkt);
+}
+

testcode/readhex.h

@@ -0,0 +1,47 @@
+/*
+ * testcode/readhex.h - read hex data.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+
+#ifndef TESTCODE_READHEX_H
+#define TESTCODE_READHEX_H
+
+/** helper to convert hex string to packet buffer */
+void hex_to_buf(ldns_buffer* pkt, const char* hex);
+
+#endif /* TESTCODE_READHEX_H */

testcode/replay.c

@@ -234,11 +234,10 @@ make_scenario(char* line)
 }
 
 struct replay_scenario* 
-replay_scenario_read(FILE* in, const char* name)
+replay_scenario_read(FILE* in, const char* name, int* lineno)
 {
 	char line[MAX_LINE_LEN];
 	char *parse;
-	int lineno = 0;
 	struct replay_scenario* scen = NULL;
 	uint16_t ttl = 3600;
 	ldns_rdf* or = NULL;
@@ -247,7 +246,7 @@ replay_scenario_read(FILE* in, const char* name)
 
 	while(fgets(line, MAX_LINE_LEN-1, in)) {
 		parse=line;
-		lineno++;
+		(*lineno)++;
 		while(isspace(*parse))
 			parse++;
 		if(!*parse) 
@@ -257,26 +256,26 @@ replay_scenario_read(FILE* in, const char* name)
 		if(parse_keyword(&parse, "SCENARIO_BEGIN")) {
 			scen = make_scenario(parse);
 			if(!scen)
-				fatal_exit("%d: could not make scen", lineno);
+				fatal_exit("%d: could not make scen", *lineno);
 			continue;
 		} 
 		if(!scen)
-			fatal_exit("%d: expected SCENARIO", lineno);
+			fatal_exit("%d: expected SCENARIO", *lineno);
 		if(parse_keyword(&parse, "RANGE_BEGIN")) {
 			struct replay_range* newr = replay_range_read(parse, 
-				in, name, &lineno, line, &ttl, &or, &prev);
+				in, name, lineno, line, &ttl, &or, &prev);
 			if(!newr)
-				fatal_exit("%d: bad range", lineno);
+				fatal_exit("%d: bad range", *lineno);
 			newr->next_range = scen->range_list;
 			scen->range_list = newr;
 		} else if(parse_keyword(&parse, "STEP")) {
 			struct replay_moment* mom = replay_moment_read(parse, 
-				in, name, &lineno, &ttl, &or, &prev);
+				in, name, lineno, &ttl, &or, &prev);
 			if(!mom)
-				fatal_exit("%d: bad moment", lineno);
+				fatal_exit("%d: bad moment", *lineno);
 			if(scen->mom_last && 
 				scen->mom_last->time_step >= mom->time_step)
-				fatal_exit("%d: time goes backwards", lineno);
+				fatal_exit("%d: time goes backwards", *lineno);
 			if(scen->mom_last)
 				scen->mom_last->mom_next = mom;
 			else	scen->mom_first = mom;

testcode/replay.h

@@ -263,9 +263,11 @@ struct replay_answer {
  * Read a replay scenario from the file.
  * @param in: file to read from.
  * @param name: name to print in errors.
+ * @param lineno: incremented for every line read.
  * @return: Scenario. NULL if no scenario read.
  */
-struct replay_scenario* replay_scenario_read(FILE* in, const char* name);
+struct replay_scenario* replay_scenario_read(FILE* in, const char* name, 
+	int* lineno);
 
 /**
  * Delete scenario.

testcode/testbound.c

@@ -51,6 +51,9 @@
 #include "daemon/unbound.c"
 #undef main
 
+/** maximum line length for lines in the replay file. */
+#define MAX_LINE_LEN 1024
+
 /** give commandline usage for testbound. */
 static void
 testbound_usage()
@@ -107,25 +110,67 @@ static void
 echo_cmdline(int argc, char* argv[])
 {
 	int i;
-	printf("testbound is starting:");
+	fprintf(stderr, "testbound is starting:");
 	for(i=0; i<argc; i++) {
-		printf(" [%s]", argv[i]);
+		fprintf(stderr, " [%s]", argv[i]);
 	}
-	printf("\n");
+	fprintf(stderr, "\n");
+}
+
+/** process config elements */
+static void
+setup_config(FILE* in, char* configfile, int* lineno,
+	int* pass_argc, char* pass_argv[])
+{
+	char line[MAX_LINE_LEN];
+	char* parse;
+	FILE* cfg;
+	snprintf(configfile, MAX_LINE_LEN, "/tmp/testbound_cfg_%u.tmp", 
+		(unsigned)getpid());
+	add_opts("-c", pass_argc, pass_argv);
+	add_opts(configfile, pass_argc, pass_argv);
+	cfg = fopen(configfile, "w");
+	if(!cfg) fatal_exit("could not open %s: %s", 
+			configfile, strerror(errno));
+	line[MAX_LINE_LEN-1] = 0;
+	while(fgets(line, MAX_LINE_LEN-1, in)) {
+		parse = line;
+		(*lineno)++;
+		while(isspace(*parse))
+			parse++;
+		if(!*parse || parse[0] == ';')
+			continue;
+		if(strncmp(parse, "COMMANDLINE", 11) == 0) {
+			parse[strlen(parse)-1] = 0; /* strip off \n */
+			add_opts(parse+11, pass_argc, pass_argv);
+			continue;
+		}
+		if(strncmp(parse, "CONFIG_END", 10) == 0) {
+			fclose(cfg);
+			return;
+		}
+		fputs(line, cfg);
+	}
+	fatal_exit("No CONFIG_END in input file");
+
 }
 
 /** read playback file */
 static struct replay_scenario* 
-setup_playback(const char* filename)
+setup_playback(const char* filename, char* configfile,
+	int* pass_argc, char* pass_argv[])
 {
 	struct replay_scenario* scen = NULL;
+	int lineno = 0;
+
 	if(filename) {
 		FILE *in = fopen(filename, "r");
 		if(!in) {
 			perror(filename);
 			exit(1);
 		}
-		scen = replay_scenario_read(in, filename);
+		setup_config(in, configfile, &lineno, pass_argc, pass_argv);
+		scen = replay_scenario_read(in, filename, &lineno);
 		fclose(in);
 		if(!scen)
 			fatal_exit("Could not read: %s", filename);
@@ -150,6 +195,7 @@ main(int argc, char* argv[])
 	int init_optind = optind;
 	char* init_optarg = optarg;
 	struct replay_scenario* scen = NULL;
+	char cfgfile[MAX_LINE_LEN];
 
 	log_init(NULL);
 	log_info("Start of %s testbound program.", PACKAGE_STRING);
@@ -180,7 +226,7 @@ main(int argc, char* argv[])
 	}
 
 	/* setup test environment */
-	scen = setup_playback(playback_file);
+	scen = setup_playback(playback_file, cfgfile, &pass_argc, pass_argv);
 	/* init fake event backend */
 	fake_event_init(scen);
 
@@ -194,6 +240,7 @@ main(int argc, char* argv[])
 	/* run the normal daemon */
 	res = daemon_main(pass_argc, pass_argv);
 
+	unlink(cfgfile);
 	fake_event_cleanup();
 	for(c=1; c<pass_argc; c++)
 		free(pass_argv[c]);

testcode/unitdname.c

@@ -0,0 +1,376 @@
+/*
+ * testcode/unitdname.c - unit test for dname routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls dname unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "util/data/dname.h"
+
+/** put dname into buffer */
+static ldns_buffer*
+dname_to_buf(ldns_buffer* b, const char* str)
+{
+	ldns_rdf* rdf;
+	ldns_status status;
+	ldns_buffer_clear(b);
+	rdf = ldns_dname_new_frm_str(str);	
+	status = ldns_dname2buffer_wire(b, rdf);
+	if(status != LDNS_STATUS_OK)
+		fatal_exit("%s ldns: %s", __func__, 
+			ldns_get_errorstr_by_id(status));
+	ldns_rdf_deep_free(rdf);
+	ldns_buffer_flip(b);
+	return b;
+}
+
+/** test query_dname_len function */
+static void 
+dname_test_qdl(ldns_buffer* buff)
+{
+	unit_assert( query_dname_len(buff) == 0);
+	unit_assert( query_dname_len(dname_to_buf(buff, ".")) == 1 );
+	unit_assert( query_dname_len(dname_to_buf(buff, "bla.foo.")) == 9 );
+	unit_assert( query_dname_len(dname_to_buf(buff, "x.y.z.example.com."
+		)) == 19 );
+}
+
+/** test query_dname_tolower */
+static void
+dname_test_qdtl(ldns_buffer* buff)
+{
+	ldns_buffer_write_at(buff, 0, "\012abCDeaBCde\003cOm\000", 16);
+	query_dname_tolower(ldns_buffer_begin(buff), 16);
+	unit_assert( memcmp(ldns_buffer_begin(buff), 
+		"\012abcdeabcde\003com\000", 16) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\001+\012abC{e-ZYXe\003NET\000", 18);
+	query_dname_tolower(ldns_buffer_begin(buff), 18);
+	unit_assert( memcmp(ldns_buffer_begin(buff), 
+		"\001+\012abc{e-zyxe\003net\000", 18) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\000", 1);
+	query_dname_tolower(ldns_buffer_begin(buff), 1);
+	unit_assert( memcmp(ldns_buffer_begin(buff), "\000", 1) == 0);
+
+	ldns_buffer_write_at(buff, 0, "\002NL\000", 4);
+	query_dname_tolower(ldns_buffer_begin(buff), 4);
+	unit_assert( memcmp(ldns_buffer_begin(buff), "\002nl\000", 4) == 0);
+}
+
+/** test query_dname_compare */
+static void
+dname_test_query_dname_compare()
+{
+	unit_assert(query_dname_compare((uint8_t*)"", (uint8_t*)"") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\001a", 
+					(uint8_t*)"\001a") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc\001a") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003aBc\001a", 
+					(uint8_t*)"\003AbC\001A") == 0);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc", 
+					(uint8_t*)"\003abc\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc") == +1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"") == +1);
+	unit_assert(query_dname_compare((uint8_t*)"", 
+					(uint8_t*)"\003abc\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003xxx\001a") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003axx\001a", 
+					(uint8_t*)"\003abc\001a") == 1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001a", 
+					(uint8_t*)"\003abc\001Z") == -1);
+	unit_assert(query_dname_compare((uint8_t*)"\003abc\001Z", 
+					(uint8_t*)"\003abc\001a") == 1);
+}
+
+/** test dname_count_labels */
+static void
+dname_test_count_labels()
+{
+	unit_assert(dname_count_labels((uint8_t*)"") == 1);
+	unit_assert(dname_count_labels((uint8_t*)"\003com") == 2);
+	unit_assert(dname_count_labels((uint8_t*)"\003org") == 2);
+	unit_assert(dname_count_labels((uint8_t*)"\007example\003com") == 3);
+	unit_assert(dname_count_labels((uint8_t*)"\003bla\007example\003com") 
+		== 4);
+}
+
+/** test dname_count_size_labels */
+static void
+dname_test_count_size_labels()
+{
+	size_t sz = 0;
+	unit_assert(dname_count_size_labels((uint8_t*)"", &sz) == 1);
+	unit_assert(sz == 1);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003com", &sz) == 2);
+	unit_assert(sz == 5);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003org", &sz) == 2);
+	unit_assert(sz == 5);
+	unit_assert(dname_count_size_labels((uint8_t*)"\007example\003com", 
+		&sz) == 3);
+	unit_assert(sz == 13);
+	unit_assert(dname_count_size_labels((uint8_t*)"\003bla\007example"
+		"\003com", &sz) == 4);
+	unit_assert(sz == 17);
+}
+
+
+/** test pkt_dname_len */
+static void
+dname_test_pkt_dname_len(ldns_buffer* buff)
+{
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\000", 1);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 1 );
+	unit_assert( ldns_buffer_position(buff) == 1);
+
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\003org\000", 5);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 5 );
+	unit_assert( ldns_buffer_position(buff) == 5);
+
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\007example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 16 );
+	unit_assert( ldns_buffer_position(buff) == 16);
+
+	/* invalid compression pointer: to self */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\300\000os\007example\003org\000", 17);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* valid compression pointer */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\003com\000\040\300\000", 8);
+	ldns_buffer_flip(buff);
+	ldns_buffer_set_position(buff, 6);
+	unit_assert( pkt_dname_len(buff) == 5 );
+	unit_assert( ldns_buffer_position(buff) == 8);
+
+	/* unknown label type */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\107example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* label too long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\047example\003org\000", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* label exceeds packet */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, "\002os\007example\007org\004", 16);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+
+	/* name very long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, 
+		"\020a1cdef5555544444"
+		"\020a2cdef5555544444"
+		"\020a3cdef5555544444"
+		"\020a4cdef5555544444"
+		"\020a5cdef5555544444"
+		"\020a6cdef5555544444"
+		"\020a7cdef5555544444"
+		"\020a8cdef5555544444"
+		"\020a9cdef5555544444"
+		"\020aAcdef5555544444"
+		"\020aBcdef5555544444"
+		"\020aCcdef5555544444"
+		"\020aDcdef5555544444"
+		"\020aEcdef5555544444"	/* 238 up to here */
+		"\007aabbccd"		/* 246 up to here */
+		"\007example\000"	/* 255 to here */
+		, 255);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 255 );
+	unit_assert( ldns_buffer_position(buff) == 255);
+
+	/* name too long */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write(buff, 
+		"\020a1cdef5555544444"
+		"\020a2cdef5555544444"
+		"\020a3cdef5555544444"
+		"\020a4cdef5555544444"
+		"\020a5cdef5555544444"
+		"\020a6cdef5555544444"
+		"\020a7cdef5555544444"
+		"\020a8cdef5555544444"
+		"\020a9cdef5555544444"
+		"\020aAcdef5555544444"
+		"\020aBcdef5555544444"
+		"\020aCcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aXcdef5555544444"
+		"\020aDcdef5555544444"
+		"\020aEcdef5555544444"	/* 238 up to here */
+		"\007aabbccd"		/* 246 up to here */
+		"\007example\000"	/* 255 to here */
+		, 255);
+	ldns_buffer_flip(buff);
+	unit_assert( pkt_dname_len(buff) == 0 );
+}
+
+/** test dname_lab_cmp */
+static void
+dname_test_dname_lab_cmp()
+{
+	int ml = 0; /* number of labels that matched exactly */
+
+	/* test for equality succeeds */
+	unit_assert(dname_lab_cmp((uint8_t*)"", 1, (uint8_t*)"", 1, &ml) == 0);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003net", 2, 
+		(uint8_t*)"\003net", 2, 
+		&ml) == 0);
+	unit_assert(ml == 2);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == 0);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\004test\007example\003net", 4, 
+		(uint8_t*)"\004test\007example\003net", 4, 
+		&ml) == 0);
+	unit_assert(ml == 4);
+
+	/* root is smaller than anything else */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"", 1, 
+		(uint8_t*)"\003net", 2, 
+		&ml) == -1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003net", 2, 
+		(uint8_t*)"", 1, 
+		&ml) == 1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"", 1, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == -1);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"", 1, 
+		&ml) == 1);
+	unit_assert(ml == 1);
+
+	/* label length makes a difference */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\004neta", 2, 
+		(uint8_t*)"\003net", 2, 
+		&ml) != 0);
+	unit_assert(ml == 1);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\002ne", 2, 
+		(uint8_t*)"\004neta", 2, 
+		&ml) != 0);
+	unit_assert(ml == 1);
+
+	/* contents follow the zone apex */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		(uint8_t*)"\007example\003net", 3, 
+		&ml) == 1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\007example\003net", 3, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+
+	/* label content makes a difference */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003aag\007example\003net", 4, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003aag\007example\003net", 4, 
+		(uint8_t*)"\003bla\007example\003net", 4, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bla\003aag\007example\003net", 5, 
+		(uint8_t*)"\003aag\003bla\007example\003net", 5, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\02sn\003opt\003aag\007example\003net", 6, 
+		(uint8_t*)"\02sn\003opt\003bla\007example\003net", 6, 
+		&ml) == -1);
+	unit_assert(ml == 3);
+
+	/* but lowercase/uppercase does not make a difference. */
+	unit_assert(dname_lab_cmp(
+		(uint8_t*)"\003bLa\007examPLe\003net", 4, 
+		(uint8_t*)"\003bla\007eXAmple\003nET", 4, 
+		&ml) == 0);
+	unit_assert(ml == 4);
+}
+
+void dname_test()
+{
+	ldns_buffer* buff = ldns_buffer_new(65800);
+	ldns_buffer_flip(buff);
+	dname_test_qdl(buff);
+	dname_test_qdtl(buff);
+	dname_test_query_dname_compare();
+	dname_test_count_labels();
+	dname_test_count_size_labels();
+	dname_test_dname_lab_cmp();
+	dname_test_pkt_dname_len(buff);
+	ldns_buffer_free(buff);
+}

testcode/unitlruhash.c

@@ -0,0 +1,535 @@
+/*
+ * testcode/unitlruhash.c - unit test for lruhash table.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Tests the locking LRU keeping hash table implementation.
+ */
+
+#include "config.h"
+#include "testcode/unitmain.h"
+#include "util/log.h"
+#include "util/storage/lruhash.h"
+
+/* --- test representation --- */
+/** structure contains test key */
+struct testkey {
+	/** the key id */
+	int id;
+	/** the entry */
+	struct lruhash_entry entry;
+};
+/** structure contains test data */
+struct testdata {
+	/** data value */
+	int data;
+};
+
+/** sizefunc for lruhash */
+static size_t test_sizefunc(void*, void*);
+/** comparefunc for lruhash */
+static int test_compfunc(void*, void*);
+/** delkey for lruhash */
+static void test_delkey(void*, void*);
+/** deldata for lruhash */
+static void test_deldata(void*, void*);
+/* --- end test representation --- */
+
+/** hash func, very bad to improve collisions. */
+static hashvalue_t myhash(int id) {return (hashvalue_t)id & 0x0f;}
+/** allocate new key, fill in hash. */
+static struct testkey* newkey(int id) {
+	struct testkey* k = (struct testkey*)calloc(1, sizeof(struct testkey));
+	if(!k) fatal_exit("out of memory");
+	k->id = id;
+	k->entry.hash = myhash(id);
+	k->entry.key = k;
+	lock_rw_init(&k->entry.lock);
+	return k;
+}
+/** new data el */
+static struct testdata* newdata(int val) {
+	struct testdata* d = (struct testdata*)calloc(1, 
+		sizeof(struct testdata));
+	if(!d) fatal_exit("out of memory");
+	d->data = val;
+	return d;
+}
+/** delete key */
+static void delkey(struct testkey* k) {	
+	lock_rw_destroy(&k->entry.lock); free(k);}
+/** delete data */
+static void deldata(struct testdata* d) {free(d);}
+
+/** test bin_find_entry function and bin_overflow_remove */
+static void
+test_bin_find_entry(struct lruhash* table)
+{
+	struct testkey* k = newkey(12);
+	struct testdata* d = newdata(128);
+	struct testkey* k2 = newkey(12 + 1024);
+	struct testkey* k3 = newkey(14);
+	struct testkey* k4 = newkey(12 + 1024*2);
+	hashvalue_t h = myhash(12);
+	struct lruhash_bin bin;
+	memset(&bin, 0, sizeof(bin));
+	bin_init(&bin, 1);
+
+	/* remove from empty list */
+	bin_overflow_remove(&bin, &k->entry);
+
+	/* find in empty list */
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	/* insert */
+	lock_quick_lock(&bin.lock);
+	bin.overflow_list = &k->entry;
+	lock_quick_unlock(&bin.lock);
+
+	/* find, hash not OK. */
+	unit_assert( bin_find_entry(table, &bin, myhash(13), k) == NULL );
+
+	/* find, hash OK, but cmp not */
+	unit_assert( k->entry.hash == k2->entry.hash );
+	unit_assert( bin_find_entry(table, &bin, h, k2) == NULL );
+
+	/* find, hash OK, and cmp too */
+	unit_assert( bin_find_entry(table, &bin, h, k) == &k->entry );
+
+	/* remove the element */
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	/* prepend two different elements; so the list is long */
+	/* one has the same hash, but different cmp */
+	lock_quick_lock(&bin.lock);
+	unit_assert( k->entry.hash == k4->entry.hash );
+	k4->entry.overflow_next = &k->entry;
+	k3->entry.overflow_next = &k4->entry;
+	bin.overflow_list = &k3->entry;
+	lock_quick_unlock(&bin.lock);
+
+	/* find, hash not OK. */
+	unit_assert( bin_find_entry(table, &bin, myhash(13), k) == NULL );
+
+	/* find, hash OK, but cmp not */
+	unit_assert( k->entry.hash == k2->entry.hash );
+	unit_assert( bin_find_entry(table, &bin, h, k2) == NULL );
+
+	/* find, hash OK, and cmp too */
+	unit_assert( bin_find_entry(table, &bin, h, k) == &k->entry );
+
+	/* remove middle element */
+	unit_assert( bin_find_entry(table, &bin, k4->entry.hash, k4) 
+		== &k4->entry );
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k4->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, k4->entry.hash, k4) == NULL);
+
+	/* remove last element */
+	lock_quick_lock(&bin.lock);
+	bin_overflow_remove(&bin, &k->entry);
+	lock_quick_unlock(&bin.lock);
+	unit_assert( bin_find_entry(table, &bin, h, k) == NULL );
+
+	lock_quick_destroy(&bin.lock);
+	delkey(k);
+	delkey(k2);
+	delkey(k3);
+	delkey(k4);
+	deldata(d);
+}
+
+/** test lru_front lru_remove */
+static void test_lru(struct lruhash* table)
+{
+	struct testkey* k = newkey(12);
+	struct testkey* k2 = newkey(14);
+	lock_quick_lock(&table->lock);
+
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+
+	/* add one */
+	lru_front(table, &k->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	/* remove it */
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+
+	/* add two */
+	lru_front(table, &k->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	lru_front(table, &k2->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k->entry);
+	/* remove first in list */
+	lru_remove(table, &k2->entry);
+	unit_assert( table->lru_start == &k->entry && 
+		table->lru_end == &k->entry);
+	lru_front(table, &k2->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k->entry);
+	/* remove last in list */
+	lru_remove(table, &k->entry);
+	unit_assert( table->lru_start == &k2->entry && 
+		table->lru_end == &k2->entry);
+
+	/* empty the list */
+	lru_remove(table, &k2->entry);
+	unit_assert( table->lru_start == NULL && table->lru_end == NULL);
+	lock_quick_unlock(&table->lock);
+	delkey(k);
+	delkey(k2);
+}
+
+/** test hashtable using short sequence */
+static void
+test_short_table(struct lruhash* table) 
+{
+	struct testkey* k = newkey(12);
+	struct testkey* k2 = newkey(14);
+	struct testdata* d = newdata(128);
+	struct testdata* d2 = newdata(129);
+	
+	k->entry.data = d;
+	k2->entry.data = d2;
+
+	lruhash_insert(table, myhash(12), &k->entry, d, NULL);
+	lruhash_insert(table, myhash(14), &k2->entry, d2, NULL);
+	
+	unit_assert( lruhash_lookup(table, myhash(12), k, 0) == &k->entry);
+	lock_rw_unlock( &k->entry.lock );
+	unit_assert( lruhash_lookup(table, myhash(14), k2, 0) == &k2->entry);
+	lock_rw_unlock( &k2->entry.lock );
+	lruhash_remove(table, myhash(12), k);
+	lruhash_remove(table, myhash(14), k2);
+}
+
+/** number of hash test max */
+#define HASHTESTMAX 25
+
+/** test adding a random element */
+static void
+testadd(struct lruhash* table, struct testdata* ref[])
+{
+	int numtoadd = random() % HASHTESTMAX;
+	struct testdata* data = newdata(numtoadd);
+	struct testkey* key = newkey(numtoadd);
+	key->entry.data = data;
+	lruhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	ref[numtoadd] = data;
+}
+
+/** test adding a random element */
+static void
+testremove(struct lruhash* table, struct testdata* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	struct testkey* key = newkey(num);
+	lruhash_remove(table, myhash(num), key);
+	ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element */
+static void
+testlookup(struct lruhash* table, struct testdata* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	struct testkey* key = newkey(num);
+	struct lruhash_entry* en = lruhash_lookup(table, myhash(num), key, 0);
+	struct testdata* data = en? (struct testdata*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0) log_info("lookup %d got %d, expect %d", num, en? data->data :-1,
+		ref[num]? ref[num]->data : -1);
+	unit_assert( data == ref[num] );
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** check integrity of hash table */
+static void
+check_table(struct lruhash* table)
+{
+	struct lruhash_entry* p;
+	size_t c = 0;
+	lock_quick_lock(&table->lock);
+	unit_assert( table->num <= table->size);
+	unit_assert( table->size_mask == (int)table->size-1 );
+	unit_assert( (table->lru_start && table->lru_end) ||
+		(!table->lru_start && !table->lru_end) );
+	unit_assert( table->space_used <= table->space_max );
+	/* check lru list integrity */
+	if(table->lru_start)
+		unit_assert(table->lru_start->lru_prev == NULL);
+	if(table->lru_end)
+		unit_assert(table->lru_end->lru_next == NULL);
+	p = table->lru_start;
+	while(p) {
+		if(p->lru_prev) {
+			unit_assert(p->lru_prev->lru_next == p);
+		}
+		if(p->lru_next) {
+			unit_assert(p->lru_next->lru_prev == p);
+		}
+		c++;
+		p = p->lru_next;
+	}
+	unit_assert(c == table->num);
+
+	/* this assertion is specific to the unit test */
+	unit_assert( table->space_used == 
+		table->num * test_sizefunc(NULL, NULL) );
+	lock_quick_unlock(&table->lock);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testadd_unlim(struct lruhash* table, struct testdata** ref)
+{
+	int numtoadd = random() % (HASHTESTMAX * 10);
+	struct testdata* data = newdata(numtoadd);
+	struct testkey* key = newkey(numtoadd);
+	key->entry.data = data;
+	lruhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	if(ref)
+		ref[numtoadd] = data;
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testremove_unlim(struct lruhash* table, struct testdata** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	struct testkey* key = newkey(num);
+	lruhash_remove(table, myhash(num), key);
+	if(ref)
+		ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testlookup_unlim(struct lruhash* table, struct testdata** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	struct testkey* key = newkey(num);
+	struct lruhash_entry* en = lruhash_lookup(table, myhash(num), key, 0);
+	struct testdata* data = en? (struct testdata*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0 && ref) log_info("lookup unlim %d got %d, expect %d", num, en ? 
+		data->data :-1, ref[num] ? ref[num]->data : -1);
+	if(data && ref) {
+		/* its okay for !data, it fell off the lru */
+		unit_assert( data == ref[num] );
+	}
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** test with long sequence of adds, removes and updates, and lookups */
+static void
+test_long_table(struct lruhash* table) 
+{
+	/* assuming it all fits in the hastable, this check will work */
+	struct testdata* ref[HASHTESTMAX * 100];
+	size_t i;
+	memset(ref, 0, sizeof(ref));
+	/* test assumption */
+	if(0) log_info(" size %d x %d < %d", (int)test_sizefunc(NULL, NULL), 
+		(int)HASHTESTMAX, (int)table->space_max);
+	unit_assert( test_sizefunc(NULL, NULL)*HASHTESTMAX < table->space_max);
+	if(0) lruhash_status(table, "unit test", 1);
+	srandom(48);
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd(table, ref);
+				break;
+			case 1:
+				testremove(table, ref);
+				break;
+			case 2:
+				testlookup(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(table, "unit test", 1);
+		check_table(table);
+		unit_assert( table->num <= HASHTESTMAX );
+	}
+
+	/* test more, but 'ref' assumption does not hold anymore */
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(table, ref);
+				break;
+			case 1:
+				testremove_unlim(table, ref);
+				break;
+			case 2:
+				testlookup_unlim(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(table, "unlim", 1);
+		check_table(table);
+	}
+}
+
+/** structure to threaded test the lru hash table */
+struct test_thr {
+	/** thread num, first entry. */
+	int num;
+	/** id */
+	ub_thread_t id;
+	/** hash table */
+	struct lruhash* table;
+};
+
+/** main routine for threaded hash table test */
+static void*
+test_thr_main(void* arg) 
+{
+	struct test_thr* t = (struct test_thr*)arg;
+	int i;
+	log_thread_set(&t->num);
+	for(i=0; i<1000; i++) {
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(t->table, NULL);
+				break;
+			case 1:
+				testremove_unlim(t->table, NULL);
+				break;
+			case 2:
+				testlookup_unlim(t->table, NULL);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) lruhash_status(t->table, "hashtest", 1);
+		if(i % 100 == 0) /* because of locking, not all the time */
+			check_table(t->table);
+	}
+	check_table(t->table);
+	return NULL;
+}
+
+/** test hash table access by multiple threads. */
+static void
+test_threaded_table(struct lruhash* table)
+{
+	int numth = 10;
+	struct test_thr t[100];
+	int i;
+
+	for(i=1; i<numth; i++) {
+		t[i].num = i;
+		t[i].table = table;
+		ub_thread_create(&t[i].id, test_thr_main, &t[i]);
+	}
+
+	for(i=1; i<numth; i++) {
+		ub_thread_join(t[i].id);
+	}
+	if(0) lruhash_status(table, "hashtest", 1);
+}
+
+void lruhash_test()
+{
+	/* start very very small array, so it can do lots of table_grow() */
+	/* also small in size so that reclaim has to be done quickly. */
+	struct lruhash* table ;
+	printf("lruhash test\n");
+	table = lruhash_create(2, 4096, 
+		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+	test_bin_find_entry(table);
+	test_lru(table);
+	test_short_table(table);
+	test_long_table(table);
+	lruhash_delete(table);
+	table = lruhash_create(2, 4096, 
+		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+	test_threaded_table(table);
+	lruhash_delete(table);
+}
+
+static size_t test_sizefunc(void* ATTR_UNUSED(key), void* ATTR_UNUSED(data))
+{
+	return sizeof(struct testkey) + sizeof(struct testdata);
+}
+
+static int test_compfunc(void* key1, void* key2)
+{
+	struct testkey* k1 = (struct testkey*)key1;
+	struct testkey* k2 = (struct testkey*)key2;
+	if(k1->id == k2->id)
+		return 0;
+	if(k1->id > k2->id)
+		return 1;
+	return -1;
+}
+
+static void test_delkey(void* key, void* ATTR_UNUSED(arg))
+{
+	delkey((struct testkey*)key);
+}
+
+static void test_deldata(void* data, void* ATTR_UNUSED(arg))
+{
+	deldata((struct testdata*)data);
+}

testcode/unitmain.c

@@ -41,11 +41,10 @@
 
 #include "config.h"
 #include "util/log.h"
+#include "testcode/unitmain.h"
 
 /** number of tests done */
 int testcount = 0;
-/** test bool x, exits on failure, increases testcount. */
-#define unit_assert(x) testcount++; log_assert(x);
 
 #include "util/alloc.h"
 /** test alloc code */
@@ -55,10 +54,9 @@ alloc_test() {
 	struct alloc_cache major, minor1, minor2;
 	int i;
 
-	checklock_start();
-	alloc_init(&major, NULL);
-	alloc_init(&minor1, &major);
-	alloc_init(&minor2, &major);
+	alloc_init(&major, NULL, 0);
+	alloc_init(&minor1, &major, 0);
+	alloc_init(&minor2, &major, 1);
 
 	t1 = alloc_special_obtain(&minor1);
 	alloc_clear(&minor1);
@@ -84,7 +82,6 @@ alloc_test() {
 	alloc_clear(&minor2);
 	unit_assert(major.num_quar == 11);
 	alloc_clear(&major);
-	checklock_stop();
 }
 
 #include "util/net_help.h"
@@ -100,6 +97,26 @@ net_test()
 	unit_assert( !str_is_ip6("213.154.224.12") );
 	unit_assert( !str_is_ip6("213.154.224.255") );
 	unit_assert( !str_is_ip6("255.255.255.0") );
+	unit_assert( is_pow2(0) );
+	unit_assert( is_pow2(1) );
+	unit_assert( is_pow2(2) );
+	unit_assert( is_pow2(4) );
+	unit_assert( is_pow2(8) );
+	unit_assert( is_pow2(16) );
+	unit_assert( is_pow2(1024) );
+	unit_assert( is_pow2(1024*1024) );
+	unit_assert( is_pow2(1024*1024*1024) );
+	unit_assert( !is_pow2(3) );
+	unit_assert( !is_pow2(5) );
+	unit_assert( !is_pow2(6) );
+	unit_assert( !is_pow2(7) );
+	unit_assert( !is_pow2(9) );
+	unit_assert( !is_pow2(10) );
+	unit_assert( !is_pow2(11) );
+	unit_assert( !is_pow2(17) );
+	unit_assert( !is_pow2(23) );
+	unit_assert( !is_pow2(257) );
+	unit_assert( !is_pow2(259) );
 }
 
 /**
@@ -117,8 +134,14 @@ main(int argc, char* argv[])
 		return 1;
 	}
 	printf("Start of %s unit test.\n", PACKAGE_STRING);
+	checklock_start();
 	net_test();
+	dname_test();
 	alloc_test();
-	printf("%d tests succeeded\n", testcount);
+	lruhash_test();
+	slabhash_test();
+	msgparse_test();
+	checklock_stop();
+	printf("%d checks ok.\n", testcount);
 	return 0;
 }

testcode/unitmain.h

@@ -0,0 +1,59 @@
+/*
+ * testcode/unitmain.h - unit test main program for unbound.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Declarations useful for the unit tests.
+ */
+
+#ifndef TESTCODE_UNITMAIN_H
+#define TESTCODE_UNITMAIN_H
+#include "util/log.h"
+
+/** number of tests done */
+extern int testcount;
+/** test bool x, exits on failure, increases testcount. */
+#define unit_assert(x) do {testcount++; log_assert(x);} while(0)
+
+/** unit test lruhashtable implementation */
+void lruhash_test();
+/** unit test slabhashtable implementation */
+void slabhash_test();
+/** unit test for msgreply and msgparse */
+void msgparse_test();
+/** unit test dname handling functions */
+void dname_test();
+
+#endif /* TESTCODE_UNITMAIN_H */

testcode/unitmsgparse.c

@@ -0,0 +1,439 @@
+/*
+ * testcode/unitmsgparse.c - unit test for msg parse routines.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Calls msg parse unit tests. Exits with code 1 on a failure. 
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "testcode/unitmain.h"
+#include "util/data/msgparse.h"
+#include "util/data/msgreply.h"
+#include "util/alloc.h"
+#include "util/region-allocator.h"
+#include "util/net_help.h"
+#include "testcode/readhex.h"
+
+/** verbose message parse unit test */
+static int vbmp = 0;
+
+/** match two rr lists */
+static int
+match_list(ldns_rr_list* q, ldns_rr_list *p)
+{
+	size_t i;
+	if(ldns_rr_list_rr_count(q) != ldns_rr_list_rr_count(p)) {
+		verbose(3, "rrlistcount different %d %d", 
+			(int)ldns_rr_list_rr_count(q), 
+			(int)ldns_rr_list_rr_count(p));
+		return 0;
+	}
+	for(i=0; i<ldns_rr_list_rr_count(q); i++)
+	{
+		if(ldns_rr_compare(ldns_rr_list_rr(q, i),
+			ldns_rr_list_rr(p, i)) != 0) {
+			verbose(3, "rr %d different", i);
+			return 0;
+		}
+		/* and check the ttl */
+		if(ldns_rr_ttl(ldns_rr_list_rr(q, i)) !=
+			ldns_rr_ttl(ldns_rr_list_rr(p, i))) {
+			verbose(3, "rr %d ttl different", i);
+			return 0;
+		}
+
+	}
+	return 1;
+}
+
+/** match edns sections */
+static int
+match_edns(ldns_pkt* q, ldns_pkt* p)
+{
+	if(ldns_pkt_edns_udp_size(q) != ldns_pkt_edns_udp_size(p))
+		return 0;
+	if(ldns_pkt_edns_extended_rcode(q) != ldns_pkt_edns_extended_rcode(p))
+		return 0;
+	if(ldns_pkt_edns_version(q) != ldns_pkt_edns_version(p))
+		return 0;
+	if(ldns_pkt_edns_do(q) != ldns_pkt_edns_do(p))
+		return 0;
+	if(ldns_pkt_edns_z(q) != ldns_pkt_edns_z(p))
+		return 0;
+	if(ldns_rdf_compare(ldns_pkt_edns_data(q), ldns_pkt_edns_data(p)) != 0)
+		return 0;
+	return 1;
+}
+
+/** compare two booleans */
+static int
+cmp_bool(int x, int y)
+{
+	if(!x && !y) return 0;
+	if(x && y) return 0;
+	if(!x) return -1;
+	return 1;
+}
+
+/** match all of the packet */
+static int
+match_all(ldns_pkt* q, ldns_pkt* p)
+{
+	if(ldns_pkt_get_opcode(q) != ldns_pkt_get_opcode(p))
+	{ verbose(3, "allmatch: opcode different"); return 0;}
+	if(ldns_pkt_get_rcode(q) != ldns_pkt_get_rcode(p))
+	{ verbose(3, "allmatch: rcode different"); return 0;}
+	if(ldns_pkt_id(q) != ldns_pkt_id(p))
+	{ verbose(3, "allmatch: id different"); return 0;}
+	if(cmp_bool(ldns_pkt_qr(q), ldns_pkt_qr(p)) != 0)
+	{ verbose(3, "allmatch: qr different"); return 0;}
+	if(cmp_bool(ldns_pkt_aa(q), ldns_pkt_aa(p)) != 0)
+	{ verbose(3, "allmatch: aa different"); return 0;}
+	if(cmp_bool(ldns_pkt_tc(q), ldns_pkt_tc(p)) != 0)
+	{ verbose(3, "allmatch: tc different"); return 0;}
+	if(cmp_bool(ldns_pkt_rd(q), ldns_pkt_rd(p)) != 0)
+	{ verbose(3, "allmatch: rd different"); return 0;}
+	if(cmp_bool(ldns_pkt_cd(q), ldns_pkt_cd(p)) != 0)
+	{ verbose(3, "allmatch: cd different"); return 0;}
+	if(cmp_bool(ldns_pkt_ra(q), ldns_pkt_ra(p)) != 0)
+	{ verbose(3, "allmatch: ra different"); return 0;}
+	if(cmp_bool(ldns_pkt_ad(q), ldns_pkt_ad(p)) != 0)
+	{ verbose(3, "allmatch: ad different"); return 0;}
+	if(ldns_pkt_qdcount(q) != ldns_pkt_qdcount(p))
+	{ verbose(3, "allmatch: qdcount different"); return 0;}
+	if(ldns_pkt_ancount(q) != ldns_pkt_ancount(p))
+	{ verbose(3, "allmatch: ancount different"); return 0;}
+	if(ldns_pkt_nscount(q) != ldns_pkt_nscount(p))
+	{ verbose(3, "allmatch: nscount different"); return 0;}
+	if(ldns_pkt_arcount(q) != ldns_pkt_arcount(p))
+	{ verbose(3, "allmatch: arcount different"); return 0;}
+	if(!match_list(ldns_pkt_question(q), ldns_pkt_question(p)))
+	{ verbose(3, "allmatch: qd section different"); return 0;}
+	if(!match_list(ldns_pkt_answer(q), ldns_pkt_answer(p)))
+	{ verbose(3, "allmatch: an section different"); return 0;}
+	if(!match_list(ldns_pkt_authority(q), ldns_pkt_authority(p)))
+	{ verbose(3, "allmatch: ns section different"); return 0;}
+	if(!match_list(ldns_pkt_additional(q), ldns_pkt_additional(p)))
+	{ verbose(3, "allmatch: ar section different"); return 0;}
+	if(!match_edns(q, p))
+	{ verbose(3, "edns different."); return 0;}
+	return 1;
+}
+
+/** see if buffers contain the same packet */
+static int
+test_buffers(ldns_buffer* pkt, ldns_buffer* out)
+{
+	ldns_pkt* p1=0, *p2=0;
+	ldns_status s1, s2;
+	/* check binary same */
+	if(ldns_buffer_limit(pkt) == ldns_buffer_limit(out) &&
+		memcmp(ldns_buffer_begin(pkt), ldns_buffer_begin(out),
+			ldns_buffer_limit(pkt)) == 0) {
+		if(vbmp) printf("binary the same (length=%u)\n",
+				(unsigned)ldns_buffer_limit(pkt));
+		return 1;
+	}
+
+	if(vbmp) {
+		size_t sz = 16;
+		size_t count;
+		size_t lim = ldns_buffer_limit(out);
+		if(ldns_buffer_limit(pkt) < lim)
+			lim = ldns_buffer_limit(pkt);
+		for(count=0; count<lim; count+=sz) {
+			size_t rem = sz;
+			if(lim-count < sz) rem = lim-count;
+			if(memcmp(ldns_buffer_at(pkt, count), 
+				ldns_buffer_at(out, count), rem) == 0) {
+				log_info("same %d %d", count, rem);
+				log_hex("same: ", ldns_buffer_at(pkt, count),
+					rem);
+			} else {
+				log_info("diff %d %d", count, rem);
+				log_hex("difp: ", ldns_buffer_at(pkt, count),
+					rem);
+				log_hex("difo: ", ldns_buffer_at(out, count),
+					rem);
+			}
+		}
+	}
+	/* check if it 'means the same' */
+	s1 = ldns_buffer2pkt_wire(&p1, pkt);
+	s2 = ldns_buffer2pkt_wire(&p2, out);
+	if(vbmp) {
+		log_hex("orig in hex", ldns_buffer_begin(pkt),
+			ldns_buffer_limit(pkt));
+		log_hex("unbound out in hex", ldns_buffer_begin(out),
+			ldns_buffer_limit(out));
+		printf("\npacket from unbound (%d):\n", 
+			(int)ldns_buffer_limit(out));
+		ldns_pkt_print(stdout, p2);
+
+		printf("\npacket original (%d):\n", 
+			(int)ldns_buffer_limit(pkt));
+		ldns_pkt_print(stdout, p1);
+		printf("\n");
+	}
+	if(s1 != s2) {
+		/* oops! */
+		printf("input ldns parse: %s, output ldns parse: %s.\n",
+			ldns_get_errorstr_by_id(s1), 
+			ldns_get_errorstr_by_id(s2));
+		unit_assert(0);
+	}
+	/* compare packets */
+	unit_assert(match_all(p1, p2));
+	ldns_pkt_free(p1);
+	ldns_pkt_free(p2);
+	return 0;
+}
+
+/** check if unbound formerr equals ldns formerr. */
+static void
+checkformerr(ldns_buffer* pkt)
+{
+	ldns_pkt* p;
+	ldns_status status = ldns_buffer2pkt_wire(&p, pkt);
+	if(vbmp) printf("formerr, ldns parse is: %s\n",
+			ldns_get_errorstr_by_id(status));
+	if(status == LDNS_STATUS_OK) {
+		printf("Formerr, but ldns gives packet:\n");
+		ldns_pkt_print(stdout, p);
+		exit(1);
+	}
+	unit_assert(status != LDNS_STATUS_OK);
+}
+
+/** test a packet */
+static void
+testpkt(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out, 
+	const char* hex)
+{
+	struct query_info qi;
+	struct reply_info* rep = 0;
+	int ret;
+	uint16_t id;
+	uint16_t flags;
+	uint32_t timenow = 0;
+	region_type *region = region_create(malloc, free);
+	struct edns_data edns;
+
+	hex_to_buf(pkt, hex);
+	memmove(&id, ldns_buffer_begin(pkt), sizeof(id));
+	if(ldns_buffer_limit(pkt) < 2)
+		flags = 0;
+	else	memmove(&flags, ldns_buffer_at(pkt, 2), sizeof(flags));
+	flags = ntohs(flags);
+	ret = reply_info_parse(pkt, alloc, &qi, &rep, region, &edns);
+	if(ret != 0) {
+		if(vbmp) printf("parse code %d: %s\n", ret, 
+			ldns_lookup_by_id(ldns_rcodes, ret)->name);
+		if(ret == LDNS_RCODE_FORMERR)
+			checkformerr(pkt);
+		unit_assert(ret != LDNS_RCODE_SERVFAIL);
+	} else {
+		const size_t lim = 512;
+		ret = reply_info_encode(&qi, rep, id, flags, out, timenow,
+			region, 65535);
+		unit_assert(ret != 0); /* udp packets should fit */
+		attach_edns_record(out, &edns);
+		if(vbmp) printf("inlen %u outlen %u\n", 
+			(unsigned)ldns_buffer_limit(pkt),
+			(unsigned)ldns_buffer_limit(out));
+		test_buffers(pkt, out);
+
+		if(ldns_buffer_limit(out) > lim) {
+			ret = reply_info_encode(&qi, rep, id, flags, out, 
+				timenow, region, 
+				lim - calc_edns_field_size(&edns));
+			unit_assert(ret != 0); /* should fit, but with TC */
+			attach_edns_record(out, &edns);
+			if( LDNS_QDCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_QDCOUNT(ldns_buffer_begin(pkt)) ||
+				LDNS_ANCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_ANCOUNT(ldns_buffer_begin(pkt)) ||
+				LDNS_NSCOUNT(ldns_buffer_begin(out)) !=
+				LDNS_NSCOUNT(ldns_buffer_begin(pkt)))
+				unit_assert(
+				LDNS_TC_WIRE(ldns_buffer_begin(out)));
+				/* must set TC bit if shortened */
+			unit_assert(ldns_buffer_limit(out) <= lim);
+		}
+	} 
+
+	query_info_clear(&qi);
+	reply_info_parsedelete(rep, alloc);
+	region_destroy(region);
+}
+
+/** simple test of parsing. */
+static void
+simpletest(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out)
+{
+	/* a root query  drill -q - */
+	testpkt(pkt, alloc, out, 
+		" c5 40 01 00 00 01 00 00 00 00 00 00 00 00 02 00 01 ");
+
+	/* very small packet */
+	testpkt(pkt, alloc, out, 
+"; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19\n"
+";-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n"
+"74 0c 85 83 00 01 00 00 00 01 00 00 03 62 6c 61 09 6e 6c 6e    ;          1-  20\n"
+"65 74 6c 61 62 73 02 6e 6c 00 00 0f 00 01 09 6e 6c 6e 65 74    ;         21-  40\n"
+"6c 61 62 73 02 6e 6c 00 00 06 00 01 00 00 46 50 00 40 04 6f    ;         41-  60\n"
+"70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 0a 68 6f    ;         61-  80\n"
+"73 74 6d 61 73 74 65 72 09 6e 6c 6e 65 74 6c 61 62 73 02 6e    ;         81- 100\n"
+"6c 00 77 a1 02 58 00 00 70 80 00 00 1c 20 00 09 3a 80 00 00    ;        101- 120\n"
+"46 50\n");
+	
+	/* a root reply  drill -w - */
+	testpkt(pkt, alloc, out, 
+	" ; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19\n"
+	" ;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --\n"
+	" 97 3f 81 80 00 01 00 0d 00 00 00 02 00 00 02 00 01 00 00 02    ;          1-  20\n"
+	" 00 01 00 06 6d 38 00 14 01 49 0c 52 4f 4f 54 2d 53 45 52 56    ;         21-  40\n"
+	" 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01    ;         41-  60\n"
+	" 4a 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00    ;         61-  80\n"
+	" 00 02 00 01 00 06 6d 38 00 14 01 4b 0c 52 4f 4f 54 2d 53 45    ;         81- 100\n"
+	" 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00    ;        101- 120\n"
+	" 14 01 4c 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54    ;        121- 140\n"
+	" 00 00 00 02 00 01 00 06 6d 38 00 14 01 4d 0c 52 4f 4f 54 2d    ;        141- 160\n"
+	" 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d    ;        161- 180\n"
+	" 38 00 14 01 41 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e    ;        181- 200\n"
+	" 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 42 0c 52 4f 4f    ;        201- 220\n"
+	" 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00    ;        221- 240\n"
+	" 06 6d 38 00 14 01 43 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53    ;        241- 260\n"
+	" 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 44 0c 52    ;        261- 280\n"
+	" 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00 02 00    ;        281- 300\n"
+	" 01 00 06 6d 38 00 14 01 45 0c 52 4f 4f 54 2d 53 45 52 56 45    ;        301- 320\n"
+	" 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14 01 46    ;        321- 340\n"
+	" 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 00    ;        341- 360\n"
+	" 02 00 01 00 06 6d 38 00 14 01 47 0c 52 4f 4f 54 2d 53 45 52    ;        361- 380\n"
+	" 56 45 52 53 03 4e 45 54 00 00 00 02 00 01 00 06 6d 38 00 14    ;        381- 400\n"
+	" 01 48 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00    ;        401- 420\n"
+	" 01 41 0c 52 4f 4f 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00    ;        421- 440\n"
+	" 00 01 00 01 00 02 64 b9 00 04 c6 29 00 04 01 4a 0c 52 4f 4f    ;        441- 460\n"
+	" 54 2d 53 45 52 56 45 52 53 03 4e 45 54 00 00 01 00 01 00 02    ;        461- 480\n"
+	" 64 b9 00 04 c0 3a 80 1e  ");
+}
+
+/** simple test of parsing, pcat file. */
+static void
+testfromfile(ldns_buffer* pkt, struct alloc_cache* alloc, ldns_buffer* out,
+	const char* fname)
+{
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	int no=0;
+	if(!in) {
+		perror("fname");
+		return;
+	}
+	while(fgets(buf, (int)sizeof(buf), in)) {
+		if(buf[0] == ';') /* comment */
+			continue;
+		if(strlen(buf) < 10) /* skip pcat line numbers. */
+			continue;
+		if(vbmp) {
+			printf("test no %d: %s", no, buf);
+			fflush(stdout);
+		}
+		testpkt(pkt, alloc, out, buf);
+		no++;
+	}
+	fclose(in);
+}
+
+/** simple test of parsing, drill file. */
+static void
+testfromdrillfile(ldns_buffer* pkt, struct alloc_cache* alloc, 
+	ldns_buffer* out, const char* fname)
+{
+	/*  ;-- is used to indicate a new message */
+	FILE* in = fopen(fname, "r");
+	char buf[102400];
+	char* np = buf;
+	buf[0]=0;
+	if(!in) {
+		perror("fname");
+		return;
+	}
+	while(fgets(np, (int)sizeof(buf) - (np-buf), in)) {
+		if(strncmp(np, ";--", 3) == 0) {
+			/* new entry */
+			/* test previous */
+			if(np != buf)
+				testpkt(pkt, alloc, out, buf);
+			/* set for new entry */
+			np = buf;
+			buf[0]=0;
+			continue;
+		}
+		if(np[0] == ';') /* comment */
+			continue;
+		np = &np[strlen(np)];
+	}
+	testpkt(pkt, alloc, out, buf);
+	fclose(in);
+}
+
+void msgparse_test()
+{
+	ldns_buffer* pkt = ldns_buffer_new(65553);
+	ldns_buffer* out = ldns_buffer_new(65553);
+	struct alloc_cache super_a, alloc;
+	/* init */
+	alloc_init(&super_a, NULL, 0);
+	alloc_init(&alloc, &super_a, 2);
+
+	printf("testmsgparse\n");
+	simpletest(pkt, &alloc, out);
+	/* plain hex dumps, like pcat */
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.1");
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.2");
+	testfromfile(pkt, &alloc, out, "testdata/test_packets.3");
+	/* like from drill -w - */
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.4");
+	testfromdrillfile(pkt, &alloc, out, "testdata/test_packets.5");
+
+	/* cleanup */
+	alloc_clear(&alloc);
+	alloc_clear(&super_a);
+	ldns_buffer_free(pkt);
+	ldns_buffer_free(out);
+}

testcode/unitslabhash.c

@@ -0,0 +1,415 @@
+/*
+ * testcode/unitslabhash.c - unit test for slabhash table.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/**
+ * \file
+ * Tests the locking LRU keeping hash table implementation.
+ */
+
+#include "config.h"
+#include "testcode/unitmain.h"
+#include "util/log.h"
+#include "util/storage/slabhash.h"
+
+/* --- test representation --- */
+/** structure contains test key */
+struct slabtestkey {
+	/** the key id */
+	int id;
+	/** the entry */
+	struct lruhash_entry entry;
+};
+/** structure contains test data */
+struct slabtestdata {
+	/** data value */
+	int data;
+};
+
+/** sizefunc for lruhash */
+static size_t test_sizefunc(void*, void*);
+/** comparefunc for lruhash */
+static int test_compfunc(void*, void*);
+/** delkey for lruhash */
+static void test_delkey(void*, void*);
+/** deldata for lruhash */
+static void test_deldata(void*, void*);
+/* --- end test representation --- */
+
+/** hash func, very bad to improve collisions, both high and low bits. */
+static hashvalue_t myhash(int id) {
+	hashvalue_t h = (hashvalue_t)id & 0x0f;
+	h |= (h << 28);
+	return h;
+}
+
+/** allocate new key, fill in hash. */
+static struct slabtestkey* newkey(int id) {
+	struct slabtestkey* k = (struct slabtestkey*)calloc(1, sizeof(struct slabtestkey));
+	if(!k) fatal_exit("out of memory");
+	k->id = id;
+	k->entry.hash = myhash(id);
+	k->entry.key = k;
+	lock_rw_init(&k->entry.lock);
+	return k;
+}
+/** new data el */
+static struct slabtestdata* newdata(int val) {
+	struct slabtestdata* d = (struct slabtestdata*)calloc(1, 
+		sizeof(struct slabtestdata));
+	if(!d) fatal_exit("out of memory");
+	d->data = val;
+	return d;
+}
+/** delete key */
+static void delkey(struct slabtestkey* k) {	
+	lock_rw_destroy(&k->entry.lock); free(k);}
+/** delete data */
+static void deldata(struct slabtestdata* d) {free(d);}
+
+/** test hashtable using short sequence */
+static void
+test_short_table(struct slabhash* table) 
+{
+	struct slabtestkey* k = newkey(12);
+	struct slabtestkey* k2 = newkey(14);
+	struct slabtestdata* d = newdata(128);
+	struct slabtestdata* d2 = newdata(129);
+	
+	k->entry.data = d;
+	k2->entry.data = d2;
+
+	slabhash_insert(table, myhash(12), &k->entry, d, NULL);
+	slabhash_insert(table, myhash(14), &k2->entry, d2, NULL);
+	
+	unit_assert( slabhash_lookup(table, myhash(12), k, 0) == &k->entry);
+	lock_rw_unlock( &k->entry.lock );
+	unit_assert( slabhash_lookup(table, myhash(14), k2, 0) == &k2->entry);
+	lock_rw_unlock( &k2->entry.lock );
+	slabhash_remove(table, myhash(12), k);
+	slabhash_remove(table, myhash(14), k2);
+}
+
+/** number of hash test max */
+#define HASHTESTMAX 32
+
+/** test adding a random element */
+static void
+testadd(struct slabhash* table, struct slabtestdata* ref[])
+{
+	int numtoadd = random() % HASHTESTMAX;
+	struct slabtestdata* data = newdata(numtoadd);
+	struct slabtestkey* key = newkey(numtoadd);
+	key->entry.data = data;
+	slabhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	ref[numtoadd] = data;
+}
+
+/** test adding a random element */
+static void
+testremove(struct slabhash* table, struct slabtestdata* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	struct slabtestkey* key = newkey(num);
+	slabhash_remove(table, myhash(num), key);
+	ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element */
+static void
+testlookup(struct slabhash* table, struct slabtestdata* ref[])
+{
+	int num = random() % HASHTESTMAX;
+	struct slabtestkey* key = newkey(num);
+	struct lruhash_entry* en = slabhash_lookup(table, myhash(num), key, 0);
+	struct slabtestdata* data = en? (struct slabtestdata*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0) log_info("lookup %d got %d, expect %d", num, en? data->data :-1,
+		ref[num]? ref[num]->data : -1);
+	unit_assert( data == ref[num] );
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** check integrity of hash table */
+static void
+check_lru_table(struct lruhash* table)
+{
+	struct lruhash_entry* p;
+	size_t c = 0;
+	lock_quick_lock(&table->lock);
+	unit_assert( table->num <= table->size);
+	unit_assert( table->size_mask == (int)table->size-1 );
+	unit_assert( (table->lru_start && table->lru_end) ||
+		(!table->lru_start && !table->lru_end) );
+	unit_assert( table->space_used <= table->space_max );
+	/* check lru list integrity */
+	if(table->lru_start)
+		unit_assert(table->lru_start->lru_prev == NULL);
+	if(table->lru_end)
+		unit_assert(table->lru_end->lru_next == NULL);
+	p = table->lru_start;
+	while(p) {
+		if(p->lru_prev) {
+			unit_assert(p->lru_prev->lru_next == p);
+		}
+		if(p->lru_next) {
+			unit_assert(p->lru_next->lru_prev == p);
+		}
+		c++;
+		p = p->lru_next;
+	}
+	unit_assert(c == table->num);
+
+	/* this assertion is specific to the unit test */
+	unit_assert( table->space_used == 
+		table->num * test_sizefunc(NULL, NULL) );
+	lock_quick_unlock(&table->lock);
+}
+
+/** check integrity of hash table */
+static void
+check_table(struct slabhash* table)
+{
+	size_t i;
+	for(i=0; i<table->size; i++)
+		check_lru_table(table->array[i]);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testadd_unlim(struct slabhash* table, struct slabtestdata** ref)
+{
+	int numtoadd = random() % (HASHTESTMAX * 10);
+	struct slabtestdata* data = newdata(numtoadd);
+	struct slabtestkey* key = newkey(numtoadd);
+	key->entry.data = data;
+	slabhash_insert(table, myhash(numtoadd), &key->entry, data, NULL);
+	if(ref)
+		ref[numtoadd] = data;
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testremove_unlim(struct slabhash* table, struct slabtestdata** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	struct slabtestkey* key = newkey(num);
+	slabhash_remove(table, myhash(num), key);
+	if(ref)
+		ref[num] = NULL;
+	delkey(key);
+}
+
+/** test adding a random element (unlimited range) */
+static void
+testlookup_unlim(struct slabhash* table, struct slabtestdata** ref)
+{
+	int num = random() % (HASHTESTMAX*10);
+	struct slabtestkey* key = newkey(num);
+	struct lruhash_entry* en = slabhash_lookup(table, myhash(num), key, 0);
+	struct slabtestdata* data = en? (struct slabtestdata*)en->data : NULL;
+	if(en) {
+		unit_assert(en->key);
+		unit_assert(en->data);
+	}
+	if(0 && ref) log_info("lookup unlim %d got %d, expect %d", num, en ? 
+		data->data :-1, ref[num] ? ref[num]->data : -1);
+	if(data && ref) {
+		/* its okay for !data, it fell off the lru */
+		unit_assert( data == ref[num] );
+	}
+	if(en) { lock_rw_unlock(&en->lock); }
+	delkey(key);
+}
+
+/** test with long sequence of adds, removes and updates, and lookups */
+static void
+test_long_table(struct slabhash* table) 
+{
+	/* assuming it all fits in the hastable, this check will work */
+	struct slabtestdata* ref[HASHTESTMAX * 100];
+	size_t i;
+	memset(ref, 0, sizeof(ref));
+	/* test assumption */
+	if(0) slabhash_status(table, "unit test", 1);
+	srandom(48);
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd(table, ref);
+				break;
+			case 1:
+				testremove(table, ref);
+				break;
+			case 2:
+				testlookup(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(table, "unit test", 1);
+		check_table(table);
+	}
+
+	/* test more, but 'ref' assumption does not hold anymore */
+	for(i=0; i<1000; i++) {
+		/* what to do? */
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(table, ref);
+				break;
+			case 1:
+				testremove_unlim(table, ref);
+				break;
+			case 2:
+				testlookup_unlim(table, ref);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(table, "unlim", 1);
+		check_table(table);
+	}
+}
+
+/** structure to threaded test the lru hash table */
+struct slab_test_thr {
+	/** thread num, first entry. */
+	int num;
+	/** id */
+	ub_thread_t id;
+	/** hash table */
+	struct slabhash* table;
+};
+
+/** main routine for threaded hash table test */
+static void*
+test_thr_main(void* arg) 
+{
+	struct slab_test_thr* t = (struct slab_test_thr*)arg;
+	int i;
+	log_thread_set(&t->num);
+	for(i=0; i<1000; i++) {
+		switch(random() % 4) {
+			case 0:
+			case 3:
+				testadd_unlim(t->table, NULL);
+				break;
+			case 1:
+				testremove_unlim(t->table, NULL);
+				break;
+			case 2:
+				testlookup_unlim(t->table, NULL);
+				break;
+			default:
+				unit_assert(0);
+		}
+		if(0) slabhash_status(t->table, "hashtest", 1);
+		if(i % 100 == 0) /* because of locking, not all the time */
+			check_table(t->table);
+	}
+	check_table(t->table);
+	return NULL;
+}
+
+/** test hash table access by multiple threads. */
+static void
+test_threaded_table(struct slabhash* table)
+{
+	int numth = 10;
+	struct slab_test_thr t[100];
+	int i;
+
+	for(i=1; i<numth; i++) {
+		t[i].num = i;
+		t[i].table = table;
+		ub_thread_create(&t[i].id, test_thr_main, &t[i]);
+	}
+
+	for(i=1; i<numth; i++) {
+		ub_thread_join(t[i].id);
+	}
+	if(0) slabhash_status(table, "hashtest", 1);
+}
+
+void slabhash_test()
+{
+	/* start very very small array, so it can do lots of table_grow() */
+	/* also small in size so that reclaim has to be done quickly. */
+	struct slabhash* table;
+	printf("slabhash test\n");
+	table = slabhash_create(4, 2, 5200, 
+		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+	test_short_table(table);
+	test_long_table(table);
+	slabhash_delete(table);
+	table = slabhash_create(4, 2, 5200, 
+		test_sizefunc, test_compfunc, test_delkey, test_deldata, NULL);
+	test_threaded_table(table);
+	slabhash_delete(table);
+}
+
+static size_t test_sizefunc(void* ATTR_UNUSED(key), void* ATTR_UNUSED(data))
+{
+	return sizeof(struct slabtestkey) + sizeof(struct slabtestdata);
+}
+
+static int test_compfunc(void* key1, void* key2)
+{
+	struct slabtestkey* k1 = (struct slabtestkey*)key1;
+	struct slabtestkey* k2 = (struct slabtestkey*)key2;
+	if(k1->id == k2->id)
+		return 0;
+	if(k1->id > k2->id)
+		return 1;
+	return -1;
+}
+
+static void test_delkey(void* key, void* ATTR_UNUSED(arg))
+{
+	delkey((struct slabtestkey*)key);
+}
+
+static void test_deldata(void* data, void* ATTR_UNUSED(arg))
+{
+	deldata((struct slabtestdata*)data);
+}

testdata/fwd.rpl

@@ -1,10 +1,13 @@
 ; This is a comment.
+; config options go here.
+CONFIG_END
 
 SCENARIO_BEGIN Sample of a valid query
 RANGE_BEGIN 0 100
 	ENTRY_BEGIN
 	MATCH opcode qtype qname
 	ADJUST copy_id
+	REPLY QR RD RA NOERROR
 	SECTION QUESTION
 www.example.com. IN A
 	SECTION ANSWER

testdata/fwd_cached.rpl

@@ -0,0 +1,73 @@
+; This is a comment.
+; config options go here.
+CONFIG_END
+
+SCENARIO_BEGIN Query receives answer from the cache
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all 
+	; first reply, have AA set.
+	REPLY QR AA RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+; another query, same, so it must be answered from the cache
+STEP 5 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; immediate answer without an OUT_QUERY happening (checked on exit)
+; also, the answer does not have AA set
+STEP 6 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all
+	REPLY QR RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+SCENARIO_END

testdata/fwd_error.rpl

@@ -1,3 +1,5 @@
+; config options go here.
+CONFIG_END
 SCENARIO_BEGIN Forwarder and an error happens on server query.
 STEP 1 QUERY
 ENTRY_BEGIN

testdata/fwd_lrudrop.rpl

@@ -0,0 +1,118 @@
+; This is a comment.
+; config options go here.
+; extremely small cache to force dropping old records.
+server:
+	msg-cache-size: 1 # one whole byte!
+	msg-cache-slabs: 1
+CONFIG_END
+
+SCENARIO_BEGIN Old answer is dropped from the cache
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+ENTRY_END
+
+; another query to force the cache to drop the example.com entry.
+STEP 11 QUERY
+ENTRY_BEGIN
+	SECTION QUESTION
+	www.example.net. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 12 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.net. IN A
+ENTRY_END
+STEP 13 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.net. IN A
+	SECTION ANSWER
+	www.example.net. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.net. IN NS ns.example.net.
+	SECTION ADDITIONAL
+	ns.example.net. IN A 10.20.30.50
+ENTRY_END
+STEP 14 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.net. IN A
+	SECTION ANSWER
+	www.example.net. IN A 10.20.30.40
+ENTRY_END
+
+
+; query, same as first, but it fell out of the cache.
+STEP 21 QUERY
+ENTRY_BEGIN
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 22 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 23 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 24 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+ENTRY_END
+
+SCENARIO_END

testdata/fwd_notcached.rpl

@@ -0,0 +1,82 @@
+; This is a comment.
+
+; can set commandline options using something like this:
+; COMMANDLINE -v 
+; here config file options:
+server:
+	msg-cache-size: 1024
+CONFIG_END
+
+SCENARIO_BEGIN Query receives answer not from the cache
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+ENTRY_END
+
+; another query, different, so not from cache.
+STEP 5 QUERY
+ENTRY_BEGIN
+	SECTION QUESTION
+	www.example.net. IN A
+ENTRY_END
+STEP 6 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.net. IN A
+ENTRY_END
+STEP 7 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	REPLY QR RD RA NOERROR
+	SECTION QUESTION
+	www.example.net. IN A
+	SECTION ANSWER
+	www.example.net. IN A 10.20.30.60
+	SECTION AUTHORITY
+	www.example.net. IN NS ns.example.net.
+	SECTION ADDITIONAL
+	ns.example.net. IN A 10.20.30.50
+ENTRY_END
+STEP 8 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH opcode qname qtype
+	SECTION QUESTION
+	www.example.net. IN A
+	SECTION ANSWER
+	www.example.net. IN A 10.20.30.60
+ENTRY_END
+
+SCENARIO_END

testdata/fwd_timeout.rpl

@@ -1,3 +1,5 @@
+; config options go here.
+CONFIG_END
 SCENARIO_BEGIN Forwarder and a timeout happens on server query.
 STEP 1 QUERY
 ENTRY_BEGIN

testdata/fwd_trunc.rpl

@@ -0,0 +1,69 @@
+; This is a comment.
+; config options go here.
+CONFIG_END
+
+SCENARIO_BEGIN Query forwarded, receives truncated reply.
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - UDP
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode UDP
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; reply TC bit
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname UDP
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA TC NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+ENTRY_END
+; retry in TCP mode.
+STEP 4 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode TCP
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 5 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname TCP
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 6 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all 
+	; first reply, have AA set.
+	REPLY QR AA RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	www.example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+SCENARIO_END

testdata/fwd_two.rpl

@@ -1,3 +1,7 @@
+; config options go here.
+server:
+	num-queries-per-thread: 1
+CONFIG_END
 SCENARIO_BEGIN Sample of a valid query
 
 ; query responses from authority servers.
@@ -5,6 +9,7 @@ RANGE_BEGIN 0 100
 ENTRY_BEGIN
 	MATCH opcode qtype qname
 	ADJUST copy_id
+	REPLY QR RD RA NOERROR
 	SECTION QUESTION
 www.example.net. IN A
 	SECTION ANSWER
@@ -44,6 +49,7 @@ STEP 5 REPLY
 ENTRY_BEGIN
 	MATCH opcode qtype qname
 	ADJUST copy_id
+	REPLY QR RD RA NOERROR
 	SECTION QUESTION
 www.example.com. IN A
 	SECTION ANSWER

testdata/rrset_rettl.rpl

@@ -0,0 +1,119 @@
+; This is a comment.
+; config options go here.
+CONFIG_END
+
+SCENARIO_BEGIN RRset TTL is updated from message.
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. 100 IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all ttl
+	; first reply, have AA set.
+	REPLY QR AA RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. 100 IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+; another query passes along
+STEP 6 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	bla.example.com. IN A
+ENTRY_END
+STEP 7 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	bla.example.com. IN A
+ENTRY_END
+STEP 8 REPLY
+; This answer has a fresh TTL
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	bla.example.com. IN A
+	SECTION ANSWER
+	bla.example.com. IN A 10.20.30.140
+	SECTION AUTHORITY
+	example.com. 200 IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 9 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all ttl
+	; first reply, have AA set.
+	REPLY QR AA RD RA
+	SECTION QUESTION
+	bla.example.com. IN A
+	SECTION ANSWER
+	bla.example.com. IN A 10.20.30.140
+	SECTION AUTHORITY
+	example.com. 200 IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+
+; original www.example.com query 
+STEP 10 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; immediate answer without an OUT_QUERY happening (checked on exit)
+; also, the answer does not have AA set
+; NS rrset has been updated.
+STEP 11 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all ttl
+	REPLY QR RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. 200 IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+SCENARIO_END

testdata/rrset_untrusted.rpl

@@ -0,0 +1,124 @@
+; This is a comment.
+; config options go here.
+CONFIG_END
+
+SCENARIO_BEGIN Untrusted rrset not used for update
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all 
+	; first reply, have AA set.
+	REPLY QR AA RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+; another query passes along
+STEP 6 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	bla.example.com. IN A
+ENTRY_END
+STEP 7 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	bla.example.com. IN A
+ENTRY_END
+STEP 8 REPLY
+; This answer has updated information on ns2.example.com.
+; But it is nerfed into the additional. Must be ignored per rfc2181.
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	bla.example.com. IN A
+	SECTION ANSWER
+	bla.example.com. IN A 10.20.30.140
+	SECTION AUTHORITY
+	SECTION ADDITIONAL
+	example.com. IN NS ns.eeeek.com.
+	example.com. IN NS ns2.eeeek.com.
+	ns.eeeek.com. IN A 55.44.33.22
+	ns2.eeeek.com. IN A 55.44.33.24
+ENTRY_END
+STEP 9 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all 
+	; first reply, have AA set.
+	REPLY QR AA RD RA
+	SECTION QUESTION
+	bla.example.com. IN A
+	SECTION ANSWER
+	bla.example.com. IN A 10.20.30.140
+	SECTION AUTHORITY
+	SECTION ADDITIONAL
+	example.com. IN NS ns.eeeek.com.
+	example.com. IN NS ns2.eeeek.com.
+	ns.eeeek.com. IN A 55.44.33.22
+	ns2.eeeek.com. IN A 55.44.33.24
+ENTRY_END
+
+
+; original www.example.com query 
+STEP 10 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; immediate answer without an OUT_QUERY happening (checked on exit)
+; also, the answer does not have AA set
+; NS rrset has not been updated from the additional seen before.
+STEP 11 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all
+	REPLY QR RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+SCENARIO_END

testdata/rrset_updated.rpl

@@ -0,0 +1,124 @@
+; This is a comment.
+; config options go here.
+CONFIG_END
+
+SCENARIO_BEGIN RRset is updated from other message that passes by.
+
+STEP 1 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; the query is sent to the forwarder - no cache yet.
+STEP 2 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+STEP 3 REPLY
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+STEP 4 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all 
+	; first reply, have AA set.
+	REPLY QR AA RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. IN NS ns.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+; another query passes along
+STEP 6 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	bla.example.com. IN A
+ENTRY_END
+STEP 7 CHECK_OUT_QUERY
+ENTRY_BEGIN
+	MATCH qname qtype opcode
+	SECTION QUESTION
+	bla.example.com. IN A
+ENTRY_END
+STEP 8 REPLY
+; This answer has updated information on ns2.example.com.
+ENTRY_BEGIN
+	MATCH opcode qtype qname
+	ADJUST copy_id
+	; authoritative answer
+	REPLY QR AA RD RA NOERROR
+	SECTION QUESTION
+	bla.example.com. IN A
+	SECTION ANSWER
+	bla.example.com. IN A 10.20.30.140
+	SECTION AUTHORITY
+	example.com. IN NS ns.example.com.
+	example.com. IN NS ns2.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+	ns2.example.com. IN A 10.20.30.53
+ENTRY_END
+STEP 9 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all 
+	; first reply, have AA set.
+	REPLY QR AA RD RA
+	SECTION QUESTION
+	bla.example.com. IN A
+	SECTION ANSWER
+	bla.example.com. IN A 10.20.30.140
+	SECTION AUTHORITY
+	example.com. IN NS ns.example.com.
+	example.com. IN NS ns2.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+	ns2.example.com. IN A 10.20.30.53
+ENTRY_END
+
+
+; original www.example.com query 
+STEP 10 QUERY
+ENTRY_BEGIN
+	REPLY RD
+	SECTION QUESTION
+	www.example.com. IN A
+ENTRY_END
+; immediate answer without an OUT_QUERY happening (checked on exit)
+; also, the answer does not have AA set
+; NS rrset has been updated.
+STEP 11 CHECK_ANSWER
+ENTRY_BEGIN
+	MATCH all
+	REPLY QR RD RA
+	SECTION QUESTION
+	www.example.com. IN A
+	SECTION ANSWER
+	www.example.com. IN A 10.20.30.40
+	SECTION AUTHORITY
+	example.com. IN NS ns.example.com.
+	example.com. IN NS ns2.example.com.
+	SECTION ADDITIONAL
+	ns.example.com. IN A 10.20.30.50
+ENTRY_END
+
+SCENARIO_END

testdata/test_packets.2

@@ -0,0 +1,50 @@
+b29092340000000000000000
+
+289
+98d50000000100000000000003626c3402626c06787334616c6c026e6c0000010001
+98d58000000100000003000203626c3402626c06787334616c6c026e6c0000010001c01300020001000151800005026e73c013c01300020001000151800006036e7332c013c0130002000100015180000d026e730472697065036e657400c02e00010001000151800004c26d0642c03f00010001000151800004c26d0963
+
+290
+b28f6a4e5011431228310000000000000000
+b28fea440000000000000000
+
+291
+de6028000001000000010000026e6c0000060001055f6c646170045f746370026463065f6d736463730b6f7372646f6d65696e2d31026e6c00002100010000025800230000006401850c504f574552454447452d53420b6f7372646f6d65696e2d31026e6c00
+de60a8040000000000000000
+
+292
+b353000000010000000000000450434b4e087963646f6d65696e026e6c0000010001
+b353840300010000000100000450434b4e087963646f6d65696e026e6c0000010001c01a00060001000038400036026e730f646f6d61696e2d7265676973747279c01a0a686f73746d6173746572c0310bf1e540000070800000384000278d0000003840
+
+293
+b2901243501140a2340b0000000000000000
+b29092440000000000000000
+
+294
+b28f6a4f5010431228300000000000000000
+b28fea440000000000000000
+
+295
+e0f600000001000000000000066b6d742d6570026e6c0000020001
+e0f680000001000000020002066b6d742d6570026e6c0000020001c00c0002000100015180000c036e73310564656d6f6ec013c00c00020001000151800006036e7332c02bc02700010001000151800004c29f4906c03f00010001000151800004c29fb004
+
+296
+0000000070024000a52500000204056401010402
+000080010000000000000000
+
+297
+000000007002400088e600000204059801010402
+000080010000000000000000
+
+298
+00000000700240001f5900000204059801010402
+000080010000000000000000
+
+299
+b2901244501040a2340a0000000000000000
+b29092440000000000000000
+
+300
+b293149b50104320eb2f0000000000000000
+b29394940000000000000000
+

testdata/test_packets.3

@@ -0,0 +1,496 @@
+1576
+1ca000000001000000000000036e73310762617365776562026e6c0000010001
+1ca080000001000000020002036e73310762617365776562026e6c0000010001c01000020001000151800002c00cc01000020001000151800006036e7332c010c00c00010001000151800004d535d035c03a00010001000151800004d535d034
+
+1577
+221c0000000100000000000009652d6c756e61746963026e6c0000010001
+221c8000000100000002000009652d6c756e61746963026e6c0000010001c00c0002000100015180000e036e73330767617574656e67c016c00c00020001000151800006036e7334c02e
+
+1578
+b9ab000000010000000000000962626469676974616c026e6c0000010001
+b9ab800000010000000200000962626469676974616c026e6c0000010001c00c0002000100015180001105646e732d33056c61646f7403636f6d00c00c0002000100015180000805646e732d34c030
+
+1579
+b2c21baf5011fae2e9c00000000000000000
+b2c29ba40000000000000000
+
+1580
+e1be0000000100000000000008686f75736561647303696467026e6c0000010001
+e1be8000000100000003000308686f75736561647303696467026e6c0000010001c01500020001000151800006036e7331c015c01500020001000151800006036e7332c015c01500020001000151800006036e7333c015c02d0001000100015180000451af4646c03f0001000100015180000451af4647c05100010001000151800004d5a0fde6
+
+1581
+000000007002faf00a970000020405b401010402
+000080010000000000000000
+
+1582
+02ac00000001000000000000036e73310563736e6574026e6c0000010001
+02ac80000001000000020002036e73310563736e6574026e6c0000010001c01000020001000151800002c00cc01000020001000151800006036e7332c010c00c00010001000151800004c2451012c03800010001000151800004c2451212
+
+1583
+153e00000001000000000000095f6b65726265726f73045f746370026463065f6d736463730644494b534954024e4c0000210001
+153e84030001000000010000095f6b65726265726f73045f746370026463065f6d736463730644494b534954024e4c0000210001c02c00060001000038400036026e730f646f6d61696e2d7265676973747279c02c0a686f73746d6173746572c0430bf1e540000070800000384000278d0000003840
+
+1584
+b2c21bb05010fae2e9bf0000000000000000
+b2c29bb40000000000000000
+
+1585
+b2c2d04f5010faf0b4380000000000000000
+
+
+1586
+b2c2d04f5018faf0747c0000008f552d000000010001000000000f313133333837313336363136322d320000f900010f313133333837313336363136322d320000f900ff00000000005303677373096d6963726f736f667403636f6d00403afc84403c4e040003000000304e544c4d535350000100000097b208e00b000b002500000005000500200000004e54465331544543484e4f50414e454c0000
+
+
+1587
+bf28000000010000000000000962696a656e6b6f7266026e6c0000010001
+bf28800000010000000300030962696a656e6b6f7266026e6c0000010001c00c00020001000151800010036e7331096d61726b6772616166c016c00c00020001000151800006036e7332c02ec00c00020001000151800006036e7333c02ec02a00010001000151800004c3f13ceec04600010001000151800004c372eb43c058000100010001518000043e320e8f
+
+1588
+f95900000001000000000000037777770766636f72696f6e026e6c0000010001
+f95984030001000000010000037777770766636f72696f6e026e6c0000010001c01800060001000038400036026e730f646f6d61696e2d7265676973747279c0180a686f73746d6173746572c02f0bf1e540000070800000384000278d0000003840
+
+1589
+b2c2d05d5011fae2b3a60000000000000000
+
+
+1590
+000000007002faf0d9770000020405b401010402
+000080010000000000000000
+
+1591
+b2c2d05e5010fae2b3a50000000000000000
+
+
+1592
+a18b00000001000000000000036e733107626265796f6e64026e6c0000010001
+a18b80000001000000020002036e733107626265796f6e64026e6c0000010001c01000020001000151800002c00cc01000020001000151800006036e7332c010c00c000100010001518000043eb1900bc03a000100010001518000043eb19002
+
+1593
+b2c379ae5010faf0d9b90000000000000000
+b2c3f9a40000000000000000
+
+1594
+410400000001000000000000036e733107626265796f6e64026e6c0000260001
+410480000001000000020002036e733107626265796f6e64026e6c0000260001c01000020001000151800002c00cc01000020001000151800006036e7332c010c00c000100010001518000043eb1900bc03a000100010001518000043eb19002
+
+1595
+852300000001000000000000037777770f64616d696174652d636f6c6c656765026e6c0000010001
+852380000001000000020002037777770f64616d696174652d636f6c6c656765026e6c0000010001c0100002000100015180000d036e7331066e6c74726565c020c01000020001000151800006036e7332c038c03400010001000151800004d4b20403c04d00010001000151800004d5881e45
+
+1596
+208200000001000000000000036e733207626265796f6e64026e6c0000010001
+208280000001000000020002036e733207626265796f6e64026e6c0000010001c01000020001000151800006036e7331c010c01000020001000151800002c00cc02c000100010001518000043eb1900bc00c000100010001518000043eb19002
+
+1597
+883a00000001000000000000036e733207626265796f6e64026e6c0000260001
+883a80000001000000020002036e733207626265796f6e64026e6c0000260001c01000020001000151800006036e7331c010c01000020001000151800002c00cc02c000100010001518000043eb1900bc00c000100010001518000043eb19002
+
+1598
+b2c379ae5018faf0d58e0000008f199c000000010001000000000f313133333837313336363136322d320000f900010f313133333837313336363136322d320000f900ff00000000005303677373096d6963726f736f667403636f6d00403afc84403c4e040003000000304e544c4d535350000100000097b208e00b000b002500000005000500200000004e54465331544543484e4f50414e454c0000
+b2c3f9a40000000000000000
+
+1599
+2a640000000100000000000009652d6c756e61746963026e6c0000010001
+2a648000000100000002000009652d6c756e61746963026e6c0000010001c00c0002000100015180000e036e73330767617574656e67c016c00c00020001000151800006036e7334c02e
+
+1600
+99a400000001000000000000037777770e7765736c6579736e65696a646572026e6c0000010001
+99a480000001000000020000037777770e7765736c6579736e65696a646572026e6c0000010001c01000020001000151800014046e733031096772616669782d697303636f6d00c01000020001000151800007046e733033c038
+
+1601
+7ab500000001000000000000096d61696c686f7374320b66726565686f7374696e67026e6c0000010001
+7ab580000001000000020000096d61696c686f7374320b66726565686f7374696e67026e6c0000010001c0160002000100015180001105646e732d33056c61646f7403636f6d00c0160002000100015180000805646e732d34c03c
+
+1602
+c3e800000001000000000000037777770f64616d696174652d636f6c6c656765026e6c0000010001
+c3e880000001000000020002037777770f64616d696174652d636f6c6c656765026e6c0000010001c0100002000100015180000d036e7331066e6c74726565c020c01000020001000151800006036e7332c038c03400010001000151800004d4b20403c04d00010001000151800004d5881e45
+
+1603
+95af00100001000000000001046d61696c03777873026e6c00000100010000290800000080000000
+95af80000001000000030004046d61696c03777873026e6c0000010001c01100020001000151800007046e733038c011c01100020001000151800007046e733039c011c01100020001000151800006036e7334c011c02900010001000151800004c3790128c03c00010001000151800004c3790143c04f00010001000151800004d5ef9a650000291000000000000000
+
+1604
+b2c379bc5011fae2d9270000000000000000
+b2c3f9b40000000000000000
+
+1605
+61be00000001000000000000037777771a676f6c666261616e68657472696a6b76616e6e696a6d6567656e026e6c0000010001
+61be80000001000000020000037777771a676f6c666261616e68657472696a6b76616e6e696a6d6567656e026e6c0000010001c0100002000100015180000b026e73026e6c036e657400c0100002000100015180001206617574683630026e73026e6c027575c045
+
+1606
+f99c00000001000000000000095f6b65726265726f73045f746370026463065f6d736463730b746563686e6f70616e656c026e6c0000060001
+f99c84030001000000010000095f6b65726265726f73045f746370026463065f6d736463730b746563686e6f70616e656c026e6c0000060001c03100060001000038400036026e730f646f6d61696e2d7265676973747279c0310a686f73746d6173746572c0480bf1e540000070800000384000278d0000003840
+
+1607
+d8bd00000001000000000000037777770c636170636974796368657679026e6c0000010001
+d8bd84030001000000010000037777770c636170636974796368657679026e6c0000010001c01d00060001000038400036026e730f646f6d61696e2d7265676973747279c01d0a686f73746d6173746572c0340bf1e540000070800000384000278d0000003840
+
+1608
+f05f00000001000000000000087261626f62616e6b026e6c0000010001
+f05f80000001000000020001087261626f62616e6b026e6c0000010001c00c00020001000151800005026e73c00cc00c0002000100015180000b026e73026e6c036e657400c0290001000100015180000491484fde
+
+1609
+b2c379bd5010fae2d9260000000000000000
+b2c3f9b40000000000000000
+
+1610
+175928000001000000010000026e6c0000060001095f6b65726265726f73045f746370026463065f6d736463730b746563686e6f70616e656c026e6c000021000100000258001c000000640058056e746673310b746563686e6f70616e656c026e6c00
+1759a8040000000000000000
+
+1611
+4d9a0000000100000000000005646e732d31056c61646f74026e6c0000010001
+4d9a8000000100000002000005646e732d31056c61646f74026e6c0000010001c0120002000100015180001105646e732d33056c61646f7403636f6d00c0120002000100015180000805646e732d34c032
+
+1612
+adb60000000100000000000005646e732d32056c61646f74026e6c0000010001
+adb68000000100000002000005646e732d32056c61646f74026e6c0000010001c0120002000100015180001105646e732d33056c61646f7403636f6d00c0120002000100015180000805646e732d34c032
+
+1613
+b22d00000001000000000000037777771777696a6b766572656e6967696e672d68616e65766f6574026e6c0000010001
+b22d84030001000000010000037777771777696a6b766572656e6967696e672d68616e65766f6574026e6c0000010001c02800060001000038400036026e730f646f6d61696e2d7265676973747279c0280a686f73746d6173746572c03f0bf1e540000070800000384000278d0000003840
+
+1614
+ba9a00000001000000000000037777771076616b616e746965776f6e696e67656e0f76616b616e7469652d61616e626f64026e6c0000010001
+ba9a80000001000000020000037777771076616b616e746965776f6e696e67656e0f76616b616e7469652d61616e626f64026e6c0000010001c02100020001000151800011036e733005736572766502636f02756b00c02100020001000151800010036e7330057365727665036e6574c052
+
+1615
+000000007002faf0ec1e0000020405b401010402
+000080010000000000000000
+
+1616
+8f9e00000001000000000000097463676e777331313608686e65746f6e7477026e6c0000010001
+8f9e84030001000000010000097463676e777331313608686e65746f6e7477026e6c0000010001c01f00060001000038400036026e730f646f6d61696e2d7265676973747279c01f0a686f73746d6173746572c0360bf1e540000070800000384000278d0000003840
+
+1617
+230b0000000100000000000004646e73310471776562026e6c0000260001
+230b8000000100000002000204646e73310471776562026e6c0000260001c01100020001000151800002c00cc0110002000100015180000704646e7332c011c00c00010001000151800004d994ac0cc03800010001000151800004d5c42f6c
+
+1618
+f8200000000100000000000004646e73320471776562026e6c0000260001
+f8208000000100000002000204646e73320471776562026e6c0000260001c0110002000100015180000704646e7331c011c01100020001000151800002c00cc02a00010001000151800004d994ac0cc00c00010001000151800004d5c42f6c
+
+1619
+825800000001000000000000046d61696c0761627374726163026e6c0000010001
+825880000001000000020002046d61696c0761627374726163026e6c0000010001c01100020001000151800013036e73310c706f77657273657276657233c019c01100020001000151800006036e7332c031c02d000100010001518000045104741ec04c000100010001518000045104741f
+
+1620
+f8f000000001000000000000037777770d6165726f736f6c64657369676e026e6c0000010001
+f8f080000001000000020002037777770d6165726f736f6c64657369676e026e6c0000010001c01000020001000151800006036e7331c010c01000020001000151800006036e7332c010c0320001000100015180000451044432c0440001000100015180000451044433
+
+1621
+cd46000000010000000000000c626c65696a656e6265726768026e6c0000ff0001
+cd46800000010000000200020c626c65696a656e6265726768026e6c0000ff0001c00c00020001000151800010036e7331096e657467726f756e64c019c00c00020001000151800006036e7332c031c02d000100010001518000045054e415c0490001000100015180000451046014
+
+1622
+b2c4383f5010faf02dcf0000000000000000
+b2c4b8340000000000000000
+
+1623
+5e1200000001000000000000037777770864616e69656c6c61026e6c0000010001
+5e1280000001000000020002037777770864616e69656c6c61026e6c0000010001c01000020001000151800012026e730c6d65676170726f7669646572c019c01000020001000151800006036e7332c030c02d0001000100015180000450474002c04b0001000100015180000450474102
+
+1624
+b2c4383f5018faf0d3750000008f6ec9000000010001000000000f313133333837313336363136322d330000f900010f313133333837313336363136322d330000f900ff00000000005303677373096d6963726f736f667403636f6d00403afc84403c4e040003000000304e544c4d535350000100000097b208e00b000b002500000005000500200000004e54465331544543484e4f50414e454c0000
+b2c4b8340000000000000000
+
+1625
+e81f000000010000000000000d7469636b6574736576656e7473026e6c00000f0001
+e81f800000010000000200020d7469636b6574736576656e7473026e6c00000f0001c00c0002000100015180000c026e7306787334616c6cc01ac00c00020001000151800006036e7332c031c02e00010001000151800004c26d0642c04600010001000151800004c26d0963
+
+1626
+6bf800100001000000000001034e53320743495354524f4e024e4c00000100010000290800000080000000
+6bf880000001000000020003034e53320743495354524f4e024e4c0000010001c01000020001000151800005026e73c010c01000020001000151800002c00cc02c000100010001518000043ed81f37c00c000100010001518000043ed81f380000291000000000000000
+
+1627
+b2c4384d5011fae22d3d0000000000000000
+b2c4b8440000000000000000
+
+1628
+000000007002faf042610000020405b401010402
+000080010000000000000000
+
+1629
+39f400000001000000000000036e73320a6461632d706c616e6574026e6c0000010001
+39f480000001000000030003036e73320a6461632d706c616e6574026e6c0000010001c0100002000100015180000c036e7331056264726567c01bc01000020001000151800006036e7332c033c01000020001000151800006036e7333c033c02f00010001000151800004d44ff4dac04700010001000151800004d44ff642c05900010001000151800004c3f5c70a
+
+1630
+b2c4384e5010fae22d3c0000000000000000
+b2c4b8440000000000000000
+
+1631
+6adb00000001000000000000037777770773657870657274026e6c0000010001
+6adb80000001000000030003037777770773657870657274026e6c0000010001c0100002000100015180000e04646e733106626c6978656dc018c0100002000100015180000704646e7332c031c0100002000100015180000704646e7333c031c02c00010001000151800004d44fe803c04600010001000151800004c3f5c782c05900010001000151800004d44ff303
+
+1632
+b2c4e8a95010faf0d3a60000000000000000
+
+
+1633
+3a440000000100000000000006504152495445024e4c00000f0001
+3a448000000100000002000206504152495445024e4c00000f0001c00c0002000100015180000c036e7331056f72696f6ec013c00c00020001000151800006036e7332c02bc02700010001000151800004c2862282c03f00010001000151800004c2862b02
+
+1634
+e9e400000001000000000000037777770b617564696f636f76657273026e6c0000010001
+e9e480000001000000020000037777770b617564696f636f76657273026e6c0000010001c0100002000100015180001105646e732d33056c61646f7403636f6d00c0100002000100015180000805646e732d34c036
+
+1635
+b2c4e8a95018faf0b5df0000008f3338000000010001000000000f313133333837313336363136322d320000f900010f313133333837313336363136322d320000f900ff00000000005303677373096d6963726f736f667403636f6d00403afc84403c4e040003000000304e544c4d535350000100000097b208e00b000b002500000005000500200000004e54465331544543484e4f50414e454c0000
+
+
+1636
+02eb00000001000000000000046d61696c03777873026e6c0000010001
+02eb80000001000000030003046d61696c03777873026e6c0000010001c01100020001000151800007046e733038c011c01100020001000151800007046e733039c011c01100020001000151800006036e7334c011c02900010001000151800004c3790128c03c00010001000151800004c3790143c04f00010001000151800004d5ef9a65
+
+1637
+e16d00000001000000000000086a756c69657474650762657374776562026e6c0000010001
+e16d80000001000000020002086a756c69657474650762657374776562026e6c0000010001c0150002000100015180000e08706f7765722d6133026133c01dc0150002000100015180000c09706f7765722d697078c03ac0310001000100015180000451113b42c04b00010001000151800004d5ab40e6
+
+1638
+04920000000100000000000003323034033139320236310232340770726f786965730a626c61636b686f6c657307656173796e6574026e6c00001c0001
+04928000000100000002000203323034033139320236310232340770726f786965730a626c61636b686f6c657307656173796e6574026e6c00001c0001c02d00020001000151800006036e7330c02dc02d00020001000151800006036e7331c02dc04900010001000151800004c2a55e01c05b00010001000151800004c2a55e05
+
+1639
+b2c4e8b75011fae2d3140000000000000000
+
+
+1640
+000000007002faf00f9e0000020405b401010402
+000080010000000000000000
+
+1641
+613000000001000000000000046d61696c086e65646c696e7578026e6c0000010001
+613080000001000000030003046d61696c086e65646c696e7578026e6c0000010001c01100020001000151800006036e7331c011c01100020001000151800006036e7332c011c01100020001000151800006036e7333c011c02e00010001000151800004d5ef87d2c04000010001000151800004d5ef87d3c05200010001000151800004d5ef8005
+
+1642
+b2c4e8b85010fae2d3130000000000000000
+
+
+1643
+e6930000000100000000000006636861726973026e6c00000f0001
+e6938000000100000002000006636861726973026e6c00000f0001c00c0002000100015180000c026e7306766576696461c013c00c00020001000151800006036e7332c02a
+
+1644
+b2c581cb5010faf007c10000000000000000
+
+
+1645
+b2c581cb5018faf0258a0000008ff7a7000000010001000000000f313133333837313336363136322d320000f900010f313133333837313336363136322d320000f900ff00000000005303677373096d6963726f736f667403636f6d00403afc84403c4e040003000000304e544c4d535350000100000097b208e00b000b002500000005000500200000004e54465331544543484e4f50414e454c0000
+
+
+1646
+e97b00000001000000000000036e73310b7669727475616c686f7374026e6c0000010001
+e97b80000001000000020000036e73310b7669727475616c686f7374026e6c0000010001c01000020001000151800013036e733109676c6f62616c6e656403636f6d00c01000020001000151800006036e7332c034
+
+1647
+e97c00000001000000000000036e73320b7669727475616c686f7374026e6c0000010001
+e97c80000001000000020000036e73320b7669727475616c686f7374026e6c0000010001c01000020001000151800013036e733109676c6f62616c6e656403636f6d00c01000020001000151800006036e7332c034
+
+1648
+bffb0000000100000000000003626c3102626c06787334616c6c026e6c0000010001
+bffb8000000100000003000203626c3102626c06787334616c6c026e6c0000010001c01300020001000151800005026e73c013c01300020001000151800006036e7332c013c0130002000100015180000d026e730472697065036e657400c02e00010001000151800004c26d0642c03f00010001000151800004c26d0963
+
+1649
+37fd00000001000000000000036e73340767617574656e67026e6c0000010001
+37fd80000001000000020000036e73340767617574656e67026e6c0000010001c0100002000100015180000d036e7331066c696e6b7570c018c01000020001000151800006036e7332c030
+
+1650
+73360000000100000000000004646e7331057368656c6c026e6c0000010001
+73368000000100000003000004646e7331057368656c6c026e6c0000010001c0110002000100015180001004646e7331057368656c6c03636f6d00c0110002000100015180000704646e7332c030c0110002000100015180000704646e7333c030
+
+1651
+8bea00000001000000000000037777770764736c6465736b026e6c0000010001
+8bea80000001000000020000037777770764736c6465736b026e6c0000010001c01000020001000151800014036e73310a676f6c64766973696f6e03636f6d00c01000020001000151800006036e7332c030
+
+1652
+35fc00000001000000000000034e53320743495354524f4e024e4c0000010001
+35fc80000001000000020002034e53320743495354524f4e024e4c0000010001c01000020001000151800005026e73c010c01000020001000151800002c00cc02c000100010001518000043ed81f37c00c000100010001518000043ed81f38
+
+1653
+b2c581d95011fae2072f0000000000000000
+
+
+1654
+e94000000001000000000000037777770a6c696e6b746970706572026e6c0000010001
+e94080000001000000030003037777770a6c696e6b746970706572026e6c0000010001c0100002000100015180000c036e7331056d61786573c01bc01000020001000151800006036e7332c033c01000020001000151800006036e7333c033c02f000100010001518000045045402ec047000100010001518000045045402fc059000100010001518000045045440a
+
+1655
+77be0000000100000000000003777777086879706f686f6d65026e6c0000010001
+77be8000000100000002000003777777086879706f686f6d65026e6c0000010001c01000020001000151800013036e733109696d6167653264617903636f6d00c01000020001000151800006036e7332c031
+
+1656
+4fb500000001000000000000036e733208696e74726f776562026e6c0000010001
+4fb580000001000000020002036e733208696e74726f776562026e6c0000010001c01000020001000151800006036e7331c010c01000020001000151800002c00cc02d0001000100015180000450416028c00c00010001000151800004c356780c
+
+1657
+b2c581da5010fae2072e0000000000000000
+
+
+1658
+5c6a0010000100000000000007636172746d616e076e6574666c6f77026e6c0000010001
+5c6a8000000100000002000207636172746d616e076e6574666c6f77026e6c0000010001c01400020001000151800006036e7331c014c01400020001000151800006036e7332c014c030000100010001518000043eb1ef24c042000100010001518000043eb1ef25
+
+1659
+92d100000001000000000000036e7332076e656473746174026e6c0000010001
+92d180000001000000020002036e7332076e656473746174026e6c0000010001c01000020001000151800005026e73c010c01000020001000151800006036e7333c010c02c00010001000151800004d4482615c03d00010001000151800004d5c4021d
+
+1660
+921500000001000000000000036e7334076e656473746174026e6c0000010001
+921580000001000000020002036e7334076e656473746174026e6c0000010001c01000020001000151800005026e73c010c01000020001000151800006036e7333c010c02c00010001000151800004d4482615c03d00010001000151800004d5c4021d
+
+1661
+ba9500000001000000000000036e7332076e656473746174026e6c0000260001
+ba9580000001000000020002036e7332076e656473746174026e6c0000260001c01000020001000151800005026e73c010c01000020001000151800006036e7333c010c02c00010001000151800004d4482615c03d00010001000151800004d5c4021d
+
+1662
+3a9f00000001000000000000036e7334076e656473746174026e6c0000260001
+3a9f80000001000000020002036e7334076e656473746174026e6c0000260001c01000020001000151800005026e73c010c01000020001000151800006036e7333c010c02c00010001000151800004d4482615c03d00010001000151800004d5c4021d
+
+1663
+175c28000001000000010000026e6c0000060001095f6b65726265726f73045f746370155374616e64616172642d6565727374652d73697465065f7369746573026463065f6d736463730b746563686e6f70616e656c026e6c000021000100000258001c000000640058056e746673310b746563686e6f70616e656c026e6c00
+175ca8040000000000000000
+
+1664
+3cfa00000001000000000000036e733108696e74726f776562026e6c0000010001
+3cfa80000001000000020002036e733108696e74726f776562026e6c0000010001c01000020001000151800002c00cc01000020001000151800006036e7332c010c00c0001000100015180000450416028c03b00010001000151800004c356780c
+
+1665
+e0ea0000000100000000000006636f6f6b696504696c7365026e6c0000010001
+e0ea8000000100000003000306636f6f6b696504696c7365026e6c0000010001c01300020001000151800006036e7331c013c01300020001000151800006036e7332c013c01300020001000151800006036e7333c013c02c000100010001518000043e45a282c03e000100010001518000043e45a283c050000100010001518000043e45a284
+
+1666
+000000007002faf01bd90000020405b401010402
+000080010000000000000000
+
+1667
+ddaf000000010000000000000377777709766572626f7577656e026e6c0000010001
+ddaf800000010000000200020377777709766572626f7577656e026e6c0000010001c0100002000100015180000f036e7331086765656e70756e74c01ac01000020001000151800006036e7332c032c02e00010001000151800004d4ccd865c049000100010001518000045054ec34
+
+1668
+b2c62b8d5010faf06a390000000000000000
+b2c6ab840000000000000000
+
+1669
+b2c62b8d5018faf031d50000008f4cd4000000010001000000000f313133333837313336363136322d330000f900010f313133333837313336363136322d330000f900ff00000000005303677373096d6963726f736f667403636f6d00403afc84403c4e040003000000304e544c4d535350000100000097b208e00b000b002500000005000500200000004e54465331544543484e4f50414e454c0000
+b2c6ab840000000000000000
+
+1670
+897c0000000100000000000004686f6d650774697363616c69026e6c0000010001
+897c8000000100000003000204686f6d650774697363616c69026e6c0000010001c0110002000100015180000704616e7331c011c0110002000100015180000704616e7332c011c0110002000100015180001404616e73330a74697363616c696e657402626500c02d00010001000151800004c3f14d38c04000010001000151800004c3f13325
+
+1671
+4caa00000001000000000000086f76657268656964047a6f656b026e6c0000010001
+4caa80000001000000020002086f76657268656964047a6f656b026e6c0000010001c0150002000100015180000d026e73076e65746c616e64c01ac01500020001000151800006036e7332c031c02e00010001000151800004d9aa2042c04700010001000151800004d9aa2c06
+
+1672
+5eb200000001000000000000086f76657268656964047a6f656b026e6c0000010001
+5eb280000001000000020002086f76657268656964047a6f656b026e6c0000010001c0150002000100015180000d026e73076e65746c616e64c01ac01500020001000151800006036e7332c031c02e00010001000151800004d9aa2042c04700010001000151800004d9aa2c06
+
+1673
+b2c62b9b5011fae269a70000000000000000
+b2c6ab940000000000000000
+
+1674
+000000007002faf04d060000020405b401010402
+000080010000000000000000
+
+1675
+e97f00000001000000000000036e73320b7669727475616c686f7374026e6c0000010001
+e97f80000001000000020000036e73320b7669727475616c686f7374026e6c0000010001c01000020001000151800013036e733109676c6f62616c6e656403636f6d00c01000020001000151800006036e7332c034
+
+1676
+e97e00000001000000000000036e73310b7669727475616c686f7374026e6c0000010001
+e97e80000001000000020000036e73310b7669727475616c686f7374026e6c0000010001c01000020001000151800013036e733109676c6f62616c6e656403636f6d00c01000020001000151800006036e7332c034
+
+1677
+c36200000001000000000000046d61696c06737469626265026e6c0000010001
+c36280000001000000020002046d61696c06737469626265026e6c0000010001c01100020001000151800011026e730b666163696e676661637473c018c01100020001000151800006036e7332c02fc02c00010001000151800004d513b013c04900010001000151800004d513b012
+
+1678
+7637000000010000000000000777656276696577026e6c00000f0001
+7637800000010000000200000777656276696577026e6c00000f0001c00c00020001000151800012036e73310b6d6565737465726c696a6bc014c00c00020001000151800006036e7332c02c
+
+1679
+e7000000000100000000000005627261616d026e6c00000f0001
+e7008000000100000002000205627261616d026e6c00000f0001c00c0002000100015180000b036e73310461786974c012c00c00020001000151800006036e7332c02ac026000100010001518000043e3a440bc03d000100010001518000043e3a440c
+
+1680
+b2c62b9c5010fae269a60000000000000000
+b2c6ab940000000000000000
+
+1681
+e5cc000000010000000000000f6d656c6b7765672d686f6c6c616e64026e6c00000f0001
+e5cc800000010000000300030f6d656c6b7765672d686f6c6c616e64026e6c00000f0001c00c0002000100015180000e076e73617574683103626974c01cc00c0002000100015180000a076e736175746832c038c00c0002000100015180000a076e736175746833c038c03000010001000151800004d5880c33c04a00010001000151800004d5880c3bc06000010001000151800004d4ccc00b
+
+1682
+3a0f000000010000000000000964656e68656c646572026e6c00000f0001
+3a0f800000010000000200020964656e68656c646572026e6c00000f0001c00c00020001000151800011036e73310a7765626275726f646e73c016c00c00020001000151800006036e7332c02ec02a00010001000151800004d51380cec04700010001000151800004d513806a
+
+1683
+b2c6b1045010faf015ef0000000000000000
+
+
+1684
+b2c6b1045018faf01a1d0000008f1143000000010001000000000f313133333837313336363136322d320000f900010f313133333837313336363136322d320000f900ff00000000005303677373096d6963726f736f667403636f6d00403afc84403c4e040003000000304e544c4d535350000100000097b208e00b000b002500000005000500200000004e54465331544543484e4f50414e454c0000
+
+
+1685
+c22900000001000000000000036e733103727567026e6c0000010001
+c22980000001000000030003036e733103727567026e6c0000010001c01000020001000151800002c00cc01000020001000151800006036e7332c010c0100002000100015180000e036e733107737572666e6574c014c00c00010001000151800004817d0406c03600010001000151800004817d040dc04800010001000151800004c0576a65
+
+1686
+ab8400000001000000000000036e733203727567026e6c0000010001
+ab8480000001000000030003036e733203727567026e6c0000010001c01000020001000151800006036e7331c010c01000020001000151800002c00cc0100002000100015180000e036e733107737572666e6574c014c02800010001000151800004817d0406c00c00010001000151800004817d040dc04800010001000151800004c0576a65
+
+1687
+0359000000010000000000000f50432d30392054454120262052494108627275696e686f66026e6c0000010001
+0359800000010000000300020f50432d30392054454120262052494108627275696e686f66026e6c0000010001c01c0002000100015180000d036e733106776964657873c025c01c00020001000151800010036e733206776964657873036e657400c01c00020001000151800006036e7333c03dc03900010001000151800004d4ccc0fcc06e000100010001518000043efa0703
+
+1688
+38fd0000000100000000000006636861726973026e6c0000ff0001
+38fd8000000100000002000006636861726973026e6c0000ff0001c00c0002000100015180000c026e7306766576696461c013c00c00020001000151800006036e7332c02a
+
+1689
+4e9100000001000000000000076e6372766e6574026e6c00000f0001
+4e9180000001000000030003076e6372766e6574026e6c00000f0001c00c0002000100015180001208736f6c7574696f6e06736f6c636f6ec014c00c00020001000151800006036e7331c031c00c00020001000151800006036e7332c031c02800010001000151800004d42d2005c04600010001000151800004d42d2003c05800010001000151800004d42d2103
+
+1690
+964b00000001000000000000066b7669616e61036b7669026e6c0000010001
+964b80000001000000030003066b7669616e61036b7669026e6c0000010001c01300020001000151800009066b7669657870c013c01300020001000151800002c00cc0130002000100015180000e04616a6178066e696b686566c017c02b00010001000151800004817d0f01c00c00010001000151800004817d0f02c04e00010001000151800004c010c701
+
+1691
+0ba600000001000000000000066b7669617376036b7669026e6c0000010001
+0ba680000001000000030003066b7669617376036b7669026e6c0000010001c01300020001000151800009066b7669657870c013c01300020001000151800009066b7669616e61c013c0130002000100015180000e04616a6178066e696b686566c017c02b00010001000151800004817d0f01c04000010001000151800004817d0f02c05500010001000151800004c010c701
+
+1692
+85d300000001000000000000066b7669657870036b7669026e6c0000010001
+85d380000001000000030003066b7669657870036b7669026e6c0000010001c01300020001000151800002c00cc01300020001000151800009066b7669616e61c013c0130002000100015180000e04616a6178066e696b686566c017c00c00010001000151800004817d0f01c03900010001000151800004817d0f02c04e00010001000151800004c010c701
+
+1693
+11fc000000010000000000000234330332313503313733033134320770726f786965730a626c61636b686f6c657307656173796e6574026e6c00001c0001
+11fc800000010000000200020234330332313503313733033134320770726f786965730a626c61636b686f6c657307656173796e6574026e6c00001c0001c02e00020001000151800006036e7330c02ec02e00020001000151800006036e7331c02ec04a00010001000151800004c2a55e01c05c00010001000151800004c2a55e05
+
+1694
+769c000000010000000000000a6d61726b656e6865656d026e6c00000f0001
+769c800000010000000200020a6d61726b656e6865656d026e6c00000f0001c00c0002000100015180000c026e7306787334616c6cc017c00c00020001000151800006036e7332c02ec02b00010001000151800004c26d0642c04300010001000151800004c26d0963
+
+1695
+9355000000010000000000000d3231312d3233322d34372d343708696e746572746e7303636f6d06787369746573026e6c0000010001
+9355800000010000000200020d3231312d3233322d34372d343708696e746572746e7303636f6d06787369746573026e6c0000010001c02700020001000151800006036e7331c027c02700020001000151800006036e7332c027c042000100010001518000045054f526c054000100010001518000045054f527
+
+1696
+51c600000001000000000000056761726f74026e6c00000f0001
+51c680000001000000020002056761726f74026e6c00000f0001c00c0002000100015180000f026e7309656174736572766572c012c00c00020001000151800006036e7332c029c02600010001000151800004d4cb0e42c04100010001000151800004d4cb1221
+
+1697
+b2c6b1125011fae2155d0000000000000000
+
+
+1698
+000000007002faf041280000020405b401010402
+000080010000000000000000
+
+1699
+c1f700000001000000000000096e696a656e6b616d70026e6c00000f0001
+c1f780000001000000020002096e696a656e6b616d70026e6c00000f0001c00c0002000100015180000f036e733108696e74726f776562c016c00c00020001000151800006036e7332c02ec02a0001000100015180000450416028c04500010001000151800004c356780c
+

testdata/test_packets.4

@@ -0,0 +1,929 @@
+; DNSSEC test packets.
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 2e 15 85 00 00 01 00 01 00 03 00 05 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 01 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 01 00 01 00 01 51 80 00 04 d5 9a e0 01 09 6e	;	  41-  60
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80	;	  61-  80
+ 00 18 03 6e 73 37 0f 64 6f 6d 61 69 6e 2d 72 65 67 69 73 74	;	  81- 100
+ 72 79 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	 101- 120
+ 00 02 00 01 00 01 51 80 00 13 04 6f 70 65 6e 09 6e 6c 6e 65	;	 121- 140
+ 74 6c 61 62 73 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73 02	;	 141- 160
+ 6e 6c 00 00 02 00 01 00 01 51 80 00 11 05 6f 6d 76 61 6c 06	;	 161- 180
+ 74 65 64 6e 65 74 02 6e 6c 00 04 6f 70 65 6e 09 6e 6c 6e 65	;	 181- 200
+ 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 00 01 51 80 00 04 d5	;	 201- 220
+ 9a e0 01 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e	;	 221- 240
+ 6c 00 00 1c 00 01 00 01 51 80 00 10 20 01 07 b8 02 06 00 01	;	 241- 260
+ 00 00 00 00 00 00 00 53 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c	;	 261- 280
+ 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51 80 00 10 20 01 07	;	 281- 300
+ b8 02 06 00 01 00 00 00 00 00 00 00 01 05 6f 6d 76 61 6c 06	;	 301- 320
+ 74 65 64 6e 65 74 02 6e 6c 00 00 01 00 01 00 00 70 80 00 04	;	 321- 340
+ d5 9a e0 11 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c	;	 341- 360
+ 00 00 1c 00 01 00 00 70 80 00 10 20 01 07 b8 02 06 00 01 02	;	 361- 380
+ 00 39 ff fe 59 b1 87
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 4d 79 85 00 00 01 00 02 00 04 00 08 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 01 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 01 00 01 00 01 51 80 00 04 d5 9a e0 01 09 6e	;	  41-  60
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80	;	  61-  80
+ 00 a0 00 01 05 02 00 01 51 80 46 3d 18 29 46 15 8b 29 ab 0f	;	  81- 100
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 46 0f 5f 69 aa fe	;	 101- 120
+ 22 ae bf 65 52 2f 18 39 d7 3c 76 6b b0 4d a8 e4 32 46 60 e2	;	 121- 140
+ 3b 68 5f 3f b1 3a 7b 4f 3f 47 62 45 89 19 60 a8 04 53 19 9b	;	 141- 160
+ 6c 7d 5a eb fe ce b3 2e 5b 2d 7d e5 87 3f 53 f7 3a ab c3 39	;	 161- 180
+ 83 bb 1a 77 a1 1a 36 4d bc 46 4d 28 3c ba 93 70 d7 dd dc 2e	;	 181- 200
+ f9 9b ea eb a8 75 8e 6b a2 03 41 4f f6 36 77 8a 61 87 72 b3	;	 201- 220
+ 4f db 96 20 19 04 93 f8 63 cb 0a 55 cf 25 95 9c fd 94 e6 90	;	 221- 240
+ 7f 5f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01	;	 241- 260
+ 00 01 51 80 00 18 03 6e 73 37 0f 64 6f 6d 61 69 6e 2d 72 65	;	 261- 280
+ 67 69 73 74 72 79 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73	;	 281- 300
+ 02 6e 6c 00 00 02 00 01 00 01 51 80 00 13 04 6f 70 65 6e 09	;	 301- 320
+ 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 09 6e 6c 6e 65 74 6c	;	 321- 340
+ 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00 11 05 6f 6d	;	 341- 360
+ 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 09 6e 6c 6e 65 74	;	 361- 380
+ 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 02	;	 381- 400
+ 05 02 00 01 51 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e	;	 401- 420
+ 65 74 6c 61 62 73 02 6e 6c 00 ad 07 18 03 3f 18 f6 5c 35 8d	;	 421- 440
+ 5b 89 d8 cf 6a 11 fc 3c 32 6b bc 06 e3 c4 c8 ec 47 83 1d ab	;	 441- 460
+ a0 ea e3 af 43 b0 f5 ec aa 5d 4e aa fa 79 6f 1a 2d fc 1f 44	;	 461- 480
+ 81 7a 61 a2 49 76 0b 8e 9d 9f 34 95 66 56 89 99 dc f6 31 6e	;	 481- 500
+ 42 bb 16 45 93 60 44 7b 19 3a c4 e6 9a ab 69 9a b5 84 8f 86	;	 501- 520
+ b5 17 af c1 e0 4c b1 33 eb 67 cc cf 5b f6 e9 27 ff f3 d9 d0	;	 521- 540
+ c5 8e f0 91 b0 ab 77 93 18 e7 7e dc 44 1e f1 73 72 7e 04 6f	;	 541- 560
+ 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00	;	 561- 580
+ 01 00 01 51 80 00 04 d5 9a e0 01 04 6f 70 65 6e 09 6e 6c 6e	;	 581- 600
+ 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51 80 00 10	;	 601- 620
+ 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00 53 04 6f 70 65	;	 621- 640
+ 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00	;	 641- 660
+ 01 51 80 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00	;	 661- 680
+ 01 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 00 01	;	 681- 700
+ 00 01 00 00 70 80 00 04 d5 9a e0 11 05 6f 6d 76 61 6c 06 74	;	 701- 720
+ 65 64 6e 65 74 02 6e 6c 00 00 1c 00 01 00 00 70 80 00 10 20	;	 721- 740
+ 01 07 b8 02 06 00 01 02 00 39 ff fe 59 b1 87 04 6f 70 65 6e	;	 741- 760
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01	;	 761- 780
+ 51 80 00 a0 00 01 05 03 00 01 51 80 46 3d 18 29 46 15 8b 29	;	 781- 800
+ ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 33 42 f9 d1	;	 801- 820
+ 94 cb d3 27 dc e3 6b fd 80 34 a8 c0 ab 2a 76 f8 ef 9d c2 f7	;	 821- 840
+ b4 08 2f ff 3c 35 c1 b4 13 46 5b 05 38 e7 97 96 41 b9 01 ff	;	 841- 860
+ 8c 0b de 62 78 29 68 2f fe fb 53 7a 61 92 3f 89 79 3e df 49	;	 861- 880
+ cc ad a3 f7 45 13 7b 99 4c 0b 74 06 ee 1e 97 5a fc 6b 74 60	;	 881- 900
+ 6d a7 68 24 5f 01 f8 5a e2 84 ad 18 cf 52 ff f7 a5 66 84 76	;	 901- 920
+ b0 34 5c 42 8f c6 58 c2 38 c4 8c 6b 7a 72 90 4c 9c 77 90 d4	;	 921- 940
+ dc cf cf 2a 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02	;	 941- 960
+ 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 1c 05 03 00 01 51	;	 961- 980
+ 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62	;	 981-1000
+ 73 02 6e 6c 00 53 09 41 45 b5 88 59 88 6e 2a 21 8e 42 ac 80	;	1001-1020
+ 7c e2 9f 2e d4 f6 48 3f 70 40 d8 61 1d 3f ea e2 ba ad 20 83	;	1021-1040
+ 80 bc 7e 3f 40 e7 41 74 9c c3 7c 51 30 e6 f2 99 50 0a 3f d9	;	1041-1060
+ 26 5a fd d2 38 09 79 e5 27 f1 25 80 e5 13 58 a5 2b 95 70 c3	;	1061-1080
+ 53 cd ee 0e c3 07 32 f6 73 17 60 b2 1d 3a b0 a4 0c 8f ef c3	;	1081-1100
+ 65 5e 87 01 98 51 51 a9 fe c0 f7 b2 0d 02 d8 29 40 b2 a8 c4	;	1101-1120
+ 8d 52 da 36 f9 98 f9 c4 6d 6e 1b ed 42 00 00 29 10 00 00 00	;	1121-1140
+ 80 00 00 00
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ b2 88 85 03 00 01 00 00 00 01 00 00 04 7a 61 72 66 09 6e 6c	;	   1-  20
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 09 6e 6c 6e 65	;	  21-  40
+ 74 6c 61 62 73 02 6e 6c 00 00 06 00 01 00 00 46 50 00 40 04	;	  41-  60
+ 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 0a 68	;	  61-  80
+ 6f 73 74 6d 61 73 74 65 72 09 6e 6c 6e 65 74 6c 61 62 73 02	;	  81- 100
+ 6e 6c 00 77 a1 02 58 00 00 70 80 00 00 1c 20 00 09 3a 80 00	;	 101- 120
+ 00 46 50
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 1c be 85 03 00 01 00 00 00 06 00 01 04 7a 61 72 66 09 6e 6c	;	   1-  20
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 09 6e 6c 6e 65	;	  21-  40
+ 74 6c 61 62 73 02 6e 6c 00 00 06 00 01 00 00 46 50 00 40 04	;	  41-  60
+ 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 0a 68	;	  61-  80
+ 6f 73 74 6d 61 73 74 65 72 09 6e 6c 6e 65 74 6c 61 62 73 02	;	  81- 100
+ 6e 6c 00 77 a1 02 58 00 00 70 80 00 00 1c 20 00 09 3a 80 00	;	 101- 120
+ 00 46 50 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00	;	 121- 140
+ 01 00 00 46 50 00 a0 00 06 05 02 00 01 51 80 46 3d 18 29 46	;	 141- 160
+ 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 81	;	 161- 180
+ de 4b b8 1d 81 1f 95 cb ca 8e fa 04 bc b6 a4 05 7f b1 7f de	;	 181- 200
+ e7 bd cc 8f 39 63 1d 0a 9a ac 7c b7 eb b5 e5 e8 38 a0 d6 9d	;	 201- 220
+ ec 34 06 43 9b 5a 4f 61 f3 18 21 32 c7 79 43 6f 99 32 b8 25	;	 221- 240
+ 54 3b 19 6c 25 d3 3e ef ec 2a d1 1e 4a 86 f6 17 ca 84 07 44	;	 241- 260
+ 73 a2 39 df d1 66 09 c1 e2 21 83 bf 79 73 2b 8d 2d bb bf 37	;	 261- 280
+ c5 e0 95 a9 2a f3 79 75 67 1d 90 29 5c 57 06 07 4b 69 25 05	;	 281- 300
+ dd 63 05 91 c5 81 0d 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c	;	 301- 320
+ 00 00 2f 00 01 00 00 46 50 00 21 04 5f 73 69 70 04 5f 75 64	;	 321- 340
+ 70 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 07 62 01 80	;	 341- 360
+ 08 10 03 80 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e	;	 361- 380
+ 00 01 00 00 46 50 00 a0 00 2f 05 02 00 00 46 50 46 3d 18 29	;	 381- 400
+ 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	 401- 420
+ 53 04 6e cf ce b4 19 e1 40 c5 47 36 02 c5 1b fa ab 43 57 ec	;	 421- 440
+ d1 57 da 02 d6 13 16 b1 4d 15 58 d1 a6 cb 5d 27 67 71 33 c2	;	 441- 460
+ 87 16 12 af 45 99 7f 95 74 19 71 e3 ea 33 b5 60 39 21 15 11	;	 461- 480
+ af 5d df 33 f3 c4 9d 9c 62 9f ba f8 10 f6 18 4c eb 92 84 8a	;	 481- 500
+ 85 5c a3 83 aa 00 01 c8 48 b7 5b de 80 9a 38 00 3d d9 cd f5	;	 501- 520
+ 09 68 06 7f 5c 83 4e 2c a1 99 76 53 dc 01 ac 06 f1 ed 9a a4	;	 521- 540
+ d3 03 17 33 4b 71 bd 94 03 77 77 77 09 6e 6c 6e 65 74 6c 61	;	 541- 560
+ 62 73 02 6e 6c 00 00 2f 00 01 00 00 46 50 00 16 09 6e 6c 6e	;	 561- 580
+ 65 74 6c 61 62 73 02 6e 6c 00 00 06 40 00 00 08 00 03 03 77	;	 581- 600
+ 77 77 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01	;	 601- 620
+ 00 00 46 50 00 a0 00 2f 05 03 00 00 46 50 46 3d 18 29 46 15	;	 621- 640
+ 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 3f f2	;	 641- 660
+ fd 50 50 3a ea 29 bf 75 26 28 0c 22 72 29 77 5a 8f ef 2f 34	;	 661- 680
+ 96 18 27 32 79 57 1e 08 ce 04 0f 22 c8 4e 32 db 1d 90 c6 68	;	 681- 700
+ 0c 23 c8 43 44 2a ab fe e6 bb 90 3a 73 7d 79 44 63 93 e8 97	;	 701- 720
+ 4f aa 51 f5 91 c2 ee d3 13 ea 94 68 4b 1d fc 9c 8e f5 1a 7b	;	 721- 740
+ 5f f6 09 e8 94 58 7e 26 4e 00 9e 98 a5 b9 c7 4d 34 8c f0 1e	;	 741- 760
+ 8e 74 f1 6d 54 ef d0 45 70 57 5a d0 fe 3d 54 78 e0 96 70 0e	;	 761- 780
+ a7 b8 e2 be 13 00 00 00 29 10 00 00 00 80 00 00 00
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 95 7c 85 00 00 01 00 00 00 01 00 00 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 07 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 06 00 01 00 00 46 50 00 40 04 6f 70 65 6e 09	;	  41-  60
+ 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 0a 68 6f 73 74 6d 61	;	  61-  80
+ 73 74 65 72 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 77 a1	;	  81- 100
+ 02 58 00 00 70 80 00 00 1c 20 00 09 3a 80 00 00 46 50
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 78 1a 85 00 00 01 00 00 00 04 00 01 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 07 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 06 00 01 00 00 46 50 00 40 04 6f 70 65 6e 09	;	  41-  60
+ 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 0a 68 6f 73 74 6d 61	;	  61-  80
+ 73 74 65 72 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 77 a1	;	  81- 100
+ 02 58 00 00 70 80 00 00 1c 20 00 09 3a 80 00 00 46 50 09 6e	;	 101- 120
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 00 46 50	;	 121- 140
+ 00 a0 00 06 05 02 00 01 51 80 46 3d 18 29 46 15 8b 29 ab 0f	;	 141- 160
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 81 de 4b b8 1d 81	;	 161- 180
+ 1f 95 cb ca 8e fa 04 bc b6 a4 05 7f b1 7f de e7 bd cc 8f 39	;	 181- 200
+ 63 1d 0a 9a ac 7c b7 eb b5 e5 e8 38 a0 d6 9d ec 34 06 43 9b	;	 201- 220
+ 5a 4f 61 f3 18 21 32 c7 79 43 6f 99 32 b8 25 54 3b 19 6c 25	;	 221- 240
+ d3 3e ef ec 2a d1 1e 4a 86 f6 17 ca 84 07 44 73 a2 39 df d1	;	 241- 260
+ 66 09 c1 e2 21 83 bf 79 73 2b 8d 2d bb bf 37 c5 e0 95 a9 2a	;	 261- 280
+ f3 79 75 67 1d 90 29 5c 57 06 07 4b 69 25 05 dd 63 05 91 c5	;	 281- 300
+ 81 0d 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2f 00 01	;	 301- 320
+ 00 00 46 50 00 21 04 5f 73 69 70 04 5f 75 64 70 09 6e 6c 6e	;	 321- 340
+ 65 74 6c 61 62 73 02 6e 6c 00 00 07 62 01 80 08 10 03 80 09	;	 341- 360
+ 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 00 46	;	 361- 380
+ 50 00 a0 00 2f 05 02 00 00 46 50 46 3d 18 29 46 15 8b 29 ab	;	 381- 400
+ 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 53 04 6e cf ce	;	 401- 420
+ b4 19 e1 40 c5 47 36 02 c5 1b fa ab 43 57 ec d1 57 da 02 d6	;	 421- 440
+ 13 16 b1 4d 15 58 d1 a6 cb 5d 27 67 71 33 c2 87 16 12 af 45	;	 441- 460
+ 99 7f 95 74 19 71 e3 ea 33 b5 60 39 21 15 11 af 5d df 33 f3	;	 461- 480
+ c4 9d 9c 62 9f ba f8 10 f6 18 4c eb 92 84 8a 85 5c a3 83 aa	;	 481- 500
+ 00 01 c8 48 b7 5b de 80 9a 38 00 3d d9 cd f5 09 68 06 7f 5c	;	 501- 520
+ 83 4e 2c a1 99 76 53 dc 01 ac 06 f1 ed 9a a4 d3 03 17 33 4b	;	 521- 540
+ 71 bd 94 00 00 29 10 00 00 00 80 00 00 00
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 0d 5b 85 00 00 01 00 01 00 03 00 05 04 64 69 76 61 09 6e 6c	;	   1-  20
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 04 64 69 76 61	;	  21-  40
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 00 01	;	  41-  60
+ 51 80 00 04 d5 9a e0 26 09 6e 6c 6e 65 74 6c 61 62 73 02 6e	;	  61-  80
+ 6c 00 00 02 00 01 00 01 51 80 00 18 03 6e 73 37 0f 64 6f 6d	;	  81- 100
+ 61 69 6e 2d 72 65 67 69 73 74 72 79 02 6e 6c 00 09 6e 6c 6e	;	 101- 120
+ 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00 13	;	 121- 140
+ 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 09	;	 141- 160
+ 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51	;	 161- 180
+ 80 00 11 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00	;	 181- 200
+ 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00	;	 201- 220
+ 01 00 01 00 01 51 80 00 04 d5 9a e0 01 04 6f 70 65 6e 09 6e	;	 221- 240
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51 80	;	 241- 260
+ 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00 01 04 6f	;	 261- 280
+ 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00	;	 281- 300
+ 01 00 01 51 80 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00	;	 301- 320
+ 00 00 53 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00	;	 321- 340
+ 00 01 00 01 00 00 70 80 00 04 d5 9a e0 11 05 6f 6d 76 61 6c	;	 341- 360
+ 06 74 65 64 6e 65 74 02 6e 6c 00 00 1c 00 01 00 00 70 80 00	;	 361- 380
+ 10 20 01 07 b8 02 06 00 01 02 00 39 ff fe 59 b1 87
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 0e 87 85 00 00 01 00 02 00 04 00 08 04 64 69 76 61 09 6e 6c	;	   1-  20
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 04 64 69 76 61	;	  21-  40
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 00 01	;	  41-  60
+ 51 80 00 04 d5 9a e0 26 04 64 69 76 61 09 6e 6c 6e 65 74 6c	;	  61-  80
+ 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 01 05	;	  81- 100
+ 03 00 01 51 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e 65	;	 101- 120
+ 74 6c 61 62 73 02 6e 6c 00 7d 90 9b 21 31 d6 0d b5 91 2b b1	;	 121- 140
+ e8 ff 90 4a 39 6f 61 bd 50 61 ee 64 8c 50 99 4a 95 04 cd 9e	;	 141- 160
+ 48 d8 d4 13 2a 7f 3a f2 ff 5f e5 2b ec 58 56 b7 e7 95 b0 89	;	 161- 180
+ 23 84 be 57 bd fd 85 59 2e b2 a1 dd 0c 59 23 63 9d 77 68 b0	;	 181- 200
+ 1d 37 84 2b 47 1b 70 00 04 2e 27 56 3c 39 37 e9 9e 93 74 34	;	 201- 220
+ 1b d9 46 7a 8f cf b5 c2 03 b8 fe 65 97 f9 0f 09 04 f4 6e fd	;	 221- 240
+ 38 fc 47 c6 e0 b6 2f 21 77 d1 f1 05 65 6a 87 7f 26 09 6e 6c	;	 241- 260
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00	;	 261- 280
+ 11 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 09 6e	;	 281- 300
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80	;	 301- 320
+ 00 18 03 6e 73 37 0f 64 6f 6d 61 69 6e 2d 72 65 67 69 73 74	;	 321- 340
+ 72 79 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	 341- 360
+ 00 02 00 01 00 01 51 80 00 13 04 6f 70 65 6e 09 6e 6c 6e 65	;	 361- 380
+ 74 6c 61 62 73 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73 02	;	 381- 400
+ 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 02 05 02 00 01 51	;	 401- 420
+ 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62	;	 421- 440
+ 73 02 6e 6c 00 ad 07 18 03 3f 18 f6 5c 35 8d 5b 89 d8 cf 6a	;	 441- 460
+ 11 fc 3c 32 6b bc 06 e3 c4 c8 ec 47 83 1d ab a0 ea e3 af 43	;	 461- 480
+ b0 f5 ec aa 5d 4e aa fa 79 6f 1a 2d fc 1f 44 81 7a 61 a2 49	;	 481- 500
+ 76 0b 8e 9d 9f 34 95 66 56 89 99 dc f6 31 6e 42 bb 16 45 93	;	 501- 520
+ 60 44 7b 19 3a c4 e6 9a ab 69 9a b5 84 8f 86 b5 17 af c1 e0	;	 521- 540
+ 4c b1 33 eb 67 cc cf 5b f6 e9 27 ff f3 d9 d0 c5 8e f0 91 b0	;	 541- 560
+ ab 77 93 18 e7 7e dc 44 1e f1 73 72 7e 04 6f 70 65 6e 09 6e	;	 561- 580
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 00 01 51 80	;	 581- 600
+ 00 04 d5 9a e0 01 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62	;	 601- 620
+ 73 02 6e 6c 00 00 1c 00 01 00 01 51 80 00 10 20 01 07 b8 02	;	 621- 640
+ 06 00 01 00 00 00 00 00 00 00 01 04 6f 70 65 6e 09 6e 6c 6e	;	 641- 660
+ 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51 80 00 10	;	 661- 680
+ 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00 53 05 6f 6d 76	;	 681- 700
+ 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 00 01 00 01 00 00 70	;	 701- 720
+ 80 00 04 d5 9a e0 11 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74	;	 721- 740
+ 02 6e 6c 00 00 1c 00 01 00 00 70 80 00 10 20 01 07 b8 02 06	;	 741- 760
+ 00 01 02 00 39 ff fe 59 b1 87 04 6f 70 65 6e 09 6e 6c 6e 65	;	 761- 780
+ 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00	;	 781- 800
+ 01 05 03 00 01 51 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c	;	 801- 820
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 33 42 f9 d1 94 cb d3 27 dc	;	 821- 840
+ e3 6b fd 80 34 a8 c0 ab 2a 76 f8 ef 9d c2 f7 b4 08 2f ff 3c	;	 841- 860
+ 35 c1 b4 13 46 5b 05 38 e7 97 96 41 b9 01 ff 8c 0b de 62 78	;	 861- 880
+ 29 68 2f fe fb 53 7a 61 92 3f 89 79 3e df 49 cc ad a3 f7 45	;	 881- 900
+ 13 7b 99 4c 0b 74 06 ee 1e 97 5a fc 6b 74 60 6d a7 68 24 5f	;	 901- 920
+ 01 f8 5a e2 84 ad 18 cf 52 ff f7 a5 66 84 76 b0 34 5c 42 8f	;	 921- 940
+ c6 58 c2 38 c4 8c 6b 7a 72 90 4c 9c 77 90 d4 dc cf cf 2a 04	;	 941- 960
+ 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e	;	 961- 980
+ 00 01 00 01 51 80 00 a0 00 1c 05 03 00 01 51 80 46 3d 18 29	;	 981-1000
+ 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	1001-1020
+ 53 09 41 45 b5 88 59 88 6e 2a 21 8e 42 ac 80 7c e2 9f 2e d4	;	1021-1040
+ f6 48 3f 70 40 d8 61 1d 3f ea e2 ba ad 20 83 80 bc 7e 3f 40	;	1041-1060
+ e7 41 74 9c c3 7c 51 30 e6 f2 99 50 0a 3f d9 26 5a fd d2 38	;	1061-1080
+ 09 79 e5 27 f1 25 80 e5 13 58 a5 2b 95 70 c3 53 cd ee 0e c3	;	1081-1100
+ 07 32 f6 73 17 60 b2 1d 3a b0 a4 0c 8f ef c3 65 5e 87 01 98	;	1101-1120
+ 51 51 a9 fe c0 f7 b2 0d 02 d8 29 40 b2 a8 c4 8d 52 da 36 f9	;	1121-1140
+ 98 f9 c4 6d 6e 1b ed 42 00 00 29 10 00 00 00 80 00 00 00
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 5c 21 87 00 00 01 00 02 00 00 00 00 04 64 69 76 61 09 6e 6c	;	   1-  20
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 04 64 69 76 61	;	  21-  40
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01	;	  41-  60
+ 51 80 00 a0 00 01 05 03 00 01 51 80 46 3d 18 29 46 15 8b 29	;	  61-  80
+ ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 7d 90 9b 21	;	  81- 100
+ 31 d6 0d b5 91 2b b1 e8 ff 90 4a 39 6f 61 bd 50 61 ee 64 8c	;	 101- 120
+ 50 99 4a 95 04 cd 9e 48 d8 d4 13 2a 7f 3a f2 ff 5f e5 2b ec	;	 121- 140
+ 58 56 b7 e7 95 b0 89 23 84 be 57 bd fd 85 59 2e b2 a1 dd 0c	;	 141- 160
+ 59 23 63 9d 77 68 b0 1d 37 84 2b 47 1b 70 00 04 2e 27 56 3c	;	 161- 180
+ 39 37 e9 9e 93 74 34 1b d9 46 7a 8f cf b5 c2 03 b8 fe 65 97	;	 181- 200
+ f9 0f 09 04 f4 6e fd 38 fc 47 c6 e0 b6 2f 21 77 d1 f1 05 65	;	 201- 220
+ 6a 87 7f 26 04 64 69 76 61 09 6e 6c 6e 65 74 6c 61 62 73 02	;	 221- 240
+ 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 1c 05 03 00 01 51	;	 241- 260
+ 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62	;	 261- 280
+ 73 02 6e 6c 00 de 55 75 bf 56 52 fb 36 13 2a c6 1b cd 95 d1	;	 281- 300
+ 3f 28 f8 60 f8 b9 d5 e1 4b ae 51 e5 b2 3c c3 87 43 53 04 19	;	 301- 320
+ 13 64 42 86 62 9c f3 b7 af b7 52 29 c5 cd 53 d2 5e 70 cb 59	;	 321- 340
+ 4e 66 7b 02 d5 d6 29 46 1a 2a 6c 4b 22 81 b8 93 65 0d 9d 12	;	 341- 360
+ de e8 9d 59 41 17 d2 c4 fd d2 76 92 fc fb 83 3d 97 85 06 f9	;	 361- 380
+ f5 08 c8 dc 2b d4 7c e0 da eb 96 ff 5a be 15 c3 17 4c b3 a7	;	 381- 400
+ c3 54 84 55 b6 be f2 cd 05 ca 99 5d c8
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ f6 7d 85 00 00 01 00 03 00 04 00 08 04 64 69 76 61 09 6e 6c	;	   1-  20
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 04 64 69 76 61	;	  21-  40
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01	;	  41-  60
+ 51 80 00 a0 00 01 05 03 00 01 51 80 46 3d 18 29 46 15 8b 29	;	  61-  80
+ ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 7d 90 9b 21	;	  81- 100
+ 31 d6 0d b5 91 2b b1 e8 ff 90 4a 39 6f 61 bd 50 61 ee 64 8c	;	 101- 120
+ 50 99 4a 95 04 cd 9e 48 d8 d4 13 2a 7f 3a f2 ff 5f e5 2b ec	;	 121- 140
+ 58 56 b7 e7 95 b0 89 23 84 be 57 bd fd 85 59 2e b2 a1 dd 0c	;	 141- 160
+ 59 23 63 9d 77 68 b0 1d 37 84 2b 47 1b 70 00 04 2e 27 56 3c	;	 161- 180
+ 39 37 e9 9e 93 74 34 1b d9 46 7a 8f cf b5 c2 03 b8 fe 65 97	;	 181- 200
+ f9 0f 09 04 f4 6e fd 38 fc 47 c6 e0 b6 2f 21 77 d1 f1 05 65	;	 201- 220
+ 6a 87 7f 26 04 64 69 76 61 09 6e 6c 6e 65 74 6c 61 62 73 02	;	 221- 240
+ 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 1c 05 03 00 01 51	;	 241- 260
+ 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62	;	 261- 280
+ 73 02 6e 6c 00 de 55 75 bf 56 52 fb 36 13 2a c6 1b cd 95 d1	;	 281- 300
+ 3f 28 f8 60 f8 b9 d5 e1 4b ae 51 e5 b2 3c c3 87 43 53 04 19	;	 301- 320
+ 13 64 42 86 62 9c f3 b7 af b7 52 29 c5 cd 53 d2 5e 70 cb 59	;	 321- 340
+ 4e 66 7b 02 d5 d6 29 46 1a 2a 6c 4b 22 81 b8 93 65 0d 9d 12	;	 341- 360
+ de e8 9d 59 41 17 d2 c4 fd d2 76 92 fc fb 83 3d 97 85 06 f9	;	 361- 380
+ f5 08 c8 dc 2b d4 7c e0 da eb 96 ff 5a be 15 c3 17 4c b3 a7	;	 381- 400
+ c3 54 84 55 b6 be f2 cd 05 ca 99 5d c8 04 64 69 76 61 09 6e	;	 401- 420
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 00 46 50	;	 421- 440
+ 00 a0 00 2f 05 03 00 00 46 50 46 3d 18 29 46 15 8b 29 ab 0f	;	 441- 460
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 5a 75 f4 5c 4f fc	;	 461- 480
+ 4e 1b 16 09 6f a9 12 d2 ac c3 ef 00 4f 45 f8 f9 e7 d5 77 58	;	 481- 500
+ 2a 2a da cd cd 7a a3 0b 4c 18 30 d1 c2 89 f6 e3 49 64 9c 81	;	 501- 520
+ f1 a8 e9 f5 29 35 e0 f0 c9 e7 33 42 3d d5 c5 0c b8 da e6 27	;	 521- 540
+ 29 45 96 df 89 5d 32 30 d6 8b b0 ee 07 d7 63 17 17 3a 44 c5	;	 541- 560
+ 1a 13 bf 72 a8 ec fe 2d 88 82 8a 3b 43 4a 30 a0 af 87 64 33	;	 561- 580
+ 23 42 ec 46 0d 19 f2 d9 34 31 ee 6b 11 f3 10 22 49 8a d7 5f	;	 581- 600
+ 15 65 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01	;	 601- 620
+ 00 01 51 80 00 11 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02	;	 621- 640
+ 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00	;	 641- 660
+ 01 00 01 51 80 00 18 03 6e 73 37 0f 64 6f 6d 61 69 6e 2d 72	;	 661- 680
+ 65 67 69 73 74 72 79 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62	;	 681- 700
+ 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00 13 04 6f 70 65 6e	;	 701- 720
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 09 6e 6c 6e 65 74	;	 721- 740
+ 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 02	;	 741- 760
+ 05 02 00 01 51 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e	;	 761- 780
+ 65 74 6c 61 62 73 02 6e 6c 00 ad 07 18 03 3f 18 f6 5c 35 8d	;	 781- 800
+ 5b 89 d8 cf 6a 11 fc 3c 32 6b bc 06 e3 c4 c8 ec 47 83 1d ab	;	 801- 820
+ a0 ea e3 af 43 b0 f5 ec aa 5d 4e aa fa 79 6f 1a 2d fc 1f 44	;	 821- 840
+ 81 7a 61 a2 49 76 0b 8e 9d 9f 34 95 66 56 89 99 dc f6 31 6e	;	 841- 860
+ 42 bb 16 45 93 60 44 7b 19 3a c4 e6 9a ab 69 9a b5 84 8f 86	;	 861- 880
+ b5 17 af c1 e0 4c b1 33 eb 67 cc cf 5b f6 e9 27 ff f3 d9 d0	;	 881- 900
+ c5 8e f0 91 b0 ab 77 93 18 e7 7e dc 44 1e f1 73 72 7e 04 6f	;	 901- 920
+ 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00	;	 921- 940
+ 01 00 01 51 80 00 04 d5 9a e0 01 04 6f 70 65 6e 09 6e 6c 6e	;	 941- 960
+ 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51 80 00 10	;	 961- 980
+ 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00 01 04 6f 70 65	;	 981-1000
+ 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00	;	1001-1020
+ 01 51 80 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00	;	1021-1040
+ 53 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 00 01	;	1041-1060
+ 00 01 00 00 70 80 00 04 d5 9a e0 11 05 6f 6d 76 61 6c 06 74	;	1061-1080
+ 65 64 6e 65 74 02 6e 6c 00 00 1c 00 01 00 00 70 80 00 10 20	;	1081-1100
+ 01 07 b8 02 06 00 01 02 00 39 ff fe 59 b1 87 04 6f 70 65 6e	;	1101-1120
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01	;	1121-1140
+ 51 80 00 a0 00 01 05 03 00 01 51 80 46 3d 18 29 46 15 8b 29	;	1141-1160
+ ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 33 42 f9 d1	;	1161-1180
+ 94 cb d3 27 dc e3 6b fd 80 34 a8 c0 ab 2a 76 f8 ef 9d c2 f7	;	1181-1200
+ b4 08 2f ff 3c 35 c1 b4 13 46 5b 05 38 e7 97 96 41 b9 01 ff	;	1201-1220
+ 8c 0b de 62 78 29 68 2f fe fb 53 7a 61 92 3f 89 79 3e df 49	;	1221-1240
+ cc ad a3 f7 45 13 7b 99 4c 0b 74 06 ee 1e 97 5a fc 6b 74 60	;	1241-1260
+ 6d a7 68 24 5f 01 f8 5a e2 84 ad 18 cf 52 ff f7 a5 66 84 76	;	1261-1280
+ b0 34 5c 42 8f c6 58 c2 38 c4 8c 6b 7a 72 90 4c 9c 77 90 d4	;	1281-1300
+ dc cf cf 2a 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02	;	1301-1320
+ 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 1c 05 03 00 01 51	;	1321-1340
+ 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62	;	1341-1360
+ 73 02 6e 6c 00 53 09 41 45 b5 88 59 88 6e 2a 21 8e 42 ac 80	;	1361-1380
+ 7c e2 9f 2e d4 f6 48 3f 70 40 d8 61 1d 3f ea e2 ba ad 20 83	;	1381-1400
+ 80 bc 7e 3f 40 e7 41 74 9c c3 7c 51 30 e6 f2 99 50 0a 3f d9	;	1401-1420
+ 26 5a fd d2 38 09 79 e5 27 f1 25 80 e5 13 58 a5 2b 95 70 c3	;	1421-1440
+ 53 cd ee 0e c3 07 32 f6 73 17 60 b2 1d 3a b0 a4 0c 8f ef c3	;	1441-1460
+ 65 5e 87 01 98 51 51 a9 fe c0 f7 b2 0d 02 d8 29 40 b2 a8 c4	;	1461-1480
+ 8d 52 da 36 f9 98 f9 c4 6d 6e 1b ed 42 00 00 29 10 00 00 00	;	1481-1500
+ 80 00 00 00
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ f6 7d 85 00 00 01 00 01 00 03 00 05 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 30 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 30 00 01 00 01 51 80 00 86 01 01 03 05 01 03	;	  41-  60
+ f3 cd 35 8c cf ca 92 58 84 25 7d 19 cf 72 4c 76 06 25 66 91	;	  61-  80
+ 53 7a 2c fb a9 3b 79 9b cf b1 61 2d 29 e6 db a9 fa 88 6f 4d	;	  81- 100
+ 2d bc bd ec 98 b8 e7 76 96 65 8d cc e7 fc 49 05 23 92 da f5	;	 101- 120
+ ad 72 8d ad f0 2d 3e c5 e3 b5 6d cf dd e9 94 f2 df 60 c7 72	;	 121- 140
+ 1a 56 ea b0 c7 d6 fa 7c e8 b9 7e 85 ad 2d 9b 0c b2 f5 03 fb	;	 141- 160
+ cb 66 d5 c3 9c d2 6c 50 ab 79 f9 3c a8 25 01 81 f1 e6 f5 b1	;	 161- 180
+ f3 30 6d 54 05 8b 44 21 09 6e 6c 6e 65 74 6c 61 62 73 02 6e	;	 181- 200
+ 6c 00 00 02 00 01 00 01 51 80 00 18 03 6e 73 37 0f 64 6f 6d	;	 201- 220
+ 61 69 6e 2d 72 65 67 69 73 74 72 79 02 6e 6c 00 09 6e 6c 6e	;	 221- 240
+ 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00 13	;	 241- 260
+ 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 09	;	 261- 280
+ 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51	;	 281- 300
+ 80 00 11 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00	;	 301- 320
+ 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00	;	 321- 340
+ 01 00 01 00 01 51 80 00 04 d5 9a e0 01 04 6f 70 65 6e 09 6e	;	 341- 360
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51 80	;	 361- 380
+ 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00 53 04 6f	;	 381- 400
+ 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00	;	 401- 420
+ 01 00 01 51 80 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00	;	 421- 440
+ 00 00 01 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00	;	 441- 460
+ 00 01 00 01 00 00 70 80 00 04 d5 9a e0 11 05 6f 6d 76 61 6c	;	 461- 480
+ 06 74 65 64 6e 65 74 02 6e 6c 00 00 1c 00 01 00 00 70 80 00	;	 481- 500
+ 10 20 01 07 b8 02 06 00 01 02 00 39 ff fe 59 b1 87
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 7d b9 85 00 00 01 00 02 00 04 00 08 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 30 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 30 00 01 00 01 51 80 00 86 01 01 03 05 01 03	;	  41-  60
+ f3 cd 35 8c cf ca 92 58 84 25 7d 19 cf 72 4c 76 06 25 66 91	;	  61-  80
+ 53 7a 2c fb a9 3b 79 9b cf b1 61 2d 29 e6 db a9 fa 88 6f 4d	;	  81- 100
+ 2d bc bd ec 98 b8 e7 76 96 65 8d cc e7 fc 49 05 23 92 da f5	;	 101- 120
+ ad 72 8d ad f0 2d 3e c5 e3 b5 6d cf dd e9 94 f2 df 60 c7 72	;	 121- 140
+ 1a 56 ea b0 c7 d6 fa 7c e8 b9 7e 85 ad 2d 9b 0c b2 f5 03 fb	;	 141- 160
+ cb 66 d5 c3 9c d2 6c 50 ab 79 f9 3c a8 25 01 81 f1 e6 f5 b1	;	 161- 180
+ f3 30 6d 54 05 8b 44 21 09 6e 6c 6e 65 74 6c 61 62 73 02 6e	;	 181- 200
+ 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 30 05 02 00 01 51 80	;	 201- 220
+ 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62 73	;	 221- 240
+ 02 6e 6c 00 e6 20 9a 3f 0b d4 7f fe e3 aa ac 29 eb d1 4e 9c	;	 241- 260
+ eb dd 76 21 9c cc 7a 4d 42 f2 47 16 f7 06 fb 48 e5 fd f6 73	;	 261- 280
+ 91 23 e1 dc 7b 43 8b 64 72 f7 06 5e a3 08 8f bd 38 7e 16 22	;	 281- 300
+ ea a2 a2 a3 10 3c 0c d0 ec 67 26 b7 06 7e e6 71 b8 8a 3a a9	;	 301- 320
+ 7f 00 d5 14 db 2c 32 02 b0 5e 65 25 07 a3 89 71 00 f2 3c db	;	 321- 340
+ 20 86 01 1e 53 80 d9 46 46 04 ec 26 8d 35 97 cc 81 a9 fe d8	;	 341- 360
+ 5b 25 6b 2e e9 f7 0c eb e7 6e b0 37 09 6e 6c 6e 65 74 6c 61	;	 361- 380
+ 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00 11 05 6f 6d 76	;	 381- 400
+ 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 09 6e 6c 6e 65 74 6c	;	 401- 420
+ 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00 18 03 6e 73	;	 421- 440
+ 37 0f 64 6f 6d 61 69 6e 2d 72 65 67 69 73 74 72 79 02 6e 6c	;	 441- 460
+ 00 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00	;	 461- 480
+ 01 51 80 00 13 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73	;	 481- 500
+ 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e	;	 501- 520
+ 00 01 00 01 51 80 00 a0 00 02 05 02 00 01 51 80 46 3d 18 29	;	 521- 540
+ 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	 541- 560
+ ad 07 18 03 3f 18 f6 5c 35 8d 5b 89 d8 cf 6a 11 fc 3c 32 6b	;	 561- 580
+ bc 06 e3 c4 c8 ec 47 83 1d ab a0 ea e3 af 43 b0 f5 ec aa 5d	;	 581- 600
+ 4e aa fa 79 6f 1a 2d fc 1f 44 81 7a 61 a2 49 76 0b 8e 9d 9f	;	 601- 620
+ 34 95 66 56 89 99 dc f6 31 6e 42 bb 16 45 93 60 44 7b 19 3a	;	 621- 640
+ c4 e6 9a ab 69 9a b5 84 8f 86 b5 17 af c1 e0 4c b1 33 eb 67	;	 641- 660
+ cc cf 5b f6 e9 27 ff f3 d9 d0 c5 8e f0 91 b0 ab 77 93 18 e7	;	 661- 680
+ 7e dc 44 1e f1 73 72 7e 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c	;	 681- 700
+ 61 62 73 02 6e 6c 00 00 01 00 01 00 01 51 80 00 04 d5 9a e0	;	 701- 720
+ 01 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	 721- 740
+ 00 1c 00 01 00 01 51 80 00 10 20 01 07 b8 02 06 00 01 00 00	;	 741- 760
+ 00 00 00 00 00 53 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62	;	 761- 780
+ 73 02 6e 6c 00 00 1c 00 01 00 01 51 80 00 10 20 01 07 b8 02	;	 781- 800
+ 06 00 01 00 00 00 00 00 00 00 01 05 6f 6d 76 61 6c 06 74 65	;	 801- 820
+ 64 6e 65 74 02 6e 6c 00 00 01 00 01 00 00 70 80 00 04 d5 9a	;	 821- 840
+ e0 11 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 00	;	 841- 860
+ 1c 00 01 00 00 70 80 00 10 20 01 07 b8 02 06 00 01 02 00 39	;	 861- 880
+ ff fe 59 b1 87 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73	;	 881- 900
+ 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 01 05 03 00 01	;	 901- 920
+ 51 80 46 3d 18 29 46 15 8b 29 ab 0f 09 6e 6c 6e 65 74 6c 61	;	 921- 940
+ 62 73 02 6e 6c 00 33 42 f9 d1 94 cb d3 27 dc e3 6b fd 80 34	;	 941- 960
+ a8 c0 ab 2a 76 f8 ef 9d c2 f7 b4 08 2f ff 3c 35 c1 b4 13 46	;	 961- 980
+ 5b 05 38 e7 97 96 41 b9 01 ff 8c 0b de 62 78 29 68 2f fe fb	;	 981-1000
+ 53 7a 61 92 3f 89 79 3e df 49 cc ad a3 f7 45 13 7b 99 4c 0b	;	1001-1020
+ 74 06 ee 1e 97 5a fc 6b 74 60 6d a7 68 24 5f 01 f8 5a e2 84	;	1021-1040
+ ad 18 cf 52 ff f7 a5 66 84 76 b0 34 5c 42 8f c6 58 c2 38 c4	;	1041-1060
+ 8c 6b 7a 72 90 4c 9c 77 90 d4 dc cf cf 2a 04 6f 70 65 6e 09	;	1061-1080
+ 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51	;	1081-1100
+ 80 00 a0 00 1c 05 03 00 01 51 80 46 3d 18 29 46 15 8b 29 ab	;	1101-1120
+ 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 53 09 41 45 b5	;	1121-1140
+ 88 59 88 6e 2a 21 8e 42 ac 80 7c e2 9f 2e d4 f6 48 3f 70 40	;	1141-1160
+ d8 61 1d 3f ea e2 ba ad 20 83 80 bc 7e 3f 40 e7 41 74 9c c3	;	1161-1180
+ 7c 51 30 e6 f2 99 50 0a 3f d9 26 5a fd d2 38 09 79 e5 27 f1	;	1181-1200
+ 25 80 e5 13 58 a5 2b 95 70 c3 53 cd ee 0e c3 07 32 f6 73 17	;	1201-1220
+ 60 b2 1d 3a b0 a4 0c 8f ef c3 65 5e 87 01 98 51 51 a9 fe c0	;	1221-1240
+ f7 b2 0d 02 d8 29 40 b2 a8 c4 8d 52 da 36 f9 98 f9 c4 6d 6e	;	1241-1260
+ 1b ed 42 00 00 29 10 00 00 00 80 00 00 00
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 50 e5 81 80 00 01 00 08 00 0a 00 18 02 73 65 00 00 30 00 01	;	   1-  20
+ 02 73 65 00 00 30 00 01 00 00 0e 10 00 86 01 00 03 05 01 03	;	  21-  40
+ db 42 4d 16 35 1a f6 8e 87 cb dc 52 4a 49 1f 15 27 68 bc 86	;	  41-  60
+ c5 d4 0b 6b e9 e0 bf 69 ab ce 9c 6b ad fe 4b 00 fb 9a 59 74	;	  61-  80
+ f2 36 1f 61 8d 9f b2 7e 63 84 fc 72 45 8d 57 bd ae ca c2 1a	;	  81- 100
+ 77 e6 8d 4d 5f 21 c1 05 c0 66 06 45 77 af 60 46 4b d7 3a d9	;	 101- 120
+ 8c d2 8f 2b ff 44 8e 32 1e 0b 3e d3 73 55 b2 a9 4d 68 cc fd	;	 121- 140
+ 0b 0d f8 a2 fd 38 2b ae 6c e3 96 52 27 47 c5 83 4e 55 55 50	;	 141- 160
+ a0 09 49 62 de 1e a1 25 02 73 65 00 00 30 00 01 00 00 0e 10	;	 161- 180
+ 00 88 01 00 03 05 03 01 00 01 b9 a9 68 15 f6 09 f9 32 a4 7b	;	 181- 200
+ 0e c9 8f 95 f6 df d3 87 18 f3 75 9c 8b de 4f 3d f9 5b b7 6f	;	 201- 220
+ 51 a5 60 46 fb 10 c2 f0 84 5e 67 e4 49 89 11 c4 5e 55 de 26	;	 221- 240
+ 67 9e 5b 54 0a d4 43 c1 af 86 2a 17 b6 8a 19 ad d2 ba db 4c	;	 241- 260
+ 15 38 78 28 ca 1c ed 2c 98 5c 8c f4 96 19 fc c1 46 59 8d 66	;	 261- 280
+ 11 72 48 34 b7 49 e8 58 31 5e 82 17 1b 00 88 55 e2 e1 bb ae	;	 281- 300
+ 13 5d de 1a dd 92 df 3d 64 a1 b5 8c 34 74 b3 55 6e 29 02 73	;	 301- 320
+ 65 00 00 30 00 01 00 00 0e 10 00 88 01 00 03 05 03 01 00 01	;	 321- 340
+ cd e7 e2 f5 7b 7f a4 db 46 34 a9 7c 00 dc 20 19 fe 30 3a b3	;	 341- 360
+ b8 55 28 fa cd db f3 9c 02 f1 3a 7e 77 51 03 94 b3 c5 ff 91	;	 361- 380
+ 88 47 86 37 3f ef 2e d4 ea 4f 2a e0 82 ce ea ef 7e 27 2c fc	;	 381- 400
+ 78 86 41 4d c9 92 fd f0 94 53 f9 71 92 c1 4a 90 a2 c1 ac d8	;	 401- 420
+ db 9d ee 27 05 e9 22 d4 be 96 23 fa 00 af a4 38 ec 2c 11 64	;	 421- 440
+ d8 c4 6e af bc 1a 1d 9b f0 12 4d ba ac d8 5a a8 90 54 cf 2d	;	 441- 460
+ 6e f8 95 33 a9 e7 56 b9 02 73 65 00 00 30 00 01 00 00 0e 10	;	 461- 480
+ 01 08 01 01 03 05 03 01 00 01 ac 4f 31 c4 76 c7 41 db 41 5e	;	 481- 500
+ 16 79 90 7a a0 40 d7 fa bd 10 33 ae 4a 6e 14 e3 ad 6d 59 68	;	 501- 520
+ 04 66 3e 11 19 65 ed d8 f6 27 e0 12 ea bb bd 9f f3 19 95 1d	;	 521- 540
+ 10 63 56 cb e2 59 ce eb 34 95 4a 6d c2 08 c6 c2 15 12 d7 9e	;	 541- 560
+ 7c 1d be 09 29 46 db 39 37 0c f2 9c 17 96 3d 04 89 7d e8 0c	;	 561- 580
+ 55 66 8c 73 b8 e3 88 0b 90 13 94 29 47 ff 98 2e c7 bd fc 18	;	 581- 600
+ 1f f0 5e db 61 a7 0a 5a e2 9c a2 69 d4 e8 38 cb 0c ce f5 68	;	 601- 620
+ 25 32 4e e3 6d f5 b7 de 05 0a be 2a d3 1e bd b9 18 be d2 4d	;	 621- 640
+ 05 33 05 be 54 b3 1b 7c 96 09 f2 4d 41 ec d6 dd c9 c0 26 39	;	 641- 660
+ 84 a6 7a 0a a1 71 98 05 59 3f 6d 22 81 38 b1 8a 7f f5 2d ca	;	 661- 680
+ e0 5f f2 8c 10 67 7f 6e eb c5 e2 a2 42 c9 db a2 78 7b a4 b9	;	 681- 700
+ 8d c7 e1 73 00 71 1f b1 89 ad a7 21 43 ef 9b fa 32 59 1f 01	;	 701- 720
+ 5b f8 56 77 33 af 9d 20 93 86 51 df df 18 98 95 e0 db b7 fa	;	 721- 740
+ 35 39 34 34 f4 0d 02 73 65 00 00 30 00 01 00 00 0e 10 01 08	;	 741- 760
+ 01 01 03 05 03 01 00 01 be b1 45 91 c4 7f e3 f2 17 97 71 12	;	 761- 780
+ fc f4 04 71 07 6f 38 ae ea 26 5a 88 d5 bf ca 34 9a 65 c9 ab	;	 781- 800
+ 99 98 91 30 5f 05 3e 9a 43 f9 7b 8a 49 ab b4 ae 0b b8 78 1c	;	 801- 820
+ 18 76 99 04 a0 c5 51 e0 58 48 f1 fa e4 b8 f1 c8 b9 ed 8b 3f	;	 821- 840
+ 51 13 a3 ed a1 0a b7 6b eb 42 b5 e7 7f 32 2e fc db 3a 13 31	;	 841- 860
+ 5e de a4 9e 33 cd f8 e9 5a 94 c9 82 92 76 9d 6a 87 ca 81 ae	;	 861- 880
+ 45 2c a3 8f 42 cd 82 de 7c 3a 00 7e 16 50 d1 ab 72 19 83 31	;	 881- 900
+ 50 70 47 07 d9 75 05 d8 65 35 de b2 a5 0b 98 63 0a ee 6c 23	;	 901- 920
+ 40 63 b1 7a b4 43 41 6a c5 bf c3 61 36 75 55 df 53 e2 85 a6	;	 921- 940
+ 01 75 0d 53 26 3b df 19 2f c5 61 92 70 bf f0 5d 76 21 0b 88	;	 941- 960
+ b7 94 9c d5 ae dc 03 ac 44 0b 21 80 e7 03 04 e0 0f 52 f2 89	;	 961- 980
+ 34 4a ba 44 71 0b 16 6b d7 38 e7 b6 77 c1 f6 cb 3b f4 d8 43	;	 981-1000
+ 10 8a 4e 86 c1 49 b5 e8 3a d2 76 65 fe 49 67 de 95 0b 3e f4	;	1001-1020
+ 42 c5 37 a9 02 73 65 00 00 2e 00 01 00 00 0e 10 00 96 00 30	;	1021-1040
+ 05 01 00 00 0e 10 46 2e a3 b7 46 26 25 d5 95 7b 02 73 65 00	;	1041-1060
+ a5 f8 54 11 87 a1 c5 0a 62 cf 21 27 61 8f 82 4e 32 8f ca 1e	;	1061-1080
+ 25 10 34 48 4c 23 46 52 fc 7c 79 c5 91 77 8a 81 b3 08 29 d2	;	1081-1100
+ 89 64 d3 4c ad 20 b9 b3 4e e3 e2 6b bb d6 e1 56 59 c0 aa b8	;	1101-1120
+ 12 c5 b3 39 2e 8f 4a ae c1 b3 78 07 8f ac 40 4f 28 a0 ea 98	;	1121-1140
+ 6f 1f 80 f3 d3 70 11 1b 4b c0 fc 5d f3 f2 a6 32 22 10 1f e7	;	1141-1160
+ c8 40 82 9a 4c 21 d1 93 ce 17 87 b6 2a 42 67 89 f9 5c 71 a1	;	1161-1180
+ 89 a3 5f b0 e0 15 3d d0 02 73 65 00 00 2e 00 01 00 00 0e 10	;	1181-1200
+ 01 16 00 30 05 01 00 00 0e 10 46 47 a6 80 46 12 1c 59 18 16	;	1201-1220
+ 02 73 65 00 4c bd 74 15 bd 17 0b e1 34 d1 b5 57 4f 18 c8 7d	;	1221-1240
+ 7c 80 75 22 d7 f4 43 aa 7b 83 61 e8 c1 70 cd d5 8c 93 2a c7	;	1241-1260
+ b6 7f 9c 73 79 52 1a 58 68 54 ef 12 83 4a 5f f3 45 e6 25 65	;	1261-1280
+ fe 4f 18 72 b3 84 ea 9c 4b 54 39 ad 1b 18 29 ed 0b ca e2 80	;	1281-1300
+ 3c 07 98 44 09 55 95 d3 57 0f 0a 81 af 61 d0 9c 93 7c aa b3	;	1301-1320
+ 7c 36 52 f3 f2 e6 93 0c 20 06 5c 45 c6 45 fe 5c 8c 84 24 d0	;	1321-1340
+ e9 c5 87 1e 1b 83 e2 30 b8 2a 41 59 b7 5b a5 a6 b0 22 0e bf	;	1341-1360
+ 55 c9 99 5a ff 7b 43 74 a4 d1 8f d0 ed f5 20 d1 8d a3 e3 f9	;	1361-1380
+ f8 c6 db 4d e3 3d 6e 33 9c 49 c4 14 3a b8 59 db bc 02 ef 94	;	1381-1400
+ d6 d1 f3 74 f9 2c 0d 5e 98 61 8f f7 3a 22 85 18 8f 4c 36 df	;	1401-1420
+ 3e 2f 1b 5f 4a 5c a1 39 ff fd 33 52 ab 43 b9 7e 89 5e d1 ac	;	1421-1440
+ e0 bd d4 88 56 39 c1 e3 4a ef f0 41 0a ee d2 96 5b c1 d8 d3	;	1441-1460
+ 4a 55 48 d3 e9 cf dc 96 b8 22 ca f7 75 fd 6d df cf d8 b2 2e	;	1461-1480
+ 02 73 65 00 00 2e 00 01 00 00 0e 10 01 16 00 30 05 01 00 00	;	1481-1500
+ 0e 10 46 47 a6 80 46 12 1c 59 45 16 02 73 65 00 20 f7 f4 13	;	1501-1520
+ 08 99 61 ca f1 75 9f 71 9a c8 c4 06 77 1a 48 a8 d8 4f 45 e8	;	1521-1540
+ 25 92 b2 5c 16 b1 14 b4 79 44 23 23 37 20 0e 36 2f 57 8f 17	;	1541-1560
+ c3 69 65 49 11 a9 40 86 06 13 ff f6 d6 ac 5e 58 ea de aa de	;	1561-1580
+ 06 d4 19 1b 0d 58 84 83 c2 1a 01 2b 6e 7b 18 e6 38 5f 2d eb	;	1581-1600
+ c5 76 01 60 a4 52 5a 23 ec 7c d0 e6 21 ca 6f 29 9e 83 5f db	;	1601-1620
+ fb 49 0b 03 ad 09 5b 3b 28 23 1c cb 99 82 5b 1c 81 a7 7d ca	;	1621-1640
+ 00 fd 52 22 66 19 51 1d c6 ac da 2d 65 6d 53 40 e7 b7 c0 6c	;	1641-1660
+ 97 73 7d e2 df e7 75 df 7b 6b 7d a9 9e cb e2 6a d3 35 0c 31	;	1661-1680
+ ee 6b 98 44 c7 17 47 4a 3c 69 d4 1b eb 90 40 79 6f 86 bb 53	;	1681-1700
+ a6 8c e7 04 4d e4 c3 6b a6 13 5f f8 21 76 f3 ff cd de c9 88	;	1701-1720
+ 9f 2a 0b a5 4a aa 43 13 f1 cd 2c 28 98 9c eb 91 cc 0a da ca	;	1721-1740
+ d8 b5 47 78 ed c2 f4 19 78 27 f7 05 d6 4a 4f 93 7a 22 43 3d	;	1741-1760
+ 59 15 da 1c 60 7a 46 ca 86 95 c3 77 02 73 65 00 00 02 00 01	;	1761-1780
+ 00 02 7b f1 00 09 01 63 02 4e 53 02 73 65 00 02 73 65 00 00	;	1781-1800
+ 02 00 01 00 02 7b f1 00 09 01 64 02 4e 53 02 73 65 00 02 73	;	1801-1820
+ 65 00 00 02 00 01 00 02 7b f1 00 09 01 65 02 4e 53 02 73 65	;	1821-1840
+ 00 02 73 65 00 00 02 00 01 00 02 7b f1 00 09 01 66 02 4e 53	;	1841-1860
+ 02 73 65 00 02 73 65 00 00 02 00 01 00 02 7b f1 00 09 01 67	;	1861-1880
+ 02 4e 53 02 73 65 00 02 73 65 00 00 02 00 01 00 02 7b f1 00	;	1881-1900
+ 09 01 68 02 4e 53 02 73 65 00 02 73 65 00 00 02 00 01 00 02	;	1901-1920
+ 7b f1 00 09 01 69 02 4e 53 02 73 65 00 02 73 65 00 00 02 00	;	1921-1940
+ 01 00 02 7b f1 00 09 01 41 02 4e 53 02 73 65 00 02 73 65 00	;	1941-1960
+ 00 02 00 01 00 02 7b f1 00 09 01 62 02 4e 53 02 73 65 00 02	;	1961-1980
+ 73 65 00 00 2e 00 01 00 02 a3 00 00 96 00 02 05 01 00 02 a3	;	1981-2000
+ 00 46 2e ad 6a 46 27 22 f5 95 7b 02 73 65 00 d0 db 15 15 e6	;	2001-2020
+ 6f 13 58 01 d7 c2 07 d1 54 e4 bf 64 00 6d ca a6 b7 61 5e 69	;	2021-2040
+ 2f e8 b4 b1 3e 91 d6 65 21 9e 92 82 c7 e8 6b ec 2a 1c 86 60	;	2041-2060
+ 6d 52 03 9c d3 7c 29 3b 60 46 0f 41 83 86 59 66 95 76 9c 83	;	2061-2080
+ 22 fd 27 aa c6 68 ce 41 44 c0 bb bb 8e 56 79 c1 66 3c 9b 07	;	2081-2100
+ ed 50 05 88 22 24 88 0e 83 09 7f 6d 81 4c 12 ab 79 df f2 d9	;	2101-2120
+ 0b 6d 41 38 55 7b 7c ac 5d cd 28 04 a6 b0 54 9b 89 82 9e 39	;	2121-2140
+ 7e c9 5b 01 41 02 4e 53 02 73 65 00 00 01 00 01 00 02 7b f1	;	2141-2160
+ 00 04 c0 24 90 6b 01 41 02 4e 53 02 73 65 00 00 1c 00 01 00	;	2161-2180
+ 02 7b f1 00 10 20 01 06 98 00 09 03 01 00 00 00 00 00 00 00	;	2181-2200
+ 53 01 62 02 4e 53 02 73 65 00 00 01 00 01 00 02 7b f1 00 04	;	2201-2220
+ c0 24 85 6b 01 63 02 4e 53 02 73 65 00 00 01 00 01 00 02 7b	;	2221-2240
+ f1 00 04 c0 24 87 6b 01 64 02 4e 53 02 73 65 00 00 01 00 01	;	2241-2260
+ 00 02 7b f3 00 04 51 e4 08 10 01 65 02 4e 53 02 73 65 00 00	;	2261-2280
+ 01 00 01 00 02 7b f1 00 04 51 e4 0a 39 01 66 02 4e 53 02 73	;	2281-2300
+ 65 00 00 01 00 01 00 02 7b f1 00 04 c0 24 7d 35 01 66 02 4e	;	2301-2320
+ 53 02 73 65 00 00 1c 00 01 00 02 7b f1 00 10 20 01 06 b0 00	;	2321-2340
+ 07 00 00 00 00 00 00 00 00 00 53 01 67 02 4e 53 02 73 65 00	;	2341-2360
+ 00 01 00 01 00 02 7b f3 00 04 82 ef 05 72 01 67 02 4e 53 02	;	2361-2380
+ 73 65 00 00 1c 00 01 00 02 7b f3 00 10 20 01 06 b0 00 0e 00	;	2381-2400
+ 03 00 00 00 00 00 00 00 01 01 68 02 4e 53 02 73 65 00 00 01	;	2401-2420
+ 00 01 00 02 7b f1 00 04 c7 07 31 1e 01 69 02 4e 53 02 73 65	;	2421-2440
+ 00 00 01 00 01 00 02 7b f1 00 04 c2 92 6a 16 01 41 02 4e 53	;	2441-2460
+ 02 73 65 00 00 2e 00 01 00 02 7b f1 00 96 00 01 05 03 00 02	;	2461-2480
+ a3 00 46 2e bd b4 46 27 06 d4 95 7b 02 73 65 00 36 41 6c 4e	;	2481-2500
+ 6b 37 41 ba 9c 84 4c ca 8a 25 b0 f4 5e b5 38 da 51 78 f6 57	;	2501-2520
+ b3 2a b8 06 24 66 06 8f fc e6 48 f6 bd 2e eb 5c 25 f8 ca 6e	;	2521-2540
+ a2 71 52 c3 b2 1a 21 b4 d3 69 35 b8 65 96 b7 15 9e e2 fa 25	;	2541-2560
+ 23 6e da e3 20 c1 bd 15 b6 40 86 2d 50 52 33 33 75 c3 97 2c	;	2561-2580
+ 50 e5 3a ec 92 ca 32 b6 e7 7c 72 f0 8c bc dc cb 33 65 b5 0d	;	2581-2600
+ 2b 99 21 81 0a 78 74 41 0a 89 6b 87 dc 4c 31 70 2a 8c f8 68	;	2601-2620
+ 10 26 e5 2a 01 41 02 4e 53 02 73 65 00 00 2e 00 01 00 02 7b	;	2621-2640
+ f1 00 96 00 1c 05 03 00 02 a3 00 46 2f f8 c0 46 27 06 d4 95	;	2641-2660
+ 7b 02 73 65 00 8b 6e 8e 8e 07 47 bc 80 ad dc 07 a4 b3 2f 3c	;	2661-2680
+ 0e 75 a6 50 5e 8b 07 80 19 aa 11 77 74 be 9c fc a5 21 26 2f	;	2681-2700
+ 56 3f cc 0a 46 91 06 78 39 86 5d c1 df 9b b2 1f 2f f5 97 7a	;	2701-2720
+ a7 91 36 6e 12 56 8c d1 d7 e0 0a b5 1b 8f 0b ac 7d df 4b 32	;	2721-2740
+ 4d 4a e1 a4 9f 4f 3e 6a dc fe ee d4 f8 00 40 e6 34 48 98 2f	;	2741-2760
+ c7 b9 df 49 63 80 8d 91 87 8d 24 2d 6d 7f ce e6 eb 93 5d 8b	;	2761-2780
+ d5 a3 e0 00 64 df 94 51 88 f8 b2 14 c1 01 62 02 4e 53 02 73	;	2781-2800
+ 65 00 00 2e 00 01 00 02 7b f1 00 96 00 01 05 03 00 02 a3 00	;	2801-2820
+ 46 30 17 a6 46 27 06 d4 95 7b 02 73 65 00 49 a8 36 07 fd bb	;	2821-2840
+ 2e a3 d3 f0 f7 3d 66 f3 b1 78 74 3b 74 92 b0 97 3e 7a 3c 01	;	2841-2860
+ d5 35 97 67 d2 bc d0 47 70 dc cd b6 98 ba 95 b9 db 35 ed b4	;	2861-2880
+ c1 68 94 ce 52 df 9d 44 a7 22 27 66 d2 19 4a 70 80 6e 3d cc	;	2881-2900
+ 60 e2 b5 7f 37 3a c1 18 b5 01 41 4a 76 ae 21 33 03 f1 e4 23	;	2901-2920
+ 0c d7 17 d1 3f 9e 01 d2 0e f3 11 d8 f1 ef 49 ec 75 5c eb fc	;	2921-2940
+ e3 74 4e 40 a9 eb 80 25 b5 a1 39 c8 e2 b6 cb 7c e3 43 fb 18	;	2941-2960
+ f0 2d 01 63 02 4e 53 02 73 65 00 00 2e 00 01 00 02 7b f1 00	;	2961-2980
+ 96 00 01 05 03 00 02 a3 00 46 2f e5 19 46 27 06 d4 95 7b 02	;	2981-3000
+ 73 65 00 0d b2 38 33 08 15 51 95 35 8a 52 ef 0f 09 74 51 a2	;	3001-3020
+ c8 41 ee a4 d2 01 54 0b 01 1f fb 73 d3 e8 3b 36 2c c5 29 75	;	3021-3040
+ 78 49 5f 25 03 fa c2 a2 da 7c 59 6c 68 90 dc 9c 7c 6e 88 83	;	3041-3060
+ 75 a8 de a9 6c 33 13 f6 61 f5 97 89 cf b6 ff 5f 08 e7 77 04	;	3061-3080
+ 13 67 da 56 1f 88 26 6b 3e db 91 bb 9c 7a 33 ec 7c a9 18 f0	;	3081-3100
+ 55 28 a3 04 52 fb e0 3d ec 82 b1 8a fa 70 0e 76 4c 40 53 f6	;	3101-3120
+ 3c 4f e6 86 ed 7f 25 cc cf 84 a1 01 64 02 4e 53 02 73 65 00	;	3121-3140
+ 00 2e 00 01 00 02 7b f3 00 96 00 01 05 03 00 02 a3 00 46 2f	;	3141-3160
+ 66 45 46 27 06 d4 95 7b 02 73 65 00 d8 0b 9f 5a 79 a2 b5 a3	;	3161-3180
+ 03 af a6 c6 6d 46 0a c4 45 c8 2c 58 3f a8 eb a6 19 da 01 f2	;	3181-3200
+ d4 24 e4 83 99 93 6f 79 d0 c1 a1 29 9e 1f b3 e9 92 ab bd c5	;	3201-3220
+ 5f 2b 27 e1 4c 8a ec 4c 48 fb ac f7 05 34 6d d3 76 78 77 41	;	3221-3240
+ 8f 32 92 d3 38 35 4c 18 1b f9 67 7d e0 82 c8 91 69 9b f7 ec	;	3241-3260
+ 7f c1 93 21 c8 2e e7 a8 95 0f 6f 2f 1c d0 f4 93 f8 09 18 07	;	3261-3280
+ cc d5 3e e7 88 26 f8 7c c8 6e 4c 3d 29 e8 22 82 af c3 f6 3c	;	3281-3300
+ 01 65 02 4e 53 02 73 65 00 00 2e 00 01 00 02 7b f1 00 96 00	;	3301-3320
+ 01 05 03 00 02 a3 00 46 2f d6 16 46 27 06 d4 95 7b 02 73 65	;	3321-3340
+ 00 93 c6 b0 7d be 20 b2 2a da b5 20 c8 52 c7 06 3d 3f 0b 2b	;	3341-3360
+ a2 2a d8 74 85 47 7c 8c 8d 6e f1 c1 4d ea ea e9 88 ad af d2	;	3361-3380
+ 5d 31 85 36 c6 9f e5 88 f2 4b 7e 94 cc 4a b7 29 68 ef 19 55	;	3381-3400
+ eb 0a f0 f3 12 07 3f a4 72 d9 dc 86 ab f3 9c 36 e1 ec 70 cd	;	3401-3420
+ d3 cd 29 4d f4 c6 f3 b4 a6 b9 98 b7 63 66 8a 68 1a be 0c d7	;	3421-3440
+ 32 af 89 14 87 6d d0 32 62 02 69 4b 71 1c bb f9 d6 f1 72 7d	;	3441-3460
+ a8 8b c3 1e af 97 75 36 9c 01 66 02 4e 53 02 73 65 00 00 2e	;	3461-3480
+ 00 01 00 02 7b f1 00 96 00 01 05 03 00 02 a3 00 46 2f ca 31	;	3481-3500
+ 46 27 06 d4 95 7b 02 73 65 00 6f dd d9 ae 84 61 ea e3 a3 40	;	3501-3520
+ f0 dc 78 f3 db ad f8 a2 08 92 c1 f7 0a 66 65 2d 4b be d6 8d	;	3521-3540
+ e2 a5 86 24 ff ed f8 f2 a2 87 b4 5a c2 c3 ee 3c 51 36 90 16	;	3541-3560
+ e0 de c3 30 43 eb c5 c2 d8 30 fe 93 3c c5 cd 32 74 46 ff 0c	;	3561-3580
+ 32 c3 d8 1d cb 76 e0 0b 39 03 7f c8 07 58 50 2f 4d 69 20 1e	;	3581-3600
+ e1 c3 4b 78 89 d1 f7 3f 80 42 9d 32 d2 cc f9 de de f7 47 b3	;	3601-3620
+ 3f 62 98 89 f7 94 14 cf 1b 42 e7 4f 3c 50 d9 f2 05 9a 01 66	;	3621-3640
+ 02 4e 53 02 73 65 00 00 2e 00 01 00 02 7b f1 00 96 00 1c 05	;	3641-3660
+ 03 00 02 a3 00 46 2f cc f8 46 27 06 d4 95 7b 02 73 65 00 56	;	3661-3680
+ 64 a5 e6 69 52 b9 6b 57 b1 0f 2a 8a 5b c5 2f 74 39 1d 6b 63	;	3681-3700
+ dd 06 01 5d a9 84 13 0f 1b b1 73 bb 3a 6e 2d 46 f3 83 b8 d3	;	3701-3720
+ f5 a8 40 41 05 2a a4 8e 0a c6 6c 6a 26 19 fd f8 ab e7 73 ce	;	3721-3740
+ 52 a4 77 66 56 5b d4 26 42 00 30 9b 3d b0 93 4a 3d e4 70 ca	;	3741-3760
+ 6f a0 2a 20 b6 cb d1 20 b8 05 2f 7c 44 4f 50 f6 8d 70 06 1a	;	3761-3780
+ 7d 8e f5 88 f3 42 63 18 23 0c bc 5d 0a a5 ce 64 2b 4c 9f 3d	;	3781-3800
+ 17 56 c2 64 aa 9c ef 01 67 02 4e 53 02 73 65 00 00 2e 00 01	;	3801-3820
+ 00 02 7b f3 00 96 00 01 05 03 00 02 a3 00 46 30 07 7e 46 27	;	3821-3840
+ 06 d4 95 7b 02 73 65 00 39 90 c1 fe 3d b9 69 33 6b 82 7d 36	;	3841-3860
+ 13 10 1d 14 b2 ca e8 4a e9 dd 32 38 aa a2 6e bf c9 96 32 a9	;	3861-3880
+ b5 8d e6 51 be bb 43 4a b6 24 a6 68 39 38 93 0d 20 4c 38 52	;	3881-3900
+ cb 6c b1 c8 9d ba f3 cd 36 32 43 65 5f 13 44 8e 0c 01 4e fd	;	3901-3920
+ d0 ec f7 12 bd 97 8e 4a ac d9 0b 6c 4b 61 f2 17 26 ac a8 cd	;	3921-3940
+ 6b a2 fb f6 54 56 53 03 a0 79 56 e4 b1 69 1f 6a 66 c7 f0 fc	;	3941-3960
+ db c1 e2 6d 83 5f 2c 7b 9b 99 35 f5 a7 3d 4d 0a 01 67 02 4e	;	3961-3980
+ 53 02 73 65 00 00 2e 00 01 00 02 7b f3 00 96 00 1c 05 03 00	;	3981-4000
+ 02 a3 00 46 2d cf 81 46 27 06 d4 95 7b 02 73 65 00 73 cf 2a	;	4001-4020
+ 01 60 76 4a 11 2d 2d 98 62 b3 63 dc 42 4c be e9 e9 81 f3 fb	;	4021-4040
+ c1 83 8b 1e d1 ab d3 73 1d a9 bc bc 25 35 5f ea dc 07 25 99	;	4041-4060
+ 09 6d b5 ac 9a 04 b2 17 5a c4 61 22 fe 96 a9 d0 16 90 44 18	;	4061-4080
+ 1d c1 6a 06 8d 2e 48 91 cf 36 75 75 26 5c ef 82 5e 73 4f 10	;	4081-4100
+ 51 1d ea fd 7d a9 db af 23 44 c1 73 62 98 a6 1e c6 61 5e 3e	;	4101-4120
+ 20 b9 f8 ce 6a 39 3e fc 41 34 5e c7 9d fc 5e cb cb 0d e3 84	;	4121-4140
+ bb b2 a1 53 8f 01 68 02 4e 53 02 73 65 00 00 2e 00 01 00 02	;	4141-4160
+ 7b f1 00 96 00 01 05 03 00 02 a3 00 46 30 0c fe 46 27 06 d4	;	4161-4180
+ 95 7b 02 73 65 00 3f c3 f3 ff 24 18 26 47 e0 3c 6a bf 3a 96	;	4181-4200
+ 79 bc 62 52 b1 25 78 d3 71 4b b3 7d d9 60 c5 ca 89 34 75 1e	;	4201-4220
+ 20 18 ef cf 8b bd 77 2a 7c 21 83 1f d1 0e f5 2a c9 5d 77 0b	;	4221-4240
+ b2 b1 8d df d5 52 a7 b8 94 18 d4 42 70 cb 42 fc 4f ab 50 2f	;	4241-4260
+ ce ac 3e dc c4 94 d3 d1 ae ac 12 70 4a e1 f8 a5 c4 06 dd 1b	;	4261-4280
+ da f9 06 28 b3 3d 07 06 71 22 fc 3e f0 4d f0 94 b8 2d e9 40	;	4281-4300
+ 03 16 d7 96 ac 38 44 58 93 7f 8b 73 6e a3 00 00 29 10 00 00	;	4301-4320
+ 00 80 00 00 00
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ b4 e2 87 00 00 01 00 02 00 00 00 00 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 2e 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 06 05 02 00 01	;	  41-  60
+ 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61	;	  61-  80
+ 62 73 02 6e 6c 00 5c 33 2b 93 5d f9 18 8f c6 7d f8 26 d2 d3	;	  81- 100
+ da 50 a3 60 42 df 19 1b cb e7 40 4a 7b 88 bb 41 08 68 8f 27	;	 101- 120
+ 7d 4f 05 17 86 89 08 a1 a3 7f 56 1f f4 91 db cc 8b 9b 8e a0	;	 121- 140
+ 03 dc 03 d8 2a 35 f7 ed 2e 01 41 9d b2 42 de 3e 50 4a ab 45	;	 141- 160
+ 9d ac cd 1c a0 90 51 12 be 6c ae 51 20 a5 34 b0 38 e9 83 1d	;	 161- 180
+ 99 57 d2 a6 2f 53 a2 4c 4e 97 47 c7 d9 43 72 a6 4a d2 83 be	;	 181- 200
+ ed f6 e4 34 6e 48 1b ae ed 2c c5 65 d0 ec 09 6e 6c 6e 65 74	;	 201- 220
+ 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 02	;	 221- 240
+ 05 02 00 01 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e	;	 241- 260
+ 65 74 6c 61 62 73 02 6e 6c 00 81 18 16 b9 0b 65 db e0 00 19	;	 261- 280
+ 68 06 01 11 46 67 4e 8c 51 c6 66 46 78 d7 7a ec b4 1f 8a 1e	;	 281- 300
+ d7 3c 44 4a 55 b0 16 b5 25 b6 ad 25 2f 7d 24 16 d2 4e 08 67	;	 301- 320
+ ac 95 fa b3 61 cc 13 9d 72 c2 22 76 9b d4 dc 17 6b 3e 58 d5	;	 321- 340
+ c3 4b 6f b4 fa 60 17 68 46 57 b8 b5 f3 3d b7 f5 a5 df e3 e4	;	 341- 360
+ bb 8c 6f 33 07 f2 c8 8f c7 ff 6c 13 72 8d 4c 54 a4 ae 70 29	;	 361- 380
+ 24 72 ba 7d fe e7 c7 4d 18 e4 5d f1 24 17 9a 8e 79 e4
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ fe c1 85 00 00 01 00 09 00 03 00 06 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 2e 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 06 05 02 00 01	;	  41-  60
+ 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61	;	  61-  80
+ 62 73 02 6e 6c 00 5c 33 2b 93 5d f9 18 8f c6 7d f8 26 d2 d3	;	  81- 100
+ da 50 a3 60 42 df 19 1b cb e7 40 4a 7b 88 bb 41 08 68 8f 27	;	 101- 120
+ 7d 4f 05 17 86 89 08 a1 a3 7f 56 1f f4 91 db cc 8b 9b 8e a0	;	 121- 140
+ 03 dc 03 d8 2a 35 f7 ed 2e 01 41 9d b2 42 de 3e 50 4a ab 45	;	 141- 160
+ 9d ac cd 1c a0 90 51 12 be 6c ae 51 20 a5 34 b0 38 e9 83 1d	;	 161- 180
+ 99 57 d2 a6 2f 53 a2 4c 4e 97 47 c7 d9 43 72 a6 4a d2 83 be	;	 181- 200
+ ed f6 e4 34 6e 48 1b ae ed 2c c5 65 d0 ec 09 6e 6c 6e 65 74	;	 201- 220
+ 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 02	;	 221- 240
+ 05 02 00 01 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e	;	 241- 260
+ 65 74 6c 61 62 73 02 6e 6c 00 81 18 16 b9 0b 65 db e0 00 19	;	 261- 280
+ 68 06 01 11 46 67 4e 8c 51 c6 66 46 78 d7 7a ec b4 1f 8a 1e	;	 281- 300
+ d7 3c 44 4a 55 b0 16 b5 25 b6 ad 25 2f 7d 24 16 d2 4e 08 67	;	 301- 320
+ ac 95 fa b3 61 cc 13 9d 72 c2 22 76 9b d4 dc 17 6b 3e 58 d5	;	 321- 340
+ c3 4b 6f b4 fa 60 17 68 46 57 b8 b5 f3 3d b7 f5 a5 df e3 e4	;	 341- 360
+ bb 8c 6f 33 07 f2 c8 8f c7 ff 6c 13 72 8d 4c 54 a4 ae 70 29	;	 361- 380
+ 24 72 ba 7d fe e7 c7 4d 18 e4 5d f1 24 17 9a 8e 79 e4 09 6e	;	 381- 400
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80	;	 401- 420
+ 00 a0 00 01 05 02 00 01 51 80 46 4e f9 ff 46 27 6c ff ab 0f	;	 421- 440
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 ca ee ca 28 cd cf	;	 441- 460
+ f4 fd a5 6e 5c ad 3f d5 89 7c 06 69 93 30 93 f7 0f b2 43 07	;	 461- 480
+ eb ad ce 8e f0 da 75 b9 5e 46 d5 28 40 9d 51 17 9e 2d 3d f3	;	 481- 500
+ 85 1b dd 85 4a 82 9d 32 5c ff 27 c9 83 8f 36 04 89 94 52 3b	;	 501- 520
+ 01 29 01 62 36 1a 66 67 f7 43 16 5b 2a 19 77 d1 cc 73 7c 86	;	 521- 540
+ 44 4c f5 d0 db a4 f5 ea 62 98 0f 18 9f 72 2c 73 5f cb 62 c2	;	 541- 560
+ 7f 45 91 99 53 bb 70 44 93 3a f5 ea 3c 23 80 3f a6 19 06 33	;	 561- 580
+ 3f 69 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01	;	 581- 600
+ 00 01 51 80 00 a0 00 0f 05 02 00 01 51 80 46 4e f9 ff 46 27	;	 601- 620
+ 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 8f 79	;	 621- 640
+ 3e 49 68 56 05 e7 6c 01 6d f6 cf 6f f3 5d 5b cb e8 c1 ff fd	;	 641- 660
+ a2 45 de 61 25 44 0d 4f 47 ea 54 8a 46 0f 58 1a 43 dc b4 3b	;	 661- 680
+ 02 b4 a3 11 a2 c5 45 0a 4b b2 f1 7a 3b 39 8d b0 ad 02 2e 60	;	 681- 700
+ 15 dd fb b5 66 3a e1 b7 6f 5c 2d ea 99 21 a1 7e d9 8c da d8	;	 701- 720
+ 94 6f e5 fc 1c 1f 0a b2 f6 18 7f 11 66 54 37 ad 1d f1 72 32	;	 721- 740
+ dd db 48 6b 53 4a 39 54 77 9b dc 54 f0 da 0c ae 94 7d 82 22	;	 741- 760
+ 05 7f ab bb 3b 79 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	 761- 780
+ 00 2e 00 01 00 01 51 80 00 a0 00 10 05 02 00 01 51 80 46 4e	;	 781- 800
+ f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e	;	 801- 820
+ 6c 00 ce 39 43 cf 06 19 80 eb 18 77 90 6a 43 52 d6 de 3a d0	;	 821- 840
+ b3 5f 00 49 34 6e bd 2a 38 2b a1 4e 24 7a 59 03 c8 72 90 15	;	 841- 860
+ c0 59 e0 8b aa da a9 06 f5 a8 25 5d 46 fc 46 d8 63 46 56 d0	;	 861- 880
+ 34 b4 dd ae 6f f6 e6 41 bc ca cf f7 f0 66 be ba a1 cd 2e 9f	;	 881- 900
+ 06 91 fe 3c 08 b4 5d de ce 11 ee 22 6d e9 ec 2e 3b 97 60 d4	;	 901- 920
+ 0e f8 de 63 ff a8 42 7f ab a9 26 90 b8 72 21 5d fd 80 f3 64	;	 921- 940
+ 09 1f 0e 9c 89 6a 91 08 46 8b 09 6e 6c 6e 65 74 6c 61 62 73	;	 941- 960
+ 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 1c 05 02 00 01	;	 961- 980
+ 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61	;	 981-1000
+ 62 73 02 6e 6c 00 9b a2 35 6e 53 03 58 22 75 43 ae 1a e2 51	;	1001-1020
+ 09 4e fb 37 0f 60 eb 77 0f bc 4d c0 27 e7 34 a4 3a 27 30 c0	;	1021-1040
+ 4d 45 ad 4b 4d a1 82 54 92 ad 47 95 2b 78 70 6b c4 c7 f4 a3	;	1041-1060
+ d3 55 77 51 09 ce 2a 6e d9 a2 5c 91 ed c1 14 91 0e 49 b5 4b	;	1061-1080
+ 3c 08 3d c1 f0 dc 5d 82 48 97 be 3c eb d9 84 ce 7c 62 27 0c	;	1081-1100
+ b4 92 65 80 5a 3f 39 af f6 96 08 cd 40 3c e2 06 6f 36 df dc	;	1101-1120
+ ae 58 fb 9f fe b1 6a b1 fc 86 1d 7b 4d 7a 09 6e 6c 6e 65 74	;	1121-1140
+ 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 00 02 58 00 a0 00 23	;	1141-1160
+ 05 02 00 00 02 58 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e	;	1161-1180
+ 65 74 6c 61 62 73 02 6e 6c 00 dc bc 30 a6 b8 6f f6 3b 07 af	;	1181-1200
+ cb 44 32 80 97 70 f9 f0 17 a0 cf d4 f8 cd a0 0d 8f cb 31 3b	;	1201-1220
+ 82 b8 8d e7 f0 76 26 28 5e 7f c9 06 a7 53 f2 29 4a 06 18 54	;	1221-1240
+ 9b 0d 9a 18 8d c1 95 51 31 6b 4e 9e 82 c0 40 33 7e 0f d4 29	;	1241-1260
+ 19 b1 ee 96 41 97 ff f8 ea 14 0c d0 f2 fc 1c 41 95 c8 2f 82	;	1261-1280
+ b0 8c 62 1f b7 d0 ea a4 f1 4e 35 0b 85 23 2f ba 44 bd 05 c4	;	1281-1300
+ 2a 47 f8 62 e1 80 d8 76 fb 8e 89 4f 87 66 19 69 7a 57 09 6e	;	1301-1320
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 00 46 50	;	1321-1340
+ 00 a0 00 2f 05 02 00 00 46 50 46 4e f9 ff 46 27 6c ff ab 0f	;	1341-1360
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 42 6f f9 6d 68 59	;	1361-1380
+ 0c c9 c2 43 f4 1a fd eb f1 2b 42 48 9c 77 87 fe 92 d6 c4 3e	;	1381-1400
+ ba e5 03 ee 23 99 18 7f 49 46 83 d1 b0 2b f9 eb 62 5d be a5	;	1401-1420
+ 9f 74 43 d3 36 27 cb 61 54 20 c5 6f e7 62 17 d9 07 5f e5 bb	;	1421-1440
+ 73 4d 93 ca d5 35 d7 06 e2 56 78 44 65 ed 08 a7 c3 e0 c5 db	;	1441-1460
+ fc 07 56 9b e5 0d 4b ab 53 95 11 7f 50 10 3f 78 5b 58 f6 e7	;	1461-1480
+ 31 35 11 80 b1 bc 99 ec a0 e6 59 dc cd 39 4f ae 31 bc df e1	;	1481-1500
+ 6d 0d 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01	;	1501-1520
+ 00 01 51 80 00 a0 00 30 05 02 00 01 51 80 46 4e f9 ff 46 27	;	1521-1540
+ 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 4e ca	;	1541-1560
+ 77 bd 35 15 34 fc b0 14 74 ed 4e d0 c3 52 e8 36 d5 80 86 ec	;	1561-1580
+ fc 06 65 af f0 37 48 56 a1 a3 4e 25 c9 55 a6 f1 35 85 e2 75	;	1581-1600
+ c1 2d 4e 28 a0 70 7d 9c 03 9f 6c d2 94 2f 7e 20 a1 9a 80 64	;	1601-1620
+ 22 0f 95 24 e1 ad 91 eb 5a 78 73 a5 65 3a eb 29 09 fb 37 37	;	1621-1640
+ 9b 7c be 32 f7 d2 ab ba 88 28 3c 4e 92 24 8d 83 0f 3a 1e 4e	;	1641-1660
+ 60 be 00 95 6b e5 4c c2 6c df f8 93 8b da 6f f7 a7 a5 e8 11	;	1661-1680
+ 2c 25 df 8d d2 7f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	1681-1700
+ 00 02 00 01 00 01 51 80 00 13 04 6f 70 65 6e 09 6e 6c 6e 65	;	1701-1720
+ 74 6c 61 62 73 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73 02	;	1721-1740
+ 6e 6c 00 00 02 00 01 00 01 51 80 00 11 05 6f 6d 76 61 6c 06	;	1741-1760
+ 74 65 64 6e 65 74 02 6e 6c 00 09 6e 6c 6e 65 74 6c 61 62 73	;	1761-1780
+ 02 6e 6c 00 00 02 00 01 00 01 51 80 00 18 03 6e 73 37 0f 64	;	1781-1800
+ 6f 6d 61 69 6e 2d 72 65 67 69 73 74 72 79 02 6e 6c 00 03 6e	;	1801-1820
+ 73 37 0f 64 6f 6d 61 69 6e 2d 72 65 67 69 73 74 72 79 02 6e	;	1821-1840
+ 6c 00 00 01 00 01 00 00 44 09 00 04 3e 04 56 e6 04 6f 70 65	;	1841-1860
+ 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 01 00 01 00	;	1861-1880
+ 01 51 80 00 04 d5 9a e0 01 04 6f 70 65 6e 09 6e 6c 6e 65 74	;	1881-1900
+ 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51 80 00 10 20 01	;	1901-1920
+ 07 b8 02 06 00 01 00 00 00 00 00 00 00 53 04 6f 70 65 6e 09	;	1921-1940
+ 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51	;	1941-1960
+ 80 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00 01 05	;	1961-1980
+ 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 00 01 00 01	;	1981-2000
+ 00 00 70 80 00 04 d5 9a e0 11 05 6f 6d 76 61 6c 06 74 65 64	;	2001-2020
+ 6e 65 74 02 6e 6c 00 00 1c 00 01 00 00 70 80 00 10 20 01 07	;	2021-2040
+ b8 02 06 00 01 02 00 39 ff fe 59 b1 87
+; 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19
+;-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+ 56 4e 85 00 00 01 00 09 00 04 00 09 09 6e 6c 6e 65 74 6c 61	;	   1-  20
+ 62 73 02 6e 6c 00 00 2e 00 01 09 6e 6c 6e 65 74 6c 61 62 73	;	  21-  40
+ 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 06 05 02 00 01	;	  41-  60
+ 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61	;	  61-  80
+ 62 73 02 6e 6c 00 5c 33 2b 93 5d f9 18 8f c6 7d f8 26 d2 d3	;	  81- 100
+ da 50 a3 60 42 df 19 1b cb e7 40 4a 7b 88 bb 41 08 68 8f 27	;	 101- 120
+ 7d 4f 05 17 86 89 08 a1 a3 7f 56 1f f4 91 db cc 8b 9b 8e a0	;	 121- 140
+ 03 dc 03 d8 2a 35 f7 ed 2e 01 41 9d b2 42 de 3e 50 4a ab 45	;	 141- 160
+ 9d ac cd 1c a0 90 51 12 be 6c ae 51 20 a5 34 b0 38 e9 83 1d	;	 161- 180
+ 99 57 d2 a6 2f 53 a2 4c 4e 97 47 c7 d9 43 72 a6 4a d2 83 be	;	 181- 200
+ ed f6 e4 34 6e 48 1b ae ed 2c c5 65 d0 ec 09 6e 6c 6e 65 74	;	 201- 220
+ 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 02	;	 221- 240
+ 05 02 00 01 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e	;	 241- 260
+ 65 74 6c 61 62 73 02 6e 6c 00 81 18 16 b9 0b 65 db e0 00 19	;	 261- 280
+ 68 06 01 11 46 67 4e 8c 51 c6 66 46 78 d7 7a ec b4 1f 8a 1e	;	 281- 300
+ d7 3c 44 4a 55 b0 16 b5 25 b6 ad 25 2f 7d 24 16 d2 4e 08 67	;	 301- 320
+ ac 95 fa b3 61 cc 13 9d 72 c2 22 76 9b d4 dc 17 6b 3e 58 d5	;	 321- 340
+ c3 4b 6f b4 fa 60 17 68 46 57 b8 b5 f3 3d b7 f5 a5 df e3 e4	;	 341- 360
+ bb 8c 6f 33 07 f2 c8 8f c7 ff 6c 13 72 8d 4c 54 a4 ae 70 29	;	 361- 380
+ 24 72 ba 7d fe e7 c7 4d 18 e4 5d f1 24 17 9a 8e 79 e4 09 6e	;	 381- 400
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80	;	 401- 420
+ 00 a0 00 01 05 02 00 01 51 80 46 4e f9 ff 46 27 6c ff ab 0f	;	 421- 440
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 ca ee ca 28 cd cf	;	 441- 460
+ f4 fd a5 6e 5c ad 3f d5 89 7c 06 69 93 30 93 f7 0f b2 43 07	;	 461- 480
+ eb ad ce 8e f0 da 75 b9 5e 46 d5 28 40 9d 51 17 9e 2d 3d f3	;	 481- 500
+ 85 1b dd 85 4a 82 9d 32 5c ff 27 c9 83 8f 36 04 89 94 52 3b	;	 501- 520
+ 01 29 01 62 36 1a 66 67 f7 43 16 5b 2a 19 77 d1 cc 73 7c 86	;	 521- 540
+ 44 4c f5 d0 db a4 f5 ea 62 98 0f 18 9f 72 2c 73 5f cb 62 c2	;	 541- 560
+ 7f 45 91 99 53 bb 70 44 93 3a f5 ea 3c 23 80 3f a6 19 06 33	;	 561- 580
+ 3f 69 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01	;	 581- 600
+ 00 01 51 80 00 a0 00 0f 05 02 00 01 51 80 46 4e f9 ff 46 27	;	 601- 620
+ 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 8f 79	;	 621- 640
+ 3e 49 68 56 05 e7 6c 01 6d f6 cf 6f f3 5d 5b cb e8 c1 ff fd	;	 641- 660
+ a2 45 de 61 25 44 0d 4f 47 ea 54 8a 46 0f 58 1a 43 dc b4 3b	;	 661- 680
+ 02 b4 a3 11 a2 c5 45 0a 4b b2 f1 7a 3b 39 8d b0 ad 02 2e 60	;	 681- 700
+ 15 dd fb b5 66 3a e1 b7 6f 5c 2d ea 99 21 a1 7e d9 8c da d8	;	 701- 720
+ 94 6f e5 fc 1c 1f 0a b2 f6 18 7f 11 66 54 37 ad 1d f1 72 32	;	 721- 740
+ dd db 48 6b 53 4a 39 54 77 9b dc 54 f0 da 0c ae 94 7d 82 22	;	 741- 760
+ 05 7f ab bb 3b 79 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	 761- 780
+ 00 2e 00 01 00 01 51 80 00 a0 00 10 05 02 00 01 51 80 46 4e	;	 781- 800
+ f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e	;	 801- 820
+ 6c 00 ce 39 43 cf 06 19 80 eb 18 77 90 6a 43 52 d6 de 3a d0	;	 821- 840
+ b3 5f 00 49 34 6e bd 2a 38 2b a1 4e 24 7a 59 03 c8 72 90 15	;	 841- 860
+ c0 59 e0 8b aa da a9 06 f5 a8 25 5d 46 fc 46 d8 63 46 56 d0	;	 861- 880
+ 34 b4 dd ae 6f f6 e6 41 bc ca cf f7 f0 66 be ba a1 cd 2e 9f	;	 881- 900
+ 06 91 fe 3c 08 b4 5d de ce 11 ee 22 6d e9 ec 2e 3b 97 60 d4	;	 901- 920
+ 0e f8 de 63 ff a8 42 7f ab a9 26 90 b8 72 21 5d fd 80 f3 64	;	 921- 940
+ 09 1f 0e 9c 89 6a 91 08 46 8b 09 6e 6c 6e 65 74 6c 61 62 73	;	 941- 960
+ 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 1c 05 02 00 01	;	 961- 980
+ 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61	;	 981-1000
+ 62 73 02 6e 6c 00 9b a2 35 6e 53 03 58 22 75 43 ae 1a e2 51	;	1001-1020
+ 09 4e fb 37 0f 60 eb 77 0f bc 4d c0 27 e7 34 a4 3a 27 30 c0	;	1021-1040
+ 4d 45 ad 4b 4d a1 82 54 92 ad 47 95 2b 78 70 6b c4 c7 f4 a3	;	1041-1060
+ d3 55 77 51 09 ce 2a 6e d9 a2 5c 91 ed c1 14 91 0e 49 b5 4b	;	1061-1080
+ 3c 08 3d c1 f0 dc 5d 82 48 97 be 3c eb d9 84 ce 7c 62 27 0c	;	1081-1100
+ b4 92 65 80 5a 3f 39 af f6 96 08 cd 40 3c e2 06 6f 36 df dc	;	1101-1120
+ ae 58 fb 9f fe b1 6a b1 fc 86 1d 7b 4d 7a 09 6e 6c 6e 65 74	;	1121-1140
+ 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 00 02 58 00 a0 00 23	;	1141-1160
+ 05 02 00 00 02 58 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e	;	1161-1180
+ 65 74 6c 61 62 73 02 6e 6c 00 dc bc 30 a6 b8 6f f6 3b 07 af	;	1181-1200
+ cb 44 32 80 97 70 f9 f0 17 a0 cf d4 f8 cd a0 0d 8f cb 31 3b	;	1201-1220
+ 82 b8 8d e7 f0 76 26 28 5e 7f c9 06 a7 53 f2 29 4a 06 18 54	;	1221-1240
+ 9b 0d 9a 18 8d c1 95 51 31 6b 4e 9e 82 c0 40 33 7e 0f d4 29	;	1241-1260
+ 19 b1 ee 96 41 97 ff f8 ea 14 0c d0 f2 fc 1c 41 95 c8 2f 82	;	1261-1280
+ b0 8c 62 1f b7 d0 ea a4 f1 4e 35 0b 85 23 2f ba 44 bd 05 c4	;	1281-1300
+ 2a 47 f8 62 e1 80 d8 76 fb 8e 89 4f 87 66 19 69 7a 57 09 6e	;	1301-1320
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 00 46 50	;	1321-1340
+ 00 a0 00 2f 05 02 00 00 46 50 46 4e f9 ff 46 27 6c ff ab 0f	;	1341-1360
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 42 6f f9 6d 68 59	;	1361-1380
+ 0c c9 c2 43 f4 1a fd eb f1 2b 42 48 9c 77 87 fe 92 d6 c4 3e	;	1381-1400
+ ba e5 03 ee 23 99 18 7f 49 46 83 d1 b0 2b f9 eb 62 5d be a5	;	1401-1420
+ 9f 74 43 d3 36 27 cb 61 54 20 c5 6f e7 62 17 d9 07 5f e5 bb	;	1421-1440
+ 73 4d 93 ca d5 35 d7 06 e2 56 78 44 65 ed 08 a7 c3 e0 c5 db	;	1441-1460
+ fc 07 56 9b e5 0d 4b ab 53 95 11 7f 50 10 3f 78 5b 58 f6 e7	;	1461-1480
+ 31 35 11 80 b1 bc 99 ec a0 e6 59 dc cd 39 4f ae 31 bc df e1	;	1481-1500
+ 6d 0d 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01	;	1501-1520
+ 00 01 51 80 00 a0 00 30 05 02 00 01 51 80 46 4e f9 ff 46 27	;	1521-1540
+ 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 4e ca	;	1541-1560
+ 77 bd 35 15 34 fc b0 14 74 ed 4e d0 c3 52 e8 36 d5 80 86 ec	;	1561-1580
+ fc 06 65 af f0 37 48 56 a1 a3 4e 25 c9 55 a6 f1 35 85 e2 75	;	1581-1600
+ c1 2d 4e 28 a0 70 7d 9c 03 9f 6c d2 94 2f 7e 20 a1 9a 80 64	;	1601-1620
+ 22 0f 95 24 e1 ad 91 eb 5a 78 73 a5 65 3a eb 29 09 fb 37 37	;	1621-1640
+ 9b 7c be 32 f7 d2 ab ba 88 28 3c 4e 92 24 8d 83 0f 3a 1e 4e	;	1641-1660
+ 60 be 00 95 6b e5 4c c2 6c df f8 93 8b da 6f f7 a7 a5 e8 11	;	1661-1680
+ 2c 25 df 8d d2 7f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00	;	1681-1700
+ 00 02 00 01 00 01 51 80 00 18 03 6e 73 37 0f 64 6f 6d 61 69	;	1701-1720
+ 6e 2d 72 65 67 69 73 74 72 79 02 6e 6c 00 09 6e 6c 6e 65 74	;	1721-1740
+ 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00 13 04 6f	;	1741-1760
+ 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 09 6e 6c	;	1761-1780
+ 6e 65 74 6c 61 62 73 02 6e 6c 00 00 02 00 01 00 01 51 80 00	;	1781-1800
+ 11 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00 09 6e	;	1801-1820
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01 00 01 51 80	;	1821-1840
+ 00 a0 00 02 05 02 00 01 51 80 46 4e f9 ff 46 27 6c ff ab 0f	;	1841-1860
+ 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 81 18 16 b9 0b 65	;	1861-1880
+ db e0 00 19 68 06 01 11 46 67 4e 8c 51 c6 66 46 78 d7 7a ec	;	1881-1900
+ b4 1f 8a 1e d7 3c 44 4a 55 b0 16 b5 25 b6 ad 25 2f 7d 24 16	;	1901-1920
+ d2 4e 08 67 ac 95 fa b3 61 cc 13 9d 72 c2 22 76 9b d4 dc 17	;	1921-1940
+ 6b 3e 58 d5 c3 4b 6f b4 fa 60 17 68 46 57 b8 b5 f3 3d b7 f5	;	1941-1960
+ a5 df e3 e4 bb 8c 6f 33 07 f2 c8 8f c7 ff 6c 13 72 8d 4c 54	;	1961-1980
+ a4 ae 70 29 24 72 ba 7d fe e7 c7 4d 18 e4 5d f1 24 17 9a 8e	;	1981-2000
+ 79 e4 03 6e 73 37 0f 64 6f 6d 61 69 6e 2d 72 65 67 69 73 74	;	2001-2020
+ 72 79 02 6e 6c 00 00 01 00 01 00 00 43 f3 00 04 3e 04 56 e6	;	2021-2040
+ 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00	;	2041-2060
+ 01 00 01 00 01 51 80 00 04 d5 9a e0 01 04 6f 70 65 6e 09 6e	;	2061-2080
+ 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00 01 00 01 51 80	;	2081-2100
+ 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00 00 00 53 04 6f	;	2101-2120
+ 70 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 1c 00	;	2121-2140
+ 01 00 01 51 80 00 10 20 01 07 b8 02 06 00 01 00 00 00 00 00	;	2141-2160
+ 00 00 01 05 6f 6d 76 61 6c 06 74 65 64 6e 65 74 02 6e 6c 00	;	2161-2180
+ 00 01 00 01 00 00 70 80 00 04 d5 9a e0 11 05 6f 6d 76 61 6c	;	2181-2200
+ 06 74 65 64 6e 65 74 02 6e 6c 00 00 1c 00 01 00 00 70 80 00	;	2201-2220
+ 10 20 01 07 b8 02 06 00 01 02 00 39 ff fe 59 b1 87 04 6f 70	;	2221-2240
+ 65 6e 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 00 2e 00 01	;	2241-2260
+ 00 01 51 80 00 a0 00 01 05 03 00 01 51 80 46 4e f9 ff 46 27	;	2261-2280
+ 6c ff ab 0f 09 6e 6c 6e 65 74 6c 61 62 73 02 6e 6c 00 bd 11	;	2281-2300
+ d7 c8 6f 7f f1 b8 d7 6c 2e c0 eb 5b 65 6c 5e 32 bc ba 0e 50	;	2301-2320
+ c6 d2 b2 65 71 81 44 6e 2a 22 3c 98 39 16 71 f1 56 2d 8f 0d	;	2321-2340
+ 02 8a e8 7d ad 93 f1 f6 85 d0 fc 19 13 26 de 5a 1f ee 33 7d	;	2341-2360
+ f0 9b 5b 15 63 1a a6 33 55 d4 d3 23 4f c5 ac cf 48 71 f5 ed	;	2361-2380
+ 44 17 eb d3 db fe 98 ea 7e fb b3 7d 8f 2a 6b f7 87 af 15 aa	;	2381-2400
+ 78 ee cb e5 24 2b 04 46 a9 e2 96 fa 41 2a 99 42 5d 3e 22 07	;	2401-2420
+ b9 bc 99 62 ad 61 04 6f 70 65 6e 09 6e 6c 6e 65 74 6c 61 62	;	2421-2440
+ 73 02 6e 6c 00 00 2e 00 01 00 01 51 80 00 a0 00 1c 05 03 00	;	2441-2460
+ 01 51 80 46 4e f9 ff 46 27 6c ff ab 0f 09 6e 6c 6e 65 74 6c	;	2461-2480
+ 61 62 73 02 6e 6c 00 a7 45 db c2 3c 05 77 76 03 b8 7e e7 9c	;	2481-2500
+ 3d 64 12 7c 9d f9 17 f2 a0 4a 94 74 19 bd a3 6b 3e 62 da e6	;	2501-2520
+ 87 0e 07 1d fb 5c 5c 0e f9 ab 97 87 71 c1 78 d4 b5 b0 21 65	;	2521-2540
+ fe 26 95 40 ed b4 d6 0c f5 ce a1 34 32 2a 54 1e e0 b5 34 0f	;	2541-2560
+ 71 01 8f bf 66 48 d7 a7 12 85 a5 a0 25 d2 5c 5f 3a 15 87 b5	;	2561-2580
+ bd 4a 3c 0c 34 4b 48 21 48 ac d7 92 02 6b 28 55 35 28 8b b6	;	2581-2600
+ cf 72 b4 2c c3 22 88 8b b0 75 80 0c 35 13 02 00 00 29 10 00	;	2601-2620
+ 00 00 80 00 00 00

testdata/test_packets.5

@@ -0,0 +1,56 @@
+; Hand made test packets.
+; By Wouter Wijngaards.
+; These DNS packets contain interesting compression cases.
+; 
+;-- next packet --
+; 0. A valid packet (handmade)
+; id flags qd  an   ns  ar   -- header
+4242 0000 0001 0001 0000 0000
+; query: qname example.com. qtype A(1) qclass IN(1)
+07 6578616d706c65 03 636f6d 00 0001 0001
+; answer: example.com          type class ttl rdatalen 10.x address.
+07 6578616d706c65 03 636f6d 00 0001 0001  00000101 0004 0a203040
+
+;-- next packet --
+; 0b. correct compression from answer to query.
+4242 0000 0001 0001 0000 0000
+07 6578616d706c65 03 636f6d 00 0001 0001
+c00c 0001 0001  00000101 0004 0a203040
+
+;-- next packet --
+; 1. Compression from query to answer.
+4242 0000 0001 0001 0000 0000
+c012 0001 0001
+07 6578616d706c65 03 636f6d 00 0001 0001  00000101 0004 0a203040
+
+;-- next packet --
+; 2. Compression loop answer 1 to answer 2.
+4242 0000 0001 0002 0000 0000
+07 6578616d706c65 03 636f6d 00 0001 0001
+c02d 0001 0001  00000101 0004 0a203040
+07 6578616d706c65 03 636f6d 00 0001 0001  00000101 0004 0a203050
+
+;-- next packet --
+; 2b. Compression loop answer 2 to answer 1.
+4242 0000 0001 0002 0000 0000
+07 6578616d706c65 03 636f6d 00 0001 0001
+07 6578616d706c65 03 636f6d 00 0001 0001  00000101 0004 0a203050
+c01d 0001 0001  00000101 0004 0a203040
+
+;-- next packet --
+; 3. Compression loop to self (in answer section).
+4242 0000 0001 0001 0000 0000
+07 6578616d706c65 03 636f6d 00 0001 0001
+c01d 0001 0001  00000101 0004 0a203040
+
+;-- next packet --
+; 4. bad compression pointer - to header.
+4242 0000 0001 0001 0000 0000
+07 6578616d706c65 03 636f6d 00 0001 0001
+c004 0001 0001  00000101 0004 0a203040
+
+;-- next packet --
+; 5. bad compression pointer - exceeds packet.
+4242 0000 0001 0001 0000 0000
+07 6578616d706c65 03 636f6d 00 0001 0001
+c0bb 0001 0001  00000101 0004 0a203040

util/alloc.c

@@ -41,6 +41,18 @@
 
 #include "config.h"
 #include "util/alloc.h"
+#include "util/data/packed_rrset.h"
+
+/** number of bits for ID part of uint64, rest for number of threads. */
+#define THRNUM_SHIFT	48	/* for 65k threads, 2^48 rrsets per thr. */
+
+/** setup new special type */
+static void
+alloc_setup_special(alloc_special_t* t)
+{
+	memset(t, 0, sizeof(*t));
+	lock_rw_init(&t->entry.lock);
+}
 
 /** prealloc some entries in the cache. To minimize contention. 
  * Result is 1 lock per alloc_max newly created entries.
@@ -54,17 +66,27 @@ prealloc(struct alloc_cache* alloc)
 	for(i=0; i<ALLOC_SPECIAL_MAX; i++) {
 		if(!(p = (alloc_special_t*)malloc(sizeof(alloc_special_t))))
 			fatal_exit("prealloc: out of memory");
-		alloc_special_next(p) = alloc->quar;
+		alloc_setup_special(p);
+		alloc_set_special_next(p, alloc->quar);
 		alloc->quar = p;
 		alloc->num_quar++;
 	}
 }
 
 void 
-alloc_init(struct alloc_cache* alloc, struct alloc_cache* super)
+alloc_init(struct alloc_cache* alloc, struct alloc_cache* super,
+	int thread_num)
 {
 	memset(alloc, 0, sizeof(*alloc));
 	alloc->super = super;
+	alloc->thread_num = thread_num;
+	alloc->next_id = (uint64_t)thread_num; 	/* in steps, so that type */
+	alloc->next_id <<= THRNUM_SHIFT; 	/* of *_id is used. */
+	alloc->last_id = 1; 			/* so no 64bit constants, */
+	alloc->last_id <<= THRNUM_SHIFT; 	/* or implicit 'int' ops. */
+	alloc->last_id -= 1; 			/* for compiler portability. */
+	alloc->last_id |= alloc->next_id;
+	alloc->next_id += 1;			/* because id=0 is special. */
 	if(!alloc->super) {
 		lock_quick_init(&alloc->lock);
 		lock_protect(&alloc->lock, alloc, sizeof(*alloc));
@@ -86,7 +108,7 @@ alloc_clear(struct alloc_cache* alloc)
 		while(alloc_special_next(p)) /* find last */
 			p = alloc_special_next(p);
 		lock_quick_lock(&alloc->super->lock);
-		alloc_special_next(p) = alloc->super->quar;
+		alloc_set_special_next(p, alloc->super->quar);
 		alloc->super->quar = alloc->quar;
 		alloc->super->num_quar += alloc->num_quar;
 		lock_quick_unlock(&alloc->super->lock);
@@ -95,6 +117,8 @@ alloc_clear(struct alloc_cache* alloc)
 		p = alloc->quar;
 		while(p) {
 			np = alloc_special_next(p);
+			/* deinit special type */
+			lock_rw_destroy(&p->entry.lock);
 			free(p);
 			p = np;
 		}
@@ -103,6 +127,18 @@ alloc_clear(struct alloc_cache* alloc)
 	alloc->num_quar = 0;
 }
 
+/** get a new id */
+static void
+alloc_get_id(struct alloc_cache* alloc, alloc_special_t* t)
+{
+	t->id = alloc->next_id++;
+	if(alloc->next_id == alloc->last_id) {
+		/* TODO: clear the rrset cache */
+		log_warn("Out of ids. Clearing cache.");
+	}
+	alloc_set_special_next(t, 0);
+}
+
 alloc_special_t* 
 alloc_special_obtain(struct alloc_cache* alloc)
 {
@@ -113,7 +149,7 @@ alloc_special_obtain(struct alloc_cache* alloc)
 		p = alloc->quar;
 		alloc->quar = alloc_special_next(p);
 		alloc->num_quar--;
-		alloc_special_clean(p);
+		alloc_get_id(alloc, p);
 		return p;
 	}
 	/* see if in global cache */
@@ -127,7 +163,7 @@ alloc_special_obtain(struct alloc_cache* alloc)
 		}
 		lock_quick_unlock(&alloc->super->lock);
 		if(p) {
-			alloc_special_clean(p);
+			alloc_get_id(alloc, p);
 			return p;
 		}
 	}
@@ -135,7 +171,8 @@ alloc_special_obtain(struct alloc_cache* alloc)
 	prealloc(alloc);
 	if(!(p = (alloc_special_t*)malloc(sizeof(alloc_special_t))))
 		fatal_exit("alloc_special_obtain: out of memory");
-	alloc_special_clean(p);
+	alloc_setup_special(p);
+	alloc_get_id(alloc, p);
 	return p;
 }
 
@@ -149,7 +186,7 @@ pushintosuper(struct alloc_cache* alloc, alloc_special_t* mem)
 	log_assert(alloc && alloc->super && 
 		alloc->num_quar >= ALLOC_SPECIAL_MAX);
 	/* push ALLOC_SPECIAL_MAX/2 after mem */
-	alloc_special_next(mem) = alloc->quar;
+	alloc_set_special_next(mem, alloc->quar);
 	for(i=1; i<ALLOC_SPECIAL_MAX/2; i++) {
 		p = alloc_special_next(p);
 	}
@@ -158,7 +195,7 @@ pushintosuper(struct alloc_cache* alloc, alloc_special_t* mem)
 
 	/* dump mem+list into the super quar list */
 	lock_quick_lock(&alloc->super->lock);
-	alloc_special_next(p) = alloc->super->quar;
+	alloc_set_special_next(p, alloc->super->quar);
 	alloc->super->quar = mem;
 	alloc->super->num_quar += ALLOC_SPECIAL_MAX/2 + 1;
 	lock_quick_unlock(&alloc->super->lock);
@@ -171,6 +208,10 @@ alloc_special_release(struct alloc_cache* alloc, alloc_special_t* mem)
 	log_assert(alloc);
 	if(!mem)
 		return;
+	if(!alloc->super) { 
+		lock_quick_lock(&alloc->lock); /* superalloc needs locking */
+	}
+
 	alloc_special_clean(mem);
 	if(alloc->super && alloc->num_quar >= ALLOC_SPECIAL_MAX) {
 		/* push it to the super structure */
@@ -178,9 +219,12 @@ alloc_special_release(struct alloc_cache* alloc, alloc_special_t* mem)
 		return;
 	}
 
-	alloc_special_next(mem) = alloc->quar;
+	alloc_set_special_next(mem, alloc->quar);
 	alloc->quar = mem;
 	alloc->num_quar++;
+	if(!alloc->super) {
+		lock_quick_unlock(&alloc->lock);
+	}
 }
 
 void 

util/alloc.h

@@ -49,13 +49,17 @@
 #define UTIL_ALLOC_H
 
 #include "util/locks.h"
+struct ub_packed_rrset_key;
 
 /** The special type, packed rrset. Not allowed to be used for other memory */
-typedef uint64_t alloc_special_t;
+typedef struct ub_packed_rrset_key alloc_special_t;
 /** clean the special type. Pass pointer. */
-#define alloc_special_clean(x) memset(x, 0, sizeof(alloc_special_t))
+#define alloc_special_clean(x) (x)->id = 0;
 /** access next pointer. (in available spot). Pass pointer. */
-#define alloc_special_next(x) (*((alloc_special_t**)(x)))
+#define alloc_special_next(x) ((alloc_special_t*)((x)->entry.overflow_next))
+/** set next pointer. (in available spot). Pass pointers. */
+#define alloc_set_special_next(x, y) \
+	((x)->entry.overflow_next) = (struct lruhash_entry*)(y);
 
 /** how many blocks to cache locally. */
 #define ALLOC_SPECIAL_MAX 10
@@ -73,6 +77,12 @@ struct alloc_cache {
 	alloc_special_t* quar;
 	/** number of items in quarantine. */
 	size_t num_quar;
+	/** thread number for id creation */
+	int thread_num;
+	/** next id number to pass out */
+	uint64_t next_id;
+	/** last id number possible */
+	uint64_t last_id;
 };
 
 /**
@@ -80,8 +90,10 @@ struct alloc_cache {
  * @param alloc: this parameter is allocated by the caller.
  * @param super: super to use (init that before with super_init).
  *    Pass this argument NULL to init the toplevel alloc structure.
+ * @param thread_num: thread number for id creation of special type.
  */
-void alloc_init(struct alloc_cache* alloc, struct alloc_cache* super);
+void alloc_init(struct alloc_cache* alloc, struct alloc_cache* super,
+	int thread_num);
 
 /**
  * Free the alloc. Pushes all the cached items into the super structure.

util/config_file.c

@@ -77,17 +77,26 @@ config_create()
 	cfg->do_tcp = 1;
 	cfg->outgoing_base_port = cfg->port + 1000;
 	cfg->outgoing_num_ports = 16;
-	if(!(cfg->fwd_address = strdup(""))) {config_delete(cfg); return NULL;}
-	if(!(cfg->username = strdup(""))) {config_delete(cfg); return NULL;}
-	if(!(cfg->chrootdir = strdup(""))) {config_delete(cfg); return NULL;}
-	if(!(cfg->directory = strdup("/etc/unbound"))) {config_delete(cfg); return NULL;}
-	if(!(cfg->logfile = strdup(""))) {config_delete(cfg); return NULL;}
-	if(!(cfg->pidfile = strdup("unbound.pid"))) {config_delete(cfg); return NULL;}
+	cfg->outgoing_num_tcp = 10;
+	cfg->msg_cache_size = 4 * 1024 * 1024;
+	cfg->msg_cache_slabs = 4;
+	cfg->num_queries_per_thread = 1024;
+	cfg->rrset_cache_size = 4 * 1024 * 1024;
+	cfg->rrset_cache_slabs = 4;
+	if(!(cfg->fwd_address = strdup(""))) goto error_exit;
+	if(!(cfg->username = strdup(""))) goto error_exit;
+	if(!(cfg->chrootdir = strdup(""))) goto error_exit;
+	if(!(cfg->directory = strdup("/etc/unbound"))) goto error_exit;
+	if(!(cfg->logfile = strdup(""))) goto error_exit;
+	if(!(cfg->pidfile = strdup("unbound.pid"))) goto error_exit;
 	cfg->fwd_port = UNBOUND_DNS_PORT;
 	cfg->do_daemonize = 1;
 	cfg->num_ifs = 0;
 	cfg->ifs = NULL;
 	return cfg;
+error_exit:
+	config_delete(cfg); 
+	return NULL;
 }
 
 /** initialize the global cfg_parser object. */

util/config_file.h

@@ -68,6 +68,19 @@ struct config_file {
 	int outgoing_base_port;
 	/** outgoing port range number of ports (per thread, per if) */
 	int outgoing_num_ports;
+	/** number of outgoing tcp buffers per (per thread) */
+	size_t outgoing_num_tcp;
+
+	/** size of the message cache */
+	size_t msg_cache_size;
+	/** slabs in the message cache. */
+	size_t msg_cache_slabs;
+	/** number of queries every thread can service */
+	size_t num_queries_per_thread;
+	/** size of the rrset cache */
+	size_t rrset_cache_size;
+	/** slabs in the rrset cache */
+	size_t rrset_cache_slabs;
 
 	/** forwarder address. string. If not NULL fwder mode is enabled. */
 	char* fwd_address;

util/configlexer.lex

@@ -102,6 +102,7 @@ verbosity{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_VERBOSITY;}
 port{COLON}		{ LEXOUT(("v(%s) ", yytext)); return VAR_PORT;}
 outgoing-port{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_OUTGOING_PORT;}
 outgoing-range{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_OUTGOING_RANGE;}
+outgoing-num-tcp{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_OUTGOING_NUM_TCP;}
 do-ip4{COLON}		{ LEXOUT(("v(%s) ", yytext)); return VAR_DO_IP4;}
 do-ip6{COLON}		{ LEXOUT(("v(%s) ", yytext)); return VAR_DO_IP6;}
 do-udp{COLON}		{ LEXOUT(("v(%s) ", yytext)); return VAR_DO_UDP;}
@@ -114,6 +115,11 @@ username{COLON}		{ LEXOUT(("v(%s) ", yytext)); return VAR_USERNAME;}
 directory{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_DIRECTORY;}
 logfile{COLON}		{ LEXOUT(("v(%s) ", yytext)); return VAR_LOGFILE;}
 pidfile{COLON}		{ LEXOUT(("v(%s) ", yytext)); return VAR_PIDFILE;}
+msg-cache-size{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_MSG_CACHE_SIZE;}
+msg-cache-slabs{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_MSG_CACHE_SLABS;}
+rrset-cache-size{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_RRSET_CACHE_SIZE;}
+rrset-cache-slabs{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_RRSET_CACHE_SLABS;}
+num-queries-per-thread{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_NUM_QUERIES_PER_THREAD;}
 {NEWLINE}		{ LEXOUT(("NL\n")); cfg_parser->line++;}
 
 	/* Quoted strings. Strip leading and ending quotes */

util/configparser.y

@@ -46,6 +46,7 @@
 
 #include "util/configyyrename.h"
 #include "util/config_file.h"
+#include "util/net_help.h"
 
 int ub_c_lex(void);
 void ub_c_error(const char *message);
@@ -71,6 +72,8 @@ extern struct config_parser_state* cfg_parser;
 %token VAR_DO_IP4 VAR_DO_IP6 VAR_DO_UDP VAR_DO_TCP
 %token VAR_FORWARD_TO VAR_FORWARD_TO_PORT VAR_CHROOT
 %token VAR_USERNAME VAR_DIRECTORY VAR_LOGFILE VAR_PIDFILE
+%token VAR_MSG_CACHE_SIZE VAR_MSG_CACHE_SLABS VAR_NUM_QUERIES_PER_THREAD
+%token VAR_RRSET_CACHE_SIZE VAR_RRSET_CACHE_SLABS VAR_OUTGOING_NUM_TCP
 
 %%
 toplevelvars: /* empty */ | toplevelvars toplevelvar ;
@@ -85,12 +88,17 @@ serverstart: VAR_SERVER
 		cfg_parser->server_settings_seen = 1;
 	}
 	;
-contents_server: contents_server content_server | ;
+contents_server: contents_server content_server 
+	| ;
 content_server: server_num_threads | server_verbosity | server_port |
 	server_outgoing_port | server_outgoing_range | server_do_ip4 |
 	server_do_ip6 | server_do_udp | server_do_tcp | server_forward_to |
 	server_forward_to_port | server_interface | server_chroot | 
-	server_username | server_directory | server_logfile | server_pidfile;
+	server_username | server_directory | server_logfile | server_pidfile |
+	server_msg_cache_size | server_msg_cache_slabs |
+	server_num_queries_per_thread | server_rrset_cache_size | 
+	server_rrset_cache_slabs | server_outgoing_num_tcp
+	;
 server_num_threads: VAR_NUM_THREADS STRING 
 	{ 
 		OUTYY(("P(server_num_threads:%s)\n", $2)); 
@@ -149,6 +157,15 @@ server_outgoing_range: VAR_OUTGOING_RANGE STRING
 		free($2);
 	}
 	;
+server_outgoing_num_tcp: VAR_OUTGOING_NUM_TCP STRING
+	{
+		OUTYY(("P(server_outgoing_num_tcp:%s)\n", $2));
+		if(atoi($2) == 0 && strcmp($2, "0") != 0)
+			yyerror("number expected");
+		else cfg_parser->cfg->outgoing_num_tcp = atoi($2);
+		free($2);
+	}
+	;
 server_do_ip4: VAR_DO_IP4 STRING
 	{
 		OUTYY(("P(server_do_ip4:%s)\n", $2));
@@ -236,6 +253,59 @@ server_pidfile: VAR_PIDFILE STRING
 		cfg_parser->cfg->pidfile = $2;
 	}
 	;
+server_msg_cache_size: VAR_MSG_CACHE_SIZE STRING
+	{
+		OUTYY(("P(server_msg_cache_size:%s)\n", $2));
+		if(atoi($2) == 0)
+			yyerror("number expected");
+		else cfg_parser->cfg->msg_cache_size = atoi($2);
+		free($2);
+	}
+	;
+server_msg_cache_slabs: VAR_MSG_CACHE_SLABS STRING
+	{
+		OUTYY(("P(server_msg_cache_slabs:%s)\n", $2));
+		if(atoi($2) == 0)
+			yyerror("number expected");
+		else {
+			cfg_parser->cfg->msg_cache_slabs = atoi($2);
+			if(!is_pow2(cfg_parser->cfg->msg_cache_slabs))
+				yyerror("must be a power of 2");
+		}
+		free($2);
+	}
+	;
+server_num_queries_per_thread: VAR_NUM_QUERIES_PER_THREAD STRING
+	{
+		OUTYY(("P(server_num_queries_per_thread:%s)\n", $2));
+		if(atoi($2) == 0)
+			yyerror("number expected");
+		else cfg_parser->cfg->num_queries_per_thread = atoi($2);
+		free($2);
+	}
+	;
+server_rrset_cache_size: VAR_RRSET_CACHE_SIZE STRING
+	{
+		OUTYY(("P(server_rrset_cache_size:%s)\n", $2));
+		if(atoi($2) == 0)
+			yyerror("number expected");
+		else cfg_parser->cfg->rrset_cache_size = atoi($2);
+		free($2);
+	}
+	;
+server_rrset_cache_slabs: VAR_RRSET_CACHE_SLABS STRING
+	{
+		OUTYY(("P(server_rrset_cache_slabs:%s)\n", $2));
+		if(atoi($2) == 0)
+			yyerror("number expected");
+		else {
+			cfg_parser->cfg->rrset_cache_slabs = atoi($2);
+			if(!is_pow2(cfg_parser->cfg->rrset_cache_slabs))
+				yyerror("must be a power of 2");
+		}
+		free($2);
+	}
+	;
 %%
 
 /* parse helper routines could be here */

util/data/dname.c

@@ -0,0 +1,473 @@
+/*
+ * util/data/dname.h - domain name handling
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains domain name handling functions.
+ */
+
+#include "config.h"
+#include "util/data/dname.h"
+#include "util/data/msgparse.h"
+#include "util/log.h"
+#include "util/storage/lookup3.h"
+
+/** determine length of a dname in buffer, no compression pointers allowed. */
+size_t
+query_dname_len(ldns_buffer* query)
+{
+	size_t len = 0;
+	size_t labellen;
+	while(1) {
+		if(ldns_buffer_remaining(query) < 1)
+			return 0; /* parse error, need label len */
+		labellen = ldns_buffer_read_u8(query);
+		if(labellen&0xc0)
+			return 0; /* no compression allowed in queries */
+		len += labellen + 1;
+		if(len > LDNS_MAX_DOMAINLEN)
+			return 0; /* too long */
+		if(labellen == 0)
+			return len;
+		if(ldns_buffer_remaining(query) < labellen)
+			return 0; /* parse error, need content */
+		ldns_buffer_skip(query, (ssize_t)labellen);
+	}
+}
+
+int 
+query_dname_compare(uint8_t* d1, uint8_t* d2)
+{
+	uint8_t lab1, lab2;
+	log_assert(d1 && d2);
+	lab1 = *d1++;
+	lab2 = *d2++;
+	while( lab1 != 0 || lab2 != 0 ) {
+		/* compare label length */
+		/* if one dname ends, it has labellength 0 */
+		if(lab1 != lab2) {
+			if(lab1 < lab2)
+				return -1;
+			return 1;
+		}
+		log_assert(lab1 == lab2 && lab1 != 0);
+		/* compare lowercased labels. */
+		while(lab1--) {
+			if(tolower((int)*d1) != tolower((int)*d2)) {
+				if(tolower((int)*d1) < tolower((int)*d2))
+					return -1;
+				return 1;
+			}
+			d1++;
+			d2++;
+		}
+		/* next pair of labels. */
+		lab1 = *d1++;
+		lab2 = *d2++;
+	}
+	return 0;
+}
+
+void 
+query_dname_tolower(uint8_t* dname, size_t len)
+{
+	/* the dname is stored uncompressed */
+	uint8_t labellen;
+	log_assert(len > 0);
+	labellen = *dname;
+	while(labellen) {
+		dname++;
+		while(labellen--) {
+			*dname = (uint8_t)tolower((int)*dname);
+			dname++;
+		}
+		labellen = *dname;
+	}
+}
+
+/** maximum compression pointer position pointed to */
+#define MAX_COMPRESS_POS 16384
+/** size of bitmap for loop detection */
+#define LOOP_BITMAP_SIZE (MAX_COMPRESS_POS/8)
+
+/** check bit in bitmap for loop detection, then set it for next check */
+static uint8_t 
+loopcheck(uint8_t loop[], size_t pos)
+{
+	const uint8_t bits[8] = {0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
+	uint8_t ret;
+	log_assert(pos < MAX_COMPRESS_POS);
+	ret = loop[ pos / 8 ] & bits[ pos % 8 ];
+	loop[ pos / 8 ] |= bits[ pos % 8 ];	
+	return ret;
+}
+
+
+size_t
+pkt_dname_len(ldns_buffer* pkt)
+{
+	size_t len = 0;
+	uint8_t loop[LOOP_BITMAP_SIZE]; /* loopcheck array. */
+	uint8_t labellen;
+	size_t endpos = 0;
+
+	/* read dname and determine length */
+	/* check compression pointers, loops, out of bounds */
+	memset(loop, 0, sizeof(loop));
+
+	while(1) {
+		/* read next label */
+		if(ldns_buffer_remaining(pkt) < 1)
+			return 0;
+		labellen = ldns_buffer_read_u8(pkt);
+		if(LABEL_IS_PTR(labellen)) {
+			/* compression ptr */
+			uint16_t ptr;
+			if(ldns_buffer_remaining(pkt) < 1)
+				return 0;
+			ptr = PTR_OFFSET(labellen, ldns_buffer_read_u8(pkt));
+			if(loopcheck(loop, ptr))
+				return 0; /* loop! */
+			if(ldns_buffer_limit(pkt) <= ptr)
+				return 0; /* out of bounds! */
+			if(!endpos)
+				endpos = ldns_buffer_position(pkt);
+			ldns_buffer_set_position(pkt, ptr);
+		} else {
+			/* label contents */
+			if(labellen > 0x3f)
+				return 0; /* label too long */
+			len += 1 + labellen;
+			if(len > LDNS_MAX_DOMAINLEN)
+				return 0;
+			if(labellen == 0) {
+				/* end of dname */
+				break;
+			}
+			if(ldns_buffer_remaining(pkt) < labellen)
+				return 0;
+			ldns_buffer_skip(pkt, (ssize_t)labellen);
+		}
+	}
+	if(endpos)
+		ldns_buffer_set_position(pkt, endpos);
+
+	return len;
+}
+
+int 
+dname_pkt_compare(ldns_buffer* pkt, uint8_t* d1, uint8_t* d2)
+{
+	uint8_t len1, len2;
+	log_assert(pkt && d1 && d2);
+	len1 = *d1++;
+	len2 = *d2++;
+	while( len1 != 0 || len2 != 0 ) {
+		/* resolve ptrs */
+		if(LABEL_IS_PTR(len1)) {
+			d1 = ldns_buffer_at(pkt, PTR_OFFSET(len1, *d1));
+			len1 = *d1++;
+			continue;
+		}
+		if(LABEL_IS_PTR(len2)) {
+			d2 = ldns_buffer_at(pkt, PTR_OFFSET(len2, *d2));
+			len2 = *d2++;
+			continue;
+		}
+		/* check label length */
+		log_assert(len1 <= LDNS_MAX_LABELLEN);
+		log_assert(len2 <= LDNS_MAX_LABELLEN);
+		if(len1 != len2) {
+			if(len1 < len2) return -1;
+			return 1;
+		}
+		log_assert(len1 == len2 && len1 != 0);
+		/* compare labels */
+		while(len1--) {
+			if(tolower((int)*d1++) != tolower((int)*d2++)) {
+				if(tolower((int)d1[-1]) < tolower((int)d2[-1]))
+					return -1;
+				return 1;
+			}
+		}
+		len1 = *d1++;
+		len2 = *d2++;
+	}
+	return 0;
+}
+
+hashvalue_t 
+dname_query_hash(uint8_t* dname, hashvalue_t h)
+{
+	uint8_t labuf[LDNS_MAX_LABELLEN+1];
+	uint8_t lablen;
+	int i;
+
+	/* preserve case of query, make hash label by label */
+	lablen = *dname;
+	while(lablen) {
+		log_assert(lablen <= LDNS_MAX_LABELLEN);
+		labuf[0] = lablen;
+		dname++;
+		i=0;
+		while(lablen--)
+			labuf[++i] = (uint8_t)tolower((int)*dname++);
+		h = hashlittle(labuf, labuf[0] + 1, h);
+		lablen = *dname;
+	}
+
+	return h;
+}
+
+hashvalue_t 
+dname_pkt_hash(ldns_buffer* pkt, uint8_t* dname, hashvalue_t h)
+{
+	uint8_t labuf[LDNS_MAX_LABELLEN+1];
+	uint8_t lablen;
+	int i;
+
+	/* preserve case of query, make hash label by label */
+	lablen = *dname++;
+	while(lablen) {
+		if(LABEL_IS_PTR(lablen)) {
+			/* follow pointer */
+			dname = ldns_buffer_at(pkt, PTR_OFFSET(lablen, *dname));
+			lablen = *dname++;
+			continue;
+		}
+		log_assert(lablen <= LDNS_MAX_LABELLEN);
+		labuf[0] = lablen;
+		i=0;
+		while(lablen--)
+			labuf[++i] = (uint8_t)tolower((int)*dname++);
+		h = hashlittle(labuf, labuf[0] + 1, h);
+		lablen = *dname++;
+	}
+
+	return h;
+}
+
+void dname_pkt_copy(ldns_buffer* pkt, uint8_t* to, uint8_t* dname)
+{
+	/* copy over the dname and decompress it at the same time */
+	uint8_t lablen;
+	lablen = *dname++;
+	while(lablen) {
+		if(LABEL_IS_PTR(lablen)) {
+			/* follow pointer */
+			dname = ldns_buffer_at(pkt, PTR_OFFSET(lablen, *dname));
+			lablen = *dname++;
+			continue;
+		}
+		log_assert(lablen <= LDNS_MAX_LABELLEN);
+		*to++ = lablen;
+		memmove(to, dname, lablen);
+		dname += lablen;
+		to += lablen;
+		lablen = *dname++;
+	}
+	/* copy last \0 */
+	*to = 0;
+}
+
+void dname_print(FILE* out, ldns_buffer* pkt, uint8_t* dname)
+{
+	uint8_t lablen;
+	if(!out) out = stdout;
+	if(!dname) return;
+
+	lablen = *dname++;
+	if(!lablen) 
+		fputc('.', out);
+	while(lablen) {
+		if(LABEL_IS_PTR(lablen)) {
+			/* follow pointer */
+			if(!pkt) {
+				fputs("??compressionptr??", out);
+				return;
+			}
+			dname = ldns_buffer_at(pkt, PTR_OFFSET(lablen, *dname));
+			lablen = *dname++;
+			continue;
+		}
+		if(lablen > LDNS_MAX_LABELLEN) {
+			fputs("??extendedlabel??", out);
+			return;
+		}
+		while(lablen--)
+			fputc((int)*dname++, out);
+		fputc('.', out);
+		lablen = *dname++;
+	}
+}
+
+int 
+dname_count_labels(uint8_t* dname)
+{
+	uint8_t lablen;
+	int labs = 1;
+
+	lablen = *dname++;
+	while(lablen) {
+		labs++;
+		dname += lablen;
+		lablen = *dname++;
+	}
+	return labs;
+}
+
+int 
+dname_count_size_labels(uint8_t* dname, size_t* size)
+{	
+	uint8_t lablen;
+	int labs = 1;
+	*size = 1;
+
+	lablen = *dname++;
+	while(lablen) {
+		labs++;
+		*size += lablen+1;
+		dname += lablen;
+		lablen = *dname++;
+	}
+	return labs;
+
+}
+
+/**
+ * Compare labels in memory, lowercase while comparing.
+ * @param p1: label 1
+ * @param p2: label 2
+ * @param len: number of bytes to compare.
+ * @return: 0, -1, +1 comparison result.
+ */
+static int
+memlowercmp(uint8_t* p1, uint8_t* p2, uint8_t len)
+{
+	while(len--) {
+		if(tolower((int)*p1++) != tolower((int)*p2++)) {
+			if(tolower((int)p1[-1]) < tolower((int)p2[-1]))
+				return -1;
+			return 1;
+		}
+	}
+	return 0;
+}
+
+
+int 
+dname_lab_cmp(uint8_t* d1, int labs1, uint8_t* d2, int labs2, int* mlabs)
+{
+	uint8_t len1, len2;
+	int atlabel = labs1;
+	int lastmlabs;
+	int lastdiff = 0;
+	int c;
+	/* first skip so that we compare same label. */
+	if(labs1 > labs2) {
+		while(atlabel > labs2) {
+			len1 = *d1++;
+			d1 += len1;
+			atlabel--;
+		}
+		log_assert(atlabel == labs2);
+	} else if(labs1 < labs2) {
+		atlabel = labs2;
+		while(atlabel > labs1) {
+			len2 = *d2++;
+			d2 += len2;
+			atlabel--;
+		}
+		log_assert(atlabel == labs1);
+	}
+	lastmlabs = atlabel+1;
+	/* now at same label in d1 and d2, atlabel */
+	/* www.example.com.                  */
+	/* 4   3       2  1   atlabel number */
+	/* repeat until at root label (which is always the same) */
+	while(atlabel > 1) {
+		len1 = *d1++;
+		len2 = *d2++;
+		if(len1 != len2) {
+			log_assert(len1 != 0 && len2 != 0);
+			if(len1<len2)
+				lastdiff = -1;
+			else	lastdiff = 1;
+			lastmlabs = atlabel;
+		} else if((c=memlowercmp(d1, d2, len1)) != 0) { 
+			if(c<0)
+				lastdiff = -1;
+			else	lastdiff = 1;
+			lastmlabs = atlabel;
+		}
+
+		d1 += len1;
+		d2 += len2;
+		atlabel--;
+	}
+	/* last difference atlabel number, so number of labels matching,
+	 * at the right side, is one less. */
+	*mlabs = lastmlabs-1;
+	if(lastdiff == 0) {
+		/* all labels compared were equal, check if one has more
+		 * labels, so that example.com. > com. */
+		if(labs1 > labs2)
+			return 1;
+		else if(labs1 < labs2)
+			return -1;
+	}
+	return lastdiff;
+}
+
+int 
+dname_buffer_write(ldns_buffer* pkt, uint8_t* dname)
+{
+	uint8_t lablen;
+
+	if(ldns_buffer_remaining(pkt) < 1)
+		return 0;
+	lablen = *dname++;
+	ldns_buffer_write_u8(pkt, lablen);
+	while(lablen) {
+		if(ldns_buffer_remaining(pkt) < (size_t)lablen+1)
+			return 0;
+		ldns_buffer_write(pkt, dname, lablen);
+		dname += lablen;
+		lablen = *dname++;
+		ldns_buffer_write_u8(pkt, lablen);
+	}
+	return 1;
+}

util/data/dname.h

@@ -0,0 +1,164 @@
+/*
+ * util/data/dname.h - domain name routines
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains functions to deal with domain names (dnames).
+ *
+ * Some of the functions deal with domain names as a wireformat buffer,
+ * with a length.
+ */
+
+#ifndef UTIL_DATA_DNAME_H
+#define UTIL_DATA_DNAME_H
+#include "util/storage/lruhash.h"
+
+/** 
+ * Determine length of dname in buffer, no compression ptrs allowed, 
+ * @param query: the ldns buffer, current position at start of dname.
+ *	at end, position is at end of the dname.
+ * @return: 0 on parse failure, or length including ending 0 of dname. 
+ */
+size_t query_dname_len(ldns_buffer* query);
+
+/** lowercase query dname */
+void query_dname_tolower(uint8_t* dname, size_t len);
+
+/**
+ * Compare query dnames (uncompressed storage). The Dnames passed do not
+ * have to be lowercased, comparison routine does this.
+ * Dnames have to be valid format.
+ * @param d1: dname to compare
+ * @param d2: dname to compare
+ * @return: -1, 0, or +1 depending on comparison results.
+ * 	Sort order is first difference found. not the canonical ordering.
+ */
+int query_dname_compare(uint8_t* d1, uint8_t* d2);
+
+/**
+ * Determine correct, compressed, dname present in packet.
+ * Checks for parse errors.
+ * @param pkt: packet to read from (from current start position).
+ * @return: 0 on parse error.
+ *	At exit the position is right after the (compressed) dname.
+ *	Compression pointers are followed and checked for loops.
+ *	The uncompressed wireformat length is returned.
+ */
+size_t pkt_dname_len(ldns_buffer* pkt);
+
+/**
+ * Compare dnames in packet (compressed). Dnames must be valid.
+ * routine performs lowercasing, so the packet casing is preserved.
+ * @param pkt: packet, used to resolve compression pointers.
+ * @param d1: dname to compare
+ * @param d2: dname to compare
+ * @return: -1, 0, or +1 depending on comparison results.
+ * 	Sort order is first difference found. not the canonical ordering.
+ */
+int dname_pkt_compare(ldns_buffer* pkt, uint8_t* d1, uint8_t* d2);
+
+/**
+ * Hash dname, label by label, lowercasing, into hashvalue.
+ * Dname in query format (not compressed).
+ * @param dname: dname to hash.
+ * @param h: initial hash value.
+ * @return: result hash value.
+ */
+hashvalue_t dname_query_hash(uint8_t* dname, hashvalue_t h);
+
+/**
+ * Hash dname, label by label, lowercasing, into hashvalue.
+ * Dname in pkt format (compressed).
+ * @param pkt: packet, for resolving compression pointers.
+ * @param dname: dname to hash, pointer to the pkt buffer.
+ * 	Must be valid format. No loops, etc.
+ * @param h: initial hash value.
+ * @return: result hash value.
+ * 	Result is the same as dname_query_hash, even if compression is used.
+ */
+hashvalue_t dname_pkt_hash(ldns_buffer* pkt, uint8_t* dname, hashvalue_t h);
+
+/**
+ * Copy over a valid dname and decompress it.
+ * @param pkt: packet to resolve compression pointers.
+ * @param to: buffer of size from pkt_len function to hold result.
+ * @param dname: pointer into packet where dname starts.
+ */
+void dname_pkt_copy(ldns_buffer* pkt, uint8_t* to, uint8_t* dname);
+
+/**
+ * Copy over a valid dname to a packet.
+ * @param pkt: packet to copy to.
+ * @param dname: dname to copy.
+ * @return: 0 if not enough space in buffer.
+ */
+int dname_buffer_write(ldns_buffer* pkt, uint8_t* dname);
+
+/** debug helper. Print wireformat dname to output. 
+ * @param out: like stdout or a file.
+ * @param pkt: if not NULL, the packet for resolving compression ptrs.
+ * @param dname: pointer to (start of) dname.
+ */
+void dname_print(FILE* out, ldns_buffer* pkt, uint8_t* dname);
+
+/**
+ * Count the number of labels in an uncompressed dname in memory.
+ * @param dname: pointer to uncompressed dname.
+ * @return: count of labels, including root label, "com." has 2 labels.
+ */
+int dname_count_labels(uint8_t* dname);
+
+/**
+ * Count labels and dname length both, for uncompressed dname in memory.
+ * @param dname: pointer to uncompressed dname.
+ * @param size: length of dname, including root label.
+ * @return: count of labels, including root label, "com." has 2 labels.
+ */
+int dname_count_size_labels(uint8_t* dname, size_t* size);
+
+/**
+ * Compare dnames, sorted not canonical, but by label.
+ * Such that zone contents follows zone apex.
+ * @param d1: first dname. pointer to uncompressed wireformat.
+ * @param labs1: number of labels in first dname.
+ * @param d2: second dname. pointer to uncompressed wireformat.
+ * @param labs2: number of labels in second dname.
+ * @param mlabs: number of labels that matched exactly.
+ * @return: 0 for equal, -1 smaller, or +1 d1 larger than d2.
+ */
+int dname_lab_cmp(uint8_t* d1, int labs1, uint8_t* d2, int labs2, int* mlabs);
+
+#endif /* UTIL_DATA_DNAME_H */

util/data/msgparse.c

@@ -0,0 +1,943 @@
+/* 
+ * util/data/msgparse.c - parse wireformat DNS messages.
+ * 
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ * 
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/**
+ * \file
+ * Routines for message parsing a packet buffer to a descriptive structure.
+ */
+#include "config.h"
+#include "util/data/msgparse.h"
+#include "util/net_help.h"
+#include "util/data/dname.h"
+#include "util/data/packed_rrset.h"
+#include "util/storage/lookup3.h"
+#include "util/region-allocator.h"
+
+/** smart comparison of (compressed, valid) dnames from packet. */
+static int
+smart_compare(ldns_buffer* pkt, uint8_t* dnow, 
+	uint8_t* dprfirst, uint8_t* dprlast)
+{
+	if(LABEL_IS_PTR(*dnow)) {
+		/* ptr points to a previous dname */
+		uint8_t* p = ldns_buffer_at(pkt, PTR_OFFSET(dnow[0], dnow[1]));
+		if( p == dprfirst || p == dprlast )
+			return 0;
+		/* prev dname is also a ptr, both ptrs are the same. */
+		if(LABEL_IS_PTR(*dprlast) &&
+			dprlast[0] == dnow[0] && dprlast[1] == dnow[1])
+			return 0;
+	}
+	return dname_pkt_compare(pkt, dnow, dprlast);
+}
+
+/**
+ * Allocate new rrset in region, fill with data.
+ */
+static struct rrset_parse* 
+new_rrset(struct msg_parse* msg, uint8_t* dname, size_t dnamelen, 
+	uint16_t type, uint16_t dclass, hashvalue_t hash, 
+	uint32_t rrset_flags, ldns_pkt_section section, region_type* region)
+{
+	struct rrset_parse* p = region_alloc(region, sizeof(*p));
+	if(!p) return NULL;
+	p->rrset_bucket_next = msg->hashtable[hash & (PARSE_TABLE_SIZE-1)];
+	msg->hashtable[hash & (PARSE_TABLE_SIZE-1)] = p;
+	p->rrset_all_next = 0;
+	if(msg->rrset_last)
+		msg->rrset_last->rrset_all_next = p;
+	else 	msg->rrset_first = p;
+	msg->rrset_last = p;
+	p->hash = hash;
+	p->section = section;
+	p->dname = dname;
+	p->dname_len = dnamelen;
+	p->type = type;
+	p->rrset_class = dclass;
+	p->flags = rrset_flags;
+	p->rr_count = 0;
+	p->size = 0;
+	p->rr_first = 0;
+	p->rr_last = 0;
+	p->rrsig_count = 0;
+	p->rrsig_first = 0;
+	p->rrsig_last = 0;
+	return p;
+}
+
+/** See if next rrset is nsec at zone apex. */
+static int
+nsec_at_apex(ldns_buffer* pkt)
+{
+	/* we are at ttl position in packet. */
+	size_t pos = ldns_buffer_position(pkt);
+	uint16_t rdatalen;
+	if(ldns_buffer_remaining(pkt) < 7) /* ttl+len+root */
+		return 0; /* eek! */
+	ldns_buffer_skip(pkt, 4); /* ttl */;
+	rdatalen = ldns_buffer_read_u16(pkt);
+	if(ldns_buffer_remaining(pkt) < rdatalen) {
+		ldns_buffer_set_position(pkt, pos);
+		return 0; /* parse error happens later */
+	}
+	/* must validate the nsec next domain name format */
+	if(pkt_dname_len(pkt) == 0) {
+		ldns_buffer_set_position(pkt, pos);
+		return 0; /* parse error */
+	}
+
+	/* see if SOA bit is set. */
+	if(ldns_buffer_position(pkt) < pos+rdatalen) {
+		/* nsec type bitmap contains items */
+		uint8_t win, blen, bits;
+		/* need: windownum, bitmap len, firstbyte */
+		if(ldns_buffer_position(pkt)+3 > pos+rdatalen) {
+			ldns_buffer_set_position(pkt, pos);
+			return 0; /* malformed nsec */
+		}
+		win = ldns_buffer_read_u8(pkt);
+		blen = ldns_buffer_read_u8(pkt);
+		bits = ldns_buffer_read_u8(pkt);
+		/* 0window always first window. bitlen >=1 or parse
+		   error really. bit 0x2 is SOA. */
+		if(win == 0 && blen >= 1 && (bits & 0x02)) {
+			ldns_buffer_set_position(pkt, pos);
+			return 1;
+		}
+	}
+
+	ldns_buffer_set_position(pkt, pos);
+	return 0;
+}
+
+/** Calculate rrset flags */
+static uint32_t
+pkt_rrset_flags(struct msg_parse* msg, ldns_buffer* pkt, uint16_t type)
+{
+	uint32_t f;
+	if(msg->flags & BIT_CD)
+		f = PACKED_RRSET_CD;
+	else	f = 0;
+	if(type == LDNS_RR_TYPE_NSEC && nsec_at_apex(pkt)) {
+		f |= PACKED_RRSET_NSEC_AT_APEX;
+	}
+	return f;
+}
+
+/** Calculate hash value for rrset in packet. */
+static hashvalue_t
+pkt_hash_rrset(ldns_buffer* pkt, uint8_t* dname, uint16_t type, 
+	uint16_t dclass, uint32_t rrset_flags)
+{
+	hashvalue_t h = 0xab;
+	h = hashlittle(&type, sizeof(type), h);
+	h = hashlittle(&dclass, sizeof(dclass), h);
+	h = hashlittle(&rrset_flags, sizeof(uint32_t), h);
+	h = dname_pkt_hash(pkt, dname, h);
+	return h;
+}
+
+/** compare rrset_parse with data. */
+static int
+rrset_parse_equals(struct rrset_parse* p, ldns_buffer* pkt, hashvalue_t h, 
+	uint32_t rrset_flags, uint8_t* dname, size_t dnamelen, 
+	uint16_t type, uint16_t dclass)
+{
+	if(p->hash == h && p->dname_len == dnamelen && p->type == type &&
+		p->rrset_class == dclass && p->flags == rrset_flags &&
+		dname_pkt_compare(pkt, dname, p->dname) == 0)
+		return 1;
+	return 0;
+}
+
+
+/**
+ * Lookup in msg hashtable to find a rrset
+ */
+static struct rrset_parse*
+hashtable_lookup(struct msg_parse* msg, ldns_buffer* pkt, hashvalue_t h, 
+	uint32_t rrset_flags, uint8_t* dname, size_t dnamelen, 
+	uint16_t type, uint16_t dclass)
+{
+	struct rrset_parse* p = msg->hashtable[h & (PARSE_TABLE_SIZE-1)];
+	while(p) {
+		if(rrset_parse_equals(p, pkt, h, rrset_flags, dname, dnamelen,
+			type, dclass))
+			return p;
+		p = p->rrset_bucket_next;
+	}
+	return NULL;
+}
+
+/** return type networkformat that rrsig in packet covers */
+static int
+pkt_rrsig_covered(ldns_buffer* pkt, uint8_t* here, uint16_t* type)
+{
+	size_t pos = ldns_buffer_position(pkt);
+	ldns_buffer_set_position(pkt, (size_t)(here-ldns_buffer_begin(pkt)));
+	/* ttl + len + size of small rrsig(rootlabel, no signature) */
+	if(ldns_buffer_remaining(pkt) < 4+2+19)
+		return 0;
+	ldns_buffer_skip(pkt, 4); /* ttl */
+	if(ldns_buffer_read_u16(pkt) < 19) /* too short */ {
+		ldns_buffer_set_position(pkt, pos);
+		return 0;
+	}
+	*type = ldns_buffer_read_u16(pkt);
+	ldns_buffer_set_position(pkt, pos);
+	return 1;
+}
+
+/** true if covered type equals prevtype */
+static int
+pkt_rrsig_covered_equals(ldns_buffer* pkt, uint8_t* here, uint16_t type)
+{
+	uint16_t t;
+	if(pkt_rrsig_covered(pkt, here, &t) && t == type)
+		return 1;
+	return 0;
+}
+
+/** remove rrset from hash list */
+static void
+bucket_remove(struct msg_parse* msg, struct rrset_parse* rrset)
+{
+	struct rrset_parse** p;
+	p = &msg->hashtable[ rrset->hash & (PARSE_TABLE_SIZE-1) ];
+	while(*p) {
+		if(*p == rrset) {
+			*p = rrset->rrset_bucket_next;
+			return;
+		}
+		p = &( (*p)->rrset_bucket_next );
+	}
+}
+
+/** change section of rrset from previous to current section */
+static void
+change_section(struct msg_parse* msg, struct rrset_parse* rrset,
+	ldns_pkt_section section)
+{
+	struct rrset_parse *p, *prev;
+	/* remove from list */
+	if(section == rrset->section)
+		return;
+	p = msg->rrset_first;
+	prev = 0;
+	while(p) {
+		if(p == rrset) {
+			if(prev) prev->rrset_all_next = p->rrset_all_next;
+			else	msg->rrset_first = p->rrset_all_next;
+			if(msg->rrset_last == rrset)
+				msg->rrset_last = prev;
+			break;
+		}
+		prev = p;
+		p = p->rrset_all_next;
+	}
+	/* remove from count */
+	switch(rrset->section) {
+		case LDNS_SECTION_ANSWER: msg->an_rrsets--; break;
+		case LDNS_SECTION_AUTHORITY: msg->ns_rrsets--; break;
+		case LDNS_SECTION_ADDITIONAL: msg->ar_rrsets--; break;
+		default: log_assert(0);
+	}
+	/* insert at end of list */
+	rrset->rrset_all_next = 0;
+	if(msg->rrset_last)
+		msg->rrset_last->rrset_all_next = rrset;
+	else	msg->rrset_first = rrset;
+	msg->rrset_last = rrset;
+	/* up count of new section */
+	switch(section) {
+		case LDNS_SECTION_AUTHORITY: msg->ns_rrsets++; break;
+		case LDNS_SECTION_ADDITIONAL: msg->ar_rrsets++; break;
+		default: log_assert(0);
+	}
+	rrset->section = section;
+}
+
+/** see if rrset of type RRSIG contains sig over given type */
+static int
+rrset_has_sigover(ldns_buffer* pkt, struct rrset_parse* rrset, uint16_t type,
+	int* hasother)
+{
+	int res = 0;
+	struct rr_parse* rr = rrset->rr_first;
+	log_assert( rrset->type == LDNS_RR_TYPE_RRSIG );
+	while(rr) {
+		if(pkt_rrsig_covered_equals(pkt, rr->ttl_data, type))
+			res = 1;
+		else	*hasother = 1;
+		rr = rr->next;
+	}
+	return res;
+}
+
+/** move rrsigs from sigset to dataset */
+static int
+moveover_rrsigs(ldns_buffer* pkt, region_type* region, 
+	struct rrset_parse* sigset, struct rrset_parse* dataset, int duplicate)
+{
+	struct rr_parse* sig = sigset->rr_first;
+	struct rr_parse* prev = NULL;
+	struct rr_parse* insert;
+	while(sig) {
+		if(pkt_rrsig_covered_equals(pkt, sig->ttl_data, 
+			dataset->type)) {
+			if(duplicate) {
+				/* new */
+				insert = (struct rr_parse*)region_alloc(region,
+					sizeof(struct rr_parse));
+				insert->ttl_data = sig->ttl_data;
+				insert->size = sig->size;
+			} else {
+				/* remove from sigset */
+				if(prev) prev->next = sig->next;
+				else	sigset->rr_first = sig->next;
+				if(sigset->rr_last == sig)
+					sigset->rr_last = prev;
+				sigset->rr_count--;
+				sigset->size -= sig->size;
+				insert = sig;
+			}
+			/* add to dataset */
+			dataset->rrsig_count++;
+			insert->next = 0;
+			if(dataset->rrsig_last) 
+				dataset->rrsig_last->next = insert;
+			else	dataset->rrsig_first = insert;
+			dataset->rrsig_last = insert;
+			dataset->size += insert->size;
+		}
+		prev = sig;
+		sig = sig->next;
+	}
+	return 1;
+}
+
+/** change an rrsig rrset for use as data rrset */
+static struct rrset_parse*
+change_rrsig_rrset(struct rrset_parse* sigset, struct msg_parse* msg, 
+	ldns_buffer* pkt, uint16_t datatype, uint32_t rrset_flags,
+	int hasother, ldns_pkt_section section, region_type* region)
+{
+	struct rrset_parse* dataset = sigset;
+	hashvalue_t hash = pkt_hash_rrset(pkt, sigset->dname, datatype, 
+		sigset->rrset_class, rrset_flags);
+	log_assert( sigset->type == LDNS_RR_TYPE_RRSIG );
+	log_assert( datatype != LDNS_RR_TYPE_RRSIG );
+	if(hasother) {
+		/* need to make new rrset to hold data type */
+		dataset = new_rrset(msg, sigset->dname, sigset->dname_len, 
+			datatype, sigset->rrset_class, hash, rrset_flags, 
+			section, region);
+		if(!dataset) 
+			return NULL;
+		switch(section) {
+			case LDNS_SECTION_ANSWER: msg->an_rrsets++; break;
+			case LDNS_SECTION_AUTHORITY: msg->ns_rrsets++; break;
+			case LDNS_SECTION_ADDITIONAL: msg->ar_rrsets++; break;
+			default: log_assert(0);
+		}
+		if(!moveover_rrsigs(pkt, region, sigset, dataset, 
+			msg->qtype == LDNS_RR_TYPE_RRSIG ||
+			msg->qtype == LDNS_RR_TYPE_ANY ))
+			return NULL;
+		return dataset;
+	}
+	/* changeover the type of the rrset to data set */
+	bucket_remove(msg, dataset);
+	/* insert into new hash bucket */
+	dataset->rrset_bucket_next = msg->hashtable[hash&(PARSE_TABLE_SIZE-1)];
+	msg->hashtable[hash&(PARSE_TABLE_SIZE-1)] = dataset;
+	dataset->hash = hash;
+	/* use section of data item for result */
+	change_section(msg, dataset, section);
+	dataset->type = datatype;
+	dataset->flags = rrset_flags;
+	dataset->rrsig_count += dataset->rr_count;
+	dataset->rr_count = 0;
+	/* move sigs to end of siglist */
+	if(dataset->rrsig_last)
+		dataset->rrsig_last->next = dataset->rr_first;
+	else	dataset->rrsig_first = dataset->rr_first;
+	dataset->rrsig_last = dataset->rr_last;
+	return dataset;
+}
+
+/** Find rrset. If equal to previous it is fast. hash if not so.
+ * @param msg: the message with hash table.
+ * @param pkt: the packet in wireformat (needed for compression ptrs).
+ * @param dname: pointer to start of dname (compressed) in packet.
+ * @param dnamelen: uncompressed wirefmt length of dname.
+ * @param type: type of current rr.
+ * @param dclass: class of current rr.
+ * @param hash: hash value is returned if the rrset could not be found.
+ * @param rrset_flags: is returned if the rrset could not be found.
+ * @param prev_dname_first: dname of last seen RR. First seen dname.
+ * @param prev_dname_last: dname of last seen RR. Last seen dname.
+ * @param prev_dnamelen: dname len of last seen RR.
+ * @param prev_type: type of last seen RR.
+ * @param prev_dclass: class of last seen RR.
+ * @param rrset_prev: last seen RRset.
+ * @param section: the current section in the packet.
+ * @param region: used to allocate temporary parsing data.
+ * @return 0 on out of memory.
+ */
+static int
+find_rrset(struct msg_parse* msg, ldns_buffer* pkt, uint8_t* dname, 
+	size_t dnamelen, uint16_t type, uint16_t dclass, hashvalue_t* hash, 
+	uint32_t* rrset_flags,
+	uint8_t** prev_dname_first, uint8_t** prev_dname_last,
+	size_t* prev_dnamelen, uint16_t* prev_type,
+	uint16_t* prev_dclass, struct rrset_parse** rrset_prev,
+	ldns_pkt_section section, region_type* region)
+{
+	uint16_t covtype;
+	if(rrset_prev) {
+		/* check if equal to previous item */
+		if(type == *prev_type && dclass == *prev_dclass &&
+			dnamelen == *prev_dnamelen &&
+			smart_compare(pkt, dname, *prev_dname_first, 
+				*prev_dname_last) == 0) {
+			/* same as previous */
+			*prev_dname_last = dname;
+			return 1;
+		}
+		/* check if rrsig over previous item */
+		if(type == LDNS_RR_TYPE_RRSIG && dclass == *prev_dclass &&
+			pkt_rrsig_covered_equals(pkt, ldns_buffer_current(pkt),
+				*prev_type) &&
+			smart_compare(pkt, dname, *prev_dname_first,
+				*prev_dname_last) == 0) {
+			/* covers previous */
+			*prev_dname_last = dname;
+			return 1;
+		}
+	}
+	/* find by hashing and lookup in hashtable */
+	*rrset_flags = pkt_rrset_flags(msg, pkt, type);
+	
+	/* if rrsig - try to lookup matching data set first */
+	if(type == LDNS_RR_TYPE_RRSIG && pkt_rrsig_covered(pkt, 
+		ldns_buffer_current(pkt), &covtype)) {
+		*hash = pkt_hash_rrset(pkt, dname, covtype, dclass, 
+			*rrset_flags);
+		*rrset_prev = hashtable_lookup(msg, pkt, *hash, *rrset_flags, 
+			dname, dnamelen, covtype, dclass);
+		if(!*rrset_prev && covtype == LDNS_RR_TYPE_NSEC) {
+			/* if NSEC try with NSEC apex bit twiddled */
+			*rrset_flags ^= PACKED_RRSET_NSEC_AT_APEX;
+			*hash = pkt_hash_rrset(pkt, dname, covtype, dclass, 
+				*rrset_flags);
+			*rrset_prev = hashtable_lookup(msg, pkt, *hash, 
+				*rrset_flags, dname, dnamelen, covtype, dclass);
+		}
+		if(*rrset_prev) {
+			*prev_dname_first = (*rrset_prev)->dname;
+			*prev_dname_last = dname;
+			*prev_dnamelen = dnamelen;
+			*prev_type = covtype;
+			*prev_dclass = dclass;
+			return 1;
+		}
+	}
+	if(type != LDNS_RR_TYPE_RRSIG) {
+		int hasother = 0;
+		/* find matching rrsig */
+		*hash = pkt_hash_rrset(pkt, dname, LDNS_RR_TYPE_RRSIG, 
+			dclass, *rrset_flags);
+		*rrset_prev = hashtable_lookup(msg, pkt, *hash, *rrset_flags, 
+			dname, dnamelen, LDNS_RR_TYPE_RRSIG, dclass);
+		if(*rrset_prev && rrset_has_sigover(pkt, *rrset_prev, type,
+			&hasother)) {
+			/* yes! */
+			*prev_dname_first = (*rrset_prev)->dname;
+			*prev_dname_last = dname;
+			*prev_dnamelen = dnamelen;
+			*prev_type = type;
+			*prev_dclass = dclass;
+			*rrset_prev = change_rrsig_rrset(*rrset_prev, msg, 
+				pkt, type, *rrset_flags, hasother, section, 
+				region);
+			if(!*rrset_prev) return 0;
+			return 1;
+		}
+	}
+
+	*hash = pkt_hash_rrset(pkt, dname, type, dclass, *rrset_flags);
+	*rrset_prev = hashtable_lookup(msg, pkt, *hash, *rrset_flags, 
+		dname, dnamelen, type, dclass);
+	if(*rrset_prev)
+		*prev_dname_first = (*rrset_prev)->dname;
+	else 	*prev_dname_first = dname;
+	*prev_dname_last = dname;
+	*prev_dnamelen = dnamelen;
+	*prev_type = type;
+	*prev_dclass = dclass;
+	return 1;
+}
+
+/**
+ * Parse query section. 
+ * @param pkt: packet, position at call must be at start of query section.
+ *	at end position is after query section.
+ * @param msg: store results here.
+ * @return: 0 if OK, or rcode on error.
+ */
+static int
+parse_query_section(ldns_buffer* pkt, struct msg_parse* msg)
+{
+	if(msg->qdcount == 0)
+		return 0;
+	if(msg->qdcount > 1)
+		return LDNS_RCODE_FORMERR;
+	log_assert(msg->qdcount == 1);
+	if(ldns_buffer_remaining(pkt) <= 0)
+		return LDNS_RCODE_FORMERR;
+	msg->qname = ldns_buffer_current(pkt);
+	if((msg->qname_len = pkt_dname_len(pkt)) == 0)
+		return LDNS_RCODE_FORMERR;
+	if(ldns_buffer_remaining(pkt) < sizeof(uint16_t)*2)
+		return LDNS_RCODE_FORMERR;
+	msg->qtype = ldns_buffer_read_u16(pkt);
+	msg->qclass = ldns_buffer_read_u16(pkt);
+	return 0;
+}
+
+size_t
+get_rdf_size(ldns_rdf_type rdf)
+{
+	switch(rdf) {
+		case LDNS_RDF_TYPE_CLASS:
+		case LDNS_RDF_TYPE_ALG:
+		case LDNS_RDF_TYPE_INT8:
+			return 1;
+			break;
+		case LDNS_RDF_TYPE_INT16:
+		case LDNS_RDF_TYPE_TYPE:
+		case LDNS_RDF_TYPE_CERT_ALG:
+			return 2;
+			break;
+		case LDNS_RDF_TYPE_INT32:
+		case LDNS_RDF_TYPE_TIME:
+		case LDNS_RDF_TYPE_A:
+		case LDNS_RDF_TYPE_PERIOD:
+			return 4;
+			break;
+		case LDNS_RDF_TYPE_TSIGTIME:
+			return 6;
+			break;
+		case LDNS_RDF_TYPE_AAAA:
+			return 16;
+			break;
+		default:
+			log_assert(false); /* add type above */
+			/* only types that appear before a domain  *
+			 * name are needed. rest is simply copied. */
+	}
+	return 0;
+}
+
+/** calculate the size of one rr */
+static int
+calc_size(ldns_buffer* pkt, uint16_t type, struct rr_parse* rr)
+{
+	const ldns_rr_descriptor* desc;
+	uint16_t pkt_len; /* length of rr inside the packet */
+	rr->size = sizeof(uint16_t); /* the rdatalen */
+	ldns_buffer_skip(pkt, 4); /* skip ttl */
+	pkt_len = ldns_buffer_read_u16(pkt);
+	if(ldns_buffer_remaining(pkt) < pkt_len)
+		return 0;
+	desc = ldns_rr_descript(type);
+	if(pkt_len > 0 && desc && desc->_dname_count > 0) {
+		int count = (int)desc->_dname_count;
+		int rdf = 0;
+		size_t len;
+		size_t oldpos;
+		/* skip first part. */
+		while(pkt_len > 0 && count) {
+			switch(desc->_wireformat[rdf]) {
+			case LDNS_RDF_TYPE_DNAME:
+				/* decompress every domain name */
+				oldpos = ldns_buffer_position(pkt);
+				if((len = pkt_dname_len(pkt)) == 0)
+					return 0; /* malformed dname */
+				if(ldns_buffer_position(pkt)-oldpos > pkt_len)
+					return 0; /* dname exceeds rdata */
+				pkt_len -= ldns_buffer_position(pkt)-oldpos;
+				rr->size += len;
+				count--;
+				len = 0;
+				break;
+			case LDNS_RDF_TYPE_STR:
+				if(pkt_len < 1)
+					return 0; /* len byte exceeds rdata */
+				len = ldns_buffer_current(pkt)[0] + 1;
+				break;
+			default:
+				len = get_rdf_size(desc->_wireformat[rdf]);
+			}
+			if(len) {
+				if(pkt_len < len)
+					return 0; /* exceeds rdata */
+				pkt_len -= len;
+				ldns_buffer_skip(pkt, (ssize_t)len);
+				rr->size += len;
+			}
+			rdf++;
+		}
+	}
+	/* remaining rdata */
+	rr->size += pkt_len;
+	ldns_buffer_skip(pkt, (ssize_t)pkt_len);
+	return 1;
+}
+
+/** skip rr ttl and rdata */
+static int
+skip_ttl_rdata(ldns_buffer* pkt) 
+{
+	uint16_t rdatalen;
+	if(ldns_buffer_remaining(pkt) < 6) /* ttl + rdatalen */
+		return 0;
+	ldns_buffer_skip(pkt, 4); /* ttl */
+	rdatalen = ldns_buffer_read_u16(pkt);
+	if(ldns_buffer_remaining(pkt) < rdatalen)
+		return 0;
+	ldns_buffer_skip(pkt, (ssize_t)rdatalen);
+	return 1;
+}
+
+/** see if RRSIG is a duplicate of another */
+static int
+sig_is_double(ldns_buffer* pkt, struct rrset_parse* rrset, uint8_t* ttldata)
+{
+	uint16_t rlen, siglen;
+	size_t pos = ldns_buffer_position(pkt);
+	struct rr_parse* sig;
+	if(ldns_buffer_remaining(pkt) < 6) 
+		return 0;
+	ldns_buffer_skip(pkt, 4); /* ttl */
+	rlen = ldns_buffer_read_u16(pkt);
+	if(ldns_buffer_remaining(pkt) < rlen) {
+		ldns_buffer_set_position(pkt, pos);
+		return 0;
+	}
+	ldns_buffer_set_position(pkt, pos);
+
+	sig = rrset->rrsig_first;
+	while(sig) {
+		/* check if rdatalen is same */
+		memmove(&siglen, sig->ttl_data+4, sizeof(siglen));
+		siglen = ntohs(siglen);
+		/* checks if data in packet is exactly the same, this means
+		 * also dname in rdata is the same, but rrsig is not allowed
+		 * to have compressed dnames anyway. If it is compressed anyway
+		 * it will lead to duplicate rrs for qtype=RRSIG. (or ANY).
+		 *
+		 * Cannot use sig->size because size of the other one is not 
+		 * calculated yet.
+		 */
+		if(siglen == rlen) {
+			if(siglen>0 && memcmp(sig->ttl_data+6, ttldata+6, 
+				siglen) == 0) {
+				/* same! */
+				return 1;
+			}
+		}
+		sig = sig->next;
+	}
+	return 0;
+}
+
+/** Add rr (from packet here) to rrset, skips rr */
+static int
+add_rr_to_rrset(struct rrset_parse* rrset, ldns_buffer* pkt, 
+	struct msg_parse* msg, region_type* region, 
+	ldns_pkt_section section, uint16_t type)
+{
+	struct rr_parse* rr;
+	/* check section of rrset. */
+	if(rrset->section != section && type != LDNS_RR_TYPE_RRSIG &&
+		rrset->type != LDNS_RR_TYPE_RRSIG) {
+		/* silently drop it - we drop the last part, since
+		 * trust in rr data depends on the section it is in. 
+		 * the less trustworthy part is discarded. 
+		 * also the last part is more likely to be incomplete.
+		 * RFC 2181: must put RRset only once in response. */
+		/*
+		verbose(VERB_DETAIL, "Packet contains rrset data in "
+			"multiple sections, dropped last part.");
+		log_hex("packet was: ", ldns_buffer_begin(pkt), 
+			ldns_buffer_limit(pkt));
+		*/
+		/* forwards */
+		if(!skip_ttl_rdata(pkt))
+			return LDNS_RCODE_FORMERR;
+		return 0;
+	} 
+
+	if( (msg->qtype == LDNS_RR_TYPE_RRSIG ||
+	     msg->qtype == LDNS_RR_TYPE_ANY) 
+	    && sig_is_double(pkt, rrset, ldns_buffer_current(pkt))) {
+		if(!skip_ttl_rdata(pkt))
+			return LDNS_RCODE_FORMERR;
+		return 0;
+	}
+
+	/* create rr */
+	if(!(rr = (struct rr_parse*)region_alloc(region, sizeof(*rr))))
+		return LDNS_RCODE_SERVFAIL;
+	rr->ttl_data = ldns_buffer_current(pkt);
+	rr->next = 0;
+	if(type == LDNS_RR_TYPE_RRSIG) {
+		if(rrset->rrsig_last) 
+			rrset->rrsig_last->next = rr;
+		else	rrset->rrsig_first = rr;
+		rrset->rrsig_last = rr;
+		rrset->rrsig_count++;
+	} else {
+		if(rrset->rr_last)
+			rrset->rr_last->next = rr;
+		else	rrset->rr_first = rr;
+		rrset->rr_last = rr;
+		rrset->rr_count++;
+	}
+
+	/* calc decompressed size */
+	if(!calc_size(pkt, type, rr))
+		return LDNS_RCODE_FORMERR;
+	rrset->size += rr->size;
+
+	return 0;
+}
+
+/**
+ * Parse packet RR section, for answer, authority and additional sections. 
+ * @param pkt: packet, position at call must be at start of section.
+ *	at end position is after section.
+ * @param msg: store results here.
+ * @param region: how to alloc results.
+ * @param section: section enum.
+ * @param num_rrs: how many rrs are in the section.
+ * @param num_rrsets: returns number of rrsets in the section.
+ * @return: 0 if OK, or rcode on error.
+ */
+static int
+parse_section(ldns_buffer* pkt, struct msg_parse* msg, region_type* region,
+	ldns_pkt_section section, uint16_t num_rrs, size_t* num_rrsets)
+{
+	uint16_t i;
+	uint8_t* dname, *prev_dname_f = NULL, *prev_dname_l = NULL;
+	size_t dnamelen, prev_dnamelen = 0;
+	uint16_t type, prev_type = 0;
+	uint16_t dclass, prev_dclass = 0;
+	uint32_t rrset_flags = 0;
+	hashvalue_t hash = 0;
+	struct rrset_parse* rrset = NULL;
+	int r;
+
+	if(num_rrs == 0)
+		return 0;
+	if(ldns_buffer_remaining(pkt) <= 0)
+		return LDNS_RCODE_FORMERR;
+	for(i=0; i<num_rrs; i++) {
+		/* parse this RR. */
+		dname = ldns_buffer_current(pkt);
+		if((dnamelen = pkt_dname_len(pkt)) == 0)
+			return LDNS_RCODE_FORMERR;
+		if(ldns_buffer_remaining(pkt) < 10) /* type, class, ttl, len */
+			return LDNS_RCODE_FORMERR;
+		type = ldns_buffer_read_u16(pkt);
+		ldns_buffer_read(pkt, &dclass, sizeof(dclass));
+
+		if(0) { /* debug show what is being parsed. */
+			printf("parse of %s(%d)",
+				ldns_rr_descript(type)?
+				ldns_rr_descript(type)->_name: "??",
+				(int)type);
+			printf(" %s(%d) ",
+				ldns_lookup_by_id(ldns_rr_classes, 
+				(int)ntohs(dclass))?ldns_lookup_by_id(
+				ldns_rr_classes, (int)ntohs(dclass))->name: 
+				"??", (int)ntohs(dclass));
+			dname_print(stdout, pkt, dname);
+			printf("\n");
+		}
+
+		/* see if it is part of an existing RR set */
+		if(!find_rrset(msg, pkt, dname, dnamelen, type, dclass, &hash, 
+			&rrset_flags, &prev_dname_f, &prev_dname_l, 
+			&prev_dnamelen, &prev_type, &prev_dclass, &rrset, 
+			section, region))
+			return LDNS_RCODE_SERVFAIL;
+		if(!rrset) {
+			/* it is a new RR set. hash&flags already calculated.*/
+			(*num_rrsets)++;
+			rrset = new_rrset(msg, dname, dnamelen, type, dclass,
+				hash, rrset_flags, section, region);
+			if(!rrset) 
+				return LDNS_RCODE_SERVFAIL;
+		}
+		/* add to rrset. */
+		if((r=add_rr_to_rrset(rrset, pkt, msg, region, section, 
+			type)) != 0)
+			return r;
+	}
+	return 0;
+}
+
+int
+parse_packet(ldns_buffer* pkt, struct msg_parse* msg, region_type* region)
+{
+	int ret;
+	if(ldns_buffer_remaining(pkt) < LDNS_HEADER_SIZE)
+		return LDNS_RCODE_FORMERR;
+	/* read the header */
+	ldns_buffer_read(pkt, &msg->id, sizeof(uint16_t));
+	msg->flags = ldns_buffer_read_u16(pkt);
+	msg->qdcount = ldns_buffer_read_u16(pkt);
+	msg->ancount = ldns_buffer_read_u16(pkt);
+	msg->nscount = ldns_buffer_read_u16(pkt);
+	msg->arcount = ldns_buffer_read_u16(pkt);
+	if(msg->qdcount > 1)
+		return LDNS_RCODE_FORMERR;
+	if((ret = parse_query_section(pkt, msg)) != 0)
+		return ret;
+	if((ret = parse_section(pkt, msg, region, LDNS_SECTION_ANSWER,
+		msg->ancount, &msg->an_rrsets)) != 0)
+		return ret;
+	if((ret = parse_section(pkt, msg, region, LDNS_SECTION_AUTHORITY,
+		msg->nscount, &msg->ns_rrsets)) != 0)
+		return ret;
+	if((ret = parse_section(pkt, msg, region, LDNS_SECTION_ADDITIONAL, 
+		msg->arcount, &msg->ar_rrsets)) != 0)
+		return ret;
+	/* if(ldns_buffer_remaining(pkt) > 0) { */
+		/* there is spurious data at end of packet. ignore */
+	/* } */
+	msg->rrset_count = msg->an_rrsets + msg->ns_rrsets + msg->ar_rrsets;
+	return 0;
+}
+
+int 
+parse_extract_edns(struct msg_parse* msg, struct edns_data* edns)
+{
+	struct rrset_parse* rrset = msg->rrset_first;
+	struct rrset_parse* prev = 0;
+	struct rrset_parse* found = 0;
+	struct rrset_parse* found_prev = 0;
+	/* since the class encodes the UDP size, we cannot use hash table to
+	 * find the EDNS OPT record. Scan the packet. */
+	while(rrset) {
+		if(rrset->type == LDNS_RR_TYPE_OPT) {
+			/* only one OPT RR allowed. */
+			if(found) return LDNS_RCODE_FORMERR;
+			/* found it! */
+			found_prev = prev;
+			found = rrset;
+		}
+		prev = rrset;
+		rrset = rrset->rrset_all_next;
+	}
+	if(!found) {
+		memset(edns, 0, sizeof(*edns));
+		edns->udp_size = 512;
+		return 0;
+	}
+	/* check the found RRset */
+	/* most lenient check possible. ignore dname, use last opt */
+	if(found->section != LDNS_SECTION_ADDITIONAL)
+		return LDNS_RCODE_FORMERR; 
+	if(found->rr_count == 0)
+		return LDNS_RCODE_FORMERR;
+	if(0) { /* strict checking of dname and RRcount */
+		if(found->dname_len != 1 || !found->dname 
+			|| found->dname[0] != 0) return LDNS_RCODE_FORMERR; 
+		if(found->rr_count != 1) return LDNS_RCODE_FORMERR; 
+	}
+	log_assert(found->rr_first == found->rr_last && found->rr_first);
+
+	/* remove from packet */
+	if(found_prev)	found_prev->rrset_all_next = found->rrset_all_next;
+	else	msg->rrset_first = found->rrset_all_next;
+	if(found == msg->rrset_last)
+		msg->rrset_last = found_prev;
+	msg->arcount --;
+	msg->ar_rrsets --;
+	msg->rrset_count --;
+	
+	/* take the data ! */
+	edns->edns_present = 1;
+	edns->ext_rcode = found->rr_last->ttl_data[0];
+	edns->edns_version = found->rr_last->ttl_data[1];
+	edns->bits = ldns_read_uint16(&found->rr_last->ttl_data[2]);
+	edns->udp_size = ntohs(found->rrset_class);
+	/* ignore rdata and rrsigs */
+	return 0;
+}
+
+int 
+parse_edns_from_pkt(ldns_buffer* pkt, struct edns_data* edns)
+{
+	log_assert(LDNS_QDCOUNT(ldns_buffer_begin(pkt)) == 1);
+	log_assert(LDNS_ANCOUNT(ldns_buffer_begin(pkt)) == 0);
+	log_assert(LDNS_NSCOUNT(ldns_buffer_begin(pkt)) == 0);
+	/* check edns section is present */
+	if(LDNS_ARCOUNT(ldns_buffer_begin(pkt)) > 1) {
+		return LDNS_RCODE_FORMERR;
+	}
+	if(LDNS_ARCOUNT(ldns_buffer_begin(pkt)) == 0) {
+		memset(edns, 0, sizeof(*edns));
+		edns->udp_size = 512;
+		return 0;
+	}
+	/* domain name must be the root of length 1. */
+	if(pkt_dname_len(pkt) != 1)
+		return LDNS_RCODE_FORMERR;
+	if(ldns_buffer_remaining(pkt) < 10) /* type, class, ttl, rdatalen */
+		return LDNS_RCODE_FORMERR;
+	if(ldns_buffer_read_u16(pkt) != LDNS_RR_TYPE_OPT)
+		return LDNS_RCODE_FORMERR;
+	edns->edns_present = 1;
+	edns->udp_size = ldns_buffer_read_u16(pkt); /* class is udp size */
+	edns->ext_rcode = ldns_buffer_read_u8(pkt); /* ttl used for bits */
+	edns->edns_version = ldns_buffer_read_u8(pkt);
+	edns->bits = ldns_buffer_read_u16(pkt);
+	/* ignore rdata and rrsigs */
+	return 0;
+}

util/data/msgparse.h

@@ -0,0 +1,260 @@
+/*
+ * util/data/msgparse.h - parse wireformat DNS messages.
+ * 
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ * 
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/**
+ * \file
+ * Contains message parsing data structures.
+ * These point back into the packet buffer.
+ *
+ * During parsing RRSIGS are put together with the rrsets they (claim to) sign.
+ * This process works as follows:
+ *	o if RRSIG follows the data rrset, it is added to the rrset rrsig list.
+ *	o if no matching data rrset is found, the RRSIG becomes a new rrset.
+ *	o If the data rrset later follows the RRSIG
+ *		o See if the RRSIG rrset contains multiple types, and needs to
+ *		  have the rrsig(s) for that data type split off.
+ *		o Put the data rr as data type in the rrset and rrsig in list.
+ *	o RRSIGs are allowed to move to a different section. The section of
+ *	  the data item is used for the final rrset.
+ *	o multiple signatures over an RRset are possible.
+ *
+ * For queries of qtype=RRSIG, some special handling is needed, to avoid
+ * splitting the RRSIG in the answer section.
+ *	o duplicate, not split, RRSIGs from the answer section, if qtype=RRSIG.
+ *	o check for doubles in the rrsig list when adding an RRSIG to data,
+ *	  so that a data rrset is signed by RRSIGs with different rdata.
+ *	  when qtype=RRSIG.
+ * This will move the RRSIG from the answer section to sign the data further
+ * in the packet (if possible). If then after that, more RRSIGs are found
+ * that sign the data as well, doubles are removed.
+ */
+
+#ifndef UTIL_DATA_MSGPARSE_H
+#define UTIL_DATA_MSGPARSE_H
+#include "util/storage/lruhash.h"
+struct rrset_parse;
+struct rr_parse;
+struct region;
+
+/** number of buckets in parse rrset hash table. Must be power of 2. */
+#define PARSE_TABLE_SIZE 1024
+/** Maximum TTL that is allowed. */
+#define MAX_TTL	3600*24*365*10 /* ten years */
+/** Negative cache time (for entries without any RRs.) */
+#define NORR_TTL 5 /* seconds */
+
+/**
+ * Data stored in scratch pad memory during parsing.
+ * Stores the data that will enter into the msgreply and packet result.
+ */
+struct msg_parse {
+	/** id from message, network format. */
+	uint16_t id;
+	/** flags from message, host format. */
+	uint16_t flags;
+	/** count of RRs, host format */
+	uint16_t qdcount;
+	/** count of RRs, host format */
+	uint16_t ancount;
+	/** count of RRs, host format */
+	uint16_t nscount;
+	/** count of RRs, host format */
+	uint16_t arcount;
+	/** count of RRsets per section. */
+	size_t an_rrsets;
+	/** count of RRsets per section. */
+	size_t ns_rrsets; 
+	/** count of RRsets per section. */
+	size_t ar_rrsets;
+	/** total number of rrsets found. */
+	size_t rrset_count;
+
+	/** query dname (pointer to start location in packet, NULL if none */
+	uint8_t* qname;
+	/** length of query dname in octets, 0 if none */
+	size_t qname_len;
+	/** query type, host order. 0 if qdcount=0 */
+	uint16_t qtype;
+	/** query class, host order. 0 if qdcount=0 */
+	uint16_t qclass;
+
+	/**
+	 * Hash table array used during parsing to lookup rrset types.
+	 * Based on name, type, class.  Same hash value as in rrset cache.
+	 */
+	struct rrset_parse* hashtable[PARSE_TABLE_SIZE];
+	
+	/** linked list of rrsets that have been found (in order). */
+	struct rrset_parse* rrset_first;
+	/** last element of rrset list. */
+	struct rrset_parse* rrset_last;
+};
+
+/**
+ * Data stored for an rrset during parsing.
+ */
+struct rrset_parse {
+	/** next in hash bucket */
+	struct rrset_parse* rrset_bucket_next;
+	/** next in list of all rrsets */
+	struct rrset_parse* rrset_all_next;
+	/** hash value of rrset */
+	hashvalue_t hash;
+	/** which section was it found in: one of
+	 * LDNS_SECTION_ANSWER, LDNS_SECTION_AUTHORITY, LDNS_SECTION_ADDITIONAL
+	 */
+	ldns_pkt_section section;
+	/** start of (possibly compressed) dname in packet */
+	uint8_t* dname;
+	/** length of the dname uncompressed wireformat */
+	size_t dname_len;
+	/** type, host order. */
+	uint16_t type;
+	/** class, network order. var name so that it is not a c++ keyword. */
+	uint16_t rrset_class;
+	/** the flags for the rrset, like for packedrrset */
+	uint32_t flags;
+	/** number of RRs in the rr list */
+	size_t rr_count;
+	/** sum of RR rdata sizes */
+	size_t size;
+	/** linked list of RRs in this rrset. */
+	struct rr_parse* rr_first;
+	/** last in list of RRs in this rrset. */
+	struct rr_parse* rr_last;
+	/** number of RRSIGs over this rrset. */
+	size_t rrsig_count;
+	/** linked list of RRsig RRs over this rrset. */
+	struct rr_parse* rrsig_first;
+	/** last in list of RRSIG RRs over this rrset. */
+	struct rr_parse* rrsig_last;
+};
+
+/**
+ * Data stored for an RR during parsing.
+ */
+struct rr_parse {
+	/** 
+	 * Pointer to the RR. Points to start of TTL value in the packet.
+	 * Rdata length and rdata follow it.
+	 * its dname, type and class are the same and stored for the rrset.
+	 */
+	uint8_t* ttl_data;
+	/** the length of the rdata if allocated (with no dname compression)*/
+	size_t size;
+	/** next in list of RRs. */
+	struct rr_parse* next;
+};
+
+/** Check if label length is first octet of a compression pointer, pass u8. */
+#define LABEL_IS_PTR(x) ( ((x)&0xc0) == 0xc0 )
+/** Calculate destination offset of a compression pointer. pass first and
+ * second octets of the compression pointer. */
+#define PTR_OFFSET(x, y) ( ((x)&0x3f)<<8 | (y) )
+/** create a compression pointer to the given offset. */
+#define PTR_CREATE(offset) ((uint16_t)(0xc000 | (offset)))
+
+/** error codes, extended with EDNS, so > 15. */
+#define EDNS_RCODE_BADVERS	16	/** bad EDNS version */
+/** largest valid compression offset */
+#define PTR_MAX_OFFSET 	0x3fff
+/** bits for EDNS bitfield */
+#define EDNS_DO	0x8000 /* Dnssec Ok */
+
+/**
+ * EDNS data storage
+ * EDNS rdata is ignored.
+ */
+struct edns_data {
+	/** if EDNS OPT record was present */
+	int edns_present;
+	/** Extended RCODE */
+	uint8_t ext_rcode;
+	/** The EDNS version number */
+	uint8_t edns_version;
+	/** the EDNS bits field from ttl (host order): Z */
+	uint16_t bits;
+	/** UDP reassembly size. */
+	uint16_t udp_size;
+};
+
+/**
+ * Obtain size in the packet of an rr type, that is before dname type.
+ * Do TYPE_DNAME, and type STR, yourself. Gives size for most regular types.
+ * @param rdf: the rdf type from the descriptor.
+ * @return: size in octets. 0 on failure.
+ */
+size_t get_rdf_size(ldns_rdf_type rdf);
+
+/**
+ * Parse the packet.
+ * @param pkt: packet, position at call must be at start of packet.
+ *	at end position is after packet.
+ * @param msg: where to store results.
+ * @param region: how to alloc results.
+ * @return: 0 if OK, or rcode on error.
+ */
+int parse_packet(ldns_buffer* pkt, struct msg_parse* msg, 
+	struct region* region);
+
+/**
+ * After parsing the packet, extract EDNS data from packet.
+ * If not present this is noted in the data structure.
+ * If a parse error happens, an error code is returned.
+ *
+ * Quirks:
+ *	o ignores OPT rdata.
+ *	o ignores OPT owner name.
+ *	o ignores extra OPT records, except the last one in the packet.
+ *
+ * @param msg: parsed message structure. Modified on exit, if EDNS was present
+ * 	it is removed from the additional section.
+ * @param edns: the edns data is stored here. Does not have to be initialised.
+ * @return: 0 on success. or an RCODE on an error.
+ *	RCODE formerr if OPT in wrong section, and so on.
+ */
+int parse_extract_edns(struct msg_parse* msg, struct edns_data* edns);
+
+/**
+ * If EDNS data follows a query section, extract it and initialize edns struct.
+ * @param pkt: the packet. position at start must be right after the query
+ *	section. At end, right after EDNS data or no movement if failed.
+ * @param edns: the edns data allocated by the caller. Does not have to be
+ *	initialised.
+ * @return: 0 on success, or an RCODE on error.
+ *	RCODE formerr if OPT is badly formatted and so on.
+ */
+int parse_edns_from_pkt(ldns_buffer* pkt, struct edns_data* edns);
+
+#endif /* UTIL_DATA_MSGPARSE_H */

util/data/msgreply.h

@@ -0,0 +1,324 @@
+/*
+ * util/data/msgreply.h - store message and reply data. 
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains a data structure to store a message and its reply.
+ */
+
+#ifndef UTIL_DATA_MSGREPLY_H
+#define UTIL_DATA_MSGREPLY_H
+#include "util/storage/lruhash.h"
+#include "util/data/packed_rrset.h"
+struct comm_reply;
+struct alloc_cache;
+struct iovec;
+struct region;
+struct edns_data;
+
+/**
+ * Structure to store query information that makes answers to queries
+ * different.
+ */
+struct query_info {
+	/** 
+	 * Salient data on the query: qname, in wireformat. 
+	 * can be allocated or a pointer to outside buffer.
+	 * User has to keep track on the status of this.
+	 */
+	uint8_t* qname;
+	/** length of qname (including last 0 octet) */
+	size_t qnamesize;
+	/** qtype, host byte order */
+	uint16_t qtype;
+	/** qclass, host byte order */
+	uint16_t qclass;
+	/** does query have CD bit set */
+	int has_cd;
+};
+
+/**
+ * Information to reference an rrset
+ */
+struct rrset_ref {
+	/** the key with lock, and ptr to packed data. */
+	struct ub_packed_rrset_key* key;
+	/** id needed */
+	rrset_id_t id;
+};
+
+/**
+ * Structure to store DNS query and the reply packet.
+ * To use it, copy over the flags from reply and modify using flags from
+ * the query (RD,CD if not AA). prepend ID. 
+ *
+ * Memory layout is:
+ *	o struct
+ *	o rrset_ref array
+ *	o packed_rrset_key* array.
+ */
+struct reply_info {
+	/** the flags for the answer, host byte order. */
+	uint16_t flags;
+
+	/**
+	 * Number of RRs in the query section.
+	 * If qdcount is not 0, then it is 1, and the data that appears
+	 * in the reply is the same as the query_info.
+	 * Host byte order.
+	 */
+	uint16_t qdcount;
+
+	/** 
+	 * TTL of the entire reply (for negative caching).
+	 * only for use when there are 0 RRsets in this message.
+	 * if there are RRsets, check those instead.
+	 */
+	uint32_t ttl;
+
+	/**
+	 * Number of RRsets in each section.
+	 * The answer section. Add up the RRs in every RRset to calculate
+	 * the number of RRs, and the count for the dns packet. 
+	 * The number of RRs in RRsets can change due to RRset updates.
+	 */
+	size_t an_numrrsets;
+
+	/** Count of authority section RRsets */
+	size_t ns_numrrsets;
+
+	/** Count of additional section RRsets */
+	size_t ar_numrrsets;
+
+	/** number of RRsets: an_numrrsets + ns_numrrsets + ar_numrrsets */
+	size_t rrset_count;
+
+	/** 
+	 * List of pointers (only) to the rrsets in the order in which 
+	 * they appear in the reply message.  
+	 * Number of elements is ancount+nscount+arcount RRsets.
+	 * This is a pointer to that array. 
+	 * Use the accessor function for access.
+	 */
+	struct ub_packed_rrset_key** rrsets;
+
+	/** 
+	 * Packed array of ids (see counts) and pointers to packed_rrset_key.
+	 * The number equals ancount+nscount+arcount RRsets. 
+	 * These are sorted in ascending pointer, the locking order. So
+	 * this list can be locked (and id, ttl checked), to see if 
+	 * all the data is available and recent enough.
+	 *
+	 * This is defined as an array of size 1, so that the compiler 
+	 * associates the identifier with this position in the structure.
+	 * Array bound overflow on this array then gives access to the further
+	 * elements of the array, which are allocated after the main structure.
+	 *
+	 * It could be more pure to define as array of size 0, ref[0].
+	 * But ref[1] may be less confusing for compilers.
+	 * Use the accessor function for access.
+	 */
+	struct rrset_ref ref[1];
+};
+
+/**
+ * Structure to keep hash table entry for message replies.
+ */
+struct msgreply_entry {
+	/** the hash table key */
+	struct query_info key;
+	/** the hash table entry, data is struct reply_info* */
+	struct lruhash_entry entry;
+};
+
+/** 
+ * Parse wire query into a queryinfo structure, return 0 on parse error. 
+ * initialises the (prealloced) queryinfo structure as well.
+ * This query structure contains a pointer back info the buffer!
+ * This pointer avoids memory allocation. allocqname does memory allocation.
+ * @param m: the prealloced queryinfo structure to put query into.
+ *    must be unused, or _clear()ed.
+ * @param query: the wireformat packet query. starts with ID.
+ * @return: 0 on format error.
+ */
+int query_info_parse(struct query_info* m, ldns_buffer* query);
+
+/**
+ * Parse query reply.
+ * Fills in preallocated query_info structure (with ptr into buffer).
+ * Allocates reply_info and packed_rrsets. These are not yet added to any
+ * caches or anything, this is only parsing. Returns formerror on qdcount > 1.
+ * @param pkt: the packet buffer. Must be positioned after the query section.
+ * @param alloc: creates packed rrset key structures.
+ * @param rep: allocated reply_info is returned (only on no error).
+ * @param qinf: query_info is returned (only on no error).
+ * @param region: where to store temporary data (for parsing).
+ * @param edns: where to store edns information, does not need to be inited.
+ * @return: zero is OK, or DNS error code in case of error
+ *	o FORMERR for parse errors.
+ *	o SERVFAIL for memory allocation errors.
+ */
+int reply_info_parse(ldns_buffer* pkt, struct alloc_cache* alloc,
+	struct query_info* qinf, struct reply_info** rep, 
+	struct region* region, struct edns_data* edns);
+
+/**
+ * Sorts the ref array.
+ * @param rep: reply info. rrsets must be filled in.
+ */
+void reply_info_sortref(struct reply_info* rep);
+
+/**
+ * Set TTLs inside the replyinfo to absolute values.
+ * @param rep: reply info. rrsets must be filled in.
+ * @param timenow: the current time.
+ */
+void reply_info_set_ttls(struct reply_info* rep, uint32_t timenow);
+
+/** 
+ * Delete reply_info and packed_rrsets (while they are not yet added to the
+ * hashtables.). Returns rrsets to the alloc cache.
+ * @param rep: reply_info to delete.
+ * @param alloc: where to return rrset structures to.
+ */
+void reply_info_parsedelete(struct reply_info* rep, struct alloc_cache* alloc);
+
+/**
+ * Allocate and copy the qname (obtained from query_info_parse()).
+ * @param m: the queryinfo structure.
+ * @return: 0 on alloc failure.
+ */
+int query_info_allocqname(struct query_info* m);
+
+/**
+ * Compare two queryinfo structures, on query and type, class. 
+ * It is _not_ sorted in canonical ordering.
+ * @param m1: struct query_info* , void* here to ease use as function pointer.
+ * @param m2: struct query_info* , void* here to ease use as function pointer.
+ * @return: 0 = same, -1 m1 is smaller, +1 m1 is larger.
+ */
+int query_info_compare(void* m1, void* m2);
+
+/** clear out query info structure. */
+void query_info_clear(struct query_info* m);
+
+/** calculate size of struct query_info + reply_info */
+size_t msgreply_sizefunc(void* k, void* d);
+
+/** delete msgreply_entry key structure */
+void query_entry_delete(void *q, void* arg);
+
+/** delete reply_info data structure */
+void reply_info_delete(void* d, void* arg);
+
+/** calculate hash value of query_info, lowercases the qname. */
+hashvalue_t query_info_hash(struct query_info *q);
+
+/** 
+ * Generate answer from reply_info.
+ * @param qinf: query information that provides query section in packet.
+ * @param rep: reply to fill in.
+ * @param id: id word from the query.
+ * @param qflags: flags word from the query.
+ * @param dest: buffer to put message into; will truncate if it does not fit.
+ * @param timenow: time to subtract.
+ * @param cached: set true if a cached reply (so no AA bit).
+ *	set false for the first reply.
+ * @param region: where to allocate temp variables (for compression).
+ * @param udpsize: size of the answer, 512, from EDNS, or 64k for TCP.
+ * @param edns: EDNS data included in the answer, NULL for none.
+ *	or if edns_present = 0, it is not included.
+ * @return: 0 on error (server failure).
+ */
+int reply_info_answer_encode(struct query_info* qinf, struct reply_info* rep, 
+	uint16_t id, uint16_t qflags, ldns_buffer* dest, uint32_t timenow,
+	int cached, struct region* region, uint16_t udpsize, 
+	struct edns_data* edns);
+
+/**
+ * Regenerate the wireformat from the stored msg reply.
+ * If the buffer is too small then the message is truncated at a whole
+ * rrset and the TC bit set, or whole rrsets are left out of the additional
+ * and the TC bit is not set.
+ * @param qinfo: query info to store.
+ * @param rep: reply to store.
+ * @param id: id value to store, network order.
+ * @param flags: flags value to store, host order.
+ * @param buffer: buffer to store the packet into.
+ * @param timenow: time now, to adjust ttl values.
+ * @param region: to store temporary data in.
+ * @param udpsize: size of the answer, 512, from EDNS, or 64k for TCP.
+ * @return: nonzero is success, or 
+ *	0 on error: malloc failure (no log_err has been done).
+ */
+int reply_info_encode(struct query_info* qinfo, struct reply_info* rep, 
+	uint16_t id, uint16_t flags, ldns_buffer* buffer, uint32_t timenow, 
+	struct region* region, uint16_t udpsize);
+
+/**
+ * Encode query packet. Assumes the buffer is large enough.
+ * @param pkt: where to store the packet.
+ * @param qinfo: query info.
+ */
+void qinfo_query_encode(ldns_buffer* pkt, struct query_info* qinfo);
+
+/**
+ * Setup query info entry
+ * @param q: query info to copy. Emptied as if clear is called.
+ * @param r: reply to init data.
+ * @param h: hash value.
+ * @return: newly allocated message reply cache item.
+ */
+struct msgreply_entry* query_info_entrysetup(struct query_info* q,
+	struct reply_info* r, hashvalue_t h);
+
+/**
+ * Estimate size of EDNS record in packet. EDNS record will be no larger.
+ * @param edns: edns data or NULL.
+ * @return octets to reserve for EDNS.
+ */
+uint16_t calc_edns_field_size(struct edns_data* edns);
+
+/**
+ * Attach EDNS record to buffer. Buffer has complete packet. There must
+ * be enough room left for the EDNS record.
+ * @param pkt: packet added to.
+ * @param edns: if NULL or present=0, nothing is added to the packet.
+ */
+void attach_edns_record(ldns_buffer* pkt, struct edns_data* edns);
+
+
+#endif /* UTIL_DATA_MSGREPLY_H */

util/data/packed_rrset.c

@@ -0,0 +1,136 @@
+/*
+ * util/data/packed_rrset.c - data storage for a set of resource records.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the data storage for RRsets.
+ */
+
+#include "config.h"
+#include "util/data/packed_rrset.h"
+#include "util/log.h"
+#include "util/alloc.h"
+
+void
+ub_packed_rrset_parsedelete(struct ub_packed_rrset_key* pkey,
+        struct alloc_cache* alloc)
+{
+	if(!pkey)
+		return;
+	if(pkey->entry.data)
+		free(pkey->entry.data);
+	pkey->entry.data = NULL;
+	if(pkey->rk.dname)
+		free(pkey->rk.dname);
+	pkey->rk.dname = NULL;
+	pkey->id = 0;
+	alloc_special_release(alloc, pkey);
+}
+
+size_t 
+ub_rrset_sizefunc(void* key, void* data)
+{
+	struct ub_packed_rrset_key* k = (struct ub_packed_rrset_key*)key;
+	struct packed_rrset_data* d = (struct packed_rrset_data*)data;
+	size_t s = sizeof(struct ub_packed_rrset_key) + k->rk.dname_len;
+	if(d->rrsig_count > 0) {
+		s += ((uint8_t*)d->rr_data[d->count+d->rrsig_count-1] - 
+			(uint8_t*)d) + d->rr_len[d->count+d->rrsig_count-1];
+	} else {
+		log_assert(d->count > 0);
+		s += ((uint8_t*)d->rr_data[d->count-1] - (uint8_t*)d) + 
+			d->rr_len[d->count-1];
+	}
+	return s;
+}
+
+int 
+ub_rrset_compare(void* k1, void* k2)
+{
+	struct ub_packed_rrset_key* key1 = (struct ub_packed_rrset_key*)k1;
+	struct ub_packed_rrset_key* key2 = (struct ub_packed_rrset_key*)k2;
+	int c;
+	if(key1 == key2 || key1->id == key2->id)
+		return 0;
+	if(key1->rk.dname_len != key2->rk.dname_len) {
+		if(key1->rk.dname_len < key2->rk.dname_len)
+			return -1;
+		return 1;
+	}
+	if((c=memcmp(key1->rk.dname, key2->rk.dname, key1->rk.dname_len)) != 0)
+		return c;
+	if(key1->rk.flags != key2->rk.flags) {
+		if(key1->rk.flags < key2->rk.flags)
+			return -1;
+		return 1;
+	}
+	return 0;
+}
+
+void 
+ub_rrset_key_delete(void* key, void* userdata)
+{
+	struct ub_packed_rrset_key* k = (struct ub_packed_rrset_key*)key;
+	struct alloc_cache* a = (struct alloc_cache*)userdata;
+	k->id = 0;
+	free(k->rk.dname);
+	k->rk.dname = NULL;
+	alloc_special_release(a, k);
+}
+
+void 
+rrset_data_delete(void* data, void* ATTR_UNUSED(userdata))
+{
+	struct packed_rrset_data* d = (struct packed_rrset_data*)data;
+	free(d);
+}
+
+int 
+rrsetdata_equal(struct packed_rrset_data* d1, struct packed_rrset_data* d2)
+{
+	size_t i;
+	size_t total;
+	if(d1->count != d2->count || d1->rrsig_count != d2->rrsig_count) 
+		return 0;
+	total = d1->count + d1->rrsig_count;
+	for(i=0; i<total; i++) {
+		if(d1->rr_len[i] != d2->rr_len[i])
+			return 0;
+		if(memcmp(d1->rr_data[i], d2->rr_data[i], d1->rr_len[i]) != 0)
+			return 0;
+	}
+	return 1;
+}

util/data/packed_rrset.h

@@ -0,0 +1,267 @@
+/*
+ * util/data/packed_rrset.h - data storage for a set of resource records.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains the data storage for RRsets.
+ */
+
+#ifndef UTIL_DATA_PACKED_RRSET_H
+#define UTIL_DATA_PACKED_RRSET_H
+#include "util/storage/lruhash.h"
+struct alloc_cache;
+
+/** type used to uniquely identify rrsets. Cannot be reused without
+ * clearing the cache. */
+typedef uint64_t rrset_id_t;
+
+/** this rrset is NSEC/NSEC3 and is at zone apex (at child size of zonecut) */
+#define PACKED_RRSET_NSEC_AT_APEX 0x1
+/** this rrset was obtained and is used for checking-disabled replies. */
+#define PACKED_RRSET_CD 0x2
+
+/**
+ * The identifying information for an RRset.
+ */
+struct packed_rrset_key {
+	/**
+	 * The domain name. If not null (for id=0) it is allocated, and
+	 * contains the wireformat domain name.
+	 * This dname is not canonicalized.
+	 * After the dname uint16_t type and uint16_t class is stored 
+	 * in wireformat.
+	 * Use accessor functions to get type and class values.
+	 */
+	uint8_t* dname;
+	/** 
+	 * Length of the domain name, including last 0 root octet. 
+	 * The value+sizeof(uint16_t)*2 is actually allocated. 
+	 */
+	size_t dname_len;
+	/**
+	 * Flags. 32bit to be easy for hashing:
+	 * 	o PACKED_RRSET_NSEC_AT_APEX
+	 * 	o PACKED_RRSET_CD
+	 */
+	uint32_t flags;
+};
+
+/**
+ * This structure contains an RRset. A set of resource records that
+ * share the same domain name, type and class.
+ *
+ * Due to memory management and threading, the key structure cannot be
+ * deleted, although the data can be. The id can be set to 0 to store and the
+ * structure can be recycled with a new id.
+ */
+struct ub_packed_rrset_key {
+	/** 
+	 * entry into hashtable. Note the lock is never destroyed,
+	 *  even when this key is retired to the cache. 
+	 * the data pointer (if not null) points to a struct packed_rrset.
+	 */
+	struct lruhash_entry entry;
+	/** 
+	 * the ID of this rrset. unique, based on threadid + sequenceno. 
+	 * ids are not reused, except after flushing the cache.
+	 * zero is an unused entry, and never a valid id.
+	 * Check this value after getting entry.lock.
+	 * The other values in this struct may only be altered after changing
+	 * the id (which needs a writelock on entry.lock).
+	 */
+	rrset_id_t id;
+	/** key data: dname, type and class */
+	struct packed_rrset_key rk;
+};
+
+/**
+ * RRset trustworthiness. Bigger value is more trust. RFC 2181.
+ * The rrset_trust_add_noAA, rrset_trust_auth_noAA, rrset_trust_add_AA,
+ * are mentioned as the same trustworthiness in 2181, but split up here
+ * for ease of processing.
+ *
+ * rrset_trust_nonauth_ans_AA, rrset_trust_ans_noAA
+ * are also mentioned as the same trustworthiness in 2181, but split up here
+ * for ease of processing.
+ *
+ * Added trust_none for a sane initial value, smaller than anything else.
+ * Added validated and ultimate trust for keys and rrsig validated content.
+ */
+enum rrset_trust {
+	/** initial value for trust */
+	rrset_trust_none = 0,
+	/** Additional information from non-authoritative answers */
+	rrset_trust_add_noAA,
+	/** Data from the authority section of a non-authoritative answer */
+	rrset_trust_auth_noAA,
+	/** Additional information from an authoritative answer */
+	rrset_trust_add_AA,
+	/** non-authoritative data from the answer section of authoritative
+	 * answers */
+	rrset_trust_nonauth_ans_AA,
+	/** Data from the answer section of a non-authoritative answer */
+	rrset_trust_ans_noAA,
+	/** Glue from a primary zone, or glue from a zone transfer */
+	rrset_trust_glue,
+	/** Data from the authority section of an authoritative answer */
+	rrset_trust_auth_AA,
+	/** The authoritative data included in the answer section of an
+	 *  authoritative reply */
+	rrset_trust_ans_AA,
+	/** Data from a zone transfer, other than glue */
+	rrset_trust_sec_noglue,
+	/** Data from a primary zone file, other than glue data */
+	rrset_trust_prim_noglue,
+	/** DNSSEC(rfc4034) validated with trusted keys */
+	rrset_trust_validated,
+	/** ultimately trusted, no more trust is possible; 
+	 * trusted keys from the unbound configuration setup. */
+	rrset_trust_ultimate
+};
+
+/**
+ * RRset data.
+ *
+ * The data is packed, stored contiguously in memory.
+ * memory layout:
+ *	o base struct
+ *	o rr_len size_t array
+ *	o rr_ttl uint32_t array
+ *	o rr_data uint8_t* array
+ *	o rr_data rdata wireformats
+ *	o rrsig_data rdata wireformat(s)
+ *
+ * Rdata is stored in wireformat. The dname is stored in wireformat.
+ * TTLs are stored as absolute values (and could be expired).
+ *
+ * RRSIGs are stored in the arrays after the regular rrs.
+ *
+ * You need the packed_rrset_key to know dname, type, class of the
+ * resource records in this RRset. (if signed the rrsig gives the type too).
+ *
+ * On the wire an RR is:
+ *	name, type, class, ttl, rdlength, rdata.
+ * So we need to send the following per RR:
+ *	key.dname, ttl, rr_data[i].
+ *	since key.dname ends with type and class.
+ *	and rr_data starts with the rdlength.
+ *	the ttl value to send changes due to time.
+ */
+struct packed_rrset_data {
+	/** TTL (in seconds like time()) of the rrset.
+	 * Same for all RRs see rfc2181(5.2).  */
+	uint32_t ttl;
+	/** number of rrs. */
+	size_t count;
+	/** number of rrsigs, if 0 no rrsigs */
+	size_t rrsig_count;
+	/** the trustworthiness of the rrset data */
+	enum rrset_trust trust; 
+	/** length of every rr's rdata, rr_len[i] is size of rr_data[i]. */
+	size_t* rr_len;
+	/** ttl of every rr. rr_ttl[i] ttl of rr i. */
+	uint32_t *rr_ttl;
+	/** 
+	 * Array of pointers to every rr's rdata. 
+	 * The rr_data[i] rdata is stored in uncompressed wireformat. 
+	 * The first uint16_t of rr_data[i] is network format rdlength.
+	 *
+	 * rr_data[count] to rr_data[count+rrsig_count] contain the rrsig data.
+	 */
+	uint8_t** rr_data;
+};
+
+/**
+ * An RRset can be represented using both key and data together.
+ * Split into key and data structures to simplify implementation of
+ * caching schemes.
+ */
+struct packed_rrset {
+	/** domain name, type and class */
+	struct packed_rrset_key* k;
+	/** ttl, count and rdatas (and rrsig) */
+	struct packed_rrset_data* d;
+};
+
+/**
+ * Delete packed rrset key and data, not entered in hashtables yet.
+ * Used during parsing.
+ * @param pkey: rrset key structure with locks, key and data pointers.
+ * @param alloc: where to return the unfree-able key structure.
+ */
+void ub_packed_rrset_parsedelete(struct ub_packed_rrset_key* pkey,
+	struct alloc_cache* alloc);
+
+/**
+ * Calculate memory size of rrset entry. For hash table usage.
+ * @param key: struct ub_packed_rrset_key*.
+ * @param data: struct packed_rrset_data*.
+ * @return size in bytes.
+ */
+size_t ub_rrset_sizefunc(void* key, void* data);
+
+/**
+ * compares two rrset keys.
+ * @param k1: struct ub_packed_rrset_key*.
+ * @param k2: struct ub_packed_rrset_key*.
+ * @return 0 if equal.
+ */
+int ub_rrset_compare(void* k1, void* k2);
+
+/**
+ * compare two rrset data structures.
+ * Compared rdata and rrsigdata, not the trust or ttl value.
+ * @param d1: data to compare.
+ * @param d2: data to compare.
+ * @return 1 if equal.
+ */
+int rrsetdata_equal(struct packed_rrset_data* d1, struct packed_rrset_data* d2);
+
+/**
+ * Old key to be deleted. RRset keys are recycled via alloc.
+ * @param key: struct ub_packed_rrset_key*.
+ * @param userdata: alloc structure to use for recycling.
+ */
+void ub_rrset_key_delete(void* key, void* userdata);
+
+/**
+ * Old data to be deleted.
+ * @param data: what to delete.
+ * @param userdata: user data ptr.
+ */
+void rrset_data_delete(void* data, void* userdata);
+
+#endif /* UTIL_DATA_PACKED_RRSET_H */

util/locks.h

@@ -81,6 +81,7 @@
 
 #else /* USE_THREAD_DEBUG */
 #define lock_protect(lock, area, size) /* nop */
+#define lock_unprotect(lock, area) /* nop */
 #define checklock_start() /* nop */
 #define checklock_stop() /* nop */
 

util/log.c

@@ -44,13 +44,15 @@
 #include <time.h>
 #endif
 
-enum verbosity_value verbosity = 0;
+enum verbosity_value verbosity = 4;
 /** the file logged to. */
 static FILE* logfile = 0;
 /** if key has been created */
 static int key_created = 0;
 /** pthread key for thread ids in logfile */
 static ub_thread_key_t logkey;
+/** the identity of this executable/process */
+static const char* ident="unbound";
 
 void
 log_init(const char* filename)
@@ -85,11 +87,15 @@ void log_thread_set(int* num)
 	ub_thread_key_set(logkey, num);
 }
 
+void log_ident_set(const char* id)
+{
+	ident = id;
+}
+
 void
 log_vmsg(const char* type, const char *format, va_list args)
 {
-	char message[MAXSYSLOGMSGLEN];
-	const char* ident="unbound";
+	char message[MAXSYSLOGMSGLEN * 10];
 	unsigned int* tid = (unsigned int*)ub_thread_key_get(logkey);
 	vsnprintf(message, sizeof(message), format, args);
 	fprintf(logfile, "[%d] %s[%d:%x] %s: %s\n",
@@ -171,13 +177,13 @@ log_hex(const char* msg, void* data, size_t length)
 {
 	size_t i;
 	uint8_t* data8 = (uint8_t*)data;
-	const char const* hexchar = "0123456789ABCDEF";
+	const char* hexchar = "0123456789ABCDEF";
 	char* buf = malloc(length*2 + 1); /* alloc hex chars + \0 */
 	for(i=0; i<length; i++) {
 		buf[i*2] = hexchar[ data8[i] >> 4 ];
 		buf[i*2 + 1] = hexchar[ data8[i] & 0xF ];
 	}
 	buf[length*2] = 0;
-	log_info("%s[%d] %s", msg, length, buf);
+	log_info("%s[%u] %s", msg, (unsigned)length, buf);
 	free(buf);
 }

util/log.h

@@ -88,6 +88,12 @@ void log_init(const char* filename);
  */
 void log_thread_set(int* num);
 
+/**
+ * Set identity to print, default is 'unbound'. 
+ * @param id: string to print. Name of executable.
+ */
+void log_ident_set(const char* id);
+
 /**
  * Log informational message.
  * Pass printf formatted arguments. No trailing newline is needed.

util/net_help.c

@@ -87,3 +87,25 @@ fd_set_nonblock(int s)
 	}
 	return 1;
 }
+
+int 
+is_pow2(size_t num)
+{
+	if(num == 0) return 1;
+	return (num & (num-1)) == 0;
+}
+
+void 
+write_iov_buffer(ldns_buffer* buffer, struct iovec* iov, size_t iovlen)
+{
+	size_t i;
+	size_t s = 0;
+	ldns_buffer_clear(buffer);
+	for(i=0; i<iovlen; i++) {
+		log_assert(ldns_buffer_position(buffer)+iov[i].iov_len
+			<= ldns_buffer_capacity(buffer));
+		s += iov[i].iov_len;
+		ldns_buffer_write(buffer, iov[i].iov_base, iov[i].iov_len);
+	}
+	ldns_buffer_flip(buffer);
+}

util/net_help.h

@@ -42,6 +42,16 @@
 #ifndef NET_HELP_H
 #define NET_HELP_H
 
+/** DNS constants for uint16_t style flag manipulation. host byteorder. */
+/** AA flag */
+#define BIT_AA 0x0400
+/** RD flag */
+#define BIT_RD 0x0100
+/** CD flag */
+#define BIT_CD 0x0010
+/** QR flag */
+#define BIT_QR 0x8000
+
 /**
  * See if string is ip4 or ip6.
  * @param str: IP specification.
@@ -66,4 +76,19 @@ write_socket(int s, const void *buf, size_t size);
  */
 int fd_set_nonblock(int s); 
 
+/**
+ * See if number is a power of 2.
+ * @param num: the value.
+ * @return: true if the number is a power of 2.
+ */
+int is_pow2(size_t num);
+
+/**
+ * Write iov vector into a buffer.
+ * @param buffer: to write data into. 
+ * @param iov: iov vector
+ * @param iovlen: length of iov vector
+ */
+void write_iov_buffer(ldns_buffer* buffer, struct iovec* iov, size_t iovlen);
+
 #endif /* NET_HELP_H */

util/netevent.c

@@ -318,11 +318,13 @@ reclaim_tcp_handler(struct comm_point* c)
 {
 	log_assert(c->type == comm_tcp);
 	comm_point_close(c);
-	c->tcp_free = c->tcp_parent->tcp_free;
-	c->tcp_parent->tcp_free = c;
-	if(!c->tcp_free) {
-		/* re-enable listening on accept socket */
-		comm_point_start_listening(c->tcp_parent, -1, -1);
+	if(c->tcp_parent) {
+		c->tcp_free = c->tcp_parent->tcp_free;
+		c->tcp_parent->tcp_free = c;
+		if(!c->tcp_free) {
+			/* re-enable listening on accept socket */
+			comm_point_start_listening(c->tcp_parent, -1, -1);
+		}
 	}
 }
 
@@ -336,8 +338,10 @@ tcp_callback_writer(struct comm_point* c)
 		c->tcp_is_reading = 1;
 	c->tcp_byte_count = 0;
 	comm_point_stop_listening(c);
-	/* for listening socket */
-	reclaim_tcp_handler(c);
+	if(c->tcp_parent) /* for listening socket */
+		reclaim_tcp_handler(c);
+	else	/* its outgoing socket, start listening for reading */
+		comm_point_start_listening(c, -1, -1);
 }
 
 /** do the callback when reading is done */
@@ -421,7 +425,8 @@ comm_point_tcp_handle_read(int fd, struct comm_point* c, int short_ok)
 	return 1;
 }
 
-/** Handle tcp writing callback. 
+/** 
+ * Handle tcp writing callback. 
  * @param fd: file descriptor of socket.
  * @param c: comm point to write buffer out of.
  * @return: 0 on error
@@ -433,27 +438,52 @@ comm_point_tcp_handle_write(int fd, struct comm_point* c)
 	log_assert(c->type == comm_tcp);
 	if(c->tcp_is_reading)
 		return 0;
+	if(c->tcp_byte_count == 0 && c->tcp_check_nb_connect) {
+		/* check for pending error from nonblocking connect */
+		/* from Stevens, unix network programming, vol1, 3rd ed, p450*/
+		int error = 0;
+		socklen_t len = (socklen_t)sizeof(error);
+		if(getsockopt(fd, SOL_SOCKET, SO_ERROR, &error, &len) < 0){
+			error = errno; /* on solaris errno is error */
+		}
+		if(error == EINPROGRESS || error == EWOULDBLOCK)
+			return 1; /* try again later */
+		if(error != 0) {
+			log_err("tcp connect: %s", strerror(error));
+			return 0;
+		}
+	}
 
 	if(c->tcp_byte_count < sizeof(uint16_t)) {
 		uint16_t len = htons(ldns_buffer_limit(c->buffer));
-		r = write(fd, &len, sizeof(uint16_t)-c->tcp_byte_count);
+		struct iovec iov[2];
+		iov[0].iov_base = (uint8_t*)&len + c->tcp_byte_count;
+		iov[0].iov_len = sizeof(uint16_t) - c->tcp_byte_count;
+		iov[1].iov_base = ldns_buffer_begin(c->buffer);
+		iov[1].iov_len = ldns_buffer_limit(c->buffer);
+		r = writev(fd, iov, 2);
 		if(r == -1) {
 			if(errno == EINTR || errno == EAGAIN)
 				return 1;
-			log_err("tcp write(s): %s", strerror(errno));
+			log_err("tcp writev: %s", strerror(errno));
 			return 0;
 		}
 		c->tcp_byte_count += r;
-		if(c->tcp_byte_count != sizeof(uint16_t))
+		if(c->tcp_byte_count < sizeof(uint16_t))
 			return 1;
-		ldns_buffer_set_position(c->buffer, 0);
+		ldns_buffer_set_position(c->buffer, c->tcp_byte_count - 
+			sizeof(uint16_t));
+		if(ldns_buffer_remaining(c->buffer) == 0) {
+			tcp_callback_writer(c);
+		}
+		return 1;
 	}
 	r = write(fd, ldns_buffer_current(c->buffer), 
 		ldns_buffer_remaining(c->buffer));
 	if(r == -1) {
 		if(errno == EINTR || errno == EAGAIN)
 			return 1;
-		log_err("tcp write(w): %s", strerror(errno));
+		log_err("tcp write: %s", strerror(errno));
 		return 0;
 	}
 	ldns_buffer_skip(c->buffer, r);
@@ -543,6 +573,7 @@ comm_point_create_udp(struct comm_base *base, int fd, ldns_buffer* buffer,
 	c->tcp_do_close = 0;
 	c->do_not_close = 0;
 	c->tcp_do_toggle_rw = 0;
+	c->tcp_check_nb_connect = 0;
 	c->callback = callback;
 	c->cb_arg = callback_arg;
 	evbits = EV_READ | EV_PERSIST;
@@ -597,6 +628,7 @@ comm_point_create_tcp_handler(struct comm_base *base,
 	c->tcp_do_close = 0;
 	c->do_not_close = 0;
 	c->tcp_do_toggle_rw = 1;
+	c->tcp_check_nb_connect = 0;
 	c->callback = callback;
 	c->cb_arg = callback_arg;
 	/* add to parent free list */
@@ -652,6 +684,7 @@ comm_point_create_tcp(struct comm_base *base, int fd, int num, size_t bufsize,
 	c->tcp_do_close = 0;
 	c->do_not_close = 0;
 	c->tcp_do_toggle_rw = 0;
+	c->tcp_check_nb_connect = 0;
 	c->callback = NULL;
 	c->cb_arg = NULL;
 	evbits = EV_READ | EV_PERSIST;
@@ -678,6 +711,56 @@ comm_point_create_tcp(struct comm_base *base, int fd, int num, size_t bufsize,
 	return c;
 }
 
+struct comm_point* 
+comm_point_create_tcp_out(struct comm_base *base, size_t bufsize,
+        comm_point_callback_t* callback, void* callback_arg)
+{
+	struct comm_point* c = (struct comm_point*)calloc(1,
+		sizeof(struct comm_point));
+	short evbits;
+	if(!c)
+		return NULL;
+	c->ev = (struct internal_event*)calloc(1,
+		sizeof(struct internal_event));
+	if(!c->ev) {
+		free(c);
+		return NULL;
+	}
+	c->fd = -1;
+	c->buffer = ldns_buffer_new(bufsize);
+	if(!c->buffer) {
+		free(c->ev);
+		free(c);
+		return NULL;
+	}
+	c->timeout = NULL;
+	c->tcp_is_reading = 0;
+	c->tcp_byte_count = 0;
+	c->tcp_parent = NULL;
+	c->max_tcp_count = 0;
+	c->tcp_handlers = NULL;
+	c->tcp_free = NULL;
+	c->type = comm_tcp;
+	c->tcp_do_close = 0;
+	c->do_not_close = 0;
+	c->tcp_do_toggle_rw = 1;
+	c->tcp_check_nb_connect = 1;
+	c->callback = callback;
+	c->cb_arg = callback_arg;
+	evbits = EV_PERSIST | EV_WRITE;
+	event_set(&c->ev->ev, c->fd, evbits, comm_point_tcp_handle_callback, c);
+	if(event_base_set(base->eb->base, &c->ev->ev) != 0)
+	{
+		log_err("could not basetset tcpout event");
+		ldns_buffer_free(c->buffer);
+		free(c->ev);
+		free(c);
+		return NULL;
+	}
+
+	return c;
+}
+
 struct comm_point* 
 comm_point_create_local(struct comm_base *base, int fd, size_t bufsize,
         comm_point_callback_t* callback, void* callback_arg)
@@ -711,6 +794,7 @@ comm_point_create_local(struct comm_base *base, int fd, size_t bufsize,
 	c->tcp_do_close = 0;
 	c->do_not_close = 1;
 	c->tcp_do_toggle_rw = 0;
+	c->tcp_check_nb_connect = 0;
 	c->callback = callback;
 	c->cb_arg = callback_arg;
 	/* libevent stuff */
@@ -733,9 +817,10 @@ comm_point_close(struct comm_point* c)
 {
 	if(!c)
 		return;
-	if(event_del(&c->ev->ev) != 0) {
-		log_err("could not event_del on close");
-	}
+	if(c->fd != -1)
+		if(event_del(&c->ev->ev) != 0) {
+			log_err("could not event_del on close");
+		}
 	/* close fd after removing from event lists, or epoll.. is messed up */
 	if(c->fd != -1 && !c->do_not_close)
 		close(c->fd);
@@ -780,6 +865,67 @@ comm_point_send_reply(struct comm_reply *repinfo)
 	}
 }
 
+void 
+comm_point_send_reply_iov(struct comm_reply* repinfo, struct iovec* iov,
+        size_t iovlen)
+{
+	log_assert(repinfo && repinfo->c);
+	if(repinfo->c->type == comm_udp) {
+		struct msghdr hdr;
+		memset(&hdr, 0, sizeof(hdr));
+		hdr.msg_name = &repinfo->addr;
+		hdr.msg_namelen = repinfo->addrlen;
+		hdr.msg_iov = iov + 1;
+		hdr.msg_iovlen = (TYPE_MSGIOVLEN)(iovlen - 1);
+		/* note that number of characters sent is not checked. */
+		if(sendmsg(repinfo->c->fd, &hdr, 0) == -1)
+			log_err("sendmsg: %s", strerror(errno));
+	} else {
+		/* try if it can be sent in writev right now */
+		size_t i;
+		uint16_t len = 0;
+		ssize_t done;
+		for(i=1; i<iovlen; i++)
+			len += iov[i].iov_len;
+		len = htons(len);
+		iov[0].iov_base = (void*)&len;
+		iov[0].iov_len = sizeof(uint16_t);
+		if((done=writev(repinfo->c->fd, iov, (int)iovlen)) == -1) {
+#ifdef S_SPLINT_S
+			/* don't complain about returning stack references */
+			iov[0].iov_base = NULL;
+#endif
+			if(errno != EINTR && errno != EAGAIN) {
+				log_err("writev: %s", strerror(errno));
+				comm_point_drop_reply(repinfo);
+				return;
+			}
+			done = 0;
+		}
+#ifdef S_SPLINT_S
+		/* don't complain about returning stack references */
+		iov[0].iov_base = NULL;
+#endif
+		if((size_t)done == ntohs(len) + sizeof(uint16_t)) {
+			/* done in one call */
+			comm_point_drop_reply(repinfo);
+		} else {
+			/* sending remaining bytes */
+			ldns_buffer_clear(repinfo->c->buffer);
+			repinfo->c->tcp_byte_count = (size_t)done;
+			for(i=1; i<iovlen; i++)
+				ldns_buffer_write(repinfo->c->buffer,
+					iov[i].iov_base, iov[i].iov_len);
+			ldns_buffer_flip(repinfo->c->buffer);
+			if((size_t)done >= sizeof(uint16_t))
+				ldns_buffer_set_position(repinfo->c->buffer,
+					(size_t)done - sizeof(uint16_t));
+			comm_point_start_listening(repinfo->c, -1, 
+				TCP_QUERY_TIMEOUT);
+		}
+	}
+}
+
 void 
 comm_point_drop_reply(struct comm_reply* repinfo)
 {

util/netevent.h

@@ -155,6 +155,9 @@ struct comm_point {
 	    So that when that is done the callback is called. */
 	int tcp_do_toggle_rw;
 
+	/** if set, checks for pending error from nonblocking connect() call.*/
+	int tcp_check_nb_connect;
+
 	/** callback when done.
 	    tcp_accept does not get called back, is NULL then.
 	    If a timeout happens, callback with timeout=1 is called.
@@ -288,6 +291,17 @@ struct comm_point* comm_point_create_tcp(struct comm_base* base,
 	int fd, int num, size_t bufsize, 
 	comm_point_callback_t* callback, void* callback_arg);
 
+/**
+ * Create an outgoing TCP commpoint. No file descriptor is opened, left at -1.
+ * @param base: in which base to alloc the commpoint.
+ * @param bufsize: size of buffer to create for handlers.
+ * @param callback: callback function pointer for the handler.
+ * @param callback_arg: will be passed to your callback function.
+ * @return: the commpoint or NULL on error.
+ */
+struct comm_point* comm_point_create_tcp_out(struct comm_base* base,
+	size_t bufsize, comm_point_callback_t* callback, void* callback_arg);
+
 /**
  * Create commpoint to listen to a local domain file descriptor.
  * @param base: in which base to alloc the commpoint.
@@ -327,6 +341,18 @@ void comm_point_set_cb_arg(struct comm_point* c, void* arg);
  */
 void comm_point_send_reply(struct comm_reply* repinfo);
 
+/**
+ * Send reply. Message is not put into commpoint buffer, but in iovec.
+ * If it cannot be sent immediately (TCP) the message is copied to the buffer.
+ * @param repinfo: reply info copied from commpoint callback call.
+ * @param iov: iovector, array of base, len parts to send out.
+ *	caller must keep entry 0 free for use by tcp handler. Start at entry 1.
+ * @param iovlen: number of iov items to concatenate and send out.
+ *	this includes the entry 0, which is not filled in by caller.
+ */
+void comm_point_send_reply_iov(struct comm_reply* repinfo, struct iovec* iov,
+	size_t iovlen);
+
 /**
  * Drop reply. Cleans up.
  * @param repinfo: The reply info copied from a commpoint callback call.

util/region-allocator.c

@@ -0,0 +1,510 @@
+/*
+ * region-allocator.c -- region based memory allocator.
+ *
+ * Copyright (c) 2001-2006, NLnet Labs. All rights reserved.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ * 
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ * Region allocator. Allocates small portions of of larger chunks.
+ */
+
+#include "config.h"
+#include "util/log.h"
+#include "util/region-allocator.h"
+
+#ifdef ALIGNMENT
+#  undef ALIGNMENT
+#endif
+/** increase size until it fits alignment of s bytes */
+#define ALIGN_UP(x, s)     (((x) + s - 1) & (~(s - 1)))
+/** what size to align on */
+#define ALIGNMENT          (sizeof(void *))
+/** set to 1 to perform expensive check for double recycle() */
+#define CHECK_DOUBLE_FREE 0 
+
+/** typedef for cleanup structure */
+typedef struct cleanup cleanup_type;
+/** store chunks and user cleanup actions */
+struct cleanup
+{
+	/** action to call (such as free) */
+	void (*action)(void *);
+	/** pointer to pass to action. */
+	void *data;
+};
+
+/** linked list of recycle elements of a certain size. */
+struct recycle_elem {
+	/** next in recycle list. First bytes of block is used for this ptr */
+	struct recycle_elem* next;
+};
+
+/** hidden type of the region. */
+struct region
+{
+	/** total bytes allocated */
+	size_t        total_allocated;
+	/** number of small objects allocated */
+	size_t        small_objects;
+	/** number of large objects allocated */
+	size_t        large_objects;
+	/** number of chunks allocated */
+	size_t        chunk_count;
+	/** Unused space due to alignment, etc. */
+	size_t        unused_space; 
+	
+	/** number of bytes allocated in the current chunk. */
+	size_t        allocated;
+	/** initial chunk */
+	char         *initial_data;
+	/** current chunk */
+	char         *data;
+
+	/** how to allocate memory (for chunks) */
+	void         *(*allocator)(size_t);
+	/** how to deallocate memory (for chunks) */
+	void          (*deallocator)(void *);
+    
+	/** current max size of growing cleanup array */
+	size_t        maximum_cleanup_count;
+	/** number used inside the cleanup array */
+	size_t        cleanup_count;
+	/** cleanup array, chunks and user actions */
+	cleanup_type *cleanups;
+
+	/** size of chunks */
+	size_t        chunk_size;
+	/** large object size */
+	size_t        large_object_size;
+
+	/** if not NULL recycling is enabled.
+	 * It is an array of linked lists of parts held for recycle.
+	 * The parts are all pointers to within the allocated chunks.
+	 * Array [i] points to elements of size i. */
+	struct recycle_elem** recycle_bin;
+	/** amount of memory in recycle storage */
+	size_t		recycle_size;
+};
+
+
+/** common code to initialize a region */
+static region_type *
+alloc_region_base(void *(*allocator)(size_t size),
+		  void (*deallocator)(void *),
+		  size_t initial_cleanup_count)
+{
+	region_type *result = (region_type *) allocator(sizeof(region_type));
+	if (!result) return NULL;
+
+	result->total_allocated = 0;
+	result->small_objects = 0;
+	result->large_objects = 0;
+	result->chunk_count = 1;
+	result->unused_space = 0;
+	result->recycle_bin = NULL;
+	result->recycle_size = 0;
+	
+	result->allocated = 0;
+	result->data = NULL;
+	result->initial_data = NULL;
+
+	result->allocator = allocator;
+	result->deallocator = deallocator;
+
+	log_assert(initial_cleanup_count > 0);
+	result->maximum_cleanup_count = initial_cleanup_count;
+	result->cleanup_count = 0;
+	result->cleanups = (cleanup_type *) allocator(
+		result->maximum_cleanup_count * sizeof(cleanup_type));
+	if (!result->cleanups) {
+		deallocator(result);
+		return NULL;
+	}
+
+	result->chunk_size = DEFAULT_CHUNK_SIZE;
+	result->large_object_size = DEFAULT_LARGE_OBJECT_SIZE;
+	return result;
+}
+
+region_type *
+region_create(void *(*allocator)(size_t), void (*deallocator)(void *))
+{
+	region_type* result = alloc_region_base(allocator, deallocator, 
+		DEFAULT_INITIAL_CLEANUP_SIZE);
+	if(!result)
+		return NULL;
+	result->data = (char *) allocator(result->chunk_size);
+	if (!result->data) {
+		deallocator(result->cleanups);
+		deallocator(result);
+		return NULL;
+	}
+	result->initial_data = result->data;
+    
+	return result;
+}
+
+
+region_type *region_create_custom(void *(*allocator)(size_t),
+				  void (*deallocator)(void *),
+				  size_t chunk_size,
+				  size_t large_object_size,
+				  size_t initial_cleanup_size,
+				  int recycle)
+{
+	region_type* result = alloc_region_base(allocator, deallocator, 
+		initial_cleanup_size);
+	if(!result)
+		return NULL;
+	log_assert(large_object_size <= chunk_size);
+	result->chunk_size = chunk_size;
+	result->large_object_size = large_object_size;
+	if(result->chunk_size > 0) {
+		result->data = (char *) allocator(result->chunk_size);
+		if (!result->data) {
+			deallocator(result->cleanups);
+			deallocator(result);
+			return NULL;
+		}
+		result->initial_data = result->data;
+	}
+	if(recycle) {
+		result->recycle_bin = allocator(sizeof(struct recycle_elem*)
+			* result->large_object_size);
+		if(!result->recycle_bin) {
+			region_destroy(result);
+			return NULL;
+		}
+		memset(result->recycle_bin, 0, sizeof(struct recycle_elem*)
+			* result->large_object_size);
+	}
+	return result;
+}
+
+
+void
+region_destroy(region_type *region)
+{
+	void (*deallocator)(void *);
+	if (!region)
+		return;
+
+	deallocator = region->deallocator;
+
+	region_free_all(region);
+	deallocator(region->cleanups);
+	deallocator(region->initial_data);
+	if(region->recycle_bin)
+		deallocator(region->recycle_bin);
+	deallocator(region);
+}
+
+
+size_t
+region_add_cleanup(region_type *region, void (*action)(void *), void *data)
+{
+	log_assert(action);
+    
+	if (region->cleanup_count >= region->maximum_cleanup_count) {
+		cleanup_type *cleanups = (cleanup_type *) region->allocator(
+			2 * region->maximum_cleanup_count * sizeof(cleanup_type));
+		if (!cleanups) return 0;
+
+		memcpy(cleanups, region->cleanups,
+		       region->cleanup_count * sizeof(cleanup_type));
+		region->deallocator(region->cleanups);
+
+		region->cleanups = cleanups;
+		region->maximum_cleanup_count *= 2;
+	}
+
+	region->cleanups[region->cleanup_count].action = action;
+	region->cleanups[region->cleanup_count].data = data;
+
+	++region->cleanup_count;
+	return region->cleanup_count;
+}
+
+void *
+region_alloc(region_type *region, size_t size)
+{
+	size_t aligned_size;
+	void *result;
+
+	if (size == 0) {
+		size = 1;
+	}
+	aligned_size = ALIGN_UP(size, ALIGNMENT);
+
+	if (aligned_size >= region->large_object_size) {
+		result = region->allocator(size);
+		if (!result) return NULL;
+        
+		if (!region_add_cleanup(region, region->deallocator, result)) {
+			region->deallocator(result);
+			return NULL;
+		}
+        
+		region->total_allocated += size;
+		++region->large_objects;
+		
+		return result;
+	}
+
+	if (region->recycle_bin && region->recycle_bin[aligned_size]) {
+		result = (void*)region->recycle_bin[aligned_size];
+		region->recycle_bin[aligned_size] = region->recycle_bin[aligned_size]->next; 
+		region->recycle_size -= aligned_size;
+		region->unused_space += aligned_size - size;
+		return result;
+	}
+    
+	if (region->allocated + aligned_size > region->chunk_size) {
+		void *chunk = region->allocator(region->chunk_size);
+		size_t wasted;
+		if (!chunk) return NULL;
+
+		wasted = (region->chunk_size - region->allocated) & (~(ALIGNMENT-1));
+		if(wasted >= ALIGNMENT) {
+			/* put wasted part in recycle bin for later use */
+			region->total_allocated += wasted;
+			++region->small_objects;
+			region_recycle(region, region->data+region->allocated, wasted);
+			region->allocated += wasted;
+		}
+		++region->chunk_count;
+		region->unused_space += region->chunk_size - region->allocated;
+		
+		if(!region_add_cleanup(region, region->deallocator, chunk)) {
+			region->deallocator(chunk);
+			region->chunk_count--;
+			region->unused_space -= 
+				region->chunk_size - region->allocated;
+			return NULL;
+		}
+		region->allocated = 0;
+		region->data = (char *) chunk;
+	}
+
+	result = region->data + region->allocated;
+	region->allocated += aligned_size;
+
+	region->total_allocated += aligned_size;
+	region->unused_space += aligned_size - size;
+	++region->small_objects;
+	
+	return result;
+}
+
+void *
+region_alloc_init(region_type *region, const void *init, size_t size)
+{
+	void *result = region_alloc(region, size);
+	if (!result) return NULL;
+	memcpy(result, init, size);
+	return result;
+}
+
+void *
+region_alloc_zero(region_type *region, size_t size)
+{
+	void *result = region_alloc(region, size);
+	if (!result) return NULL;
+	memset(result, 0, size);
+	return result;
+}
+
+void
+region_free_all(region_type *region)
+{
+	size_t i;
+	log_assert(region);
+	log_assert(region->cleanups);
+    
+	i = region->cleanup_count;
+	while (i > 0) {
+		--i;
+		log_assert(region->cleanups[i].action);
+		region->cleanups[i].action(region->cleanups[i].data);
+	}
+
+	if(region->recycle_bin) {
+		memset(region->recycle_bin, 0, sizeof(struct recycle_elem*)
+			* region->large_object_size);
+		region->recycle_size = 0;
+	}
+
+	region->data = region->initial_data;
+	region->cleanup_count = 0;
+	region->allocated = 0;
+
+	region->total_allocated = 0;
+	region->small_objects = 0;
+	region->large_objects = 0;
+	region->chunk_count = 1;
+	region->unused_space = 0;
+}
+
+
+char *
+region_strdup(region_type *region, const char *string)
+{
+	return (char *) region_alloc_init(region, string, strlen(string) + 1);
+}
+
+void 
+region_recycle(region_type *region, void *block, size_t size)
+{
+	size_t aligned_size;
+	size_t i;
+
+	if(!block || !region->recycle_bin)
+		return;
+
+	if (size == 0) {
+		size = 1;
+	}
+	aligned_size = ALIGN_UP(size, ALIGNMENT);
+
+	if(aligned_size < region->large_object_size) {
+		struct recycle_elem* elem = (struct recycle_elem*)block;
+		/* we rely on the fact that ALIGNMENT is void* so the next will fit */
+		log_assert(aligned_size >= sizeof(struct recycle_elem));
+
+		if(CHECK_DOUBLE_FREE) {
+			/* make sure the same ptr is not freed twice. */
+			struct recycle_elem *p = region->recycle_bin[aligned_size];
+			while(p) {
+				log_assert(p != elem); 
+				p = p->next;
+			}
+		}
+
+		elem->next = region->recycle_bin[aligned_size];
+		region->recycle_bin[aligned_size] = elem;
+		region->recycle_size += aligned_size;
+		region->unused_space -= aligned_size - size;
+		return;
+	}
+
+	/* a large allocation */
+	region->total_allocated -= size;
+	--region->large_objects;
+	for(i=0; i<region->cleanup_count; i++) {
+		while(region->cleanups[i].data == block) {
+			/* perform action (deallocator) on block */
+			region->cleanups[i].action(block);
+			region->cleanups[i].data = NULL;
+			/* remove cleanup - move last entry here, check this one again */
+			--region->cleanup_count;
+			region->cleanups[i].action = 
+				region->cleanups[region->cleanup_count].action;
+			region->cleanups[i].data = 
+				region->cleanups[region->cleanup_count].data;
+		}
+	}
+}
+
+void
+region_dump_stats(region_type *region, FILE *out)
+{
+	fprintf(out, "%lu objects (%lu small/%lu large), %lu bytes allocated (%lu wasted) in %lu chunks, %lu cleanups, %lu in recyclebin",
+		(unsigned long) (region->small_objects + region->large_objects),
+		(unsigned long) region->small_objects,
+		(unsigned long) region->large_objects,
+		(unsigned long) region->total_allocated,
+		(unsigned long) region->unused_space,
+		(unsigned long) region->chunk_count,
+		(unsigned long) region->cleanup_count,
+		(unsigned long) region->recycle_size);
+	if(1 && region->recycle_bin) {
+		/* print details of the recycle bin */
+		size_t i;
+		for(i=0; i<region->large_object_size; i++) {
+			size_t count = 0;
+			struct recycle_elem* el = region->recycle_bin[i];
+			while(el) {
+				count++;
+				el = el->next;
+			}
+			if(i%ALIGNMENT == 0 && i!=0)
+				fprintf(out, " %lu", (unsigned long)count);
+		}
+	}
+}
+
+size_t region_get_recycle_size(region_type* region)
+{
+	return region->recycle_size;
+}
+
+/* debug routine, includes here to keep base region-allocator independent */
+#undef ALIGN_UP
+#include "util/log.h"
+void
+region_log_stats(region_type *region)
+{
+	char buf[10240], *str=buf;
+	int len=0;
+	snprintf(str, 10240, "%lu objects (%lu small/%lu large), %lu bytes allocated (%lu wasted) in %lu chunks, %lu cleanups, %lu in recyclebin%n",
+		(unsigned long) (region->small_objects + region->large_objects),
+		(unsigned long) region->small_objects,
+		(unsigned long) region->large_objects,
+		(unsigned long) region->total_allocated,
+		(unsigned long) region->unused_space,
+		(unsigned long) region->chunk_count,
+		(unsigned long) region->cleanup_count,
+		(unsigned long) region->recycle_size,
+		&len);
+	str+=len;
+	if(1 && region->recycle_bin) {
+		/* print details of the recycle bin */
+		size_t i;
+		for(i=0; i<region->large_object_size; i++) {
+			size_t count = 0;
+			struct recycle_elem* el = region->recycle_bin[i];
+			while(el) {
+				count++;
+				el = el->next;
+			}
+			if(i%ALIGNMENT == 0 && i!=0) {
+				snprintf(str, 10240, " %lu%n", 
+					(unsigned long)count, &len);
+				str+=len;
+			}
+		}
+	}
+	log_info("memory: %s", buf);
+}

util/region-allocator.h

@@ -0,0 +1,176 @@
+/*
+ * region-allocator.h -- region based memory allocator.
+ *
+ * Copyright (c) 2001-2006, NLnet Labs. All rights reserved.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ * 
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ * Region allocator. Allocates small portions of of larger chunks.
+ */
+
+#ifndef _REGION_ALLOCATOR_H_
+#define _REGION_ALLOCATOR_H_
+
+#include <stdio.h>
+
+/** The region type */
+typedef struct region region_type;
+
+/** Default reasonable size for chunks */
+#define DEFAULT_CHUNK_SIZE         4096
+/** Default size for large objects - allocated outside of chunks. */
+#define DEFAULT_LARGE_OBJECT_SIZE  (DEFAULT_CHUNK_SIZE / 8)
+/** Default size for cleanup array initial size. */
+#define DEFAULT_INITIAL_CLEANUP_SIZE 16
+
+/**
+ * Create a new region.
+ * @param allocator: 'malloc' or another memory allocator.
+ * @param deallocator: 'free' or another memory deallocator.
+ */
+region_type *region_create(void *(*allocator)(size_t),
+			   void (*deallocator)(void *));
+
+
+/**
+ * Create a new region, with chunk size and large object size.
+ * @param allocator: 'malloc' or another memory allocator.
+ * @param deallocator: 'free' or another memory deallocator.
+ * @param chunk_size: size of chunks to allocate.
+ * @param large_object_size:
+ * Note that large_object_size must be <= chunk_size.
+ * Anything larger than the large object size is individually alloced.
+ * large_object_size = chunk_size/8 is reasonable;
+ * @param initial_cleanup_size:
+ * initial_cleanup_size is the number of prealloced ptrs for cleanups.
+ * The cleanups are in a growing array, and it must start larger than zero.
+ * @param recycle:
+ * If recycle is true, environmentally friendly memory recycling is enabled.
+ */
+region_type *region_create_custom(void *(*allocator)(size_t),
+				  void (*deallocator)(void *),
+				  size_t chunk_size,
+				  size_t large_object_size,
+				  size_t initial_cleanup_size,
+				  int recycle);
+
+
+/**
+ * Destroy REGION.  All memory associated with REGION is freed as if
+ * region_free_all was called.
+ * @param region: to delete.
+ */
+void region_destroy(region_type *region);
+
+
+/**
+ * Add a cleanup to REGION.  
+ * @param region: the region.
+ * @param action: ACTION will be called with DATA as
+ * parameter when the region is freed or destroyed.
+ * @param data: argument to action.
+ * @return: 0 on failure.
+ */
+size_t region_add_cleanup(region_type *region,
+			  void (*action)(void *),
+			  void *data);
+
+
+/**
+ * Allocate SIZE bytes of memory inside REGION.  The memory is
+ * deallocated when region_free_all is called for this region.
+ * @param region: the region.
+ * @param size: number of bytes.
+ * @return: pointer to memory allocated.
+ */
+void *region_alloc(region_type *region, size_t size);
+
+
+/**
+ * Allocate SIZE bytes of memory inside REGION and copy INIT into it.
+ * The memory is deallocated when region_free_all is called for this
+ * region.
+ * @param region: the region.
+ * @param init: to copy.
+ * @param size: number of bytes.
+ * @return: pointer to memory allocated.
+ */
+void *region_alloc_init(region_type *region, const void *init, size_t size);
+
+
+/**
+ * Allocate SIZE bytes of memory inside REGION that are initialized to
+ * 0.  The memory is deallocated when region_free_all is called for
+ * this region.
+ * @param region: the region.
+ * @param size: number of bytes.
+ * @return: pointer to memory allocated.
+ */
+void *region_alloc_zero(region_type *region, size_t size);
+
+
+/**
+ * Run the cleanup actions and free all memory associated with REGION.
+ * @param region: the region.
+ */
+void region_free_all(region_type *region);
+
+
+/**
+ * Duplicate STRING and allocate the result in REGION.
+ */
+char *region_strdup(region_type *region, const char *string);
+
+/**
+ * Recycle an allocated memory block. Pass size used to alloc it.
+ * Does nothing if recycling is not enabled for the region.
+ * @param region: the region.
+ * @param block: pointer to memory from region_alloc call.
+ * @param size: number of bytes, same as passed to region_alloc call.
+ */
+void region_recycle(region_type *region, void *block, size_t size);
+
+/**
+ * Print some REGION statistics to OUT.
+ */
+void region_dump_stats(region_type *region, FILE *out);
+
+/** get size of recyclebin */
+size_t region_get_recycle_size(region_type* region);
+
+/** Debug print REGION statistics to LOG. */
+void region_log_stats(region_type *region);
+
+#endif /* _REGION_ALLOCATOR_H_ */

util/storage/lookup3.c

@@ -0,0 +1,994 @@
+/*
+  March 2007(Wouter) adapted from lookup3.c original, add config.h include.
+     added #ifdef VALGRIND to remove 298,384,660 'unused variable k8' warnings.
+     added include of lookup3.h to check definitions match declarations.
+     removed include of stdint - config.h takes care of platform independence.
+  url http://burtleburtle.net/bob/hash/index.html.
+*/
+/*
+-------------------------------------------------------------------------------
+lookup3.c, by Bob Jenkins, May 2006, Public Domain.
+
+These are functions for producing 32-bit hashes for hash table lookup.
+hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and final() 
+are externally useful functions.  Routines to test the hash are included 
+if SELF_TEST is defined.  You can use this free for any purpose.  It's in
+the public domain.  It has no warranty.
+
+You probably want to use hashlittle().  hashlittle() and hashbig()
+hash byte arrays.  hashlittle() is is faster than hashbig() on
+little-endian machines.  Intel and AMD are little-endian machines.
+On second thought, you probably want hashlittle2(), which is identical to
+hashlittle() except it returns two 32-bit hashes for the price of one.  
+You could implement hashbig2() if you wanted but I haven't bothered here.
+
+If you want to find a hash of, say, exactly 7 integers, do
+  a = i1;  b = i2;  c = i3;
+  mix(a,b,c);
+  a += i4; b += i5; c += i6;
+  mix(a,b,c);
+  a += i7;
+  final(a,b,c);
+then use c as the hash value.  If you have a variable length array of
+4-byte integers to hash, use hashword().  If you have a byte array (like
+a character string), use hashlittle().  If you have several byte arrays, or
+a mix of things, see the comments above hashlittle().  
+
+Why is this so big?  I read 12 bytes at a time into 3 4-byte integers, 
+then mix those integers.  This is fast (you can do a lot more thorough
+mixing with 12*3 instructions on 3 integers than you can with 3 instructions
+on 1 byte), but shoehorning those bytes into integers efficiently is messy.
+-------------------------------------------------------------------------------
+*/
+/*#define SELF_TEST 1*/
+
+#include "config.h"
+#include "util/storage/lookup3.h"
+#include <stdio.h>      /* defines printf for tests */
+#include <time.h>       /* defines time_t for timings in the test */
+/*#include <stdint.h>     defines uint32_t etc  (from config.h) */
+#include <sys/param.h>  /* attempt to define endianness */
+#ifdef linux
+# include <endian.h>    /* attempt to define endianness */
+#endif
+
+/*
+ * My best guess at if you are big-endian or little-endian.  This may
+ * need adjustment.
+ */
+#if (defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
+     __BYTE_ORDER == __LITTLE_ENDIAN) || \
+    (defined(i386) || defined(__i386__) || defined(__i486__) || \
+     defined(__i586__) || defined(__i686__) || defined(vax) || defined(MIPSEL))
+# define HASH_LITTLE_ENDIAN 1
+# define HASH_BIG_ENDIAN 0
+#elif (defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && \
+       __BYTE_ORDER == __BIG_ENDIAN) || \
+      (defined(sparc) || defined(POWERPC) || defined(mc68000) || defined(sel))
+# define HASH_LITTLE_ENDIAN 0
+# define HASH_BIG_ENDIAN 1
+#else
+# define HASH_LITTLE_ENDIAN 0
+# define HASH_BIG_ENDIAN 0
+#endif
+
+#define hashsize(n) ((uint32_t)1<<(n))
+#define hashmask(n) (hashsize(n)-1)
+#define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k))))
+
+/*
+-------------------------------------------------------------------------------
+mix -- mix 3 32-bit values reversibly.
+
+This is reversible, so any information in (a,b,c) before mix() is
+still in (a,b,c) after mix().
+
+If four pairs of (a,b,c) inputs are run through mix(), or through
+mix() in reverse, there are at least 32 bits of the output that
+are sometimes the same for one pair and different for another pair.
+This was tested for:
+* pairs that differed by one bit, by two bits, in any combination
+  of top bits of (a,b,c), or in any combination of bottom bits of
+  (a,b,c).
+* "differ" is defined as +, -, ^, or ~^.  For + and -, I transformed
+  the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
+  is commonly produced by subtraction) look like a single 1-bit
+  difference.
+* the base values were pseudorandom, all zero but one bit set, or 
+  all zero plus a counter that starts at zero.
+
+Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that
+satisfy this are
+    4  6  8 16 19  4
+    9 15  3 18 27 15
+   14  9  3  7 17  3
+Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing
+for "differ" defined as + with a one-bit base and a two-bit delta.  I
+used http://burtleburtle.net/bob/hash/avalanche.html to choose 
+the operations, constants, and arrangements of the variables.
+
+This does not achieve avalanche.  There are input bits of (a,b,c)
+that fail to affect some output bits of (a,b,c), especially of a.  The
+most thoroughly mixed value is c, but it doesn't really even achieve
+avalanche in c.
+
+This allows some parallelism.  Read-after-writes are good at doubling
+the number of bits affected, so the goal of mixing pulls in the opposite
+direction as the goal of parallelism.  I did what I could.  Rotates
+seem to cost as much as shifts on every machine I could lay my hands
+on, and rotates are much kinder to the top and bottom bits, so I used
+rotates.
+-------------------------------------------------------------------------------
+*/
+#define mix(a,b,c) \
+{ \
+  a -= c;  a ^= rot(c, 4);  c += b; \
+  b -= a;  b ^= rot(a, 6);  a += c; \
+  c -= b;  c ^= rot(b, 8);  b += a; \
+  a -= c;  a ^= rot(c,16);  c += b; \
+  b -= a;  b ^= rot(a,19);  a += c; \
+  c -= b;  c ^= rot(b, 4);  b += a; \
+}
+
+/*
+-------------------------------------------------------------------------------
+final -- final mixing of 3 32-bit values (a,b,c) into c
+
+Pairs of (a,b,c) values differing in only a few bits will usually
+produce values of c that look totally different.  This was tested for
+* pairs that differed by one bit, by two bits, in any combination
+  of top bits of (a,b,c), or in any combination of bottom bits of
+  (a,b,c).
+* "differ" is defined as +, -, ^, or ~^.  For + and -, I transformed
+  the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
+  is commonly produced by subtraction) look like a single 1-bit
+  difference.
+* the base values were pseudorandom, all zero but one bit set, or 
+  all zero plus a counter that starts at zero.
+
+These constants passed:
+ 14 11 25 16 4 14 24
+ 12 14 25 16 4 14 24
+and these came close:
+  4  8 15 26 3 22 24
+ 10  8 15 26 3 22 24
+ 11  8 15 26 3 22 24
+-------------------------------------------------------------------------------
+*/
+#define final(a,b,c) \
+{ \
+  c ^= b; c -= rot(b,14); \
+  a ^= c; a -= rot(c,11); \
+  b ^= a; b -= rot(a,25); \
+  c ^= b; c -= rot(b,16); \
+  a ^= c; a -= rot(c,4);  \
+  b ^= a; b -= rot(a,14); \
+  c ^= b; c -= rot(b,24); \
+}
+
+/*
+--------------------------------------------------------------------
+ This works on all machines.  To be useful, it requires
+ -- that the key be an array of uint32_t's, and
+ -- that the length be the number of uint32_t's in the key
+
+ The function hashword() is identical to hashlittle() on little-endian
+ machines, and identical to hashbig() on big-endian machines,
+ except that the length has to be measured in uint32_ts rather than in
+ bytes.  hashlittle() is more complicated than hashword() only because
+ hashlittle() has to dance around fitting the key bytes into registers.
+--------------------------------------------------------------------
+*/
+uint32_t hashword(
+const uint32_t *k,                   /* the key, an array of uint32_t values */
+size_t          length,               /* the length of the key, in uint32_ts */
+uint32_t        initval)         /* the previous hash, or an arbitrary value */
+{
+  uint32_t a,b,c;
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + (((uint32_t)length)<<2) + initval;
+
+  /*------------------------------------------------- handle most of the key */
+  while (length > 3)
+  {
+    a += k[0];
+    b += k[1];
+    c += k[2];
+    mix(a,b,c);
+    length -= 3;
+    k += 3;
+  }
+
+  /*------------------------------------------- handle the last 3 uint32_t's */
+  switch(length)                     /* all the case statements fall through */
+  { 
+  case 3 : c+=k[2];
+  case 2 : b+=k[1];
+  case 1 : a+=k[0];
+    final(a,b,c);
+  case 0:     /* case 0: nothing left to add */
+    break;
+  }
+  /*------------------------------------------------------ report the result */
+  return c;
+}
+
+
+/*
+--------------------------------------------------------------------
+hashword2() -- same as hashword(), but take two seeds and return two
+32-bit values.  pc and pb must both be nonnull, and *pc and *pb must
+both be initialized with seeds.  If you pass in (*pb)==0, the output 
+(*pc) will be the same as the return value from hashword().
+--------------------------------------------------------------------
+*/
+void hashword2 (
+const uint32_t *k,                   /* the key, an array of uint32_t values */
+size_t          length,               /* the length of the key, in uint32_ts */
+uint32_t       *pc,                      /* IN: seed OUT: primary hash value */
+uint32_t       *pb)               /* IN: more seed OUT: secondary hash value */
+{
+  uint32_t a,b,c;
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + ((uint32_t)(length<<2)) + *pc;
+  c += *pb;
+
+  /*------------------------------------------------- handle most of the key */
+  while (length > 3)
+  {
+    a += k[0];
+    b += k[1];
+    c += k[2];
+    mix(a,b,c);
+    length -= 3;
+    k += 3;
+  }
+
+  /*------------------------------------------- handle the last 3 uint32_t's */
+  switch(length)                     /* all the case statements fall through */
+  { 
+  case 3 : c+=k[2];
+  case 2 : b+=k[1];
+  case 1 : a+=k[0];
+    final(a,b,c);
+  case 0:     /* case 0: nothing left to add */
+    break;
+  }
+  /*------------------------------------------------------ report the result */
+  *pc=c; *pb=b;
+}
+
+
+/*
+-------------------------------------------------------------------------------
+hashlittle() -- hash a variable-length key into a 32-bit value
+  k       : the key (the unaligned variable-length array of bytes)
+  length  : the length of the key, counting by bytes
+  initval : can be any 4-byte value
+Returns a 32-bit value.  Every bit of the key affects every bit of
+the return value.  Two keys differing by one or two bits will have
+totally different hash values.
+
+The best hash table sizes are powers of 2.  There is no need to do
+mod a prime (mod is sooo slow!).  If you need less than 32 bits,
+use a bitmask.  For example, if you need only 10 bits, do
+  h = (h & hashmask(10));
+In which case, the hash table should have hashsize(10) elements.
+
+If you are hashing n strings (uint8_t **)k, do it like this:
+  for (i=0, h=0; i<n; ++i) h = hashlittle( k[i], len[i], h);
+
+By Bob Jenkins, 2006.  bob_jenkins@burtleburtle.net.  You may use this
+code any way you wish, private, educational, or commercial.  It's free.
+
+Use for hash table lookup, or anything where one collision in 2^^32 is
+acceptable.  Do NOT use for cryptographic purposes.
+-------------------------------------------------------------------------------
+*/
+
+uint32_t hashlittle( const void *key, size_t length, uint32_t initval)
+{
+  uint32_t a,b,c;                                          /* internal state */
+  union { const void *ptr; size_t i; } u;     /* needed for Mac Powerbook G4 */
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + ((uint32_t)length) + initval;
+
+  u.ptr = key;
+  if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) {
+    const uint32_t *k = (const uint32_t *)key;         /* read 32-bit chunks */
+#ifdef VALGRIND
+    const uint8_t  *k8;
+#endif
+
+    /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      b += k[1];
+      c += k[2];
+      mix(a,b,c);
+      length -= 12;
+      k += 3;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    /* 
+     * "k[2]&0xffffff" actually reads beyond the end of the string, but
+     * then masks off the part it's not allowed to read.  Because the
+     * string is aligned, the masked-off tail is in the same word as the
+     * rest of the string.  Every machine with memory protection I've seen
+     * does it on word boundaries, so is OK with this.  But VALGRIND will
+     * still catch it and complain.  The masking trick does make the hash
+     * noticably faster for short strings (like English words).
+     */
+#ifndef VALGRIND
+
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break;
+    case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break;
+    case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break;
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=k[1]&0xffffff; a+=k[0]; break;
+    case 6 : b+=k[1]&0xffff; a+=k[0]; break;
+    case 5 : b+=k[1]&0xff; a+=k[0]; break;
+    case 4 : a+=k[0]; break;
+    case 3 : a+=k[0]&0xffffff; break;
+    case 2 : a+=k[0]&0xffff; break;
+    case 1 : a+=k[0]&0xff; break;
+    case 0 : return c;              /* zero length strings require no mixing */
+    }
+
+#else /* make valgrind happy */
+
+    k8 = (const uint8_t *)k;
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=((uint32_t)k8[10])<<16;  /* fall through */
+    case 10: c+=((uint32_t)k8[9])<<8;    /* fall through */
+    case 9 : c+=k8[8];                   /* fall through */
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=((uint32_t)k8[6])<<16;   /* fall through */
+    case 6 : b+=((uint32_t)k8[5])<<8;    /* fall through */
+    case 5 : b+=k8[4];                   /* fall through */
+    case 4 : a+=k[0]; break;
+    case 3 : a+=((uint32_t)k8[2])<<16;   /* fall through */
+    case 2 : a+=((uint32_t)k8[1])<<8;    /* fall through */
+    case 1 : a+=k8[0]; break;
+    case 0 : return c;
+    }
+
+#endif /* !valgrind */
+
+  } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) {
+    const uint16_t *k = (const uint16_t *)key;         /* read 16-bit chunks */
+    const uint8_t  *k8;
+
+    /*--------------- all but last block: aligned reads and different mixing */
+    while (length > 12)
+    {
+      a += k[0] + (((uint32_t)k[1])<<16);
+      b += k[2] + (((uint32_t)k[3])<<16);
+      c += k[4] + (((uint32_t)k[5])<<16);
+      mix(a,b,c);
+      length -= 12;
+      k += 6;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    k8 = (const uint8_t *)k;
+    switch(length)
+    {
+    case 12: c+=k[4]+(((uint32_t)k[5])<<16);
+             b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 11: c+=((uint32_t)k8[10])<<16;     /* fall through */
+    case 10: c+=k[4];
+             b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 9 : c+=k8[8];                      /* fall through */
+    case 8 : b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 7 : b+=((uint32_t)k8[6])<<16;      /* fall through */
+    case 6 : b+=k[2];
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 5 : b+=k8[4];                      /* fall through */
+    case 4 : a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 3 : a+=((uint32_t)k8[2])<<16;      /* fall through */
+    case 2 : a+=k[0];
+             break;
+    case 1 : a+=k8[0];
+             break;
+    case 0 : return c;                     /* zero length requires no mixing */
+    }
+
+  } else {                        /* need to read the key one byte at a time */
+    const uint8_t *k = (const uint8_t *)key;
+
+    /*--------------- all but the last block: affect some 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      a += ((uint32_t)k[1])<<8;
+      a += ((uint32_t)k[2])<<16;
+      a += ((uint32_t)k[3])<<24;
+      b += k[4];
+      b += ((uint32_t)k[5])<<8;
+      b += ((uint32_t)k[6])<<16;
+      b += ((uint32_t)k[7])<<24;
+      c += k[8];
+      c += ((uint32_t)k[9])<<8;
+      c += ((uint32_t)k[10])<<16;
+      c += ((uint32_t)k[11])<<24;
+      mix(a,b,c);
+      length -= 12;
+      k += 12;
+    }
+
+    /*-------------------------------- last block: affect all 32 bits of (c) */
+    switch(length)                   /* all the case statements fall through */
+    {
+    case 12: c+=((uint32_t)k[11])<<24;
+    case 11: c+=((uint32_t)k[10])<<16;
+    case 10: c+=((uint32_t)k[9])<<8;
+    case 9 : c+=k[8];
+    case 8 : b+=((uint32_t)k[7])<<24;
+    case 7 : b+=((uint32_t)k[6])<<16;
+    case 6 : b+=((uint32_t)k[5])<<8;
+    case 5 : b+=k[4];
+    case 4 : a+=((uint32_t)k[3])<<24;
+    case 3 : a+=((uint32_t)k[2])<<16;
+    case 2 : a+=((uint32_t)k[1])<<8;
+    case 1 : a+=k[0];
+             break;
+    case 0 : return c;
+    }
+  }
+
+  final(a,b,c);
+  return c;
+}
+
+
+/*
+ * hashlittle2: return 2 32-bit hash values
+ *
+ * This is identical to hashlittle(), except it returns two 32-bit hash
+ * values instead of just one.  This is good enough for hash table
+ * lookup with 2^^64 buckets, or if you want a second hash if you're not
+ * happy with the first, or if you want a probably-unique 64-bit ID for
+ * the key.  *pc is better mixed than *pb, so use *pc first.  If you want
+ * a 64-bit value do something like "*pc + (((uint64_t)*pb)<<32)".
+ */
+void hashlittle2( 
+  const void *key,       /* the key to hash */
+  size_t      length,    /* length of the key */
+  uint32_t   *pc,        /* IN: primary initval, OUT: primary hash */
+  uint32_t   *pb)        /* IN: secondary initval, OUT: secondary hash */
+{
+  uint32_t a,b,c;                                          /* internal state */
+  union { const void *ptr; size_t i; } u;     /* needed for Mac Powerbook G4 */
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + ((uint32_t)length) + *pc;
+  c += *pb;
+
+  u.ptr = key;
+  if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) {
+    const uint32_t *k = (const uint32_t *)key;         /* read 32-bit chunks */
+#ifdef VALGRIND
+    const uint8_t  *k8;
+#endif
+
+    /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      b += k[1];
+      c += k[2];
+      mix(a,b,c);
+      length -= 12;
+      k += 3;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    /* 
+     * "k[2]&0xffffff" actually reads beyond the end of the string, but
+     * then masks off the part it's not allowed to read.  Because the
+     * string is aligned, the masked-off tail is in the same word as the
+     * rest of the string.  Every machine with memory protection I've seen
+     * does it on word boundaries, so is OK with this.  But VALGRIND will
+     * still catch it and complain.  The masking trick does make the hash
+     * noticably faster for short strings (like English words).
+     */
+#ifndef VALGRIND
+
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break;
+    case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break;
+    case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break;
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=k[1]&0xffffff; a+=k[0]; break;
+    case 6 : b+=k[1]&0xffff; a+=k[0]; break;
+    case 5 : b+=k[1]&0xff; a+=k[0]; break;
+    case 4 : a+=k[0]; break;
+    case 3 : a+=k[0]&0xffffff; break;
+    case 2 : a+=k[0]&0xffff; break;
+    case 1 : a+=k[0]&0xff; break;
+    case 0 : *pc=c; *pb=b; return;  /* zero length strings require no mixing */
+    }
+
+#else /* make valgrind happy */
+
+    k8 = (const uint8_t *)k;
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=((uint32_t)k8[10])<<16;  /* fall through */
+    case 10: c+=((uint32_t)k8[9])<<8;    /* fall through */
+    case 9 : c+=k8[8];                   /* fall through */
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=((uint32_t)k8[6])<<16;   /* fall through */
+    case 6 : b+=((uint32_t)k8[5])<<8;    /* fall through */
+    case 5 : b+=k8[4];                   /* fall through */
+    case 4 : a+=k[0]; break;
+    case 3 : a+=((uint32_t)k8[2])<<16;   /* fall through */
+    case 2 : a+=((uint32_t)k8[1])<<8;    /* fall through */
+    case 1 : a+=k8[0]; break;
+    case 0 : *pc=c; *pb=b; return;  /* zero length strings require no mixing */
+    }
+
+#endif /* !valgrind */
+
+  } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) {
+    const uint16_t *k = (const uint16_t *)key;         /* read 16-bit chunks */
+    const uint8_t  *k8;
+
+    /*--------------- all but last block: aligned reads and different mixing */
+    while (length > 12)
+    {
+      a += k[0] + (((uint32_t)k[1])<<16);
+      b += k[2] + (((uint32_t)k[3])<<16);
+      c += k[4] + (((uint32_t)k[5])<<16);
+      mix(a,b,c);
+      length -= 12;
+      k += 6;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    k8 = (const uint8_t *)k;
+    switch(length)
+    {
+    case 12: c+=k[4]+(((uint32_t)k[5])<<16);
+             b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 11: c+=((uint32_t)k8[10])<<16;     /* fall through */
+    case 10: c+=k[4];
+             b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 9 : c+=k8[8];                      /* fall through */
+    case 8 : b+=k[2]+(((uint32_t)k[3])<<16);
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 7 : b+=((uint32_t)k8[6])<<16;      /* fall through */
+    case 6 : b+=k[2];
+             a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 5 : b+=k8[4];                      /* fall through */
+    case 4 : a+=k[0]+(((uint32_t)k[1])<<16);
+             break;
+    case 3 : a+=((uint32_t)k8[2])<<16;      /* fall through */
+    case 2 : a+=k[0];
+             break;
+    case 1 : a+=k8[0];
+             break;
+    case 0 : *pc=c; *pb=b; return;  /* zero length strings require no mixing */
+    }
+
+  } else {                        /* need to read the key one byte at a time */
+    const uint8_t *k = (const uint8_t *)key;
+
+    /*--------------- all but the last block: affect some 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      a += ((uint32_t)k[1])<<8;
+      a += ((uint32_t)k[2])<<16;
+      a += ((uint32_t)k[3])<<24;
+      b += k[4];
+      b += ((uint32_t)k[5])<<8;
+      b += ((uint32_t)k[6])<<16;
+      b += ((uint32_t)k[7])<<24;
+      c += k[8];
+      c += ((uint32_t)k[9])<<8;
+      c += ((uint32_t)k[10])<<16;
+      c += ((uint32_t)k[11])<<24;
+      mix(a,b,c);
+      length -= 12;
+      k += 12;
+    }
+
+    /*-------------------------------- last block: affect all 32 bits of (c) */
+    switch(length)                   /* all the case statements fall through */
+    {
+    case 12: c+=((uint32_t)k[11])<<24;
+    case 11: c+=((uint32_t)k[10])<<16;
+    case 10: c+=((uint32_t)k[9])<<8;
+    case 9 : c+=k[8];
+    case 8 : b+=((uint32_t)k[7])<<24;
+    case 7 : b+=((uint32_t)k[6])<<16;
+    case 6 : b+=((uint32_t)k[5])<<8;
+    case 5 : b+=k[4];
+    case 4 : a+=((uint32_t)k[3])<<24;
+    case 3 : a+=((uint32_t)k[2])<<16;
+    case 2 : a+=((uint32_t)k[1])<<8;
+    case 1 : a+=k[0];
+             break;
+    case 0 : *pc=c; *pb=b; return;  /* zero length strings require no mixing */
+    }
+  }
+
+  final(a,b,c);
+  *pc=c; *pb=b;
+}
+
+
+
+/*
+ * hashbig():
+ * This is the same as hashword() on big-endian machines.  It is different
+ * from hashlittle() on all machines.  hashbig() takes advantage of
+ * big-endian byte ordering. 
+ */
+uint32_t hashbig( const void *key, size_t length, uint32_t initval)
+{
+  uint32_t a,b,c;
+  union { const void *ptr; size_t i; } u; /* to cast key to (size_t) happily */
+
+  /* Set up the internal state */
+  a = b = c = 0xdeadbeef + ((uint32_t)length) + initval;
+
+  u.ptr = key;
+  if (HASH_BIG_ENDIAN && ((u.i & 0x3) == 0)) {
+    const uint32_t *k = (const uint32_t *)key;         /* read 32-bit chunks */
+#ifdef VALGRIND
+    const uint8_t  *k8;
+#endif
+
+    /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += k[0];
+      b += k[1];
+      c += k[2];
+      mix(a,b,c);
+      length -= 12;
+      k += 3;
+    }
+
+    /*----------------------------- handle the last (probably partial) block */
+    /* 
+     * "k[2]<<8" actually reads beyond the end of the string, but
+     * then shifts out the part it's not allowed to read.  Because the
+     * string is aligned, the illegal read is in the same word as the
+     * rest of the string.  Every machine with memory protection I've seen
+     * does it on word boundaries, so is OK with this.  But VALGRIND will
+     * still catch it and complain.  The masking trick does make the hash
+     * noticably faster for short strings (like English words).
+     */
+#ifndef VALGRIND
+
+    switch(length)
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=k[2]&0xffffff00; b+=k[1]; a+=k[0]; break;
+    case 10: c+=k[2]&0xffff0000; b+=k[1]; a+=k[0]; break;
+    case 9 : c+=k[2]&0xff000000; b+=k[1]; a+=k[0]; break;
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=k[1]&0xffffff00; a+=k[0]; break;
+    case 6 : b+=k[1]&0xffff0000; a+=k[0]; break;
+    case 5 : b+=k[1]&0xff000000; a+=k[0]; break;
+    case 4 : a+=k[0]; break;
+    case 3 : a+=k[0]&0xffffff00; break;
+    case 2 : a+=k[0]&0xffff0000; break;
+    case 1 : a+=k[0]&0xff000000; break;
+    case 0 : return c;              /* zero length strings require no mixing */
+    }
+
+#else  /* make valgrind happy */
+
+    k8 = (const uint8_t *)k;
+    switch(length)                   /* all the case statements fall through */
+    {
+    case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
+    case 11: c+=((uint32_t)k8[10])<<8;  /* fall through */
+    case 10: c+=((uint32_t)k8[9])<<16;  /* fall through */
+    case 9 : c+=((uint32_t)k8[8])<<24;  /* fall through */
+    case 8 : b+=k[1]; a+=k[0]; break;
+    case 7 : b+=((uint32_t)k8[6])<<8;   /* fall through */
+    case 6 : b+=((uint32_t)k8[5])<<16;  /* fall through */
+    case 5 : b+=((uint32_t)k8[4])<<24;  /* fall through */
+    case 4 : a+=k[0]; break;
+    case 3 : a+=((uint32_t)k8[2])<<8;   /* fall through */
+    case 2 : a+=((uint32_t)k8[1])<<16;  /* fall through */
+    case 1 : a+=((uint32_t)k8[0])<<24; break;
+    case 0 : return c;
+    }
+
+#endif /* !VALGRIND */
+
+  } else {                        /* need to read the key one byte at a time */
+    const uint8_t *k = (const uint8_t *)key;
+
+    /*--------------- all but the last block: affect some 32 bits of (a,b,c) */
+    while (length > 12)
+    {
+      a += ((uint32_t)k[0])<<24;
+      a += ((uint32_t)k[1])<<16;
+      a += ((uint32_t)k[2])<<8;
+      a += ((uint32_t)k[3]);
+      b += ((uint32_t)k[4])<<24;
+      b += ((uint32_t)k[5])<<16;
+      b += ((uint32_t)k[6])<<8;
+      b += ((uint32_t)k[7]);
+      c += ((uint32_t)k[8])<<24;
+      c += ((uint32_t)k[9])<<16;
+      c += ((uint32_t)k[10])<<8;
+      c += ((uint32_t)k[11]);
+      mix(a,b,c);
+      length -= 12;
+      k += 12;
+    }
+
+    /*-------------------------------- last block: affect all 32 bits of (c) */
+    switch(length)                   /* all the case statements fall through */
+    {
+    case 12: c+=k[11];
+    case 11: c+=((uint32_t)k[10])<<8;
+    case 10: c+=((uint32_t)k[9])<<16;
+    case 9 : c+=((uint32_t)k[8])<<24;
+    case 8 : b+=k[7];
+    case 7 : b+=((uint32_t)k[6])<<8;
+    case 6 : b+=((uint32_t)k[5])<<16;
+    case 5 : b+=((uint32_t)k[4])<<24;
+    case 4 : a+=k[3];
+    case 3 : a+=((uint32_t)k[2])<<8;
+    case 2 : a+=((uint32_t)k[1])<<16;
+    case 1 : a+=((uint32_t)k[0])<<24;
+             break;
+    case 0 : return c;
+    }
+  }
+
+  final(a,b,c);
+  return c;
+}
+
+
+#ifdef SELF_TEST
+
+/* used for timings */
+void driver1()
+{
+  uint8_t buf[256];
+  uint32_t i;
+  uint32_t h=0;
+  time_t a,z;
+
+  time(&a);
+  for (i=0; i<256; ++i) buf[i] = 'x';
+  for (i=0; i<1; ++i) 
+  {
+    h = hashlittle(&buf[0],1,h);
+  }
+  time(&z);
+  if (z-a > 0) printf("time %d %.8x\n", z-a, h);
+}
+
+/* check that every input bit changes every output bit half the time */
+#define HASHSTATE 1
+#define HASHLEN   1
+#define MAXPAIR 60
+#define MAXLEN  70
+void driver2()
+{
+  uint8_t qa[MAXLEN+1], qb[MAXLEN+2], *a = &qa[0], *b = &qb[1];
+  uint32_t c[HASHSTATE], d[HASHSTATE], i=0, j=0, k, l, m=0, z;
+  uint32_t e[HASHSTATE],f[HASHSTATE],g[HASHSTATE],h[HASHSTATE];
+  uint32_t x[HASHSTATE],y[HASHSTATE];
+  uint32_t hlen;
+
+  printf("No more than %d trials should ever be needed \n",MAXPAIR/2);
+  for (hlen=0; hlen < MAXLEN; ++hlen)
+  {
+    z=0;
+    for (i=0; i<hlen; ++i)  /*----------------------- for each input byte, */
+    {
+      for (j=0; j<8; ++j)   /*------------------------ for each input bit, */
+      {
+	for (m=1; m<8; ++m) /*------------ for serveral possible initvals, */
+	{
+	  for (l=0; l<HASHSTATE; ++l)
+	    e[l]=f[l]=g[l]=h[l]=x[l]=y[l]=~((uint32_t)0);
+
+      	  /*---- check that every output bit is affected by that input bit */
+	  for (k=0; k<MAXPAIR; k+=2)
+	  { 
+	    uint32_t finished=1;
+	    /* keys have one bit different */
+	    for (l=0; l<hlen+1; ++l) {a[l] = b[l] = (uint8_t)0;}
+	    /* have a and b be two keys differing in only one bit */
+	    a[i] ^= (k<<j);
+	    a[i] ^= (k>>(8-j));
+	     c[0] = hashlittle(a, hlen, m);
+	    b[i] ^= ((k+1)<<j);
+	    b[i] ^= ((k+1)>>(8-j));
+	     d[0] = hashlittle(b, hlen, m);
+	    /* check every bit is 1, 0, set, and not set at least once */
+	    for (l=0; l<HASHSTATE; ++l)
+	    {
+	      e[l] &= (c[l]^d[l]);
+	      f[l] &= ~(c[l]^d[l]);
+	      g[l] &= c[l];
+	      h[l] &= ~c[l];
+	      x[l] &= d[l];
+	      y[l] &= ~d[l];
+	      if (e[l]|f[l]|g[l]|h[l]|x[l]|y[l]) finished=0;
+	    }
+	    if (finished) break;
+	  }
+	  if (k>z) z=k;
+	  if (k==MAXPAIR) 
+	  {
+	     printf("Some bit didn't change: ");
+	     printf("%.8x %.8x %.8x %.8x %.8x %.8x  ",
+	            e[0],f[0],g[0],h[0],x[0],y[0]);
+	     printf("i %d j %d m %d len %d\n", i, j, m, hlen);
+	  }
+	  if (z==MAXPAIR) goto done;
+	}
+      }
+    }
+   done:
+    if (z < MAXPAIR)
+    {
+      printf("Mix success  %2d bytes  %2d initvals  ",i,m);
+      printf("required  %d  trials\n", z/2);
+    }
+  }
+  printf("\n");
+}
+
+/* Check for reading beyond the end of the buffer and alignment problems */
+void driver3()
+{
+  uint8_t buf[MAXLEN+20], *b;
+  uint32_t len;
+  uint8_t q[] = "This is the time for all good men to come to the aid of their country...";
+  uint32_t h;
+  uint8_t qq[] = "xThis is the time for all good men to come to the aid of their country...";
+  uint32_t i;
+  uint8_t qqq[] = "xxThis is the time for all good men to come to the aid of their country...";
+  uint32_t j;
+  uint8_t qqqq[] = "xxxThis is the time for all good men to come to the aid of their country...";
+  uint32_t ref,x,y;
+  uint8_t *p;
+
+  printf("Endianness.  These lines should all be the same (for values filled in):\n");
+  printf("%.8x                            %.8x                            %.8x\n",
+         hashword((const uint32_t *)q, (sizeof(q)-1)/4, 13),
+         hashword((const uint32_t *)q, (sizeof(q)-5)/4, 13),
+         hashword((const uint32_t *)q, (sizeof(q)-9)/4, 13));
+  p = q;
+  printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
+         hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13),
+         hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13),
+         hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13),
+         hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13),
+         hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13),
+         hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13));
+  p = &qq[1];
+  printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
+         hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13),
+         hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13),
+         hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13),
+         hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13),
+         hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13),
+         hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13));
+  p = &qqq[2];
+  printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
+         hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13),
+         hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13),
+         hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13),
+         hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13),
+         hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13),
+         hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13));
+  p = &qqqq[3];
+  printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n",
+         hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13),
+         hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13),
+         hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13),
+         hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13),
+         hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13),
+         hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13));
+  printf("\n");
+
+  /* check that hashlittle2 and hashlittle produce the same results */
+  i=47; j=0;
+  hashlittle2(q, sizeof(q), &i, &j);
+  if (hashlittle(q, sizeof(q), 47) != i)
+    printf("hashlittle2 and hashlittle mismatch\n");
+
+  /* check that hashword2 and hashword produce the same results */
+  len = 0xdeadbeef;
+  i=47, j=0;
+  hashword2(&len, 1, &i, &j);
+  if (hashword(&len, 1, 47) != i)
+    printf("hashword2 and hashword mismatch %x %x\n", 
+	   i, hashword(&len, 1, 47));
+
+  /* check hashlittle doesn't read before or after the ends of the string */
+  for (h=0, b=buf+1; h<8; ++h, ++b)
+  {
+    for (i=0; i<MAXLEN; ++i)
+    {
+      len = i;
+      for (j=0; j<i; ++j) *(b+j)=0;
+
+      /* these should all be equal */
+      ref = hashlittle(b, len, (uint32_t)1);
+      *(b+i)=(uint8_t)~0;
+      *(b-1)=(uint8_t)~0;
+      x = hashlittle(b, len, (uint32_t)1);
+      y = hashlittle(b, len, (uint32_t)1);
+      if ((ref != x) || (ref != y)) 
+      {
+	printf("alignment error: %.8x %.8x %.8x %d %d\n",ref,x,y,
+               h, i);
+      }
+    }
+  }
+}
+
+/* check for problems with nulls */
+ void driver4()
+{
+  uint8_t buf[1];
+  uint32_t h,i,state[HASHSTATE];
+
+
+  buf[0] = ~0;
+  for (i=0; i<HASHSTATE; ++i) state[i] = 1;
+  printf("These should all be different\n");
+  for (i=0, h=0; i<8; ++i)
+  {
+    h = hashlittle(buf, 0, h);
+    printf("%2ld  0-byte strings, hash is  %.8x\n", i, h);
+  }
+}
+
+
+int main()
+{
+  driver1();   /* test that the key is hashed: used for timings */
+  driver2();   /* test that whole key is hashed thoroughly */
+  driver3();   /* test that nothing but the key is hashed */
+  driver4();   /* test hashing multiple buffers (all buffers are null) */
+  return 1;
+}
+
+#endif  /* SELF_TEST */

util/storage/lookup3.h

@@ -0,0 +1,64 @@
+/*
+ * util/storage/lookup3.h - header file for hashing functions.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains header definitions for the hash functions we use.
+ * The hash functions are public domain (see lookup3.c).
+ */
+
+#ifndef UTIL_STORAGE_LOOKUP3_H
+#define UTIL_STORAGE_LOOKUP3_H
+
+/**
+ * Hash key made of 4byte chunks.
+ * @param k: the key, an array of uint32_t values
+ * @param length: the length of the key, in uint32_ts
+ * @param initval: the previous hash, or an arbitrary value
+ * @return: hash value.
+ */
+uint32_t hashword(const uint32_t *k, size_t length, uint32_t initval);
+
+/**
+ * Hash key data.
+ * @param k: the key, array of uint8_t
+ * @param length: the length of the key, in uint8_ts
+ * @param initval: the previous hash, or an arbitrary value
+ * @return: hash value.
+ */
+uint32_t hashlittle(const void *k, size_t length, uint32_t initval);
+
+#endif /* UTIL_STORAGE_LOOKUP3_H */

util/storage/lruhash.c

@@ -0,0 +1,425 @@
+/*
+ * util/storage/lruhash.c - hashtable, hash function, LRU keeping.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains a hashtable with LRU keeping of entries.
+ *
+ */
+
+#include "config.h"
+#include "util/storage/lruhash.h"
+
+void
+bin_init(struct lruhash_bin* array, size_t size)
+{
+	size_t i;
+	for(i=0; i<size; i++) {
+		lock_quick_init(&array[i].lock);
+		lock_protect(&array[i].lock, &array[i], 
+			sizeof(struct lruhash_bin));
+	}
+}
+
+struct lruhash* 
+lruhash_create(size_t start_size, size_t maxmem, lruhash_sizefunc_t sizefunc, 
+	lruhash_compfunc_t compfunc, lruhash_delkeyfunc_t delkeyfunc, 
+	lruhash_deldatafunc_t deldatafunc, void* arg)
+{
+	struct lruhash* table = (struct lruhash*)calloc(1, 
+		sizeof(struct lruhash));
+	if(!table)
+		return NULL;
+	lock_quick_init(&table->lock);
+	table->sizefunc = sizefunc;
+	table->compfunc = compfunc;
+	table->delkeyfunc = delkeyfunc;
+	table->deldatafunc = deldatafunc;
+	table->cb_arg = arg;
+	table->size = start_size;
+	table->size_mask = (int)(start_size-1);
+	table->lru_start = NULL;
+	table->lru_end = NULL;
+	table->num = 0;
+	table->space_used = 0;
+	table->space_max = maxmem;
+	table->array = calloc(table->size, sizeof(struct lruhash_bin));
+	if(!table->array) {
+		lock_quick_destroy(&table->lock);
+		free(table);
+		return NULL;
+	}
+	bin_init(table->array, table->size);
+	lock_protect(&table->lock, table, sizeof(*table));
+	lock_protect(&table->lock, table->array, 
+		table->size*sizeof(struct lruhash_bin));
+	return table;
+}
+
+void 
+bin_delete(struct lruhash* table, struct lruhash_bin* bin)
+{
+	struct lruhash_entry* p, *np;
+	void *d;
+	if(!bin)
+		return;
+	lock_quick_destroy(&bin->lock);
+	p = bin->overflow_list;
+	bin->overflow_list = NULL;
+	while(p) {
+		np = p->overflow_next;
+		d = p->data;
+		(*table->delkeyfunc)(p->key, table->cb_arg);
+		(*table->deldatafunc)(d, table->cb_arg);
+		p = np;
+	}
+}
+
+void 
+bin_split(struct lruhash* table, struct lruhash_bin* newa, 
+	int newmask)
+{
+	size_t i;
+	struct lruhash_entry *p, *np;
+	struct lruhash_bin* newbin;
+	/* move entries to new table. Notice that since hash x is mapped to
+	 * bin x & mask, and new mask uses one more bit, so all entries in
+	 * one bin will go into the old bin or bin | newbit */
+	int newbit = newmask - table->size_mask;
+	/* so, really, this task could also be threaded, per bin. */
+	/* LRU list is not changed */
+	for(i=0; i<table->size; i++)
+	{
+		lock_quick_lock(&table->array[i].lock);
+		p = table->array[i].overflow_list;
+		/* lock both destination bins */
+		lock_quick_lock(&newa[i].lock);
+		lock_quick_lock(&newa[newbit|i].lock);
+		while(p) {
+			np = p->overflow_next;
+			/* link into correct new bin */
+			newbin = &newa[p->hash & newmask];
+			p->overflow_next = newbin->overflow_list;
+			newbin->overflow_list = p;
+			p=np;
+		}
+		lock_quick_unlock(&newa[i].lock);
+		lock_quick_unlock(&newa[newbit|i].lock);
+	}
+}
+
+void 
+lruhash_delete(struct lruhash* table)
+{
+	size_t i;
+	if(!table)
+		return;
+	/* delete lock on hashtable to force check its OK */
+	lock_quick_destroy(&table->lock);
+	for(i=0; i<table->size; i++)
+		bin_delete(table, &table->array[i]);
+	free(table->array);
+	free(table);
+}
+
+void 
+bin_overflow_remove(struct lruhash_bin* bin, struct lruhash_entry* entry)
+{
+	struct lruhash_entry* p = bin->overflow_list;
+	struct lruhash_entry** prevp = &bin->overflow_list;
+	while(p) {
+		if(p == entry) {
+			*prevp = p->overflow_next;
+			return;
+		}
+		prevp = &p->overflow_next;
+		p = p->overflow_next;
+	}
+}
+
+void 
+reclaim_space(struct lruhash* table, struct lruhash_entry** list)
+{
+	struct lruhash_entry* d;
+	struct lruhash_bin* bin;
+	log_assert(table);
+	/* does not delete MRU entry, so table will not be empty. */
+	while(table->num > 1 && table->space_used > table->space_max) {
+		/* notice that since we hold the hashtable lock, nobody
+		   can change the lru chain. So it cannot be deleted underneath
+		   us. We still need the hashbin and entry write lock to make 
+		   sure we flush all users away from the entry. 
+		   which is unlikely, since it is LRU, if someone got a rdlock
+		   it would be moved to front, but to be sure. */
+		d = table->lru_end;
+		/* specialised, delete from end of double linked list,
+		   and we know num>1, so there is a previous lru entry. */
+		log_assert(d && d->lru_prev);
+		table->lru_end = d->lru_prev;
+		d->lru_prev->lru_next = NULL;
+		/* schedule entry for deletion */
+		bin = &table->array[d->hash & table->size_mask];
+		table->num --;
+		lock_quick_lock(&bin->lock);
+		bin_overflow_remove(bin, d);
+		d->overflow_next = *list;
+		*list = d;
+		lock_rw_wrlock(&d->lock);
+		table->space_used -= table->sizefunc(d->key, d->data);
+		lock_quick_unlock(&bin->lock);
+	}
+}
+
+struct lruhash_entry* 
+bin_find_entry(struct lruhash* table, 
+	struct lruhash_bin* bin, hashvalue_t hash, void* key)
+{
+	struct lruhash_entry* p = bin->overflow_list;
+	while(p) {
+		if(p->hash == hash && table->compfunc(p->key, key) == 0)
+			return p;
+		p = p->overflow_next;
+	}
+	return NULL;
+}
+
+void 
+table_grow(struct lruhash* table)
+{
+	struct lruhash_bin* newa;
+	int newmask;
+	size_t i;
+	if(table->size_mask == (int)(((size_t)-1)>>1)) {
+		log_err("hash array malloc: size_t too small");
+		return;
+	}
+	/* try to allocate new array, if not fail */
+	newa = calloc(table->size*2, sizeof(struct lruhash_bin));
+	if(!newa) {
+		log_err("hash grow: malloc failed");
+		/* continue with smaller array. Though its slower. */
+		return;
+	}
+	bin_init(newa, table->size*2);
+	newmask = (table->size_mask << 1) | 1;
+	bin_split(table, newa, newmask);
+	/* delete the old bins */
+	lock_unprotect(&table->lock, table->array);
+	for(i=0; i<table->size; i++) {
+		lock_quick_unlock(&table->array[i].lock);
+		lock_quick_destroy(&table->array[i].lock);
+	}
+	free(table->array);
+	
+	table->size *= 2;
+	table->size_mask = newmask;
+	table->array = newa;
+	lock_protect(&table->lock, table->array, 
+		table->size*sizeof(struct lruhash_bin));
+	return;
+}
+
+void 
+lru_front(struct lruhash* table, struct lruhash_entry* entry)
+{
+	entry->lru_prev = NULL;
+	entry->lru_next = table->lru_start;
+	if(!table->lru_start)
+		table->lru_end = entry;
+	else	table->lru_start->lru_prev = entry;
+	table->lru_start = entry;
+}
+
+void 
+lru_remove(struct lruhash* table, struct lruhash_entry* entry)
+{
+	if(entry->lru_prev)
+		entry->lru_prev->lru_next = entry->lru_next;
+	else	table->lru_start = entry->lru_next;
+	if(entry->lru_next)
+		entry->lru_next->lru_prev = entry->lru_prev;
+	else	table->lru_end = entry->lru_prev;
+}
+
+void 
+lru_touch(struct lruhash* table, struct lruhash_entry* entry)
+{
+	log_assert(table && entry);
+	if(entry == table->lru_start)
+		return; /* nothing to do */
+	/* remove from current lru position */
+	lru_remove(table, entry);
+	/* add at front */
+	lru_front(table, entry);
+}
+
+void 
+lruhash_insert(struct lruhash* table, hashvalue_t hash,
+        struct lruhash_entry* entry, void* data, void* cb_arg)
+{
+	struct lruhash_bin* bin;
+	struct lruhash_entry* found, *reclaimlist=NULL;
+	size_t need_size;
+	need_size = table->sizefunc(entry->key, data);
+	if(cb_arg == NULL) cb_arg = table->cb_arg;
+
+	/* find bin */
+	lock_quick_lock(&table->lock);
+	bin = &table->array[hash & table->size_mask];
+	lock_quick_lock(&bin->lock);
+
+	/* see if entry exists already */
+	if(!(found=bin_find_entry(table, bin, hash, entry->key))) {
+		/* if not: add to bin */
+		entry->overflow_next = bin->overflow_list;
+		bin->overflow_list = entry;
+		lru_front(table, entry);
+		table->num++;
+		table->space_used += need_size;
+	} else {
+		/* if so: update data - needs a writelock */
+		table->space_used += need_size -
+			(*table->sizefunc)(found->key, found->data);
+		(*table->delkeyfunc)(entry->key, cb_arg);
+		lru_touch(table, found);
+		lock_rw_wrlock(&found->lock);
+		(*table->deldatafunc)(found->data, cb_arg);
+		found->data = data;
+		lock_rw_unlock(&found->lock);
+	}
+	lock_quick_unlock(&bin->lock);
+	if(table->space_used > table->space_max)
+		reclaim_space(table, &reclaimlist);
+	if(table->num >= table->size)
+		table_grow(table);
+	lock_quick_unlock(&table->lock);
+
+	/* finish reclaim if any (outside of critical region) */
+	while(reclaimlist) {
+		struct lruhash_entry* n = reclaimlist->overflow_next;
+		void* d = reclaimlist->data;
+		lock_rw_unlock(&reclaimlist->lock);
+		(*table->delkeyfunc)(reclaimlist->key, cb_arg);
+		(*table->deldatafunc)(d, cb_arg);
+		reclaimlist = n;
+	}
+}
+
+struct lruhash_entry* 
+lruhash_lookup(struct lruhash* table, hashvalue_t hash, void* key, int wr)
+{
+	struct lruhash_entry* entry;
+	struct lruhash_bin* bin;
+
+	lock_quick_lock(&table->lock);
+	bin = &table->array[hash & table->size_mask];
+	lock_quick_lock(&bin->lock);
+	if((entry=bin_find_entry(table, bin, hash, key)))
+		lru_touch(table, entry);
+	lock_quick_unlock(&table->lock);
+
+	if(entry) {
+		if(wr)	{ lock_rw_wrlock(&entry->lock); }
+		else	{ lock_rw_rdlock(&entry->lock); }
+	}
+	lock_quick_unlock(&bin->lock);
+	return entry;
+}
+
+void 
+lruhash_remove(struct lruhash* table, hashvalue_t hash, void* key)
+{
+	struct lruhash_entry* entry;
+	struct lruhash_bin* bin;
+	void *d;
+
+	lock_quick_lock(&table->lock);
+	bin = &table->array[hash & table->size_mask];
+	lock_quick_lock(&bin->lock);
+	if((entry=bin_find_entry(table, bin, hash, key))) {
+		bin_overflow_remove(bin, entry);
+		lru_remove(table, entry);
+	} else {
+		lock_quick_unlock(&table->lock);
+		lock_quick_unlock(&bin->lock);
+		return;
+	}
+	table->num--;
+	table->space_used -= (*table->sizefunc)(entry->key, entry->data);
+	lock_quick_unlock(&table->lock);
+	lock_rw_wrlock(&entry->lock);
+	lock_quick_unlock(&bin->lock);
+	/* finish removal */
+	lock_rw_unlock(&entry->lock);
+	d = entry->data;
+	(*table->delkeyfunc)(entry->key, table->cb_arg);
+	(*table->deldatafunc)(d, table->cb_arg);
+}
+
+
+void 
+lruhash_status(struct lruhash* table, const char* id, int extended)
+{
+	lock_quick_lock(&table->lock);
+	log_info("%s: %u entries, memory %u / %u",
+		id, (unsigned)table->num, (unsigned)table->space_used,
+		(unsigned)table->space_max);
+	log_info("  itemsize %u, array %u, mask %d",
+		(unsigned)(table->num? table->space_used/table->num : 0),
+		(unsigned)table->size, table->size_mask);
+	if(extended) {
+		size_t i;
+		int min=(int)table->size*2, max=-2;
+		for(i=0; i<table->size; i++) {
+			int here = 0;
+			struct lruhash_entry *en;
+			lock_quick_lock(&table->array[i].lock);
+			en = table->array[i].overflow_list;
+			while(en) {
+				here ++;
+				en = en->overflow_next;
+			}
+			lock_quick_unlock(&table->array[i].lock);
+			if(extended >= 2)
+				log_info("bin[%d] %d", (int)i, here);
+			if(here > max) max = here;
+			if(here < min) min = here;
+		}
+		log_info("  bin min %d, avg %.2lf, max %d", min, 
+			(double)table->num/(double)table->size, max);
+	}
+	lock_quick_unlock(&table->lock);
+}

util/storage/lruhash.h

@@ -0,0 +1,372 @@
+/*
+ * util/storage/lruhash.h - hashtable, hash function, LRU keeping.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * This file contains a hashtable with LRU keeping of entries.
+ *
+ * The hash table keeps a maximum memory size. Old entries are removed
+ * to make space for new entries.
+ *
+ * The locking strategy is as follows:
+ * 	o since (almost) every read also implies a LRU update, the
+ *	  hashtable lock is a spinlock, not rwlock.
+ *	o the idea is to move every thread through the hash lock quickly,
+ *	  so that the next thread can access the lookup table.
+ *	o User performs hash function.
+ *
+ * For read:
+ *	o lock hashtable.
+ *		o lookup hash bin.
+ *		o lock hash bin.
+ *			o find entry (if failed, unlock hash, unl bin, exit).
+ *			o swizzle pointers for LRU update.
+ *		o unlock hashtable.
+ *		o lock entry (rwlock).
+ *		o unlock hash bin.
+ *		o work on entry.
+ *	o unlock entry.
+ *
+ * To update an entry, gain writelock and change the entry.
+ * (the entry must keep the same hashvalue, so a data update.)
+ * (you cannot upgrade a readlock to a writelock, because the item may
+ *  be deleted, it would cause race conditions. So instead, unlock and
+ *  relookup it in the hashtable.)
+ *
+ * To delete an entry:
+ *	o unlock the entry if you hold the lock already.
+ *	o lock hashtable.
+ *		o lookup hash bin.
+ *		o lock hash bin.
+ *			o find entry (if failed, unlock hash, unl bin, exit).
+ *			o remove entry from hashtable bin overflow chain.
+ *		o unlock hashtable.
+ *		o lock entry (writelock).
+ *		o unlock hash bin.
+ *	o unlock entry (nobody else should be waiting for this lock,
+ *	  since you removed it from hashtable, and you got writelock while
+ *	  holding the hashbinlock so you are the only one.)
+ * 	  Note you are only allowed to obtain a lock while holding hashbinlock.
+ *	o delete entry.
+ *
+ * The above sequence is:
+ *	o race free, works with read, write and delete.
+ *	o but has a queue, imagine someone needing a writelock on an item.
+ *	  but there are still readlocks. The writelocker waits, but holds
+ *	  the hashbinlock. The next thread that comes in and needs the same
+ * 	  hashbin will wait for the lock while holding the hashtable lock.
+ *	  thus halting the entire system on hashtable.
+ *	  This is because of the delete protection. 
+ *	  Readlocks will be easier on the rwlock on entries.
+ *	  While the writer is holding writelock, similar problems happen with
+ *	  a reader or writer needing the same item.
+ *	  the scenario requires more than three threads.
+ * 	o so the queue length is 3 threads in a bad situation. The fourth is
+ *	  unable to use the hashtable.
+ */
+
+#ifndef UTIL_STORAGE_LRUHASH_H
+#define UTIL_STORAGE_LRUHASH_H
+#include "util/locks.h"
+struct lruhash_bin;
+struct lruhash_entry;
+
+/** default start size for hash arrays */
+#define HASH_DEFAULT_STARTARRAY		1024 /* entries in array */
+/** default max memory for hash arrays */
+#define HASH_DEFAULT_MAXMEM		4*1024*1024 /* bytes */
+
+/** the type of a hash value */
+typedef uint32_t hashvalue_t;
+
+/** 
+ * Type of function that calculates the size of an entry.
+ * Result must include the size of struct lruhash_entry. 
+ * Keys that are identical must also calculate to the same size.
+ * size = func(key, data).
+ */
+typedef size_t (*lruhash_sizefunc_t)(void*, void*);
+
+/** type of function that compares two keys. return 0 if equal. */
+typedef int (*lruhash_compfunc_t)(void*, void*);
+
+/** old keys is deleted. This function is called: func(key, userarg) */
+typedef void (*lruhash_delkeyfunc_t)(void*, void*);
+
+/** old data is deleted. This function is called: func(data, userarg). */
+typedef void (*lruhash_deldatafunc_t)(void*, void*);
+
+/**
+ * Hash table that keeps LRU list of entries.
+ */
+struct lruhash {
+	/** lock for exclusive access, to the lookup array */
+	lock_quick_t lock;
+	/** the size function for entries in this table */
+	lruhash_sizefunc_t sizefunc;
+	/** the compare function for entries in this table. */
+	lruhash_compfunc_t compfunc;
+	/** how to delete keys. */
+	lruhash_delkeyfunc_t delkeyfunc;
+	/** how to delete data. */
+	lruhash_deldatafunc_t deldatafunc;
+	/** user argument for user functions */
+	void* cb_arg;
+
+	/** the size of the lookup array */
+	size_t size;
+	/** size bitmask - since size is a power of 2 */
+	int size_mask;
+	/** lookup array of bins */
+	struct lruhash_bin* array;
+
+	/** the lru list, start and end, noncyclical double linked list. */
+	struct lruhash_entry* lru_start;
+	/** lru list end item (least recently used) */
+	struct lruhash_entry* lru_end;
+
+	/** the number of entries in the hash table. */
+	size_t num;
+	/** the amount of space used, roughly the number of bytes in use. */
+	size_t space_used;
+	/** the amount of space the hash table is maximally allowed to use. */
+	size_t space_max;
+};
+
+/**
+ * A single bin with a linked list of entries in it.
+ */
+struct lruhash_bin {
+	/** 
+	 * Lock for exclusive access to the linked list
+	 * This lock makes deletion of items safe in this overflow list.
+	 */
+	lock_quick_t lock;
+	/** linked list of overflow entries */
+	struct lruhash_entry* overflow_list;
+};
+
+/**
+ * An entry into the hash table.
+ * To change overflow_next you need to hold the bin lock.
+ * To change the lru items you need to hold the hashtable lock.
+ * This structure is designed as part of key struct. And key pointer helps
+ * to get the surrounding structure. Data should be allocated on its own.
+ */
+struct lruhash_entry {
+	/** 
+	 * rwlock for access to the contents of the entry
+	 * Note that it does _not_ cover the lru_ and overflow_ ptrs.
+	 * Even with a writelock, you cannot change hash and key.
+	 * You need to delete it to change hash or key.
+	 */
+	lock_rw_t lock;
+	/** next entry in overflow chain. Covered by hashlock and binlock. */
+	struct lruhash_entry* overflow_next;
+	/** next entry in lru chain. covered by hashlock. */
+	struct lruhash_entry* lru_next;
+	/** prev entry in lru chain. covered by hashlock. */
+	struct lruhash_entry* lru_prev;
+	/** hash value of the key. It may not change, until entry deleted. */
+	hashvalue_t hash;
+	/** key */
+	void* key;
+	/** data */
+	void* data;
+};
+
+/**
+ * Create new hash table.
+ * @param start_size: size of hashtable array at start, must be power of 2.
+ * @param maxmem: maximum amount of memory this table is allowed to use.
+ * @param sizefunc: calculates memory usage of entries.
+ * @param compfunc: compares entries, 0 on equality.
+ * @param delkeyfunc: deletes key.
+ *   Calling both delkey and deldata will also free the struct lruhash_entry.
+ *   Make it part of the key structure and delete it in delkeyfunc.
+ * @param deldatafunc: deletes data. 
+ * @param arg: user argument that is passed to user function calls.
+ * @return: new hash table or NULL on malloc failure.
+ */
+struct lruhash* lruhash_create(size_t start_size, size_t maxmem,
+	lruhash_sizefunc_t sizefunc, lruhash_compfunc_t compfunc,
+	lruhash_delkeyfunc_t delkeyfunc, lruhash_deldatafunc_t deldatafunc, 
+	void* arg);
+
+/**
+ * Delete hash table. Entries are all deleted.
+ * @param table: to delete.
+ */
+void lruhash_delete(struct lruhash* table);
+
+/**
+ * Insert a new element into the hashtable. 
+ * If key is already present data pointer in that entry is updated.
+ * The space calculation function is called with the key, data.
+ * If necessary the least recently used entries are deleted to make space.
+ * If necessary the hash array is grown up.
+ *
+ * @param table: hash table.
+ * @param hash: hash value. User calculates the hash.
+ * @param entry: identifies the entry.
+ * 	If key already present, this entry->key is deleted immediately.
+ *	But entry->data is set to NULL before deletion, and put into
+ * 	the existing entry. The data is then freed.
+ * @param data: the data.
+ * @param cb_override: if not null overrides the cb_arg for the deletefunc.
+ */
+void lruhash_insert(struct lruhash* table, hashvalue_t hash, 
+	struct lruhash_entry* entry, void* data, void* cb_override);
+
+/**
+ * Lookup an entry in the hashtable.
+ * At the end of the function you hold a (read/write)lock on the entry.
+ * The LRU is updated for the entry (if found).
+ * @param table: hash table.
+ * @param hash: hash of key.
+ * @param key: what to look for, compared against entries in overflow chain.
+ *    the hash value must be set, and must work with compare function.
+ * @param wr: set to true if you desire a writelock on the entry.
+ *    with a writelock you can update the data part.
+ * @return: pointer to the entry or NULL. The entry is locked.
+ *    The user must unlock the entry when done.
+ */
+struct lruhash_entry* lruhash_lookup(struct lruhash* table, hashvalue_t hash, 
+	void* key, int wr);
+
+
+/************************* Internal functions ************************/
+/*** these are only exposed for unit tests. ***/
+
+/**
+ * Remove entry from hashtable. Does nothing if not found in hashtable.
+ * Delfunc is called for the entry.
+ * @param table: hash table.
+ * @param hash: hash of key.
+ * @param key: what to look for. 
+ */
+void lruhash_remove(struct lruhash* table, hashvalue_t hash, void* key);
+
+/** init the hash bins for the table. */
+void bin_init(struct lruhash_bin* array, size_t size);
+
+/** delete the hash bin and entries inside it */
+void bin_delete(struct lruhash* table, struct lruhash_bin* bin);
+
+/** 
+ * Find entry in hash bin. You must have locked the bin.
+ * @param table: hash table with function pointers.
+ * @param bin: hash bin to look into.
+ * @param hash: hash value to look for.
+ * @param key: key to look for.
+ * @return: the entry or NULL if not found.
+ */
+struct lruhash_entry* bin_find_entry(struct lruhash* table, 
+	struct lruhash_bin* bin, hashvalue_t hash, void* key);
+
+/**
+ * Remove entry from bin overflow chain.
+ * You must have locked the bin.
+ * @param bin: hash bin to look into.
+ * @param entry: entry ptr that needs removal.
+ */
+void bin_overflow_remove(struct lruhash_bin* bin, 
+	struct lruhash_entry* entry);
+
+/**
+ * Split hash bin into two new ones. Based on increased size_mask.
+ * Caller must hold hash table lock.
+ * At the end the routine acquires all hashbin locks (in the old array).
+ * This makes it wait for other threads to finish with the bins.
+ * So the bins are ready to be deleted after this function.
+ * @param table: hash table with function pointers.
+ * @param newa: new increased array.
+ * @param newmask: new lookup mask.
+ */
+void bin_split(struct lruhash* table, struct lruhash_bin* newa, 
+	int newmask);
+
+/** 
+ * Try to make space available by deleting old entries.
+ * Assumes that the lock on the hashtable is being held by caller.
+ * Caller must not hold bin locks.
+ * @param table: hash table.
+ * @param list: list of entries that are to be deleted later.
+ *	Entries have been removed from the hash table and writelock is held.
+ */
+void reclaim_space(struct lruhash* table, struct lruhash_entry** list);
+
+/**
+ * Grow the table lookup array. Becomes twice as large.
+ * Caller must hold the hash table lock. Must not hold any bin locks.
+ * Tries to grow, on malloc failure, nothing happened.
+ * @param table: hash table.
+ */
+void table_grow(struct lruhash* table);
+
+/**
+ * Touch entry, so it becomes the most recently used in the LRU list.
+ * Caller must hold hash table lock. The entry must be inserted already.
+ * @param table: hash table.
+ * @param entry: entry to make first in LRU.
+ */
+void lru_touch(struct lruhash* table, struct lruhash_entry* entry);
+
+/**
+ * Put entry at front of lru. entry must be unlinked from lru.
+ * Caller must hold hash table lock.
+ * @param table: hash table with lru head and tail.
+ * @param entry: entry to make most recently used.
+ */
+void lru_front(struct lruhash* table, struct lruhash_entry* entry);
+
+/**
+ * Remove entry from lru list.
+ * Caller must hold hash table lock.
+ * @param table: hash table with lru head and tail.
+ * @param entry: entry to remove from lru.
+ */
+void lru_remove(struct lruhash* table, struct lruhash_entry* entry);
+
+/**
+ * Output debug info to the log as to state of the hash table.
+ * @param table: hash table.
+ * @param id: string printed with table to identify the hash table.
+ * @param extended: set to true to print statistics on overflow bin lengths.
+ */
+void lruhash_status(struct lruhash* table, const char* id, int extended);
+
+#endif /* UTIL_STORAGE_LRUHASH_H */

util/storage/slabhash.c

@@ -0,0 +1,145 @@
+/*
+ * util/storage/slabhash.c - hashtable consisting of several smaller tables.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * Implementation of hash table that consists of smaller hash tables.
+ * This results in a partitioned lruhash table.
+ * It cannot grow, but that gives it the ability to have multiple
+ * locks. Also this means there are multiple LRU lists.
+ */
+
+#include "config.h"
+#include "util/storage/slabhash.h"
+
+struct slabhash* slabhash_create(size_t numtables, size_t start_size, 
+	size_t maxmem, lruhash_sizefunc_t sizefunc, 
+	lruhash_compfunc_t compfunc, lruhash_delkeyfunc_t delkeyfunc, 
+	lruhash_deldatafunc_t deldatafunc, void* arg)
+{
+	size_t i;
+	struct slabhash* sl = (struct slabhash*)calloc(1, 
+		sizeof(struct slabhash));
+	if(!sl) return NULL;
+	sl->size = numtables;
+	log_assert(sl->size > 0);
+	sl->array = (struct lruhash**)calloc(sl->size, sizeof(struct lruhash*));
+	if(!sl->array) {
+		free(sl);
+		return NULL;
+	}
+	sl->mask = (uint32_t)(sl->size - 1);
+	if(sl->mask == 0) {
+		sl->shift = 0;
+	} else {
+		log_assert( (sl->size & sl->mask) == 0 
+			/* size must be power of 2 */ );
+		sl->shift = 0;
+		while(!(sl->mask & 0x80000000)) {
+			sl->mask <<= 1;
+			sl->shift ++;
+		}
+	}
+	for(i=0; i<sl->size; i++) {
+		sl->array[i] = lruhash_create(start_size, maxmem / sl->size,
+			sizefunc, compfunc, delkeyfunc, deldatafunc, arg);
+		if(!sl->array[i]) {
+			slabhash_delete(sl);
+			return NULL;
+		}
+	}
+	return sl;
+}
+
+void slabhash_delete(struct slabhash* sl)
+{
+	if(!sl)
+		return;
+	if(sl->array) {
+		size_t i;
+		for(i=0; i<sl->size; i++)
+			lruhash_delete(sl->array[i]);
+		free(sl->array);
+	}
+	free(sl);
+}
+
+/** helper routine to calculate the slabhash index */
+static unsigned int
+slab_idx(struct slabhash* sl, hashvalue_t hash)
+{
+	return ((hash & sl->mask) >> sl->shift);
+}
+
+void slabhash_insert(struct slabhash* sl, hashvalue_t hash, 
+	struct lruhash_entry* entry, void* data, void* arg)
+{
+	lruhash_insert(sl->array[slab_idx(sl, hash)], hash, entry, data, arg);
+}
+
+struct lruhash_entry* slabhash_lookup(struct slabhash* sl, 
+	hashvalue_t hash, void* key, int wr)
+{
+	return lruhash_lookup(sl->array[slab_idx(sl, hash)], hash, key, wr);
+}
+
+void slabhash_remove(struct slabhash* sl, hashvalue_t hash, void* key)
+{
+	lruhash_remove(sl->array[slab_idx(sl, hash)], hash, key);
+}
+
+void slabhash_status(struct slabhash* sl, const char* id, int extended)
+{
+	size_t i;
+	char num[17];
+	log_info("Slabhash %s: %u tables mask=%x shift=%d", 
+		id, (unsigned)sl->size, sl->mask, sl->shift);
+	for(i=0; i<sl->size; i++) {
+		snprintf(num, sizeof(num), "table %u", (unsigned)i);
+		lruhash_status(sl->array[i], num, extended);
+	}
+}
+
+size_t slabhash_get_size(struct slabhash* sl)
+{
+	size_t i, total = 0;
+	for(i=0; i<sl->size; i++) {
+		lock_quick_lock(&sl->array[i]->lock);
+		total += sl->array[i]->space_max;
+		lock_quick_unlock(&sl->array[i]->lock);
+	}
+	return total;
+}

util/storage/slabhash.h

@@ -0,0 +1,148 @@
+/*
+ * util/storage/slabhash.h - hashtable consisting of several smaller tables.
+ *
+ * Copyright (c) 2007, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ *
+ * Hash table that consists of smaller hash tables.
+ * It cannot grow, but that gives it the ability to have multiple
+ * locks. Also this means there are multiple LRU lists.
+ */
+
+#ifndef UTIL_STORAGE_SLABHASH_H
+#define UTIL_STORAGE_SLABHASH_H
+#include "util/storage/lruhash.h"
+
+/** default number of slabs */
+#define HASH_DEFAULT_SLABS 4
+
+/**
+ * Hash table formed from several smaller ones. 
+ * This results in a partitioned lruhash table, a 'slashtable'.
+ * None of the data inside the slabhash may be altered.
+ * Therefore, no locks are needed to access the structure.
+ */
+struct slabhash {
+	/** the size of the array - must be power of 2 */
+	size_t size;
+	/** size bitmask - uses high bits. */
+	uint32_t mask;
+	/** shift right this many bits to get index into array. */
+	unsigned int shift;
+	/** lookup array of hash tables */
+	struct lruhash** array;
+};
+
+/**
+ * Create new slabbed hash table.
+ * @param numtables: number of hash tables to use, other parameters used to
+ *	initialize these smaller hashtables.
+ * @param start_size: size of hashtable array at start, must be power of 2.
+ * @param maxmem: maximum amount of memory this table is allowed to use.
+ *	so every table gets maxmem/numtables to use for itself.
+ * @param sizefunc: calculates memory usage of entries.
+ * @param compfunc: compares entries, 0 on equality.
+ * @param delkeyfunc: deletes key.
+ * @param deldatafunc: deletes data. 
+ * @param arg: user argument that is passed to user function calls.
+ * @return: new hash table or NULL on malloc failure.
+ */
+struct slabhash* slabhash_create(size_t numtables, size_t start_size, 
+	size_t maxmem, lruhash_sizefunc_t sizefunc, 
+	lruhash_compfunc_t compfunc, lruhash_delkeyfunc_t delkeyfunc, 
+	lruhash_deldatafunc_t deldatafunc, void* arg);
+
+/**
+ * Delete hash table. Entries are all deleted.
+ * @param table: to delete.
+ */
+void slabhash_delete(struct slabhash* table);
+
+/**
+ * Insert a new element into the hashtable, uses lruhash_insert. 
+ * If key is already present data pointer in that entry is updated.
+ *
+ * @param table: hash table.
+ * @param hash: hash value. User calculates the hash.
+ * @param entry: identifies the entry.
+ * 	If key already present, this entry->key is deleted immediately.
+ *	But entry->data is set to NULL before deletion, and put into
+ * 	the existing entry. The data is then freed.
+ * @param data: the data.
+ * @param cb_override: if not NULL overrides the cb_arg for deletfunc.
+ */
+void slabhash_insert(struct slabhash* table, hashvalue_t hash, 
+	struct lruhash_entry* entry, void* data, void* cb_override);
+
+/**
+ * Lookup an entry in the hashtable. Uses lruhash_lookup.
+ * At the end of the function you hold a (read/write)lock on the entry.
+ * The LRU is updated for the entry (if found).
+ * @param table: hash table.
+ * @param hash: hash of key.
+ * @param key: what to look for, compared against entries in overflow chain.
+ *    the hash value must be set, and must work with compare function.
+ * @param wr: set to true if you desire a writelock on the entry.
+ *    with a writelock you can update the data part.
+ * @return: pointer to the entry or NULL. The entry is locked.
+ *    The user must unlock the entry when done.
+ */
+struct lruhash_entry* slabhash_lookup(struct slabhash* table, 
+	hashvalue_t hash, void* key, int wr);
+
+/**
+ * Remove entry from hashtable. Does nothing if not found in hashtable.
+ * Delfunc is called for the entry. Uses lruhash_remove.
+ * @param table: hash table.
+ * @param hash: hash of key.
+ * @param key: what to look for. 
+ */
+void slabhash_remove(struct slabhash* table, hashvalue_t hash, void* key);
+
+/**
+ * Output debug info to the log as to state of the hash table.
+ * @param table: hash table.
+ * @param id: string printed with table to identify the hash table.
+ * @param extended: set to true to print statistics on overflow bin lengths.
+ */
+void slabhash_status(struct slabhash* table, const char* id, int extended);
+
+/**
+ * Retrieve slab hash total size.
+ * @param table: hash table.
+ */
+size_t slabhash_get_size(struct slabhash* table);
+
+#endif /* UTIL_STORAGE_SLABHASH_H */