Merge branch 'master' into stream-reuse

Resolved comm_point_start_listening conflict for socket close.

testcode/delayer.c

@@ -372,11 +372,7 @@ service_send(struct ringbuf* ring, struct timeval* now, sldns_buffer* pkt,
 			sldns_buffer_limit(pkt), 0, 
 			(struct sockaddr*)srv_addr, srv_len);
 		if(sent == -1) {
-#ifndef USE_WINSOCK
-			log_err("sendto: %s", strerror(errno));
-#else
-			log_err("sendto: %s", wsa_strerror(WSAGetLastError()));
-#endif
+			log_err("sendto: %s", sock_strerror(errno));
 		} else if(sent != (ssize_t)sldns_buffer_limit(pkt)) {
 			log_err("sendto: partial send");
 		}
@@ -398,13 +394,12 @@ do_proxy(struct proxy* p, int retsock, sldns_buffer* pkt)
 #ifndef USE_WINSOCK
 			if(errno == EAGAIN || errno == EINTR)
 				return;
-			log_err("recv: %s", strerror(errno));
 #else
 			if(WSAGetLastError() == WSAEINPROGRESS ||
 				WSAGetLastError() == WSAEWOULDBLOCK)
 				return;
-			log_err("recv: %s", wsa_strerror(WSAGetLastError()));
 #endif
+			log_err("recv: %s", sock_strerror(errno));
 			return;
 		}
 		sldns_buffer_set_limit(pkt, (size_t)r);
@@ -414,11 +409,7 @@ do_proxy(struct proxy* p, int retsock, sldns_buffer* pkt)
 		r = sendto(retsock, (void*)sldns_buffer_begin(pkt), (size_t)r, 
 			0, (struct sockaddr*)&p->addr, p->addr_len);
 		if(r == -1) {
-#ifndef USE_WINSOCK
-			log_err("sendto: %s", strerror(errno));
-#else
-			log_err("sendto: %s", wsa_strerror(WSAGetLastError()));
-#endif
+			log_err("sendto: %s", sock_strerror(errno));
 		}
 	}
 }
@@ -469,11 +460,7 @@ find_create_proxy(struct sockaddr_storage* from, socklen_t from_len,
 	if(!p) fatal_exit("out of memory");
 	p->s = socket(serv_ip6?AF_INET6:AF_INET, SOCK_DGRAM, 0);
 	if(p->s == -1) {
-#ifndef USE_WINSOCK
-		fatal_exit("socket: %s", strerror(errno));
-#else
-		fatal_exit("socket: %s", wsa_strerror(WSAGetLastError()));
-#endif
+		fatal_exit("socket: %s", sock_strerror(errno));
 	}
 	fd_set_nonblock(p->s);
 	memmove(&p->addr, from, from_len);
@@ -507,14 +494,12 @@ service_recv(int s, struct ringbuf* ring, sldns_buffer* pkt,
 #ifndef USE_WINSOCK
 			if(errno == EAGAIN || errno == EINTR)
 				return;
-			fatal_exit("recvfrom: %s", strerror(errno));
 #else
 			if(WSAGetLastError() == WSAEWOULDBLOCK || 
 				WSAGetLastError() == WSAEINPROGRESS)
 				return;
-			fatal_exit("recvfrom: %s", 
-				wsa_strerror(WSAGetLastError()));
 #endif
+			fatal_exit("recvfrom: %s", sock_strerror(errno));
 		}
 		sldns_buffer_set_limit(pkt, (size_t)len);
 		/* find its proxy element */
@@ -550,15 +535,9 @@ tcp_proxy_delete(struct tcp_proxy* p)
 		free(s);
 		s = sn;
 	}
-#ifndef USE_WINSOCK
-	close(p->client_s);
+	sock_close(p->client_s);
 	if(p->server_s != -1)
-		close(p->server_s);
-#else
-	closesocket(p->client_s);
-	if(p->server_s != -1)
-		closesocket(p->server_s);
-#endif
+		sock_close(p->server_s);
 	free(p);
 }
 
@@ -577,14 +556,13 @@ service_tcp_listen(int s, fd_set* rorig, int* max, struct tcp_proxy** proxies,
 #ifndef USE_WINSOCK
 		if(errno == EAGAIN || errno == EINTR)
 			return;
-		fatal_exit("accept: %s", strerror(errno));
 #else
 		if(WSAGetLastError() == WSAEWOULDBLOCK || 
 			WSAGetLastError() == WSAEINPROGRESS ||
 			WSAGetLastError() == WSAECONNRESET)
 			return;
-		fatal_exit("accept: %s", wsa_strerror(WSAGetLastError()));
 #endif
+		fatal_exit("accept: %s", sock_strerror(errno));
 	}
 	p = (struct tcp_proxy*)calloc(1, sizeof(*p));
 	if(!p) fatal_exit("out of memory");
@@ -595,11 +573,7 @@ service_tcp_listen(int s, fd_set* rorig, int* max, struct tcp_proxy** proxies,
 	p->server_s = socket(addr_is_ip6(srv_addr, srv_len)?AF_INET6:AF_INET,
 		SOCK_STREAM, 0);
 	if(p->server_s == -1) {
-#ifndef USE_WINSOCK
-		fatal_exit("tcp socket: %s", strerror(errno));
-#else
-		fatal_exit("tcp socket: %s", wsa_strerror(WSAGetLastError()));
-#endif
+		fatal_exit("tcp socket: %s", sock_strerror(errno));
 	}
 	fd_set_nonblock(p->client_s);
 	fd_set_nonblock(p->server_s);
@@ -607,16 +581,14 @@ service_tcp_listen(int s, fd_set* rorig, int* max, struct tcp_proxy** proxies,
 #ifndef USE_WINSOCK
 		if(errno != EINPROGRESS) {
 			log_err("tcp connect: %s", strerror(errno));
-			close(p->server_s);
-			close(p->client_s);
 #else
 		if(WSAGetLastError() != WSAEWOULDBLOCK &&
 			WSAGetLastError() != WSAEINPROGRESS) {
 			log_err("tcp connect: %s", 
 				wsa_strerror(WSAGetLastError()));
-			closesocket(p->server_s);
-			closesocket(p->client_s);
 #endif
+			sock_close(p->server_s);
+			sock_close(p->client_s);
 			free(p);
 			return;
 		}
@@ -650,13 +622,12 @@ tcp_relay_read(int s, struct tcp_send_list** first,
 #ifndef USE_WINSOCK
 		if(errno == EINTR || errno == EAGAIN)
 			return 1;
-		log_err("tcp read: %s", strerror(errno));
 #else
 		if(WSAGetLastError() == WSAEINPROGRESS || 
 			WSAGetLastError() == WSAEWOULDBLOCK)
 			return 1;
-		log_err("tcp read: %s", wsa_strerror(WSAGetLastError()));
 #endif
+		log_err("tcp read: %s", sock_strerror(errno));
 		return 0;
 	} else if(r == 0) {
 		/* connection closed */
@@ -708,14 +679,12 @@ tcp_relay_write(int s, struct tcp_send_list** first,
 #ifndef USE_WINSOCK
 			if(errno == EAGAIN || errno == EINTR)
 				return 1;
-			log_err("tcp write: %s", strerror(errno));
 #else
 			if(WSAGetLastError() == WSAEWOULDBLOCK || 
 				WSAGetLastError() == WSAEINPROGRESS)
 				return 1;
-			log_err("tcp write: %s", 
-				wsa_strerror(WSAGetLastError()));
 #endif
+			log_err("tcp write: %s", sock_strerror(errno));
 			return 0;
 		} else if(r == 0) {
 			/* closed */
@@ -769,11 +738,7 @@ service_tcp_relay(struct tcp_proxy** tcp_proxies, struct timeval* now,
 			log_addr(1, "read tcp answer", &p->addr, p->addr_len);
 			if(!tcp_relay_read(p->server_s, &p->answerlist, 
 				&p->answerlast, now, delay, pkt)) {
-#ifndef USE_WINSOCK
-				close(p->server_s);
-#else
-				closesocket(p->server_s);
-#endif
+				sock_close(p->server_s);
 				FD_CLR(FD_SET_T p->server_s, worig);
 				FD_CLR(FD_SET_T p->server_s, rorig);
 				p->server_s = -1;
@@ -901,11 +866,7 @@ proxy_list_clear(struct proxy* p)
 			"%u returned\n", i++, from, port, (int)p->numreuse+1,
 			(unsigned)p->numwait, (unsigned)p->numsent, 
 			(unsigned)p->numreturn);
-#ifndef USE_WINSOCK
-		close(p->s);
-#else
-		closesocket(p->s);
-#endif
+		sock_close(p->s);
 		free(p);
 		p = np;
 	}
@@ -1034,11 +995,7 @@ service(const char* bind_str, int bindport, const char* serv_str,
 	/* bind UDP port */
 	if((s = socket(str_is_ip6(bind_str)?AF_INET6:AF_INET,
 		SOCK_DGRAM, 0)) == -1) {
-#ifndef USE_WINSOCK
-		fatal_exit("socket: %s", strerror(errno));
-#else
-		fatal_exit("socket: %s", wsa_strerror(WSAGetLastError()));
-#endif
+		fatal_exit("socket: %s", sock_strerror(errno));
 	}
 	i=0;
 	if(bindport == 0) {
@@ -1051,11 +1008,7 @@ service(const char* bind_str, int bindport, const char* serv_str,
 			exit(1);
 		}
 		if(bind(s, (struct sockaddr*)&bind_addr, bind_len) == -1) {
-#ifndef USE_WINSOCK
-			log_err("bind: %s", strerror(errno));
-#else
-			log_err("bind: %s", wsa_strerror(WSAGetLastError()));
-#endif
+			log_err("bind: %s", sock_strerror(errno));
 			if(i--==0)
 				fatal_exit("cannot bind any port");
 			bindport = 1024 + ((int)arc4random())%64000;
@@ -1065,39 +1018,22 @@ service(const char* bind_str, int bindport, const char* serv_str,
 	/* and TCP port */
 	if((listen_s = socket(str_is_ip6(bind_str)?AF_INET6:AF_INET,
 		SOCK_STREAM, 0)) == -1) {
-#ifndef USE_WINSOCK
-		fatal_exit("tcp socket: %s", strerror(errno));
-#else
-		fatal_exit("tcp socket: %s", wsa_strerror(WSAGetLastError()));
-#endif
+		fatal_exit("tcp socket: %s", sock_strerror(errno));
 	}
 #ifdef SO_REUSEADDR
 	if(1) {
 		int on = 1;
 		if(setsockopt(listen_s, SOL_SOCKET, SO_REUSEADDR, (void*)&on,
 			(socklen_t)sizeof(on)) < 0)
-#ifndef USE_WINSOCK
 			fatal_exit("setsockopt(.. SO_REUSEADDR ..) failed: %s",
-				strerror(errno));
-#else
-			fatal_exit("setsockopt(.. SO_REUSEADDR ..) failed: %s",
-				wsa_strerror(WSAGetLastError()));
-#endif
+				sock_strerror(errno));
 	}
 #endif
 	if(bind(listen_s, (struct sockaddr*)&bind_addr, bind_len) == -1) {
-#ifndef USE_WINSOCK
-		fatal_exit("tcp bind: %s", strerror(errno));
-#else
-		fatal_exit("tcp bind: %s", wsa_strerror(WSAGetLastError()));
-#endif
+		fatal_exit("tcp bind: %s", sock_strerror(errno));
 	}
 	if(listen(listen_s, 5) == -1) {
-#ifndef USE_WINSOCK
-		fatal_exit("tcp listen: %s", strerror(errno));
-#else
-		fatal_exit("tcp listen: %s", wsa_strerror(WSAGetLastError()));
-#endif
+		fatal_exit("tcp listen: %s", sock_strerror(errno));
 	}
 	fd_set_nonblock(listen_s);
 	printf("listening on port: %d\n", bindport);
@@ -1109,13 +1045,8 @@ service(const char* bind_str, int bindport, const char* serv_str,
 
 	/* cleanup */
 	verbose(1, "cleanup");
-#ifndef USE_WINSOCK
-	close(s);
-	close(listen_s);
-#else
-	closesocket(s);
-	closesocket(listen_s);
-#endif
+	sock_close(s);
+	sock_close(listen_s);
 	sldns_buffer_free(pkt);
 	ring_delete(ring);
 }

testcode/do-tests.sh

@@ -29,6 +29,9 @@ else
 	HAVE_MINGW=no
 fi
 
+# stop tests from notifying systemd, if that is compiled in.
+export -n NOTIFY_SOCKET
+
 cd testdata;
 sh ../testcode/mini_tdir.sh clean
 rm -f .perfstats.txt

testcode/dohclient.c

@@ -0,0 +1,638 @@
+/*
+ * testcode/dohclient.c - debug program. Perform multiple DNS queries using DoH.
+ *
+ * Copyright (c) 2020, NLnet Labs. All rights reserved.
+ *
+ * This software is open source.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 
+ * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * Neither the name of the NLNET LABS nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** 
+ * \file
+ *
+ * Simple DNS-over-HTTPS client. For testing and debugging purposes.
+ * No authentication of TLS cert.
+ */
+
+#include "config.h"
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+#include "sldns/wire2str.h"
+#include "sldns/sbuffer.h"
+#include "sldns/str2wire.h"
+#include "sldns/parseutil.h"
+#include "util/data/msgencode.h"
+#include "util/data/msgreply.h"
+#include "util/data/msgparse.h"
+#include "util/net_help.h"
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#ifdef HAVE_NGHTTP2
+#include <nghttp2/nghttp2.h>
+
+struct http2_session {
+	nghttp2_session* session;
+	SSL* ssl;
+	int fd;
+	int query_count;
+	/* Use POST :method if 1 */
+	int post;
+	int block_select;
+	const char* authority;
+	const char* endpoint;
+	const char* content_type;
+};
+
+struct http2_stream {
+	int32_t stream_id;
+	int res_status;
+	struct sldns_buffer* buf;
+	char* path;
+};
+
+static void usage(char* argv[])
+{
+	printf("usage: %s [options] name type class ...\n", argv[0]);
+	printf("	sends the name-type-class queries over "
+			"DNS-over-HTTPS.\n");
+	printf("-s server	IP address to send the queries to, "
+			"default: 127.0.0.1\n");
+	printf("-p		Port to connect to, default: %d\n",
+		UNBOUND_DNS_OVER_HTTPS_PORT);
+	printf("-P		Use POST method instead of default GET\n");
+	printf("-e		HTTP endpoint, default: /dns-query\n");
+	printf("-c		Content-type in request, default: "
+		"application/dns-message\n");
+	printf("-n		no-tls, TLS is disabled\n");
+	printf("-h 		This help text\n");
+	exit(1);
+}
+
+/** open TCP socket to svr */
+static int
+open_svr(const char* svr, int port)
+{
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	int fd = -1;
+	int r;
+	if(!ipstrtoaddr(svr, port, &addr, &addrlen)) {
+		printf("fatal: bad server specs '%s'\n", svr);
+		exit(1);
+	}
+
+	fd = socket(addr_is_ip6(&addr, addrlen)?PF_INET6:PF_INET,
+		SOCK_STREAM, 0);
+	if(fd == -1) {
+		perror("socket() error");
+		exit(1);
+	}
+	r = connect(fd, (struct sockaddr*)&addr, addrlen);
+	if(r < 0 && r != EINPROGRESS) {
+		perror("connect() error");
+		exit(1);
+	}
+	return fd;
+}
+
+static ssize_t http2_submit_request_read_cb(
+	nghttp2_session* ATTR_UNUSED(session),
+	int32_t ATTR_UNUSED(stream_id), uint8_t* buf, size_t length,
+	uint32_t* data_flags, nghttp2_data_source* source,
+	void* ATTR_UNUSED(cb_arg))
+{
+	if(length > sldns_buffer_remaining(source->ptr))
+		length = sldns_buffer_remaining(source->ptr);
+
+	memcpy(buf, sldns_buffer_current(source->ptr), length);
+	sldns_buffer_skip(source->ptr, length);
+
+	if(sldns_buffer_remaining(source->ptr) == 0) {
+		*data_flags |= NGHTTP2_DATA_FLAG_EOF;
+	}
+
+	return length;
+}
+
+static void
+submit_query(struct http2_session* h2_session, struct sldns_buffer* buf)
+{
+	int32_t stream_id;
+	struct http2_stream* h2_stream;
+	nghttp2_nv headers[5];
+	char* qb64;
+	size_t qb64_size;
+	size_t qb64_expected_size;
+	size_t i;
+	nghttp2_data_provider data_prd;
+
+	h2_stream = calloc(1,  sizeof(*h2_stream));
+	if(!h2_stream)
+		fatal_exit("could not malloc http2 stream");
+	h2_stream->buf = buf;
+
+	if(h2_session->post) {
+		data_prd.source.ptr = buf;
+		data_prd.read_callback = http2_submit_request_read_cb;
+		h2_stream->path = (char*)h2_session->endpoint;
+	} else {
+		qb64_expected_size = sldns_b64_ntop_calculate_size(
+				sldns_buffer_remaining(buf));
+		qb64 = malloc(qb64_expected_size);
+		if(!qb64) fatal_exit("out of memory");
+		qb64_size = sldns_b64url_ntop(sldns_buffer_begin(buf),
+			sldns_buffer_remaining(buf), qb64, qb64_expected_size);
+		h2_stream->path = malloc(strlen(
+			h2_session->endpoint)+strlen("?dns=")+qb64_size+1);
+		if(!h2_stream->path) fatal_exit("out of memory");
+		snprintf(h2_stream->path, strlen(h2_session->endpoint)+
+			strlen("?dns=")+qb64_size+1, "%s?dns=%s",
+			h2_session->endpoint, qb64);
+		free(qb64);
+	}
+
+	headers[0].name = (uint8_t*)":method";
+	if(h2_session->post)
+		headers[0].value = (uint8_t*)"POST";
+	else
+		headers[0].value = (uint8_t*)"GET";
+	headers[1].name = (uint8_t*)":path";
+	headers[1].value = (uint8_t*)h2_stream->path;
+	headers[2].name = (uint8_t*)":scheme";
+	if(h2_session->ssl)
+		headers[2].value = (uint8_t*)"https";
+	else
+		headers[2].value = (uint8_t*)"http";
+	headers[3].name = (uint8_t*)":authority";
+	headers[3].value = (uint8_t*)h2_session->authority;
+	headers[4].name = (uint8_t*)"content-type";
+	headers[4].value = (uint8_t*)h2_session->content_type;
+
+	printf("Request headers\n");
+	for(i=0; i<sizeof(headers)/sizeof(headers[0]); i++) {
+		headers[i].namelen = strlen((char*)headers[i].name);
+		headers[i].valuelen = strlen((char*)headers[i].value);
+		headers[i].flags = NGHTTP2_NV_FLAG_NONE;
+		printf("%s: %s\n", headers[i].name, headers[i].value);
+	}
+
+	stream_id = nghttp2_submit_request(h2_session->session, NULL, headers,
+		sizeof(headers)/sizeof(headers[0]),
+		(h2_session->post) ? &data_prd : NULL, h2_stream);
+	if(stream_id < 0) {
+		printf("Failed to submit nghttp2 request");
+		exit(1);
+	}
+	h2_session->query_count++;
+	h2_stream->stream_id = stream_id;
+}
+
+static sldns_buffer*
+make_query(char* qname, char* qtype, char* qclass)
+{
+	struct query_info qinfo;
+	struct edns_data edns;
+	sldns_buffer* buf = sldns_buffer_new(65553);
+	if(!buf) fatal_exit("out of memory");
+	qinfo.qname = sldns_str2wire_dname(qname, &qinfo.qname_len);
+	if(!qinfo.qname) {
+		printf("cannot parse query name: '%s'\n", qname);
+		exit(1);
+	}
+
+	qinfo.qtype = sldns_get_rr_type_by_name(qtype);
+	qinfo.qclass = sldns_get_rr_class_by_name(qclass);
+	qinfo.local_alias = NULL;
+
+	qinfo_query_encode(buf, &qinfo); /* flips buffer */
+	free(qinfo.qname);
+	sldns_buffer_write_u16_at(buf, 0, 0x0000);
+	sldns_buffer_write_u16_at(buf, 2, BIT_RD);
+	memset(&edns, 0, sizeof(edns));
+	edns.edns_present = 1;
+	edns.bits = EDNS_DO;
+	edns.udp_size = 4096;
+	if(sldns_buffer_capacity(buf) >=
+		sldns_buffer_limit(buf)+calc_edns_field_size(&edns))
+		attach_edns_record(buf, &edns);
+	return buf;
+}
+
+static ssize_t http2_recv_cb(nghttp2_session* ATTR_UNUSED(session),
+	uint8_t* buf, size_t len, int ATTR_UNUSED(flags), void* cb_arg)
+{
+	struct http2_session* h2_session = (struct http2_session*)cb_arg;
+	int r;
+	ssize_t ret;
+	struct timeval tv, *waittv;
+	fd_set rfd;
+	ERR_clear_error();
+
+	memset(&tv, 0, sizeof(tv));
+
+	if(h2_session->block_select && h2_session->query_count <= 0) {
+		return NGHTTP2_ERR_WOULDBLOCK;
+	}
+	if(h2_session->block_select)
+		waittv = NULL;
+	else
+		waittv = &tv;
+	memset(&rfd, 0, sizeof(rfd));
+	FD_ZERO(&rfd);
+	FD_SET(h2_session->fd, &rfd);
+	r = select(h2_session->fd+1, &rfd, NULL, NULL, waittv);
+	if(r <= 0) {
+		return NGHTTP2_ERR_WOULDBLOCK;
+	}
+
+	if(h2_session->ssl) {
+		r = SSL_read(h2_session->ssl, buf, len);
+		if(r <= 0) {
+			int want = SSL_get_error(h2_session->ssl, r);
+			if(want == SSL_ERROR_ZERO_RETURN) {
+				return NGHTTP2_ERR_EOF;
+			}
+			log_crypto_err("could not SSL_read");
+			return NGHTTP2_ERR_EOF;
+		}
+		return r;
+	}
+
+	ret = read(h2_session->fd, buf, len);
+	if(ret == 0) {
+		return NGHTTP2_ERR_EOF;
+	} else if(ret < 0) {
+		log_err("could not http2 read: %s", strerror(errno));
+		return NGHTTP2_ERR_EOF;
+	}
+	return ret;
+}
+
+static ssize_t http2_send_cb(nghttp2_session* ATTR_UNUSED(session),
+	const uint8_t* buf, size_t len, int ATTR_UNUSED(flags), void* cb_arg)
+{
+	struct http2_session* h2_session = (struct http2_session*)cb_arg;
+	ssize_t ret;
+
+	if(h2_session->ssl) {
+		int r;
+		ERR_clear_error();
+		r = SSL_write(h2_session->ssl, buf, len);
+		if(r <= 0) {
+			int want = SSL_get_error(h2_session->ssl, r);
+			if(want == SSL_ERROR_ZERO_RETURN) {
+				return NGHTTP2_ERR_CALLBACK_FAILURE;
+			}
+			log_crypto_err("could not SSL_write");
+			return NGHTTP2_ERR_CALLBACK_FAILURE;
+		}
+		return r;
+	}
+
+	ret = write(h2_session->fd, buf, len);
+	if(ret == 0) {
+		return NGHTTP2_ERR_CALLBACK_FAILURE;
+	} else if(ret < 0) {
+		log_err("could not http2 write: %s", strerror(errno));
+		return NGHTTP2_ERR_CALLBACK_FAILURE;
+	}
+	return ret;
+}
+
+static int http2_stream_close_cb(nghttp2_session* ATTR_UNUSED(session),
+	int32_t ATTR_UNUSED(stream_id),
+	nghttp2_error_code ATTR_UNUSED(error_code), void *cb_arg)
+{
+	struct http2_session* h2_session = (struct http2_session*)cb_arg;
+	struct http2_stream* h2_stream;
+	if(!(h2_stream = nghttp2_session_get_stream_user_data(
+		h2_session->session, stream_id))) {
+		return 0;
+	}
+	h2_session->query_count--;
+	sldns_buffer_free(h2_stream->buf);
+	if(!h2_session->post)
+		free(h2_stream->path);
+	free(h2_stream);
+	h2_stream = NULL;
+	return 0;
+}
+
+static int http2_data_chunk_recv_cb(nghttp2_session* ATTR_UNUSED(session),
+	uint8_t ATTR_UNUSED(flags), int32_t stream_id, const uint8_t* data,
+	size_t len, void* cb_arg)
+{
+	struct http2_session* h2_session = (struct http2_session*)cb_arg;
+	struct http2_stream* h2_stream;
+
+	if(!(h2_stream = nghttp2_session_get_stream_user_data(
+		h2_session->session, stream_id))) {
+		return 0;
+	}
+
+	if(sldns_buffer_remaining(h2_stream->buf) < len) {
+		log_err("received data chunck does not fit into buffer");
+		return NGHTTP2_ERR_CALLBACK_FAILURE;
+	}
+
+	sldns_buffer_write(h2_stream->buf, data, len);
+
+	return 0;
+}
+
+static int http2_frame_recv_cb(nghttp2_session *session,
+	const nghttp2_frame *frame, void* ATTR_UNUSED(cb_arg))
+{
+	struct http2_stream* h2_stream;
+
+	if(!(h2_stream = nghttp2_session_get_stream_user_data(
+		session, frame->hd.stream_id)))
+		return 0;
+	if(frame->hd.type == NGHTTP2_HEADERS &&
+		frame->headers.cat == NGHTTP2_HCAT_RESPONSE) {
+		sldns_buffer_clear(h2_stream->buf);
+	}
+	if(((frame->hd.type != NGHTTP2_DATA &&
+		frame->hd.type != NGHTTP2_HEADERS) ||
+		frame->hd.flags & NGHTTP2_FLAG_END_STREAM) &&
+			h2_stream->res_status == 200) {
+			char* pktstr;
+			sldns_buffer_flip(h2_stream->buf);
+			pktstr = sldns_wire2str_pkt(
+				sldns_buffer_begin(h2_stream->buf),
+				sldns_buffer_limit(h2_stream->buf));
+			printf("%s\n", pktstr);
+			free(pktstr);
+			return 0;
+	}
+	return 0;
+}
+static int http2_header_cb(nghttp2_session* ATTR_UNUSED(session),
+	const nghttp2_frame* frame, const uint8_t* name, size_t namelen,
+	const uint8_t* value, size_t ATTR_UNUSED(valuelen),
+	uint8_t ATTR_UNUSED(flags), void* cb_arg)
+{
+	struct http2_stream* h2_stream;
+	struct http2_session* h2_session = (struct http2_session*)cb_arg;
+	printf("%s %s\n", name, value);
+	if(namelen == 7 && memcmp(":status", name, namelen) == 0) {
+		if(!(h2_stream = nghttp2_session_get_stream_user_data(
+			h2_session->session, frame->hd.stream_id))) {
+			return 0;
+		}
+		h2_stream->res_status = atoi((char*)value);
+	}
+	return 0;
+}
+
+static struct http2_session*
+http2_session_create()
+{
+	struct http2_session* h2_session = calloc(1,
+		sizeof(struct http2_session));
+	nghttp2_session_callbacks* callbacks;
+	if(!h2_session)
+		fatal_exit("out of memory");
+
+	if(nghttp2_session_callbacks_new(&callbacks) == NGHTTP2_ERR_NOMEM) {
+		log_err("failed to initialize nghttp2 callback");
+		return NULL;
+	}
+	nghttp2_session_callbacks_set_recv_callback(callbacks, http2_recv_cb);
+	nghttp2_session_callbacks_set_send_callback(callbacks, http2_send_cb);
+	nghttp2_session_callbacks_set_on_stream_close_callback(callbacks,
+		http2_stream_close_cb);
+	nghttp2_session_callbacks_set_on_data_chunk_recv_callback(callbacks,
+		http2_data_chunk_recv_cb);
+	nghttp2_session_callbacks_set_on_frame_recv_callback(callbacks,
+		http2_frame_recv_cb);
+	nghttp2_session_callbacks_set_on_header_callback(callbacks,
+		http2_header_cb);
+	nghttp2_session_client_new(&h2_session->session, callbacks, h2_session);
+	nghttp2_session_callbacks_del(callbacks);
+	return h2_session;
+}
+
+static void
+http2_session_delete(struct http2_session* h2_session)
+{
+	nghttp2_session_del(h2_session->session);
+	free(h2_session);
+}
+
+static void
+http2_submit_setting(struct http2_session* h2_session)
+{
+	int ret;
+	nghttp2_settings_entry settings[1] = {
+		{NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS,
+		 100}};
+
+	ret = nghttp2_submit_settings(h2_session->session, NGHTTP2_FLAG_NONE,
+		settings, 1);
+	if(ret) {
+		printf("http2: submit_settings failed, "
+			"error: %s\n", nghttp2_strerror(ret));
+		exit(1);
+	}
+}
+
+static void
+http2_write(struct http2_session* h2_session)
+{
+	if(nghttp2_session_want_write(h2_session->session)) {
+		if(nghttp2_session_send(h2_session->session)) {
+			printf("nghttp2 session send failed\n");
+			exit(1);
+		}
+	}
+}
+
+static void
+http2_read(struct http2_session* h2_session)
+{
+	if(nghttp2_session_want_read(h2_session->session)) {
+		if(nghttp2_session_recv(h2_session->session)) {
+			printf("nghttp2 session mem_recv failed\n");
+			exit(1);
+		}
+	}
+}
+
+static void
+run(struct http2_session* h2_session, int port, int no_tls, int count, char** q)
+{
+	int i;
+	SSL_CTX* ctx = NULL;
+	SSL* ssl = NULL;
+	int fd;
+	struct sldns_buffer* buf = NULL;
+
+	fd = open_svr(h2_session->authority, port);
+	h2_session->fd = fd;
+
+	if(!no_tls) {
+		ctx = connect_sslctx_create(NULL, NULL, NULL, 0);
+		if(!ctx) fatal_exit("cannot create ssl ctx");
+		SSL_CTX_set_alpn_protos(ctx, (const unsigned char *)"\x02h2", 3);
+		ssl = outgoing_ssl_fd(ctx, fd);
+		if(!ssl) {
+			printf("cannot create ssl\n");
+			exit(1);
+		}
+		h2_session->ssl = ssl;
+		while(1) {
+			int r;
+			ERR_clear_error();
+			if( (r=SSL_do_handshake(ssl)) == 1)
+				break;
+			r = SSL_get_error(ssl, r);
+			if(r != SSL_ERROR_WANT_READ &&
+				r != SSL_ERROR_WANT_WRITE) {
+				log_crypto_err("could not ssl_handshake");
+				exit(1);
+			}
+		}
+	}
+
+	http2_submit_setting(h2_session);
+	http2_write(h2_session);
+	http2_read(h2_session); /* Read setting from remote peer */
+
+	h2_session->block_select = 1;
+
+	/* hande query */
+	for(i=0; i<count; i+=3) {
+		buf = make_query(q[i], q[i+1], q[i+2]);
+		submit_query(h2_session, buf);
+	}
+	http2_write(h2_session);
+	while(h2_session->query_count) {
+		http2_read(h2_session);
+		http2_write(h2_session);
+	}
+
+	/* shutdown */
+	http2_session_delete(h2_session);
+	if(ssl) {
+		SSL_shutdown(ssl);
+		SSL_free(ssl);
+	}
+	if(ctx) {
+		SSL_CTX_free(ctx);
+	}
+	close(fd);
+}
+
+/** getopt global, in case header files fail to declare it. */
+extern int optind;
+/** getopt global, in case header files fail to declare it. */
+extern char* optarg;
+int main(int argc, char** argv)
+{
+	int c;
+	int port = UNBOUND_DNS_OVER_HTTPS_PORT, no_tls = 0;
+	struct http2_session* h2_session;
+
+#ifdef USE_WINSOCK
+	WSADATA wsa_data;
+	if(WSAStartup(MAKEWORD(2,2), &wsa_data) != 0) {
+		printf("WSAStartup failed\n");
+		return 1;
+	}
+#endif
+	log_init(0, 0, 0);
+	checklock_start();
+
+	h2_session = http2_session_create();
+	if(!h2_session) fatal_exit("out of memory");
+	if(argc == 1) {
+		usage(argv);
+	}
+	
+	h2_session->authority = "127.0.0.1";
+	h2_session->post = 0;
+	h2_session->endpoint = "/dns-query";
+	h2_session->content_type = "application/dns-message";
+
+	while((c=getopt(argc, argv, "c:e:hns:p:P")) != -1) {
+		switch(c) {
+			case 'c':
+				h2_session->content_type = optarg;
+				break;
+			case 'e':
+				h2_session->endpoint = optarg;
+				break;
+			case 'n':
+				no_tls = 1;
+				break;
+			case 'p':
+				if(atoi(optarg)==0 && strcmp(optarg,"0")!=0) {
+					printf("error parsing port, "
+					    "number expected: %s\n", optarg);
+					return 1;
+				}
+				port = atoi(optarg);
+				break;
+			case 'P':
+				h2_session->post = 1;
+				break;
+			case 's':
+				h2_session->authority = optarg;
+				break;
+			case 'h':
+			case '?':
+			default:
+				usage(argv);
+		}
+	}
+	argc -= optind;
+	argv += optind;
+	if(argc%3!=0) {
+		printf("Invalid input. Specify qname, qtype, and qclass.\n");
+		return 1;
+	}
+
+
+	run(h2_session, port, no_tls, argc, argv);
+
+	checklock_stop();
+#ifdef USE_WINSOCK
+	WSACleanup();
+#endif
+	return 0;
+}
+#else
+int main(int ATTR_UNUSED(argc), char** ATTR_UNUSED(argv))
+{
+	printf("Compiled without nghttp2, cannot run test.\n");
+	return 1;
+}
+#endif /*  HAVE_NGHTTP2 */

testcode/fake_event.c

@@ -52,6 +52,7 @@
 #include "util/data/msgreply.h"
 #include "util/data/msgencode.h"
 #include "util/data/dname.h"
+#include "util/edns.h"
 #include "util/config_file.h"
 #include "services/listen_dnsport.h"
 #include "services/outside_network.h"
@@ -868,9 +869,13 @@ struct listen_dnsport*
 listen_create(struct comm_base* base, struct listen_port* ATTR_UNUSED(ports),
 	size_t bufsize, int ATTR_UNUSED(tcp_accept_count),
 	int ATTR_UNUSED(tcp_idle_timeout),
+	int ATTR_UNUSED(harden_large_queries),
+	uint32_t ATTR_UNUSED(http_max_streams),
+	char* ATTR_UNUSED(http_endpoint),
+	int ATTR_UNUSED(http_notls),
 	struct tcl_list* ATTR_UNUSED(tcp_conn_limit),
 	void* ATTR_UNUSED(sslctx), struct dt_env* ATTR_UNUSED(dtenv),
-	comm_point_callback_type* cb, void* cb_arg)
+	comm_point_callback_type* cb, void *cb_arg)
 {
 	struct replay_runtime* runtime = (struct replay_runtime*)base;
 	struct listen_dnsport* l= calloc(1, sizeof(struct listen_dnsport));
@@ -1180,7 +1185,7 @@ struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
 	socklen_t addrlen, uint8_t* zone, size_t zonelen,
 	struct module_qstate* qstate, comm_point_callback_type* callback,
 	void* callback_arg, sldns_buffer* ATTR_UNUSED(buff),
-	struct module_env* ATTR_UNUSED(env))
+	struct module_env* env)
 {
 	struct replay_runtime* runtime = (struct replay_runtime*)outnet->base;
 	struct fake_pending* pend = (struct fake_pending*)calloc(1,
@@ -1209,6 +1214,7 @@ struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
 	sldns_buffer_flip(pend->buffer);
 	if(1) {
 		struct edns_data edns;
+		struct edns_tag_addr* client_tag_addr;
 		if(!inplace_cb_query_call(env, qinfo, flags, addr, addrlen,
 			zone, zonelen, qstate, qstate->region)) {
 			free(pend);
@@ -1220,9 +1226,17 @@ struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
 		edns.edns_version = EDNS_ADVERTISED_VERSION;
 		edns.udp_size = EDNS_ADVERTISED_SIZE;
 		edns.bits = 0;
-		edns.opt_list = qstate->edns_opts_back_out;
 		if(dnssec)
 			edns.bits = EDNS_DO;
+		if((client_tag_addr = edns_tag_addr_lookup(
+			&env->edns_tags->client_tags,
+			addr, addrlen))) {
+			uint16_t client_tag = htons(client_tag_addr->tag_data);
+			edns_opt_list_append(&qstate->edns_opts_back_out,
+				env->edns_tags->client_tag_opcode, 2,
+				(uint8_t*)&client_tag, qstate->region);
+		}
+		edns.opt_list = qstate->edns_opts_back_out;
 		attach_edns_record(pend->buffer, &edns);
 	}
 	memcpy(&pend->addr, addr, addrlen);
@@ -1290,7 +1304,14 @@ void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
 	log_info("double delete of pending serviced query");
 }
 
+int resolve_interface_names(struct config_file* ATTR_UNUSED(cfg),
+	char*** ATTR_UNUSED(resif), int* ATTR_UNUSED(num_resif))
+{
+	return 1;
+}
+
 struct listen_port* listening_ports_open(struct config_file* ATTR_UNUSED(cfg),
+	char** ATTR_UNUSED(ifs), int ATTR_UNUSED(num_ifs),
 	int* ATTR_UNUSED(reuseport))
 {
 	return calloc(1, 1);
@@ -1837,4 +1858,21 @@ tcp_req_info_get_stream_buffer_size(void)
 	return 0;
 }
 
+size_t
+http2_get_query_buffer_size(void)
+{
+	return 0;
+}
+
+size_t
+http2_get_response_buffer_size(void)
+{
+	return 0;
+}
+
+void http2_stream_add_meshstate(struct http2_stream* ATTR_UNUSED(h2_stream),
+	struct mesh_area* ATTR_UNUSED(mesh), struct mesh_state* ATTR_UNUSED(m))
+{
+}
+
 /*********** End of Dummy routines ***********/

testcode/perf.c

@@ -233,12 +233,7 @@ perfsetup(struct perfinfo* info)
 			addr_is_ip6(&info->dest, info->destlen)?
 			AF_INET6:AF_INET, SOCK_DGRAM, 0);
 		if(info->io[i].fd == -1) {
-#ifndef USE_WINSOCK
-			fatal_exit("socket: %s", strerror(errno));
-#else
-			fatal_exit("socket: %s", 
-				wsa_strerror(WSAGetLastError()));
-#endif
+			fatal_exit("socket: %s", sock_strerror(errno));
 		}
 		if(info->io[i].fd > info->maxfd)
 			info->maxfd = info->io[i].fd;
@@ -260,11 +255,7 @@ perffree(struct perfinfo* info)
 	if(!info) return;
 	if(info->io) {
 		for(i=0; i<info->io_num; i++) {
-#ifndef USE_WINSOCK
-			close(info->io[i].fd);
-#else
-			closesocket(info->io[i].fd);
-#endif
+			sock_close(info->io[i].fd);
 		}
 		free(info->io);
 	}
@@ -285,11 +276,7 @@ perfsend(struct perfinfo* info, size_t n, struct timeval* now)
 	/*log_hex("send", info->qlist_data[info->qlist_idx],
 		info->qlist_len[info->qlist_idx]);*/
 	if(r == -1) {
-#ifndef USE_WINSOCK
-		log_err("sendto: %s", strerror(errno));
-#else
-		log_err("sendto: %s", wsa_strerror(WSAGetLastError()));
-#endif
+		log_err("sendto: %s", sock_strerror(errno));
 	} else if(r != (ssize_t)info->qlist_len[info->qlist_idx]) {
 		log_err("partial sendto");
 	}
@@ -309,11 +296,7 @@ perfreply(struct perfinfo* info, size_t n, struct timeval* now)
 	r = recv(info->io[n].fd, (void*)sldns_buffer_begin(info->buf),
 		sldns_buffer_capacity(info->buf), 0);
 	if(r == -1) {
-#ifndef USE_WINSOCK
-		log_err("recv: %s", strerror(errno));
-#else
-		log_err("recv: %s", wsa_strerror(WSAGetLastError()));
-#endif
+		log_err("recv: %s", sock_strerror(errno));
 	} else {
 		info->by_rcode[LDNS_RCODE_WIRE(sldns_buffer_begin(
 			info->buf))]++;

testcode/run_vm.sh

@@ -40,6 +40,8 @@ cleanup() {
 	exit 0
 }
 trap cleanup INT
+# stop tests from notifying systemd, if that is compiled in.
+export -n NOTIFY_SOCKET
 
 for t in $RUNLIST
 do

testcode/streamtcp.c

@@ -388,11 +388,7 @@ send_em(const char* svr, int udp, int usessl, int noanswer, int onarrival,
 		SSL_free(ssl);
 		SSL_CTX_free(ctx);
 	}
-#ifndef USE_WINSOCK
-	close(fd);
-#else
-	closesocket(fd);
-#endif
+	sock_close(fd);
 	sldns_buffer_free(buf);
 	printf("orderly exit\n");
 }

testcode/testbound.c

@@ -362,6 +362,10 @@ main(int argc, char* argv[])
 	/* we do not want the test to depend on the timezone */
 	(void)putenv("TZ=UTC");
 	memset(pass_argv, 0, sizeof(pass_argv));
+#ifdef HAVE_SYSTEMD
+	/* we do not want the test to use systemd daemon startup notification*/
+	(void)unsetenv("NOTIFY_SOCKET");
+#endif /* HAVE_SYSTEMD */
 
 	log_init(NULL, 0, NULL);
 	/* determine commandline options for the daemon */

testcode/testpkts.c

@@ -501,7 +501,7 @@ add_edns(uint8_t* pktbuf, size_t pktsize, int do_flag, uint8_t *ednsdata,
 {
 	uint8_t edns[] = {0x00, /* root label */
 		0x00, LDNS_RR_TYPE_OPT, /* type */
-		0x10, 0x00, /* class is UDPSIZE 4096 */
+		0x04, 0xD0, /* class is UDPSIZE 1232 */
 		0x00, /* TTL[0] is ext rcode */
 		0x00, /* TTL[1] is edns version */
 		(uint8_t)(do_flag?0x80:0x00), 0x00, /* TTL[2-3] is edns flags, DO */

testcode/unitldns.c

@@ -44,6 +44,7 @@
 #include "sldns/sbuffer.h"
 #include "sldns/str2wire.h"
 #include "sldns/wire2str.h"
+#include "sldns/parseutil.h"
 
 /** verbose this unit test */
 static int vbmp = 0; 
@@ -220,9 +221,60 @@ rr_tests(void)
 		SRCDIRSTR "/testdata/test_ldnsrr.c5");
 }
 
+/** test various base64 decoding options */
+static void
+b64_test(void)
+{
+	/* "normal" b64 alphabet, with padding */
+	char* p1 = "aGVsbG8="; /* "hello" */
+	char* p2 = "aGVsbG8+"; /* "hello>" */
+	char* p3 = "aGVsbG8/IQ=="; /* "hello?!" */
+	char* p4 = "aGVsbG8"; /* "hel" + extra garbage */
+
+	/* base64 url, without padding */
+	char* u1 = "aGVsbG8"; /* "hello" */
+	char* u2 = "aGVsbG8-"; /* "hello>" */
+	char* u3 = "aGVsbG8_IQ"; /* "hello?!" */
+	char* u4 = "aaaaa"; /* garbage */
+
+	char target[128];
+	size_t tarsize = 128;
+	int result;
+
+	memset(target, 0, sizeof(target));
+	result = sldns_b64_pton(p1, (uint8_t*)target, tarsize);
+	unit_assert(result == strlen("hello") && strcmp(target, "hello") == 0);
+	memset(target, 0, sizeof(target));
+	result = sldns_b64_pton(p2, (uint8_t*)target, tarsize);
+	unit_assert(result == strlen("hello>") && strcmp(target, "hello>") == 0);
+	memset(target, 0, sizeof(target));
+	result = sldns_b64_pton(p3, (uint8_t*)target, tarsize);
+	unit_assert(result == strlen("hello?!") && strcmp(target, "hello?!") == 0);
+	memset(target, 0, sizeof(target));
+	result = sldns_b64_pton(p4, (uint8_t*)target, tarsize);
+	/* when padding is used everything that is not a block of 4 will be
+	 * ignored */
+	unit_assert(result == strlen("hel") && strcmp(target, "hel") == 0);
+
+	memset(target, 0, sizeof(target));
+	result = sldns_b64url_pton(u1, strlen(u1), (uint8_t*)target, tarsize);
+	unit_assert(result == strlen("hello") && strcmp(target, "hello") == 0);
+	memset(target, 0, sizeof(target));
+	result = sldns_b64url_pton(u2, strlen(u2), (uint8_t*)target, tarsize);
+	unit_assert(result == strlen("hello>") && strcmp(target, "hello>") == 0);
+	memset(target, 0, sizeof(target));
+	result = sldns_b64url_pton(u3, strlen(u3), (uint8_t*)target, tarsize);
+	unit_assert(result == strlen("hello+/") && strcmp(target, "hello?!") == 0);
+	/* one item in block of four is not allowed */
+	memset(target, 0, sizeof(target));
+	result = sldns_b64url_pton(u4, strlen(u4), (uint8_t*)target, tarsize);
+	unit_assert(result == -1);
+}
+
 void
 ldns_test(void)
 {
 	unit_show_feature("sldns");
 	rr_tests();
+	b64_test();
 }