From 62e9801acba2f866f6d9961607743bf7fbaf7045 Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Tue, 15 Mar 2016 10:27:41 -0700
Subject: [PATCH 1/9] lkl: Add support for thread_join host op

This will allow us to more gracefully shutdown and wait for host threads
to release resources.

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 arch/lkl/include/uapi/asm/host_ops.h |  6 +++++-
 tools/lkl/lib/nt-host.c              |  8 ++++++++
 tools/lkl/lib/posix-host.c           | 10 +++++++++-
 tools/lkl/tests/boot.c               | 19 +++++++++++++++++++
 4 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/arch/lkl/include/uapi/asm/host_ops.h b/arch/lkl/include/uapi/asm/host_ops.h
index aeb76a55169cf4..1530e9b2b0e1c4 100644
--- a/arch/lkl/include/uapi/asm/host_ops.h
+++ b/arch/lkl/include/uapi/asm/host_ops.h
@@ -36,6 +36,8 @@ typedef unsigned long lkl_thread_t;
  * @thread_detach - on POSIX systems, free up resources held by
  * pthreads. Noop on Win32.
  * @thread_exit - terminates the current thread
+ * @thread_join - wait for the given thread to terminate. Returns 0
+ * for success, -1 otherwise
  *
  * @tls_alloc - allocate a thread local storage key; returns 0 if succesful
  * @tls_free - frees a thread local storage key; returns 0 if succesful
@@ -58,7 +60,8 @@ typedef unsigned long lkl_thread_t;
  * @iomem_acess - reads or writes to and I/O memory region; addr must be in the
  * range returned by ioremap
  *
- * @gettid - returns the host thread id of the caller
+ * @gettid - returns the host thread id of the caller, which need not
+ * be the same as the handle returned by thread_create
  */
 struct lkl_host_operations {
 	const char *virtio_devices;
@@ -80,6 +83,7 @@ struct lkl_host_operations {
 	lkl_thread_t (*thread_create)(void (*f)(void *), void *arg);
 	void (*thread_detach)(void);
 	void (*thread_exit)(void);
+	int (*thread_join)(lkl_thread_t tid);
 
 	int (*tls_alloc)(unsigned int *key);
 	int (*tls_free)(unsigned int key);
diff --git a/tools/lkl/lib/nt-host.c b/tools/lkl/lib/nt-host.c
index 3307ee7897a374..575e9cf5fa175a 100644
--- a/tools/lkl/lib/nt-host.c
+++ b/tools/lkl/lib/nt-host.c
@@ -79,6 +79,13 @@ static void thread_exit(void)
 	ExitThread(0);
 }
 
+static int thread_join(lkl_thread_t tid)
+{
+	/* TODO: error handling */
+	WaitForSingleObject(tid, INFINITE);
+	return 0;
+}
+
 static int tls_alloc(unsigned int *key)
 {
 	*key = TlsAlloc();
@@ -202,6 +209,7 @@ struct lkl_host_operations lkl_host_ops = {
 	.thread_create = thread_create,
 	.thread_detach = thread_detach,
 	.thread_exit = thread_exit,
+	.thread_join = thread_join,
 	.sem_alloc = sem_alloc,
 	.sem_free = sem_free,
 	.sem_up = sem_up,
diff --git a/tools/lkl/lib/posix-host.c b/tools/lkl/lib/posix-host.c
index f3cd0c2a63a4f9..328859b3fba4bb 100644
--- a/tools/lkl/lib/posix-host.c
+++ b/tools/lkl/lib/posix-host.c
@@ -199,6 +199,14 @@ static void thread_exit(void)
 	pthread_exit(NULL);
 }
 
+static int thread_join(lkl_thread_t tid)
+{
+	if (WARN_PTHREAD(pthread_join(tid, NULL)))
+		return -1;
+	else
+		return 0;
+}
+
 static int tls_alloc(unsigned int *key)
 {
 	return pthread_key_create((pthread_key_t*)key, NULL);
@@ -219,7 +227,6 @@ static void *tls_get(unsigned int key)
 	return pthread_getspecific(key);
 }
 
-
 static unsigned long long time_ns(void)
 {
 	struct timeval tv;
@@ -286,6 +293,7 @@ struct lkl_host_operations lkl_host_ops = {
 	.thread_create = thread_create,
 	.thread_detach = thread_detach,
 	.thread_exit = thread_exit,
+	.thread_join = thread_join,
 	.sem_alloc = sem_alloc,
 	.sem_free = sem_free,
 	.sem_up = sem_up,
diff --git a/tools/lkl/tests/boot.c b/tools/lkl/tests/boot.c
index 96f6987af18334..994f0e7b3bbe36 100644
--- a/tools/lkl/tests/boot.c
+++ b/tools/lkl/tests/boot.c
@@ -684,6 +684,24 @@ static int test_syscall_thread(char *str, int len)
 	return TEST_SUCCESS;
 }
 
+void thread_quit_immediately(void *unused)
+{
+}
+
+static int test_join(char *str, int len)
+{
+	lkl_thread_t tid = lkl_host_ops.thread_create(thread_quit_immediately, NULL);
+	int ret = lkl_host_ops.thread_join(tid);
+
+	if (ret == 0) {
+		snprintf(str, len, "joined %ld", tid);
+		return TEST_SUCCESS;
+	} else {
+		snprintf(str, len, "failed joining %ld", tid);
+		return TEST_FAILURE;
+	}
+}
+
 static struct cl_option *find_short_opt(char name)
 {
 	struct cl_option *opt;
@@ -792,6 +810,7 @@ int main(int argc, char **argv)
 	TEST(semaphore);
 	TEST(gettid);
 	TEST(syscall_thread);
+	TEST(join);
 
 	lkl_sys_halt();
 

From f613d54757aec703c908ecb5aabd3cd93c1763bf Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Tue, 15 Mar 2016 11:25:10 -0700
Subject: [PATCH 2/9] lkl: Track registered virtio_net devices

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 tools/lkl/lib/virtio_net.c | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/tools/lkl/lib/virtio_net.c b/tools/lkl/lib/virtio_net.c
index 71b3b85ff9b6c9..cded7b91892194 100644
--- a/tools/lkl/lib/virtio_net.c
+++ b/tools/lkl/lib/virtio_net.c
@@ -14,6 +14,10 @@
 #define NUM_QUEUES (TX_QUEUE_IDX + 1)
 #define QUEUE_DEPTH 32
 
+/* In fact, we'll hit the limit on the devs string below long before
+ * we hit this, but it's good enough for now. */
+#define MAX_NET_DEVS 16
+
 #ifdef DEBUG
 #define bad_request(s) do {			\
 		lkl_printf("%s\n", s);		\
@@ -124,6 +128,22 @@ void poll_thread(void *arg)
 	}
 }
 
+struct virtio_net_dev *registered_devs[MAX_NET_DEVS];
+static int registered_dev_idx = 0;
+
+static int dev_register(struct virtio_net_dev *dev)
+{
+	if (registered_dev_idx == MAX_NET_DEVS) {
+		lkl_printf("Too many virtio_net devices!\n");
+		/* This error code is a little bit of a lie */
+		return -LKL_ENOMEM;
+	} else {
+		/* registered_dev_idx is incremented by the caller */
+		registered_devs[registered_dev_idx] = dev;
+		return 0;
+	}
+}
+
 static void free_queue_locks(struct lkl_mutex_t **queues, int num_queues)
 {
 	int i = 0;
@@ -158,7 +178,6 @@ static struct lkl_mutex_t **init_queue_locks(int num_queues)
 int lkl_netdev_add(struct lkl_netdev *nd, void *mac)
 {
 	struct virtio_net_dev *dev;
-	static int count;
 	int ret = -LKL_ENOMEM;
 
 	dev = lkl_host_ops.mem_alloc(sizeof(*dev));
@@ -204,9 +223,11 @@ int lkl_netdev_add(struct lkl_netdev *nd, void *mac)
 	if (lkl_host_ops.thread_create(poll_thread, &dev->tx_poll) == 0)
 		goto out_cleanup_dev;
 
-	/* RX/TX thread polls will exit when the host netdev handle is closed */
+	ret = dev_register(dev);
+	if (ret < 0)
+		goto out_cleanup_dev;
 
-	return count++;
+	return registered_dev_idx++;
 
 out_cleanup_dev:
 	virtio_dev_cleanup(&dev->dev);

From 240ed3773a710725fba52711cb7a39d82e638115 Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Tue, 15 Mar 2016 11:58:16 -0700
Subject: [PATCH 3/9] lkl: Add support for netdev cleanup

This commit sets up a framework that allows the various kinds of net
devices to release their resources, provided that they define a "close"
operation. It does not define any close operations, which will be
included in future commits.

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 tools/lkl/include/lkl.h        | 11 ++++++++++
 tools/lkl/include/lkl_host.h   |  4 ++++
 tools/lkl/lib/hijack/init.c    |  1 +
 tools/lkl/lib/virtio_net.c     | 37 ++++++++++++++++++++++++++++++++++
 tools/lkl/tests/hijack-test.sh |  3 +++
 5 files changed, 56 insertions(+)

diff --git a/tools/lkl/include/lkl.h b/tools/lkl/include/lkl.h
index c17f6d6018090c..bff150cd3d9943 100644
--- a/tools/lkl/include/lkl.h
+++ b/tools/lkl/include/lkl.h
@@ -228,6 +228,17 @@ struct lkl_netdev {
  */
 int lkl_netdev_add(struct lkl_netdev *nd, void *mac);
 
+/**
+* lkl_netdevs_remove - destroy all network devices
+*
+* Attempts to release all resources held by network devices created
+* via lkl_netdev_add.
+*
+* @returns 0 if all devices are successfully removed, -1 if at least
+* one fails.
+*/
+int lkl_netdevs_remove(void);
+
 /**
  * lkl_netdev_get_ifindex - retrieve the interface index for a given network
  * device id
diff --git a/tools/lkl/include/lkl_host.h b/tools/lkl/include/lkl_host.h
index eded31762e9c24..198605c421bf38 100644
--- a/tools/lkl/include/lkl_host.h
+++ b/tools/lkl/include/lkl_host.h
@@ -53,6 +53,10 @@ struct lkl_dev_net_ops {
 #define LKL_DEV_NET_POLL_RX		1
 #define LKL_DEV_NET_POLL_TX		2
 	int (*poll)(struct lkl_netdev *nd, int events);
+	/* Release all resources acquired --- in particular, kill the
+	 * polling threads and close any open handles. Not implemented
+	 * by all netdev types. 0 for success, -1 for failure. */
+	int (*close)(struct lkl_netdev *nd);
 };
 
 #ifdef __cplusplus
diff --git a/tools/lkl/lib/hijack/init.c b/tools/lkl/lib/hijack/init.c
index 19e15eece49232..31da515f47ef53 100644
--- a/tools/lkl/lib/hijack/init.c
+++ b/tools/lkl/lib/hijack/init.c
@@ -276,4 +276,5 @@ hijack_fini(void)
 
 
 	lkl_sys_halt();
+	lkl_netdevs_remove();
 }
diff --git a/tools/lkl/lib/virtio_net.c b/tools/lkl/lib/virtio_net.c
index cded7b91892194..748915117b6740 100644
--- a/tools/lkl/lib/virtio_net.c
+++ b/tools/lkl/lib/virtio_net.c
@@ -239,3 +239,40 @@ int lkl_netdev_add(struct lkl_netdev *nd, void *mac)
 
 	return ret;
 }
+
+/* Return 0 for success, -1 for failure. */
+static int lkl_netdev_remove(struct virtio_net_dev *dev)
+{
+	if (!dev->nd->ops->close)
+		/* Can't kill the poll threads, so we can't do
+		 * anything safely. */
+		return -1;
+
+	if (dev->nd->ops->close(dev->nd) < 0)
+		/* Something went wrong */
+		return -1;
+
+	virtio_dev_cleanup(&dev->dev);
+
+	lkl_host_ops.mem_free(dev->nd);
+	free_queue_locks(dev->queue_locks, NUM_QUEUES);
+	lkl_host_ops.mem_free(dev);
+
+	return 0;
+}
+
+int lkl_netdevs_remove(void)
+{
+	int i = 0, failure_count = 0;
+
+	for (; i < registered_dev_idx; i++)
+		failure_count -= lkl_netdev_remove(registered_devs[i]);
+
+	if (failure_count) {
+		lkl_printf("WARN: failed to free %d of %d netdevs.\n",
+			failure_count, registered_dev_idx);
+		return -1;
+	}
+
+	return 0;
+}
diff --git a/tools/lkl/tests/hijack-test.sh b/tools/lkl/tests/hijack-test.sh
index c787b039264b6b..1c410c96a60b98 100755
--- a/tools/lkl/tests/hijack-test.sh
+++ b/tools/lkl/tests/hijack-test.sh
@@ -53,6 +53,9 @@ ans=$(LKL_HIJACK_MOUNT=proc,sysfs\
 echo "$ans" | tail -n 15 | grep "65536" # lo's MTU
 # lo's dev id
 echo "$ans" | grep "0x0"        # lo's dev_id
+# Doesn't really belong in this section, but might as well check for
+# it here.
+! echo "$ans" | grep "WARN: failed to free"
 
 echo "== TAP tests =="
 if [ ! -c /dev/net/tun ]; then

From df4fb628dfdc9a9d1d499a1bedb1c02e63987f2a Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Tue, 15 Mar 2016 13:51:36 -0700
Subject: [PATCH 4/9] lkl: Track tids for poll threads

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 tools/lkl/include/lkl.h        | 7 ++-----
 tools/lkl/include/lkl_host.h   | 5 +++++
 tools/lkl/lib/virtio_net.c     | 9 +++++----
 tools/lkl/lib/virtio_net_tap.c | 2 ++
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/tools/lkl/include/lkl.h b/tools/lkl/include/lkl.h
index bff150cd3d9943..483badee6f9e32 100644
--- a/tools/lkl/include/lkl.h
+++ b/tools/lkl/include/lkl.h
@@ -209,12 +209,9 @@ int lkl_if_set_ipv4(int ifindex, unsigned int addr, unsigned int netmask_len);
 int lkl_set_ipv4_gateway(unsigned int addr);
 
 /**
- * lkl_netdev - host network device handle
+ * lkl_netdev - host network device handle, defined in virtio.h.
  */
-struct lkl_netdev {
-	struct lkl_dev_net_ops *ops;
-};
-
+struct lkl_netdev;
 
 /**
  * lkl_netdev_add - add a new network device
diff --git a/tools/lkl/include/lkl_host.h b/tools/lkl/include/lkl_host.h
index 198605c421bf38..36b2f546bb2e06 100644
--- a/tools/lkl/include/lkl_host.h
+++ b/tools/lkl/include/lkl_host.h
@@ -47,6 +47,11 @@ struct lkl_dev_blk_ops {
 	int (*request)(union lkl_disk disk, struct lkl_blk_req *req);
 };
 
+struct lkl_netdev {
+	struct lkl_dev_net_ops *ops;
+	lkl_thread_t rx_tid, tx_tid;
+};
+
 struct lkl_dev_net_ops {
 	int (*tx)(struct lkl_netdev *nd, void *data, int len);
 	int (*rx)(struct lkl_netdev *nd, void *data, int *len);
diff --git a/tools/lkl/lib/virtio_net.c b/tools/lkl/lib/virtio_net.c
index 748915117b6740..4da1e66ad84e88 100644
--- a/tools/lkl/lib/virtio_net.c
+++ b/tools/lkl/lib/virtio_net.c
@@ -27,7 +27,6 @@
 #define bad_request(s) lkl_printf("virtio_net: %s\n", s);
 #endif /* DEBUG */
 
-
 struct virtio_net_poll {
 	struct virtio_net_dev *dev;
 	int event;
@@ -213,14 +212,16 @@ int lkl_netdev_add(struct lkl_netdev *nd, void *mac)
 	 * this, but netdevs are the only flavor that need these
 	 * locks, so it's better to do it here. */
 	ret = virtio_dev_setup(&dev->dev, NUM_QUEUES, QUEUE_DEPTH);
-
+ 
 	if (ret)
 		goto out_free;
 
-	if (lkl_host_ops.thread_create(poll_thread, &dev->rx_poll) == 0)
+	nd->rx_tid = lkl_host_ops.thread_create(poll_thread, &dev->rx_poll);
+	if (nd->rx_tid == 0)
 		goto out_cleanup_dev;
 
-	if (lkl_host_ops.thread_create(poll_thread, &dev->tx_poll) == 0)
+	nd->tx_tid = lkl_host_ops.thread_create(poll_thread, &dev->tx_poll);
+	if (nd->tx_tid == 0)
 		goto out_cleanup_dev;
 
 	ret = dev_register(dev);
diff --git a/tools/lkl/lib/virtio_net_tap.c b/tools/lkl/lib/virtio_net_tap.c
index 8e87e0d74c2193..cdb780d5c88f6d 100644
--- a/tools/lkl/lib/virtio_net_tap.c
+++ b/tools/lkl/lib/virtio_net_tap.c
@@ -24,6 +24,8 @@
 
 #include <lkl_host.h>
 
+#include "virtio.h"
+
 struct lkl_netdev_tap {
 	struct lkl_netdev dev;
 	int fd;

From 7e04c94205e45007bca28ce9d359b2a8c90427b0 Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Wed, 16 Mar 2016 10:40:55 -0700
Subject: [PATCH 5/9] lkl: Support closing TAP netdevs under Linux

Because this process depends heavily on device-specific and
host-specific operations, this commit only supports full cleanup for a
small subset of possible host/device combinations.

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 tools/lkl/include/lkl.h        |   2 +-
 tools/lkl/lib/hijack/init.c    |  14 ++++
 tools/lkl/lib/virtio_net.c     |   2 +-
 tools/lkl/lib/virtio_net_tap.c | 144 ++++++++++++++++++++++++++++-----
 tools/lkl/lib/virtio_net_tap.h |  10 +++
 tools/lkl/tests/hijack-test.sh |   4 +-
 6 files changed, 155 insertions(+), 21 deletions(-)
 create mode 100644 tools/lkl/lib/virtio_net_tap.h

diff --git a/tools/lkl/include/lkl.h b/tools/lkl/include/lkl.h
index 483badee6f9e32..b000286d0dc33c 100644
--- a/tools/lkl/include/lkl.h
+++ b/tools/lkl/include/lkl.h
@@ -209,7 +209,7 @@ int lkl_if_set_ipv4(int ifindex, unsigned int addr, unsigned int netmask_len);
 int lkl_set_ipv4_gateway(unsigned int addr);
 
 /**
- * lkl_netdev - host network device handle, defined in virtio.h.
+ * lkl_netdev - host network device handle, defined in lkl_host.h.
  */
 struct lkl_netdev;
 
diff --git a/tools/lkl/lib/hijack/init.c b/tools/lkl/lib/hijack/init.c
index 31da515f47ef53..b46927f1d1b955 100644
--- a/tools/lkl/lib/hijack/init.c
+++ b/tools/lkl/lib/hijack/init.c
@@ -27,6 +27,10 @@
 #include <lkl_host.h>
 
 #include "xlate.h"
+#include "../virtio_net_tap.h"
+
+#define __USE_GNU
+#include <dlfcn.h>
 
 /* Mount points are named after filesystem types so they should never
  * be longer than ~6 characters. */
@@ -139,6 +143,13 @@ static void mount_cmds_exec(char *_cmds, int (*callback)(char*))
 	free(cmds);
 }
 
+void fixup_netdev_tap_ops(void)
+{
+	/* It's okay if this is NULL, because then netdev close will
+	 * fall back onto an uncloseable implementation. */
+	lkl_netdev_tap_ops.eventfd = dlsym(RTLD_NEXT, "eventfd");
+}
+
 void __attribute__((constructor(102)))
 hijack_init(void)
 {
@@ -157,6 +168,9 @@ hijack_init(void)
 	char *mount = getenv("LKL_HIJACK_MOUNT");
 	struct lkl_netdev *nd = NULL;
 
+	/* Must be run before lkl_netdev_tap_create */
+	fixup_netdev_tap_ops();
+
 	if (tap) {
 		fprintf(stderr,
 			"WARN: variable LKL_HIJACK_NET_TAP is now obsoleted.\n"
diff --git a/tools/lkl/lib/virtio_net.c b/tools/lkl/lib/virtio_net.c
index 4da1e66ad84e88..349f11656daa67 100644
--- a/tools/lkl/lib/virtio_net.c
+++ b/tools/lkl/lib/virtio_net.c
@@ -212,7 +212,7 @@ int lkl_netdev_add(struct lkl_netdev *nd, void *mac)
 	 * this, but netdevs are the only flavor that need these
 	 * locks, so it's better to do it here. */
 	ret = virtio_dev_setup(&dev->dev, NUM_QUEUES, QUEUE_DEPTH);
- 
+
 	if (ret)
 		goto out_free;
 
diff --git a/tools/lkl/lib/virtio_net_tap.c b/tools/lkl/lib/virtio_net_tap.c
index cdb780d5c88f6d..575df983eb78a4 100644
--- a/tools/lkl/lib/virtio_net_tap.c
+++ b/tools/lkl/lib/virtio_net_tap.c
@@ -21,14 +21,30 @@
 #include <net/if.h>
 #include <linux/if_tun.h>
 #include <sys/ioctl.h>
+#include <sys/eventfd.h>
+#include <linux/unistd.h>
 
 #include <lkl_host.h>
 
 #include "virtio.h"
+#include "virtio_net_tap.h"
 
 struct lkl_netdev_tap {
 	struct lkl_netdev dev;
+	/* TAP device */
 	int fd;
+	/* Needed to initiate shutdown */
+	int eventfd;
+};
+
+struct lkl_netdev_tap_ops lkl_netdev_tap_ops = {
+	/* /dev/net/tun is Linux specific so we know our host is some
+	 * flavor of Linux, but this allows graceful support if we're
+	 * on a kernel that's < 2.6.22. */
+	#ifdef __NR_eventfd
+	/* This sigature was recently (9/2014) changed in glibc. */
+	.eventfd = (int (*)(unsigned int, int))eventfd,
+	#endif /* __NR_eventfd */
 };
 
 static int net_tx(struct lkl_netdev *nd, void *data, int len)
@@ -54,40 +70,115 @@ static int net_rx(struct lkl_netdev *nd, void *data, int *len)
 	return 0;
 }
 
-static int net_poll(struct lkl_netdev *nd, int events)
+static inline int pfds_should_close(struct pollfd *pfds, int closeable)
 {
-	struct lkl_netdev_tap *nd_tap = (struct lkl_netdev_tap *) nd;
+	if (pfds[0].revents & (POLLHUP | POLLNVAL))
+		return 1;
 
-	struct pollfd pfd = {
-		.fd = nd_tap->fd,
-	};
-	int ret = 0;
+	if (closeable && pfds[1].revents & POLLIN)
+		return 1;
+
+	return 0;
+}
+
+/* pfds must be zeroed out by the caller */
+static inline void config_pfds(struct pollfd *pfds, struct lkl_netdev_tap *nd_tap,
+			int events, int closeable)
+{
+	pfds[0].fd = nd_tap->fd;
 
 	if (events & LKL_DEV_NET_POLL_RX)
-		pfd.events |= POLLIN | POLLPRI;
+		pfds[0].events |= POLLIN | POLLPRI;
 	if (events & LKL_DEV_NET_POLL_TX)
-		pfd.events |= POLLOUT;
+		pfds[0].events |= POLLOUT;
 
-	while (poll(&pfd, 1, -1) < 0 && errno == EINTR)
-		;
+	if (closeable) {
+		pfds[1].fd = nd_tap->eventfd;
+		pfds[1].events = POLLIN;
+	}
+}
 
-	if (pfd.revents & (POLLHUP | POLLNVAL))
+static inline void poll_pfds(struct pollfd *pfds, int closeable)
+{
+	while (poll(pfds, closeable ? 2 : 1, -1) < 0 && errno == EINTR)
+		; 		/* spin */
+}
+
+static int net_poll(struct lkl_netdev *nd, int events, int closeable)
+{
+	struct lkl_netdev_tap *nd_tap = container_of(nd, struct lkl_netdev_tap, dev);
+	struct pollfd pfds[2] = {{0}};
+	int ret = 0;
+
+	config_pfds(pfds, nd_tap, events, closeable);
+
+	poll_pfds(pfds, closeable);
+
+	if (pfds_should_close(pfds, closeable))
 		return -1;
 
-	if (pfd.revents & POLLIN)
+	if (pfds[0].revents & POLLIN)
 		ret |= LKL_DEV_NET_POLL_RX;
-	if (pfd.revents & POLLOUT)
+	if (pfds[0].revents & POLLOUT)
 		ret |= LKL_DEV_NET_POLL_TX;
 
 	return ret;
 }
 
-struct lkl_dev_net_ops tap_net_ops = {
+static int net_poll_uncloseable(struct lkl_netdev *nd, int events)
+{
+	return net_poll(nd, events, 0);
+}
+
+static int net_poll_closeable(struct lkl_netdev *nd, int events)
+{
+	return net_poll(nd, events, 1);
+}
+
+
+/* Passing the virtio_net_dev in here is a little bit of a hack, but
+ * we need access to those tids and this is the easiest way to get
+ * it. */
+static int net_close(struct lkl_netdev *nd)
+{
+	long buf = 1;
+	struct lkl_netdev_tap *nd_tap = container_of(nd, struct lkl_netdev_tap, dev);
+
+	if (write(nd_tap->eventfd, &buf, sizeof(buf)) < 0) {
+		perror("tap: failed to close tap");
+		/* This should never happen. */
+		return -1;
+	}
+
+	/* The order that we join in doesn't matter. */
+	if (lkl_host_ops.thread_join(nd->rx_tid) ||
+		lkl_host_ops.thread_join(nd->tx_tid))
+		return -1;
+
+	/* nor does the order that we close */
+	if (close(nd_tap->fd) || close(nd_tap->eventfd)) {
+		perror("tap net_close TAP fd");
+		return -1;
+	}
+
+	return 0;
+}
+
+struct lkl_dev_net_ops tap_net_ops_uncloseable = {
+	.tx = net_tx,
+	.rx = net_rx,
+	.poll = net_poll_uncloseable,
+};
+
+
+struct lkl_dev_net_ops tap_net_ops_closeable = {
 	.tx = net_tx,
 	.rx = net_rx,
-	.poll = net_poll,
+	.poll = net_poll_closeable,
+	.close = net_close,
 };
 
+
 struct lkl_netdev *lkl_netdev_tap_create(const char *ifname)
 {
 	struct lkl_netdev_tap *nd;
@@ -99,7 +190,6 @@ struct lkl_netdev *lkl_netdev_tap_create(const char *ifname)
 		/* TODO: propagate the error state, maybe use errno for that? */
 		return NULL;
 	}
-	nd->dev.ops = &tap_net_ops;
 
 	struct ifreq ifr = {
 		.ifr_flags = IFF_TAP | IFF_NO_PI,
@@ -109,8 +199,7 @@ struct lkl_netdev *lkl_netdev_tap_create(const char *ifname)
 
 	nd->fd = open("/dev/net/tun", O_RDWR|O_NONBLOCK);
 	if (nd->fd < 0) {
-		fprintf(stderr, "tap: failed to open tap: %s\n",
-			strerror(errno));
+		perror("tap: failed to open tap");
 		free(nd);
 		return NULL;
 	}
@@ -124,5 +213,24 @@ struct lkl_netdev *lkl_netdev_tap_create(const char *ifname)
 		return NULL;
 	}
 
+	if (lkl_netdev_tap_ops.eventfd) {
+		/* eventfd is supported by the host, all is well */
+		nd->dev.ops = &tap_net_ops_closeable;
+		nd->eventfd = lkl_netdev_tap_ops.eventfd(
+			0, EFD_NONBLOCK | EFD_SEMAPHORE);
+
+		if (nd->eventfd < 0) {
+			perror("lkl_netdev_tap_create eventfd");
+			close(nd->fd);
+			free(nd);
+			return NULL;
+		}
+	} else {
+		/* no host eventfd support */
+		nd->dev.ops = &tap_net_ops_uncloseable;
+		nd->eventfd = -1;
+	}
+
+
 	return (struct lkl_netdev *)nd;
 }
diff --git a/tools/lkl/lib/virtio_net_tap.h b/tools/lkl/lib/virtio_net_tap.h
new file mode 100644
index 00000000000000..4069c7c4fd0ccd
--- /dev/null
+++ b/tools/lkl/lib/virtio_net_tap.h
@@ -0,0 +1,10 @@
+#ifndef _VIRTIO_NET_TAP_H
+#define _VIRTIO_NET_TAP_H
+
+extern struct lkl_netdev_tap_ops {
+	/* We need this so that we can "unhijack" this function in
+	 * case we decided to hijack it. */
+	int (*eventfd)(unsigned int initval, int flags);
+} lkl_netdev_tap_ops;
+
+#endif /* _VIRTIO_NET_TAP_H*/
diff --git a/tools/lkl/tests/hijack-test.sh b/tools/lkl/tests/hijack-test.sh
index 1c410c96a60b98..9c9f0a041eb114 100755
--- a/tools/lkl/tests/hijack-test.sh
+++ b/tools/lkl/tests/hijack-test.sh
@@ -75,10 +75,12 @@ sudo ip link set dev lkl_ptt0 up
 sudo ip addr add dev lkl_ptt0 192.168.13.1/24
 
 # Make sure our device has the addresses we expect
-addr=$(LKL_HIJACK_NET_MAC="aa:bb:cc:dd:ee:ff" ${hijack_script} ip addr) 
+addr=$(LKL_HIJACK_DEBUG=1\
+  LKL_HIJACK_NET_MAC="aa:bb:cc:dd:ee:ff" ${hijack_script} ip addr) 
 echo "$addr" | grep eth0
 echo "$addr" | grep 192.168.13.2
 echo "$addr" | grep "aa:bb:cc:dd:ee:ff"
+! echo "$addr" | grep "WARN: failed to free"
 
 # Copy ping so we're allowed to run it under LKL
 cp `which ping` .

From 2d040105220e13f2db2a077a97266d3dd3b1ef91 Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Tue, 15 Mar 2016 15:54:13 -0700
Subject: [PATCH 6/9] lkl: Free initial memory

Now that we have a way to release the host resources and kill the host
threads tied up in our netdevs, we can undo a previous regression that
prevented us from freeing the memory allocated in init_mem.

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 arch/lkl/include/uapi/asm/host_ops.h |  4 ++--
 arch/lkl/kernel/setup.c              | 14 ++++++++------
 tools/lkl/lib/hijack/init.c          |  1 -
 tools/lkl/lib/virtio_net_tap.c       |  5 -----
 4 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/arch/lkl/include/uapi/asm/host_ops.h b/arch/lkl/include/uapi/asm/host_ops.h
index 1530e9b2b0e1c4..dabb7295886e29 100644
--- a/arch/lkl/include/uapi/asm/host_ops.h
+++ b/arch/lkl/include/uapi/asm/host_ops.h
@@ -117,7 +117,7 @@ struct lkl_host_operations {
  * generate the Linux kernel command line
  */
 int lkl_start_kernel(struct lkl_host_operations *lkl_ops,
-		     unsigned long mem_size,
-		     const char *cmd_line, ...);
+		    unsigned long mem_size,
+		    const char *cmd_line, ...);
 
 #endif
diff --git a/arch/lkl/kernel/setup.c b/arch/lkl/kernel/setup.c
index 5e3e859f14c7f2..be38edd13d47eb 100644
--- a/arch/lkl/kernel/setup.c
+++ b/arch/lkl/kernel/setup.c
@@ -50,8 +50,8 @@ static void __init lkl_run_kernel(void *arg)
 }
 
 int __init lkl_start_kernel(struct lkl_host_operations *ops,
-			    unsigned long _mem_size,
-			    const char *fmt, ...)
+			unsigned long _mem_size,
+			const char *fmt, ...)
 {
 	va_list ap;
 	int ret;
@@ -117,6 +117,8 @@ void machine_restart(char *unused)
 	machine_halt();
 }
 
+extern int lkl_netdevs_remove(void);
+
 long lkl_sys_halt(void)
 {
 	long err;
@@ -139,6 +141,10 @@ long lkl_sys_halt(void)
 	lkl_ops->sem_free(init_sem);
 
 	free_initial_syscall_thread();
+	if (lkl_netdevs_remove() == 0)
+		/* We know that there is nothing else touching our
+		 * memory. */
+		free_mem();
 
 	return 0;
 }
@@ -147,10 +153,6 @@ void arch_cpu_idle(void)
 {
 	if (halt) {
 		threads_cleanup();
-		/* TODO(pscollins): If we free here, it causes a
-		 * segfault because the tx/rx threads are still
-		 * running in parallel. */
-		/* free_mem(); */
 
 		/* Shutdown the clockevents source. */
 		tick_suspend_local();
diff --git a/tools/lkl/lib/hijack/init.c b/tools/lkl/lib/hijack/init.c
index b46927f1d1b955..8a63aa6a9c1951 100644
--- a/tools/lkl/lib/hijack/init.c
+++ b/tools/lkl/lib/hijack/init.c
@@ -290,5 +290,4 @@ hijack_fini(void)
 
 
 	lkl_sys_halt();
-	lkl_netdevs_remove();
 }
diff --git a/tools/lkl/lib/virtio_net_tap.c b/tools/lkl/lib/virtio_net_tap.c
index 575df983eb78a4..64de831c8efafa 100644
--- a/tools/lkl/lib/virtio_net_tap.c
+++ b/tools/lkl/lib/virtio_net_tap.c
@@ -135,10 +135,6 @@ static int net_poll_closeable(struct lkl_netdev *nd, int events)
 	return net_poll(nd, events, 1);
 }
 
-
-/* Passing the virtio_net_dev in here is a little bit of a hack, but
- * we need access to those tids and this is the easiest way to get
- * it. */
 static int net_close(struct lkl_netdev *nd)
 {
 	long buf = 1;
@@ -178,7 +174,6 @@ struct lkl_dev_net_ops tap_net_ops_closeable = {
 	.close = net_close,
 };
 
-
 struct lkl_netdev *lkl_netdev_tap_create(const char *ifname)
 {
 	struct lkl_netdev_tap *nd;

From f2347c77f36c2a33e89a5a4edb5faa9a817a60e9 Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Fri, 18 Mar 2016 10:59:39 -0700
Subject: [PATCH 7/9] lkl: Remove excess whitespace from hijack/init.c

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 tools/lkl/lib/hijack/init.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tools/lkl/lib/hijack/init.c b/tools/lkl/lib/hijack/init.c
index 8a63aa6a9c1951..40b907be7dc1cf 100644
--- a/tools/lkl/lib/hijack/init.c
+++ b/tools/lkl/lib/hijack/init.c
@@ -1,5 +1,3 @@
-
-
 /*
  * system calls hijack code
  * Copyright (c) 2015 Hajime Tazaki

From b957f8015b33674ed78f0830fcab6bc6b6f14d36 Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Fri, 18 Mar 2016 13:52:11 -0700
Subject: [PATCH 8/9] lkl: Add explicit cast from HANDLE to lkl_thread_t

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 tools/lkl/lib/nt-host.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/lkl/lib/nt-host.c b/tools/lkl/lib/nt-host.c
index 575e9cf5fa175a..77830133eb32e7 100644
--- a/tools/lkl/lib/nt-host.c
+++ b/tools/lkl/lib/nt-host.c
@@ -67,7 +67,7 @@ static lkl_thread_t thread_create(void (*fn)(void *), void *arg)
 {
 	DWORD WINAPI (*win_fn)(LPVOID arg) = (DWORD WINAPI (*)(LPVOID))fn;
 
-	return CreateThread(NULL, 0, win_fn, arg, 0, NULL);
+	return (lkl_thread_t)CreateThread(NULL, 0, win_fn, arg, 0, NULL);
 }
 
 static void thread_detach(void)

From 61475a363dc73412d3c594b432f9d22ad6fe90df Mon Sep 17 00:00:00 2001
From: Patrick Collins <pscollins@google.com>
Date: Fri, 18 Mar 2016 13:55:48 -0700
Subject: [PATCH 9/9] lkl: Resolve netdevs_remove linker error on Win32

Signed-off-by: Patrick Collins <pscollins@google.com>
---
 tools/lkl/lib/nt-host.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tools/lkl/lib/nt-host.c b/tools/lkl/lib/nt-host.c
index 77830133eb32e7..20374cb43dd643 100644
--- a/tools/lkl/lib/nt-host.c
+++ b/tools/lkl/lib/nt-host.c
@@ -309,3 +309,10 @@ struct lkl_dev_blk_ops lkl_dev_blk_ops = {
 	.get_capacity = handle_get_capacity,
 	.request = blk_request,
 };
+
+/* Needed to resolve linker error on Win32. We don't really support
+ * any network IO on Windows, anyway, so there's no loss here. */
+int lkl_netdevs_remove(void)
+{
+	return -1;
+}