Replace the service by extra hub handlers

CONTRIBUTING.md

@@ -42,7 +42,7 @@ npm -g install configurable-http-proxy
 User environments are built with `repo2docker` running in a Docker container. To pull the Docker image:
 
 ```bash
-docker pull jupyter/repo2docker
+docker pull jupyter/repo2docker:master
 ```
 
 ## Run

setup.py

@@ -6,5 +6,5 @@
     entry_points={"tljh": ["tljh_repo2docker = tljh_repo2docker"]},
     packages=find_packages(),
     include_package_data=True,
-    install_requires=["dockerspawner", "jupyter_client", "docker"],
+    install_requires=["dockerspawner", "jupyter_client", "aiodocker"],
 )

tljh_repo2docker/__init__.py

@@ -1,19 +1,17 @@
 import os
-import sys
-
-from concurrent.futures import ThreadPoolExecutor
 
 from dockerspawner import DockerSpawner
 from jinja2 import Environment, BaseLoader
 from jupyter_client.localinterfaces import public_ips
+from jupyterhub.handlers.static import CacheControlStaticFilesHandler
 from jupyterhub.traitlets import ByteSpecification
 from tljh.hooks import hookimpl
 from tljh.configurer import load_config
-from tornado.ioloop import IOLoop
 from traitlets import Unicode
 from traitlets.config import Configurable
 
-from .images import list_images, client
+from .builder import BuildHandler
+from .images import list_images, docker, ImagesHandler
 
 # Default CPU period
 # See: https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory#configure-the-default-cfs-scheduler
@@ -22,14 +20,6 @@
 
 class SpawnerMixin(Configurable):
 
-    _docker_executor = None
-
-    def _run_in_executor(self, func, *args):
-        cls = self.__class__
-        if cls._docker_executor is None:
-            cls._docker_executor = ThreadPoolExecutor(1)
-        return IOLoop.current().run_in_executor(cls._docker_executor, func, *args)
-
     """
     Mixin for spawners that derive from DockerSpawner, to use local Docker images
     built with tljh-repo2docker.
@@ -103,7 +93,7 @@ async def list_images(self):
         """
         Return the list of available images
         """
-        return await self._run_in_executor(list_images)
+        return await list_images()
 
     async def get_options_form(self):
         """
@@ -138,9 +128,9 @@ async def set_limits(self):
         Set the user environment limits if they are defined in the image
         """
         imagename = self.user_options.get("image")
-        image = await self._run_in_executor(client.images.get, imagename)
-        mem_limit = image.labels.get("tljh_repo2docker.mem_limit", None)
-        cpu_limit = image.labels.get("tljh_repo2docker.cpu_limit", None)
+        image = await docker.images.inspect(imagename)
+        mem_limit = image["ContainerConfig"]["Labels"].get("tljh_repo2docker.mem_limit", None)
+        cpu_limit = image["ContainerConfig"]["Labels"].get("tljh_repo2docker.cpu_limit", None)
 
         # override the spawner limits if defined in the image
         if mem_limit:
@@ -187,21 +177,21 @@ def tljh_custom_jupyterhub_config(c):
     cpu_limit = limits["cpu"]
     mem_limit = limits["memory"]
 
-    # register the service to manage the user images
-    c.JupyterHub.services.append(
-        {
-            "name": "environments",
-            "admin": True,
-            "url": "http://127.0.0.1:9988",
-            "command": [
-                sys.executable,
-                "-m",
-                "tljh_repo2docker.images",
-                f"--default-mem-limit={mem_limit}",
-                f"--default-cpu-limit={cpu_limit}",
-            ],
-        }
-    )
+    c.JupyterHub.tornado_settings.update({
+        'default_cpu_limit': cpu_limit,
+        'default_mem_limit': mem_limit
+    })
+
+    # register the handlers to manage the user images
+    c.JupyterHub.extra_handlers.extend([
+        (r"environments", ImagesHandler),
+        (r"api/environments", BuildHandler),
+        (
+            r"environments-static/(.*)",
+            CacheControlStaticFilesHandler,
+            {"path": os.path.join(os.path.dirname(__file__), "static")},
+        ),
+    ])
 
 
 @hookimpl

tljh_repo2docker/builder.py

@@ -1,134 +1,40 @@
 import json
 import re
 
-from concurrent.futures import ThreadPoolExecutor
-from http.client import responses
 from urllib.parse import urlparse
 
-import docker
+from aiodocker import Docker, DockerError
+from jupyterhub.apihandlers import APIHandler
+from jupyterhub.utils import admin_only
+from tornado import web
 
-from jupyterhub.services.auth import HubAuthenticated
-from tornado import web, escape
-from tornado.concurrent import run_on_executor
-from tornado.log import app_log
-
-client = docker.from_env()
+docker = Docker()
 
 IMAGE_NAME_RE = r"^[a-z0-9-_]+$"
 
 
-def build_image(repo, ref, name="", memory=None, cpu=None):
-    """
-    Build an image given a repo, ref and limits
+class BuildHandler(APIHandler):
     """
-    ref = ref or "master"
-    if len(ref) >= 40:
-        ref = ref[:7]
-
-    # default to the repo name if no name specified
-    # and sanitize the name of the docker image
-    name = name or urlparse(repo).path.strip("/")
-    name = name.replace("/", "-")
-    image_name = f"{name}:{ref}"
-
-    # memory is specified in GB
-    memory = f"{memory}G" if memory else ""
-    cpu = cpu or ""
-
-    # add extra labels to set additional image properties
-    labels = [
-        f"LABEL tljh_repo2docker.display_name={name}",
-        f"LABEL tljh_repo2docker.image_name={image_name}",
-        f"LABEL tljh_repo2docker.mem_limit={memory}",
-        f"LABEL tljh_repo2docker.cpu_limit={cpu}",
-    ]
-    cmd = [
-        "jupyter-repo2docker",
-        "--ref",
-        ref,
-        "--user-name",
-        "jovyan",
-        "--user-id",
-        "1100",
-        "--no-run",
-        "--image-name",
-        image_name,
-        "--appendix",
-        "\n".join(labels),
-        repo,
-    ]
-    client.containers.run(
-        "jupyter/repo2docker:master",
-        cmd,
-        labels={
-            "repo2docker.repo": repo,
-            "repo2docker.ref": ref,
-            "repo2docker.build": image_name,
-            "tljh_repo2docker.display_name": name,
-            "tljh_repo2docker.mem_limit": memory,
-            "tljh_repo2docker.cpu_limit": cpu,
-        },
-        volumes={
-            "/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"}
-        },
-        detach=True,
-        remove=True,
-    )
-
-
-def remove_image(name):
+    Handle requests to build user environments as Docker images
     """
-    Remove an image by name
-    """
-    client.images.remove(name)
-
-
-class BuildHandler(HubAuthenticated, web.RequestHandler):
-
-    executor = ThreadPoolExecutor(max_workers=5)
-
-    def initialize(self):
-        self.log = app_log
-
-    def write_error(self, status_code, **kwargs):
-        exc_info = kwargs.get("exc_info")
-        message = ""
-        exception = None
-        status_message = responses.get(status_code, "Unknown Error")
-        if exc_info:
-            exception = exc_info[1]
-            try:
-                message = exception.log_message % exception.args
-            except Exception:
-                pass
-
-            reason = getattr(exception, "reason", "")
-            if reason:
-                status_message = reason
-
-        self.set_header("Content-Type", "application/json")
-        self.write(
-            json.dumps({"status": status_code, "message": message or status_message})
-        )
 
     @web.authenticated
-    @run_on_executor
-    def delete(self):
-        data = escape.json_decode(self.request.body)
+    @admin_only
+    async def delete(self):
+        data = self.get_json_body()
         name = data["name"]
         try:
-            remove_image(name)
-        except docker.errors.ImageNotFound:
-            raise web.HTTPError(400, f"Image {name} does not exist")
-        except docker.errors.APIError as e:
-            raise web.HTTPError(500, str(e))
+            await docker.images.delete(name)
+        except DockerError as e:
+            raise web.HTTPError(e.status, e.message)
 
         self.set_status(200)
+        self.finish(json.dumps({"status": "ok"}))
 
     @web.authenticated
-    @run_on_executor
-    def post(self):
-        data = escape.json_decode(self.request.body)
+    @admin_only
+    async def post(self):
+        data = self.get_json_body()
         repo = data["repo"]
         ref = data["ref"]
         name = data["name"].lower()
@@ -141,13 +47,13 @@ def post(self):
         if memory:
             try:
                 float(memory)
-            except:
+            except ValueError:
                 raise web.HTTPError(400, "Memory Limit must be a number")
 
         if cpu:
             try:
                 float(cpu)
-            except:
+            except ValueError:
                 raise web.HTTPError(400, "CPU Limit must be a number")
 
         if name and not re.match(IMAGE_NAME_RE, name):
@@ -156,5 +62,73 @@ def post(self):
                 f"The name of the environment is restricted to the following characters: {IMAGE_NAME_RE}",
             )
 
-        build_image(repo, ref, name, memory, cpu)
+        await self._build_image(repo, ref, name, memory, cpu)
+
         self.set_status(200)
+        self.finish(json.dumps({"status": "ok"}))
+
+    async def _build_image(self, repo, ref, name="", memory=None, cpu=None):
+        """
+        Build an image given a repo, ref and limits
+        """
+        ref = ref or "master"
+        if len(ref) >= 40:
+            ref = ref[:7]
+
+        # default to the repo name if no name specified
+        # and sanitize the name of the docker image
+        name = name or urlparse(repo).path.strip("/")
+        name = name.replace("/", "-")
+        image_name = f"{name}:{ref}"
+
+        # memory is specified in GB
+        memory = f"{memory}G" if memory else ""
+        cpu = cpu or ""
+
+        # add extra labels to set additional image properties
+        labels = [
+            f"LABEL tljh_repo2docker.display_name={name}",
+            f"LABEL tljh_repo2docker.image_name={image_name}",
+            f"LABEL tljh_repo2docker.mem_limit={memory}",
+            f"LABEL tljh_repo2docker.cpu_limit={cpu}",
+        ]
+        cmd = [
+            "jupyter-repo2docker",
+            "--ref",
+            ref,
+            "--user-name",
+            "jovyan",
+            "--user-id",
+            "1100",
+            "--no-run",
+            "--image-name",
+            image_name,
+            "--appendix",
+            "\n".join(labels),
+            repo,
+        ]
+        await docker.containers.run(
+            config={
+                "Cmd": cmd,
+                "Image": "jupyter/repo2docker:master",
+                "Labels": {
+                    "repo2docker.repo": repo,
+                    "repo2docker.ref": ref,
+                    "repo2docker.build": image_name,
+                    "tljh_repo2docker.display_name": name,
+                    "tljh_repo2docker.mem_limit": memory,
+                    "tljh_repo2docker.cpu_limit": cpu,
+                },
+                "Volumes": {
+                    "/var/run/docker.sock": {
+                        "bind": "/var/run/docker.sock",
+                        "mode": "rw",
+                    }
+                },
+                "HostConfig": {"Binds": ["/var/run/docker.sock:/var/run/docker.sock"],},
+                "Tty": False,
+                "AttachStdout": False,
+                "AttachStderr": False,
+                "OpenStdin": False,
+            }
+        )