From 55775ee0ac3c19a9c09b24fb8ded28e5c69bd219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Srokosz?= Date: Wed, 28 Sep 2022 18:58:15 +0200 Subject: [PATCH] Deploy v4.0.0 (#3) --- .dockerignore | 4 + .github/workflows/release.yml | 1 + .github/workflows/test.yml | 5 +- README.md | 35 ++- deploy/docker/Dockerfile | 7 +- pyproject.toml | 5 + setup.py | 3 +- src/__init__.py | 24 +- src/__main__.py | 3 + src/deploy.py | 497 +++++++--------------------------- src/docker.py | 42 +++ src/git.py | 22 ++ src/k8s.py | 64 +++++ src/schema.py | 39 +++ src/services.py | 251 +++++++++++++++++ src/utils.py | 48 ++++ 16 files changed, 614 insertions(+), 436 deletions(-) create mode 100644 .dockerignore create mode 100644 src/__main__.py create mode 100644 src/docker.py create mode 100644 src/git.py create mode 100644 src/k8s.py create mode 100644 src/schema.py create mode 100644 src/services.py create mode 100644 src/utils.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8ee5664 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +venv/ +.github/ +.mypy_cache/ +build/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9be6cb..af101de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,7 @@ jobs: - name: Build and push the image uses: docker/build-push-action@v2.2.1 with: + file: "./deploy/docker/Dockerfile" tags: | certpl/deploy:${{ github.sha }} certpl/deploy:${{ github.event.release.tag_name }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9d4546..2b6201a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,8 +10,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: CERT-Polska/lint-python-action@v1 + - uses: CERT-Polska/lint-python-action@v2 with: source: src/ - use-mypy: false - install-requirements: false + install-requirements: 'false' diff --git a/README.md b/README.md index c6d25c7..836e77d 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,8 @@ Then you can use `deploy build` to build your Docker image. Your image will be t ``` $ deploy build -[INFO] Building image certpl/fancy-service:a2f3d45a4ccfcee03a8a3941ec7177e394626cf2 +[INFO] Building image for fancy-service +[INFO] Building certpl/fancy-service:cb85eb9d38c407e462a6681351dfd36331635329 ``` If you want to provide alternative version tag, use `--version`. You can provide any tag you want, but few of them are special: @@ -41,8 +42,10 @@ Use `deploy push` if you want to push (and build) image to Docker Registry or Do ``` $ deploy push -[INFO] Building image certpl/fancy-service:a2f3d45a4ccfcee03a8a3941ec7177e394626cf2 -[INFO] Pushing image certpl/fancy-service:a2f3d45a4ccfcee03a8a3941ec7177e394626cf2 +[INFO] Building image for fancy-service +[INFO] Building certpl/fancy-service:cb85eb9d38c407e462a6681351dfd36331635329 +[INFO] Pushing image for fancy-service +[INFO] Pushing certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 ``` ### Make your k8s deployment deploy-enabled @@ -75,6 +78,9 @@ A complete example of a `deploy.json` file is presented below: } ``` +Starting from v4.0.0 you can provide `cronjob` instead of `deployment`. `init-container` is also supported if it uses +the same image. + > It's recommended to place your Kubernetes configuration files in the `deploy/k8s` and `deploy/k8s-staging` subdirectories. This enables you to use `deploy staging` and `deploy production` commands. @@ -86,11 +92,13 @@ to new version. ``` $ deploy production -[INFO] Building image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 -[INFO] Pushing image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 -[INFO] Tagging image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 as certpl/fancy-service:master -[INFO] Tagging image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 as certpl/fancy-service:latest -[INFO] Setting image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 for k8s environment +[INFO] Building image for fancy-service +[INFO] Building certpl/fancy-service:cb85eb9d38c407e462a6681351dfd36331635329 +[INFO] Pushing image for fancy-service +[INFO] Pushing certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 +[INFO] Deploying image to k8s +[INFO] Tagging certpl/fancy-service:e12840da50a9426b36de7c0be6dc553cde9923e8 as certpl/fancy-service::latest +[INFO] Pushing certpl/fancy-service:latest ``` If you don't want to rebuild your Docker images and need just to pull them from the Docker Registry, you can use @@ -98,11 +106,12 @@ If you don't want to rebuild your Docker images and need just to pull them from ``` $ docker pull -[INFO] Pulling image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 +[INFO] Pulling image for fancy-service +[INFO] Pulling certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 $ deploy production --deploy-only -[INFO] Tagging image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 as certpl/fancy-service:master -[INFO] Tagging image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 as certpl/fancy-service:latest -[INFO] Setting image certpl/fancy-service:7f6dd7010dba1ffdaeb32875e0f71c30c9810df7 for k8s environment +[INFO] Deploying image to k8s +[INFO] Tagging certpl/fancy-service:e12840da50a9426b36de7c0be6dc553cde9923e8 as certpl/fancy-service::latest +[INFO] Pushing certpl/fancy-service:latest ``` ### Support for multiple services @@ -127,7 +136,7 @@ Deploy will build, push and deploy them all (unless you explicitly select servic You can automate all these steps with CI/CD. Example `.gitlab-ci.yml` file is presented below: ```yaml -image: certpl/deploy +image: certpl/deploy:v4.0.0 services: - docker:dind diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 0e995c4..5fd6dfe 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,11 +1,12 @@ FROM docker:stable RUN apk update \ - && apk add python3 curl openssh-client git openssl bash python-dev libffi-dev openssl-dev gcc libc-dev make py-pip \ + && apk add python3 curl openssh-client git openssl bash python3-dev libffi-dev openssl-dev gcc libc-dev make py-pip \ && curl -L https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl \ + && curl -SL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose \ && chmod +x /usr/local/bin/kubectl \ - && pip install docker-compose + && chmod +x /usr/local/bin/docker-compose COPY . /app/deploy RUN cd /app/deploy \ - && python3 setup.py install + && pip install . \ diff --git a/pyproject.toml b/pyproject.toml index 118dd16..01ba7ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,8 @@ force_grid_wrap = 0 use_parentheses = true ensure_newline_before_comments = true line_length = 88 + +[tool.lint-python] +lint-version = "2" +source = "src/" +extra-requirements = "types-PyYAML" diff --git a/setup.py b/setup.py index 8ed576a..5709628 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="python-deploy", - version="3.0.2", + version="4.0.0", author="msm, psrok1", author_email="info@cert.pl", description="Build, push and deploy k8s services with single " @@ -17,6 +17,7 @@ long_description=long_description, long_description_content_type="text/markdown", install_requires=requirements, + python_requires=">=3.8", package_dir={"deploy": "src"}, url="https://github.com/CERT-Polska/python-deploy", packages=["deploy"], diff --git a/src/__init__.py b/src/__init__.py index 62c98e4..7924d70 100755 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,25 +1,5 @@ #!/usr/bin/python3 -from .deploy import ( - build_image, - get_current_git_hash, - get_current_image, - is_clean_repo, - main, - pull_image, - push_image, - set_current_image, - tag_docker_image, -) +from .deploy import Deploy, load_deploy_json, main -__all__ = [ - "main", - "get_current_image", - "set_current_image", - "push_image", - "pull_image", - "is_clean_repo", - "get_current_git_hash", - "tag_docker_image", - "build_image", -] +__all__ = ["main", "Deploy", "load_deploy_json"] diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..12ad56c --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,3 @@ +from .deploy import main + +main() diff --git a/src/deploy.py b/src/deploy.py index bcfa376..c4aa2b5 100755 --- a/src/deploy.py +++ b/src/deploy.py @@ -4,428 +4,143 @@ import json import logging import os +import pathlib import subprocess from datetime import datetime -from typing import Callable, Dict, List, Optional, Union +from typing import List, Optional -import yaml +from .docker import run_image_command +from .git import get_current_git_hash, has_git_repo, is_clean_repo +from .schema import KubernetesEnv +from .services import DeployService, DeployServiceSet +from .utils import DeployError, logger -log = logging.getLogger("python-deploy") -CallErrorFilter = Callable[[subprocess.CalledProcessError], bool] - - -class DeployError(RuntimeError): - pass - - -def flatten(items: List[Union[str, List[str]]]) -> List[str]: - return [ - item - for sublist in items - for item in (sublist if isinstance(sublist, list) else [sublist]) - ] - - -def check_call( - params: List[Union[str, List[str]]], - input: Optional[bytes] = None, - error_filter: Optional[CallErrorFilter] = None, -) -> Optional[bytes]: - try: - params = flatten(params) - log.debug(f"> {' '.join(params)}") - result = subprocess.check_output(params, input=input, stderr=subprocess.STDOUT) - log.debug(f"$ {result.decode()}") - return result - except subprocess.CalledProcessError as exc: - if error_filter and error_filter(exc): - return exc.output - log.error("Called process error:") - log.error(exc.output.decode()) - raise DeployError(f"Command {params[0]} failed.") - - -def has_git_repo() -> bool: - proc = subprocess.Popen( - ["git", "status"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - proc.communicate() - return proc.returncode == 0 - - -def is_clean_repo() -> bool: - status = check_call(["git", "status", "--short"]) - return status.strip() == b"" - - -def get_current_git_hash() -> str: - return check_call(["git", "rev-parse", "HEAD"]).strip().decode("utf-8") - - -def tag_docker_image(existing_image: str, new_tag: str, push: bool = False) -> None: - check_call(["docker", "tag", existing_image, new_tag]) - if push: - push_image(new_tag) - - -def build_image(image_cfg: dict, tags: List[str], no_cache: bool): - check_call( - [ - "docker", - "image", - "build", - image_cfg.get("dir", "src"), - flatten([["-t", tag] for tag in tags]), - "-f", - image_cfg.get("dockerfile", "deploy/docker/Dockerfile"), - ["--no-cache"] if no_cache else [], - ] - ) - - -def push_image(tag): - check_call(["docker", "image", "push", tag]) - - -def pull_image(tag: str) -> None: - check_call(["docker", "image", "pull", tag]) - - -def run_image_command(tag: str, params: List[str]) -> None: - check_call(["docker", "run", tag, params]) - - -def get_k8s_config_path( - service_name: str, environment: str, service_config: Dict -) -> str: - if service_config[environment].get("configuration"): - return service_config[environment]["configuration"] - if environment == "k8s-staging": - environment = "k8s-st" - return f"deploy/{environment}/{service_name}.yml" - - -def get_k8s_config(k8s_config_path: str) -> List[Dict]: - if not os.path.exists(k8s_config_path): - raise DeployError( - f"Configuration file {k8s_config_path} not found. " - f"Check whether deploy/deploy.json is configured properly." - ) - with open(k8s_config_path, "r") as f: - return list(yaml.safe_load_all(f)) - - -def get_k8s_deployment_config(k8s_config: List[Dict]): - for config in k8s_config: - if config.get("kind") == "Deployment": - return config - else: +def load_deploy_json(load_only: Optional[List[str]] = None) -> DeployServiceSet: + deploy_path = pathlib.Path("deploy/deploy.json") + if not deploy_path.is_file(): raise DeployError( - "Deployment configuration not found in " "Kubernetes configuration file." + "Configuration file deploy/deploy.json not found. " "Check your CWD." ) + deploy_json = deploy_path.read_text() + return DeployServiceSet(json.loads(deploy_json), load_only=load_only) -def get_k8s_config_key(k8s_depl_config: Dict, key_path: str): - tokens = key_path.split(".") - current = k8s_depl_config - for level, key in enumerate(tokens): - if key not in current: +def get_version_tag(source: str, check_dirty: bool = True) -> str: + if check_dirty: + if not has_git_repo(): raise DeployError( - f"Missing key {'.'.join(tokens[:level + 1])} in " - f"Kubernetes configuration file." + "There is no Git repository available. " + "Use --force if you don't care about it" + ) + if not is_clean_repo(): + raise DeployError( + "Git repository is dirty. Commit your changes " + "or use --force if you don't care about it" ) - current = current[key] - return current - - -def get_k8s_container_spec(k8s_depl_config: Dict, container_name: str): - containers_spec = get_k8s_config_key( - k8s_depl_config, "spec.template.spec.containers" - ) - for container_spec in containers_spec: - if container_spec.get("name") == container_name: - return container_spec else: - raise DeployError( - f"Kubernetes configuration file doesn't apply to the container " - f"{container_name} provided in deploy/deploy.json" - ) - - -def get_validated_k8s_config( - service_name: str, environment: str, service_config: Dict -) -> List[Dict]: - k8s_config_path = get_k8s_config_path(service_name, environment, service_config) - k8s_config = get_k8s_config(k8s_config_path) - k8s_depl_config = get_k8s_deployment_config(k8s_config) - deploy_config = service_config[environment] - - def from_k8s(key_path): - return get_k8s_config_key(k8s_depl_config, key_path) - - k8s_config_namespace = from_k8s("metadata.namespace") - deploy_namespace = deploy_config["namespace"] - - if k8s_config_namespace != deploy_namespace: - raise DeployError( - f"Kubernetes configuration file doesn't apply to the namespace " - f"provided in deploy/deploy.json (expected: {deploy_namespace}, " - f"found: {k8s_config_namespace})" - ) - - k8s_config_deployment = from_k8s("metadata.name") - deploy_deployment = deploy_config["deployment"] - - if k8s_config_deployment != deploy_deployment: - raise DeployError( - f"Kubernetes configuration file doesn't apply to the deployment " - f"provided in deploy/deploy.json (expected: {deploy_deployment}, " - f"found: {k8s_config_deployment})" - ) - - container_spec = get_k8s_container_spec(k8s_depl_config, deploy_config["container"]) - - k8s_config_image = container_spec["image"].split(":")[0] - deploy_image = service_config["docker"]["image"] - - if k8s_config_image != deploy_image: - raise DeployError( - f"Configuration file {k8s_config_path} doesn't refer to the image " - f"provided in deploy/deploy.json (expected: {deploy_image}, " - f"found: {k8s_config_image})" - ) - return k8s_config - - -def set_k8s_config_image( - k8s_config: List[Dict], service_config: Dict, environment: str, tag: str -): - deploy_config = service_config[environment] - k8s_depl_config = get_k8s_deployment_config(k8s_config) - container_spec = get_k8s_container_spec(k8s_depl_config, deploy_config["container"]) - container_spec["image"] = tag - - -def diff_k8s_configuration(k8s_config: List[Dict], deploy_config: Dict) -> bytes: - def error_filter(exc: subprocess.CalledProcessError): - return exc.returncode in [0, 1] - - return check_call( - ["kubectl", "diff", "--namespace", deploy_config["namespace"], "-f", "-"], - input=yaml.dump_all(k8s_config, encoding="utf-8"), - error_filter=error_filter, - ) - - -def apply_k8s_configuration(k8s_config: List[Dict], deploy_config: Dict): - return check_call( - ["kubectl", "apply", "--namespace", deploy_config["namespace"], "-f", "-"], - input=yaml.dump_all(k8s_config, encoding="utf-8"), - ) - - -def set_current_image(deploy_config, tag): - check_call( - [ - "kubectl", - "set", - "image", - "--namespace", - deploy_config["namespace"], - "deployment/{}".format(deploy_config["deployment"]), - "{}={}".format(deploy_config["container"], tag), - ] - ) - - -def get_current_image(deploy_config): - container = deploy_config["container"] - jsonpath = f'{{..containers[?(@.name=="{container}")].image}}' - - return check_call( - [ - "kubectl", - "get", - "deployment", - "--namespace", - deploy_config["namespace"], - f"-o=jsonpath={jsonpath}", - ] - ).decode("utf-8") - - -def get_k8s_namespaces(): - ns_bytes = check_call(["kubectl", "get", "namespaces"]) - ns_text = ns_bytes.decode("utf-8") - namespaces = [] - for line in ns_text.split("\n")[1:]: - if not line.strip(): - continue - namespaces.append(line.split()[0]) - return namespaces + logger.warning("Used --force. Git won't be checked.") + if source == "commit": + try: + return get_current_git_hash() + except subprocess.CalledProcessError: + ci_commit_sha = os.getenv("CI_COMMIT_SHA") + if ci_commit_sha: + logger.warning( + "Can't determine commit hash: " + "getting version from $CI_COMMIT_SHA." + ) + return ci_commit_sha + else: + raise DeployError( + "Can't determine commit hash. Use alternative " "--version variant." + ) + elif source == "date": + return datetime.utcnow().strftime("v%Y%m%d%H%M%S") + else: + return source class Deploy(object): def __init__(self, args): self.args = args - if not os.path.isfile("deploy/deploy.json"): - raise DeployError( - "Configuration file deploy/deploy.json not found. " "Check your CWD." - ) - with open("deploy/deploy.json", "r") as configfile: - self.config = json.loads(configfile.read()) + self.config = load_deploy_json(load_only=self.args.service) if self.args.service: for srv in self.args.service: - if srv not in self.config.keys(): + if srv not in self.config.services.keys(): raise DeployError(f"Unknown service: {srv}") - self.config = { - k: v for k, v in self.config.items() if k in self.args.service - } - self.version = self._check_version() - - def _check_version(self): - if not self.args.force: - if not has_git_repo(): - raise DeployError( - "There is no Git repository available. " - "Use --force if you don't care about it" - ) - if not is_clean_repo(): - raise DeployError( - "Git repository is dirty. Commit your changes " - "or use --force if you don't care about it" - ) - else: - log.warning("Used --force. I hope you know what you are doing.") - if self.args.version == "commit": - try: - return get_current_git_hash() - except subprocess.CalledProcessError: - ci_commit_sha = os.getenv("CI_COMMIT_SHA") - if ci_commit_sha: - log.warning( - "Can't determine commit hash: " - "getting version from $CI_COMMIT_SHA." - ) - return ci_commit_sha - else: - raise DeployError( - "Can't determine commit hash. Use alternative " - "--version variant." - ) - elif self.args.version == "date": - return datetime.utcnow().strftime("v%Y%m%d%H%M%S") - else: - return self.args.version - - def _version_tag(self, config): - return f"{config['image']}:{self.version}" - - def _tags(self, config): - return [self._version_tag(config)] + [ - f"{config['image']}:{tag}" - for tag in (self.args.tag or []) - if ":" not in tag - ] - - def _build(self, service): - config = self.config[service]["docker"] - log.info(f"Building image {self._version_tag(config)}") - build_image(config, self._tags(config), self.args.no_cache) - - def build(self): - for service in self.config.keys(): - self._build(service) - - def _push(self, service): - config = self.config[service]["docker"] - for tag in self._tags(config): - log.info(f"Pushing image {tag}") - push_image(tag) - - def push(self): - self.build() - for service in self.config.keys(): - self._push(service) - - def pull(self): - for service in self.config.keys(): - config = self.config[service]["docker"] - version_tag = self._version_tag(config) - log.info(f"Pulling image {version_tag}") - pull_image(version_tag) - - def _deploy(self, service, environment): - if environment not in self.config[service]: - raise DeployError( - f"There is no {environment} key defined " f"for {service} service" - ) - config = self.config[service]["docker"] - version_tag = self._version_tag(config) - - k8s_config = get_validated_k8s_config( - service, environment, self.config[service] + self.version_tag: str = get_version_tag( + self.args.version, check_dirty=not self.args.force ) - set_k8s_config_image(k8s_config, self.config[service], environment, version_tag) - - if self.args.validate: - diff = diff_k8s_configuration( - k8s_config, self.config[service][environment] - ).decode() - if diff: - log.info(f"Found difference for {version_tag}") - print(diff) - else: - log.info(f"No changes found for {version_tag}") - return + def build(self) -> None: + extra_tags = self.args.tag or [] + for service in self.config.get_services(): + logger.info(f"Building image for {service.service_name}") + service.build_docker([self.version_tag] + extra_tags, self.args.no_cache) + + def push(self) -> None: + extra_tags = self.args.tag or [] + for service in self.config.get_services(): + logger.info(f"Pushing image for {service.service_name}") + service.push_docker([self.version_tag] + extra_tags) + + def pull(self) -> None: + for service in self.config.get_services(): + logger.info(f"Pulling image for {service.service_name}") + service.pull_docker([self.version_tag]) + + def _validate(self, service: DeployService, environment: KubernetesEnv) -> None: + diff = service.diff_k8s_config(self.version_tag, environment) + if diff: + logger.info(f"Found difference for {service.service_name}") + print(diff) + else: + logger.info(f"No changes found for {service.service_name}") + def _deploy(self, service: DeployService, environment: KubernetesEnv) -> None: if environment == "k8s": - tag = f"{config['image']}:master" - log.info(f"Tagging image {version_tag} as {tag}") - tag_docker_image(version_tag, tag, push=True) - tag = f"{config['image']}:latest" - log.info(f"Tagging image {version_tag} as {tag}") - tag_docker_image(version_tag, tag, push=True) - log.info(f"Setting image {version_tag} for {environment} environment") - if self.args.set_image_only: - set_current_image(self.config[service][environment], version_tag) + service.tag_docker(self.version_tag, ["master", "latest"]) else: - apply_k8s_configuration(k8s_config, self.config[service][environment]) - - def production(self): - if not self.args.deploy_only and not self.args.validate: - self.push() - for service in self.config.keys(): + service.tag_docker(self.version_tag, ["latest"]) + service.apply_k8s_config(self.version_tag, environment) + + def production(self) -> None: + for service in self.config.get_services(): + if self.args.validate: + self._validate(service, "k8s") + continue + if not self.args.deploy_only: + self.push() + logger.info(f"Deploying {service.service_name} to k8s") self._deploy(service, "k8s") - def staging(self): - if not self.args.deploy_only and not self.args.validate: - self.push() - for service in self.config.keys(): + def staging(self) -> None: + for service in self.config.get_services(): + if self.args.validate: + self._validate(service, "k8s-staging") + continue + if not self.args.deploy_only: + self.push() + logger.info(f"Deploying {service.service_name} to k8s-staging") self._deploy(service, "k8s-staging") - def image(self): - for service in self.config.keys(): - config = self.config[service]["docker"] - version_tag = self._version_tag(config) + def image(self) -> None: + for service in self.config.get_services(): + version_tag = service.get_docker_tags([self.version_tag])[0] print(version_tag) - def run(self): - for service in self.config.keys(): - config = self.config[service]["docker"] - version_tag = self._version_tag(config) + def run(self) -> None: + for service in self.config.get_services(): + version_tag = service.get_docker_tags([self.version_tag])[0] run_image_command(version_tag, self.args.cmd) - def list(self): - for service in self.config.keys(): - print(f"{service} ({','.join(self.config[service].keys())})") + def list(self) -> None: + for service in self.config.get_services(): + print(f"{service.service_name} ({','.join(service.spec.keys())})") - def perform(self): - return getattr(self, self.args.command)() + def perform(self) -> None: + getattr(self, self.args.command)() def main(): @@ -473,12 +188,6 @@ def main(): action="store_true", help="Don't build and push, just apply k8s configuration", ) - deploy_subparser.add_argument( - "--set-image-only", - action="store_true", - help="Only set image in existing deployment," - " without applying k8s configuration", - ) deploy_subparser.add_argument( "--validate", action="store_true", diff --git a/src/docker.py b/src/docker.py new file mode 100644 index 0000000..3ccae20 --- /dev/null +++ b/src/docker.py @@ -0,0 +1,42 @@ +from typing import List + +from .utils import check_call, flatten, logger + + +def tag_image(existing_image: str, new_tag: str, push: bool = False) -> None: + logger.info(f"Tagging {existing_image} as {new_tag}") + check_call(["docker", "tag", existing_image, new_tag]) + if push: + push_image(new_tag) + + +def build_image( + dockerfile: str, context_dir: str, tags: List[str], no_cache: bool +) -> None: + logger.info(f"Building {', '.join(tags)}") + check_call( + [ + "docker", + "image", + "build", + context_dir, + flatten([["-t", tag] for tag in tags]), + "-f", + dockerfile, + ["--no-cache"] if no_cache else [], + ] + ) + + +def push_image(tag: str) -> None: + logger.info(f"Pushing {tag}") + check_call(["docker", "image", "push", tag]) + + +def pull_image(tag: str) -> None: + logger.info(f"Pulling {tag}") + check_call(["docker", "image", "pull", tag]) + + +def run_image_command(tag: str, params: List[str]) -> None: + check_call(["docker", "run", tag, params]) diff --git a/src/git.py b/src/git.py new file mode 100644 index 0000000..a53aa09 --- /dev/null +++ b/src/git.py @@ -0,0 +1,22 @@ +import subprocess + +from .utils import check_call + + +def has_git_repo() -> bool: + proc = subprocess.Popen( + ["git", "status"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + proc.communicate() + return proc.returncode == 0 + + +def is_clean_repo() -> bool: + status = check_call(["git", "status", "--short"]) + return status.strip() == b"" + + +def get_current_git_hash() -> str: + return check_call(["git", "rev-parse", "HEAD"]).strip().decode("utf-8") diff --git a/src/k8s.py b/src/k8s.py new file mode 100644 index 0000000..747a3a4 --- /dev/null +++ b/src/k8s.py @@ -0,0 +1,64 @@ +import subprocess +from typing import Dict, List + +import yaml + +from .utils import check_call + + +def diff_k8s_configuration(k8s_config: List[Dict], namespace: str) -> str: + def error_filter(exc: subprocess.CalledProcessError): + return exc.returncode in [0, 1] + + return check_call( + ["kubectl", "diff", "--namespace", namespace, "-f", "-"], + input=yaml.dump_all(k8s_config, encoding="utf-8"), + error_filter=error_filter, + ).decode() + + +def apply_k8s_configuration(k8s_config: List[Dict], namespace: str) -> bytes: + return check_call( + ["kubectl", "apply", "--namespace", namespace, "-f", "-"], + input=yaml.dump_all(k8s_config, encoding="utf-8"), + ) + + +def set_current_image(namespace: str, object: str, container: str, image: str) -> None: + check_call( + [ + "kubectl", + "set", + "image", + "--namespace", + namespace, + object, + f"{container}={image}", + ] + ) + + +def get_current_image(namespace: str, container: str) -> str: + jsonpath = f'{{..containers[?(@.name=="{container}")].image}}' + + return check_call( + [ + "kubectl", + "get", + "deployment", + "--namespace", + namespace, + f"-o=jsonpath={jsonpath}", + ] + ).decode() + + +def get_k8s_namespaces() -> List[str]: + ns_bytes = check_call(["kubectl", "get", "namespaces"]) + ns_text = ns_bytes.decode("utf-8") + namespaces = [] + for line in ns_text.split("\n")[1:]: + if not line.strip(): + continue + namespaces.append(line.split()[0]) + return namespaces diff --git a/src/schema.py b/src/schema.py new file mode 100644 index 0000000..2b69a7c --- /dev/null +++ b/src/schema.py @@ -0,0 +1,39 @@ +""" +Types for docker/deploy.json files +""" + +from typing import Dict, Literal, TypedDict, Union + + +class DockerSpec(TypedDict, total=False): + image: str + dockerfile: str + dir: str + + +KubernetesSpec = TypedDict( + "KubernetesSpec", + { + "namespace": str, + "deployment": str, + "cronjob": str, + "container": str, + "init-container": str, + "configuration": str, + }, + total=False, +) + + +DeployServiceSpec = TypedDict( + "DeployServiceSpec", + { + "docker": DockerSpec, + "k8s": KubernetesSpec, + "k8s-staging": KubernetesSpec, + }, + total=False, +) + +DeploySpec = Dict[str, DeployServiceSpec] +KubernetesEnv = Union[Literal["k8s"], Literal["k8s-staging"]] diff --git a/src/services.py b/src/services.py new file mode 100644 index 0000000..b011b2e --- /dev/null +++ b/src/services.py @@ -0,0 +1,251 @@ +import copy +import pathlib +from typing import Dict, List, Optional + +import yaml + +from .docker import build_image, pull_image, push_image, tag_image +from .k8s import apply_k8s_configuration, diff_k8s_configuration +from .schema import DeployServiceSpec, DeploySpec, KubernetesEnv, KubernetesSpec +from .utils import DeployError, get_dict_key + + +class DeployK8S: + def __init__( + self, spec: KubernetesSpec, environment: KubernetesEnv, service_name: str + ) -> None: + self.spec = spec + self.environment = environment + self.service_name = service_name + self.config_path = self.get_config_path() + self.config = self.load_config(self.config_path) + + def get_config_path(self) -> pathlib.Path: + if "configuration" in self.spec: + return pathlib.Path(self.spec["configuration"]) + if self.environment == "k8s-staging": + environment_dir = "k8s-st" + else: + environment_dir = "k8s" + return pathlib.Path(f"deploy/{environment_dir}/{self.service_name}.yml") + + def load_config(self, config_path: pathlib.Path) -> List[Dict]: + if not config_path.exists(): + raise DeployError( + f"Configuration file {config_path.as_posix()} not found. " + f"Check whether deploy/deploy.json is configured properly." + ) + with config_path.open("r") as f: + return list(yaml.safe_load_all(f)) + + def get_service_config(self, k8s_config: List[Dict]) -> Dict: + """ + Returns config that matches deployment/cronjob + """ + if "deployment" in self.spec: + kind = "deployment" + name = self.spec["deployment"] + elif "cronjob" in self.spec: + kind = "cronjob" + name = self.spec["cronjob"] + else: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"'deployment' or 'cronjob' must be specified." + ) + + for svc_config in k8s_config: + svc_kind = svc_config.get("kind") + if not svc_kind or svc_kind.lower() != kind: + continue + metadata = svc_config.get("metadata", {}) + if metadata.get("name") == name: + break + else: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"{kind}/{name} not found in {self.config_path.as_posix()}." + ) + + namespace = self.spec.get("namespace") + if not namespace: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"'namespace' must be specified." + ) + + if metadata.get("namespace") != namespace: + raise DeployError( + f"Kubernetes configuration file doesn't apply to the namespace " + f"provided in deploy/deploy.json (expected: {namespace}, " + f"found: {metadata.get('namespace')})" + ) + return svc_config + + def get_from_template_spec(self, service_config: Dict, key: str) -> Dict: + if service_config["kind"].lower() == "deployment": + path = "spec.template.spec." + key + elif service_config["kind"].lower() == "cronjob": + path = "spec.jobTemplate.spec.template.spec." + key + else: + raise NotImplementedError + spec = get_dict_key(service_config, path) + if not spec: + raise DeployError( + f"Missing key {path} in " f"Kubernetes configuration file." + ) + return spec + + def get_container_spec(self, service_config: Dict) -> Dict: + if "container" not in self.spec: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"'container' must be specified." + ) + container_name = self.spec["container"] + containers = self.get_from_template_spec(service_config, "containers") + for container in containers: + if container.get("name") == container_name: + return container + else: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"container {container_name} not found in " + f"{self.config_path.as_posix()}." + ) + + def get_init_container_spec(self, service_config: Dict) -> Optional[Dict]: + if "init-container" not in self.spec: + return None + init_container_name = self.spec["init-container"] + containers = self.get_from_template_spec(service_config, "initContainers") + for container in containers: + if container.get("name") == init_container_name: + return container + else: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"init container {init_container_name} not found " + f"in {self.config_path.as_posix()}." + ) + + def _set_image_tag(self, container_config: Dict, image: str) -> None: + if "image" not in container_config: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"container doesn't have 'image' specified " + f"in {self.config_path.as_posix()}." + ) + container_image_name, container_tag = container_config["image"].split(":") + image_name, tag = image.split(":") + if container_image_name != image_name: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"container image doesn't match with '{image_name}' " + f"(found '{container_image_name}' specified " + f"in {self.config_path.as_posix()})." + ) + container_config["image"] = image + + def set_image_tag(self, image: str) -> List[Dict]: + k8s_config = copy.deepcopy(self.config) + service_config = self.get_service_config(k8s_config) + container = self.get_container_spec(service_config) + self._set_image_tag(container, image) + init_container = self.get_init_container_spec(service_config) + if init_container: + self._set_image_tag(init_container, image) + return k8s_config + + +class DeployService: + def __init__(self, spec: DeployServiceSpec, service_name: str) -> None: + self.service_name = service_name + self.spec = spec + self.docker_spec = spec.get("docker") + self.k8s_spec = spec.get("k8s") + self.k8s_staging_spec = spec.get("k8s-staging") + + def get_docker_tags(self, tags: List[str]) -> List[str]: + if not self.docker_spec: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"Missing docker image specification." + ) + if "image" not in self.docker_spec: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"'docker.image' must be specified." + ) + return [ + tag if ":" in tag else f'{self.docker_spec["image"]}:{tag}' for tag in tags + ] + + def build_docker(self, tags: List[str], no_cache: bool = False) -> None: + if not self.docker_spec: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"Missing docker image specification." + ) + dockerfile = self.docker_spec.get("dockerfile", "./Dockerfile") + context_dir = self.docker_spec.get("dir", ".") + build_image(dockerfile, context_dir, self.get_docker_tags(tags), no_cache) + + def push_docker(self, tags: List[str]) -> None: + for tag in self.get_docker_tags(tags): + push_image(tag) + + def tag_docker(self, existing_tag: str, tags: List[str]) -> None: + existing_docker_tag = self.get_docker_tags([existing_tag])[0] + for tag in self.get_docker_tags(tags): + tag_image(existing_docker_tag, tag, push=True) + + def pull_docker(self, tags: List[str]) -> None: + for tag in self.get_docker_tags(tags): + pull_image(tag) + + def get_k8s(self, environment: KubernetesEnv) -> DeployK8S: + if environment == "k8s": + if not self.k8s_spec: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"Missing k8s specification." + ) + k8s = DeployK8S(self.k8s_spec, "k8s", service_name=self.service_name) + elif environment == "k8s-staging": + if not self.k8s_staging_spec: + raise DeployError( + f"Invalid specification of {self.service_name}. " + f"Missing k8s-staging specification." + ) + k8s = DeployK8S( + self.k8s_staging_spec, "k8s-staging", service_name=self.service_name + ) + else: + raise NotImplementedError + return k8s + + def apply_k8s_config(self, tag: str, environment: KubernetesEnv) -> None: + image = self.get_docker_tags([tag])[0] + k8s = self.get_k8s(environment) + patched_config = k8s.set_image_tag(image) + apply_k8s_configuration(patched_config, k8s.spec["namespace"]) + + def diff_k8s_config(self, tag: str, environment: KubernetesEnv) -> str: + image = self.get_docker_tags([tag])[0] + k8s = self.get_k8s(environment) + patched_config = k8s.set_image_tag(image) + return diff_k8s_configuration(patched_config, k8s.spec["namespace"]) + + +class DeployServiceSet: + def __init__(self, spec: DeploySpec, load_only: Optional[List[str]] = None) -> None: + self.spec = spec + self.services = { + service_name: DeployService(service_spec, service_name=service_name) + for service_name, service_spec in spec.items() + if not load_only or service_name in load_only + } + + def get_services(self) -> List[DeployService]: + return list(self.services.values()) diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..629a038 --- /dev/null +++ b/src/utils.py @@ -0,0 +1,48 @@ +import logging +import subprocess +from typing import Any, Callable, Dict, List, Optional, Sequence, Union + +logger = logging.getLogger("python-deploy") + +CallErrorFilter = Callable[[subprocess.CalledProcessError], bool] + + +class DeployError(RuntimeError): + pass + + +def flatten(items: Sequence[Union[str, List[str]]]) -> List[str]: + return [ + item + for sublist in items + for item in (sublist if isinstance(sublist, list) else [sublist]) + ] + + +def check_call( + params: List[Union[str, List[str]]], + input: Optional[bytes] = None, + error_filter: Optional[CallErrorFilter] = None, +) -> bytes: + try: + args = flatten(params) + logger.debug(f"> {' '.join(args)}") + result = subprocess.check_output(args, input=input, stderr=subprocess.STDOUT) + logger.debug(f"$ {result.decode()}") + return result + except subprocess.CalledProcessError as exc: + if error_filter and error_filter(exc): + return exc.output + logger.error("Called process error:") + logger.error(exc.output.decode()) + raise DeployError(f"Command {params[0]} failed.") + + +def get_dict_key(obj: Dict, key_path: str) -> Any: + tokens = key_path.split(".") + current = obj + for level, key in enumerate(tokens): + if key not in current: + return None + current = current[key] + return current