Fixes #253 adds reproducible testing container

This PR adds a new container and also uses it in the CircleCI
to test reproducible wheels and Debian packages. The container
is based on standard Debian Buster, thus having the same version
of Python and environment.

.circleci/config.yml

@@ -234,29 +234,27 @@ jobs:
 
   reprotest-wheels:
     docker:
-      - image: circleci/python:3.7-buster
+      - image: quay.io/kushaldas/packaging-debian-buster@sha256:bcdd8fd7ffcf89a4e252c33445791f6cd3e07f8216d98cb1c0b84dab44df698e
     steps:
       - checkout
       - run:
           name: install test requirements and run tests
           command: |
             make install-deps
-            virtualenv -p /usr/bin/python3 .venv
             source .venv/bin/activate
             pip install -r test-requirements.txt
             sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
             pytest -vvs tests/test_reproducible_wheels.py
 
   reprotest-debs:
     docker:
-      - image: circleci/python:3.7-buster
+      - image: quay.io/kushaldas/packaging-debian-buster@sha256:bcdd8fd7ffcf89a4e252c33445791f6cd3e07f8216d98cb1c0b84dab44df698e
     steps:
       - checkout
       - run:
           name: install test requirements and run tests
           command: |
             make install-deps
-            virtualenv -p /usr/bin/python3 .venv
             source .venv/bin/activate
             pip install -r test-requirements.txt
             # Patch reprotest in-place to skip 'setarch' prefix, which fails under containers.

.gitignore

@@ -2,3 +2,4 @@ tests/__pycache__/
 debhelper-build-stamp
 *.debhelper.log
 build/
+.venv

dockerfiles/Dockerfile

@@ -0,0 +1,41 @@
+# We want to do things using Debian Buster's own Python
+FROM debian:buster
+
+# make Apt non-interactive
+RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \
+  && echo 'DPkg::Options "--force-confnew";' >> /etc/apt/apt.conf.d/90circleci
+
+ENV DEBIAN_FRONTEND=noninteractive
+# Make sure PATH includes ~/.local/bin
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=839155
+# This only works for root. The circleci user is done near the end of this Dockerfile
+RUN echo 'PATH="$HOME/.local/bin:$PATH"' >> /etc/profile.d/user-local-path.sh
+
+# man directory is missing in some base images
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+RUN apt-get update \
+  && mkdir -p /usr/share/man/man1 \
+  && apt-get install -y \
+    git mercurial xvfb apt \
+    locales sudo openssh-client ca-certificates tar gzip parallel \
+    net-tools netcat unzip zip bzip2 gnupg curl wget make python3 python3-venv python3-pip
+
+
+# Set timezone to UTC by default
+RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime
+
+# Use unicode
+RUN locale-gen C.UTF-8 || true
+ENV LANG=C.UTF-8
+
+RUN groupadd --gid 3434 ci \
+  && useradd --uid 3434 --gid ci --shell /bin/bash --create-home ci \
+  && echo 'ci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-ci \
+  && echo 'Defaults    env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep
+
+
+
+USER ci
+ENV PATH /home/ci/.local/bin:/home/ci/bin:${PATH}
+
+CMD ["/bin/sh"]

dockerfiles/Makefile

@@ -0,0 +1,20 @@
+DATE_STR := $(shell date +"%Y_%m_%d")
+BUILDER_IMAGE ?= "quay.io/freedomofpress/packaging-debian-buster:$(DATE_STR)"
+
+.PHONY: build-container
+build-container: ## Build Docker image for Debian Buster wheel and package creation
+	@echo "███Building Docker image $(BUILDER_IMAGE) for Debian Buster wheel and package creation"
+	@docker build --no-cache -t $(BUILDER_IMAGE) .
+
+.PHONY: push-container
+push-container: ## Push the Docker image for Debian Buster wheel and package creation to quay.io
+	@echo "███Pushing Docker image for Debian package creation to quay.io..."
+	@./push.sh
+
+.PHONY: help
+help: ## Print this message and exit.
+	@printf "Molecule scenario for building a Docker container for Debian package creation.\n"
+	@printf "Subcommands:\n\n"
+	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
+		| sort \
+		| column -s ':' -t

dockerfiles/push.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DATE_STR=$(date +"%Y_%m_%d")
+QUAY_REPO=quay.io/freedomofpress/packaging-debian-buster
+
+set -e
+set -x
+
+docker push "${QUAY_REPO}:${DATE_STR}"
+
+echo "# sha256 digest ${QUAY_REPO}:${DATE_STR}" > image_hash
+docker inspect --format='{{index .RepoDigests 0}}' "${QUAY_REPO}:${DATE_STR}" \
+    | sed 's/.*://g' >> image_hash

scripts/install-deps

@@ -2,11 +2,6 @@
 # Installs required dependencies for building SecureDrop Worsktation packages.
 # Assumes a Debian 10 machine, ideally a Qubes AppVM.
 
-# If running in CI, we need to add the Ubuntu Bionic repo to download dh-virtualenv
-if [[ -v CIRCLE_BUILD_URL ]]; then
-  echo "deb http://archive.ubuntu.com/ubuntu/ bionic universe" | sudo tee -a /etc/apt/sources.list
-fi
-
 sudo apt-get update
 sudo apt-get install  \
     build-essential \
@@ -21,9 +16,10 @@ sudo apt-get install  \
     libyaml-dev \
     python3-all \
     python3-pip \
+    python3-venv \
     python3-setuptools \
     reprotest \
-    desktop-file-utils
+    desktop-file-utils -y
 
 # Inspect the wheel files present locally. If repo was cloned
 # without git-lfs, they'll be "text/plain", rather than "application/zip".