Backport main into develop-scikitlearn (#847)

* Containerized CI Pipeline (#836)

* Revert "Remove Python 2.7 and pypy2 testing (#835)"

This reverts commit abb6405.

* Containerize CI process

* Publish new docker container for CI images

* Rename github actions job

* Copyright tag scripts

* Drop debug line

* Swap to new CI image

* Move pip install to just main python

* Remove libcurl special case from tox

* Install special case packages into main image

* Remove unused packages

* Remove all other triggers besides manual

* Add make run command

* Cleanup small bugs

* Fix CI Image Tagging (#838)

* Correct templated CI image name

* Pin pypy2.7 in image

* Fix up scripting

* Temporarily Restore Old CI Pipeline (#841)

* Restore old pipelines

* Remove python 2 from setup-python

* Rework CI Pipeline (#839)

Change pypy to pypy27 in tox.

Fix checkout logic

Pin tox requires

* Fix Tests on New CI (#843)

* Remove non-root user

* Test new CI image

* Change pypy to pypy27 in tox.

* Fix checkout logic

* Fetch git tags properly

* Pin tox requires

* Adjust default db settings for github actions

* Rename elasticsearch services

* Reset to new pipelines

* [Mega-Linter] Apply linters fixes

* Fix timezone

* Fix docker networking

* Pin dev image to new sha

* Standardize gearman DB settings

* Fix elasticsearch settings bug

* Fix gearman bug

* Add missing odbc headers

* Add more debug messages

* Swap out dev ci image

* Fix required virtualenv version

* Swap out dev ci image

* Swap out dev ci image

* Remove aioredis v1 for EOL

* Add coverage paths for docker container

* Unpin ci container

---------

Co-authored-by: TimPansino <TimPansino@users.noreply.github.com>

* Fix pypy27 dependency

* Add skip for OTLP on py27

---------

Co-authored-by: TimPansino <TimPansino@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.github/containers/Dockerfile

@@ -0,0 +1,83 @@
+
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:20.04
+
+# Install OS packages
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+        bash \
+        build-essential \
+        curl \
+        expat \
+        gcc \
+        git \
+        libbz2-dev \
+        libcurl4-openssl-dev \
+        libffi-dev \
+        libgmp-dev \
+        liblzma-dev \
+        libmpfr-dev \
+        libncurses-dev \
+        libpq-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        libssl-dev \
+        locales \
+        make \
+        odbc-postgresql \
+        openssl \
+        python2-dev \
+        python3-dev \
+        python3-pip \
+        tzdata \
+        unixodbc-dev \
+        unzip \
+        wget \
+        zip \
+        zlib1g \
+        zlib1g-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+# Setup ODBC config
+RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini && \
+    sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini && \
+    sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini
+
+# Set the locale
+RUN locale-gen --no-purge en_US.UTF-8
+ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8
+ENV TZ="Etc/UTC"
+RUN ln -fs "/usr/share/zoneinfo/${TZ}" /etc/localtime && \
+    dpkg-reconfigure -f noninteractive tzdata
+
+# Use root user
+ENV HOME /root
+WORKDIR "${HOME}"
+
+# Install pyenv
+ENV PYENV_ROOT="${HOME}/.pyenv"
+RUN curl https://pyenv.run/ | /bin/bash
+ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:${PATH}"
+RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \
+    pyenv update
+
+# Install Python
+ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.11 pypy3.7"
+COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh
+COPY ./requirements.txt /requirements.txt
+RUN /tmp/install-python.sh && \
+    rm /tmp/install-python.sh

.github/containers/Makefile

@@ -0,0 +1,44 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Repository root for mounting into container.
+MAKEFILE_DIR:=$(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+REPO_ROOT:=$(realpath $(MAKEFILE_DIR)../../)
+
+.PHONY: default
+default: test
+
+.PHONY: build
+build:
+	@# Perform a shortened build for testing
+	@docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' $(MAKEFILE_DIR) -t ghcr.io/newrelic/newrelic-python-agent-ci:local
+
+.PHONY: test
+test: build
+	@# Ensure python versions are usable
+	@docker run --rm ghcr.io/newrelic/python-agent-ci:local /bin/bash -c '\
+		python3.10 --version && \
+		python2.7 --version && \
+		touch tox.ini && tox --version && \
+		echo "Success! Python versions installed."'
+
+.PHONY: run
+run: build
+	@docker run --rm -it \
+		--mount type=bind,source="$(REPO_ROOT)",target=/home/github/python-agent \
+		--workdir=/home/github/python-agent \
+		-e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \
+		-e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \
+		-e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \
+		ghcr.io/newrelic/newrelic-python-agent-ci:local /bin/bash

.github/containers/install-python.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+SCRIPT_DIR=$(dirname "$0")
+PIP_REQUIREMENTS=$(cat /requirements.txt)
+
+main() {
+    # Coerce space separated string to array
+    if [[ ${#PYTHON_VERSIONS[@]} -eq 1 ]]; then
+        PYTHON_VERSIONS=($PYTHON_VERSIONS)
+    fi
+
+    if [[ -z "${PYTHON_VERSIONS[@]}" ]]; then
+        echo "No python versions specified. Make sure PYTHON_VERSIONS is set." 1>&2
+        exit 1
+    fi
+
+    # Find all latest pyenv supported versions for requested python versions
+    PYENV_VERSIONS=()
+    for v in "${PYTHON_VERSIONS[@]}"; do
+        LATEST=$(pyenv latest -k "$v" || pyenv latest -k "$v-dev")
+        if [[ -z "$LATEST" ]]; then
+            echo "Latest version could not be found for ${v}." 1>&2
+            exit 1
+        fi
+        PYENV_VERSIONS+=($LATEST)
+    done
+
+    # Install each specific version
+    for v in "${PYENV_VERSIONS[@]}"; do
+        pyenv install "$v" &
+    done
+    wait
+
+    # Set all installed versions as globally accessible
+    pyenv global ${PYENV_VERSIONS[@]}
+
+    # Install dependencies for main python installation
+    pyenv exec pip install --upgrade $PIP_REQUIREMENTS
+}
+
+main

.github/containers/requirements.txt

@@ -0,0 +1,5 @@
+pip
+setuptools
+wheel
+virtualenv<20.22.0
+tox

.github/scripts/retry.sh

@@ -1,4 +1,18 @@
 #!/bin/bash
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # Time in seconds to backoff after the initial attempt.
 INITIAL_BACKOFF=10

.github/workflows/build-ci-image.yml

@@ -0,0 +1,68 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build CI Image
+
+on:
+  workflow_dispatch:  # Allow manual trigger
+
+concurrency: 
+  group: ${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate Docker Metadata (Tags and Labels)
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ghcr.io/${{ github.repository }}-ci
+          flavor: |
+            prefix=
+            suffix=
+            latest=false
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=schedule,pattern={{date 'YYYY-MM-DD'}}
+            type=sha,format=short,prefix=sha-
+            type=sha,format=long,prefix=sha-
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and Publish Image
+        uses: docker/build-push-action@v3
+        with:
+          push: ${{ github.event_name != 'pull_request' }}
+          context: .github/containers
+          platforms: linux/amd64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/get-envs.py

@@ -1,4 +1,18 @@
 #!/usr/bin/env python3.8
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import fileinput
 import os