chore: 2nd try - simplify python dependencies (apache#27505)

sfirke · Mar 22, 2024 · 4088cb2 · 4088cb2
1 parent 9a233f1
commit 4088cb2
Show file tree

Hide file tree

Showing 28 changed files with 621 additions and 610 deletions.
diff --git a/.github/actions/setup-backend/action.yml b/.github/actions/setup-backend/action.yml
@@ -0,0 +1,33 @@
+name: 'Setup Python Environment'
+description: 'Set up Python and install dependencies with optional configurations.'
+inputs:
+  python-version:
+    description: 'Python version to set up.'
+    required: true
+    default: '3.9'
+  cache:
+    description: 'Cache dependencies. Options: pip'
+    required: false
+    default: 'pip'
+  requirements-type:
+    description: 'Type of requirements to install. Options: base, development, default'
+    required: false
+    default: 'dev'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: ${{ inputs.cache }}
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip setuptools wheel
+        if [ "${{ inputs.requirements-type }}" = "dev" ]; then
+          pip install -r requirements/development.txt
+        elif [ "${{ inputs.requirements-type }}" = "base" ]; then
+          pip install -r requirements/base.txt
+        fi
+      shell: bash
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -26,24 +26,9 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: |
-            requirements/base.txt
-            requirements/integration.txt
-      - name: Install dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/base.txt
-            pip install -r requirements/integration.txt
-      # Add brew to the path - see https://github.com/actions/runner-images/issues/6283
+        uses: ./.github/actions/setup-backend/
       - name: Enable brew and helm-docs
+        # Add brew to the path - see https://github.com/actions/runner-images/issues/6283
         run: |
           echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
           eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"

diff --git a/.github/workflows/superset-applitool-cypress.yml b/.github/workflows/superset-applitool-cypress.yml
@@ -58,23 +58,7 @@ jobs:
           submodules: recursive
           ref: master
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
-      - name: OS dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: apt-get-install
-      - name: Install python dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            pip-upgrade
-            pip install -r requirements/testing.txt
-      - name: Setup postgres
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: setup-postgres
+        uses: ./.github/actions/setup-backend/
       - name: Import test data
         uses: ./.github/actions/cached-dependencies
         with:
@@ -91,6 +75,11 @@ jobs:
         uses: ./.github/actions/cached-dependencies
         with:
           run: build-instrumented-assets
+      - name: Setup Postgres
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: setup-postgres
       - name: Install cypress
         uses: ./.github/actions/cached-dependencies
         with:

diff --git a/.github/workflows/superset-cli.yml b/.github/workflows/superset-cli.yml
@@ -52,22 +52,13 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: superset init
         if: steps.check.outcome == 'failure'
         run: |

diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml
@@ -69,21 +69,15 @@ jobs:
         run: ./scripts/ci_check_no_file_changes.sh python frontend
       - name: Setup Python
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
+        uses: ./.github/actions/setup-backend/
       - name: OS dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: apt-get-install
-      - name: Install python dependencies
+      - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            pip-upgrade
-            pip install -r requirements/testing.txt
       - name: Setup postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml
@@ -51,21 +51,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup MySQL
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
             setup-mysql
       - name: Run celery
         if: steps.check.outcome == 'failure'
@@ -78,7 +72,6 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           bash .github/workflows/codecov.sh -c -F python -F mysql
-
   test-postgres:
     runs-on: ubuntu-20.04
     strategy:
@@ -117,21 +110,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
@@ -177,21 +164,16 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
+            # sqlite needs this working directory
             mkdir ${{ github.workspace }}/.temp
       - name: Run celery
         if: steps.check.outcome == 'failure'

diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml
@@ -38,21 +38,10 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
       - name: pylint
         if: steps.check.outcome == 'failure'
         # `-j 0` run Pylint in parallel
@@ -70,18 +59,8 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: ./.github/actions/setup-backend/
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/base.txt"
-      - name: Install dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/base.txt
       - name: Test babel extraction
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml
@@ -67,22 +67,13 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
@@ -144,22 +135,13 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
+          run: setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &

diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml
@@ -46,27 +46,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      # TODO: separated requirements.txt file just for unit tests
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            mkdir ${{ github.workspace }}/.temp
       - name: Python unit tests
         if: steps.check.outcome == 'failure'
         env:
           SUPERSET_TESTENV: true
+          SUPERSET_SECRET_KEY: not-a-secret
         run: |
           pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
       - name: Upload code coverage

diff --git a/.github/workflows/superset-translations.yml b/.github/workflows/superset-translations.yml
@@ -47,7 +47,7 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
+        uses: ./.github/actions/setup-backend/
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -477,7 +477,7 @@ python3 -m venv venv # setup a python3 virtualenv
 source venv/bin/activate
 
 # Install external dependencies
-pip install -r requirements/testing.txt
+pip install -r requirements/development.txt
 
 # Install Superset in editable (development) mode
 pip install -e .
@@ -531,7 +531,7 @@ If you add a new requirement or update an existing requirement (per the `install
 ```bash
 $ python3 -m venv venv
 $ source venv/bin/activate
-$ python3 -m pip install -r requirements/integration.txt
+$ python3 -m pip install -r requirements/development.txt
 $ pip-compile-multi --no-upgrade
 ```
 
@@ -749,7 +749,7 @@ The current status of the usability of each flag (stable vs testing, etc) can be
 Superset uses Git pre-commit hooks courtesy of [pre-commit](https://pre-commit.com/). To install run the following:
 
 ```bash
-pip3 install -r requirements/integration.txt
+pip3 install -r requirements/development.txt
 pre-commit install
 ```