chore: simplify python dependencies management

.github/actions/setup-backend/action.yml

@@ -0,0 +1,36 @@
+name: 'Setup Python Environment'
+description: 'Set up Python and install dependencies with optional configurations.'
+inputs:
+  python-version:
+    description: 'Python version to set up.'
+    required: true
+    default: '3.9'
+  cache:
+    description: 'Cache dependencies. Options: pip'
+    required: false
+    default: 'pip'
+  requirements-type:
+    description: 'Type of requirements to install. Options: base, development, default'
+    required: false
+    default: 'dev'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: ${{ inputs.cache }}
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip setuptools wheel
+        if [ "${{ inputs.requirements-type }}" = "dev" ]; then
+          pip install -r requirements/development.txt
+        elif [ "${{ inputs.requirements-type }}" = "base" ]; then
+          pip install -r requirements/base.txt
+        else
+          pip install -r requirements.txt
+        fi
+        pip install -e .
+      shell: bash

.github/workflows/pre-commit.yml

@@ -26,24 +26,9 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: |
-            requirements/base.txt
-            requirements/integration.txt
-      - name: Install dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/base.txt
-            pip install -r requirements/integration.txt
-      # Add brew to the path - see https://github.com/actions/runner-images/issues/6283
+        uses: ./.github/actions/setup-backend/
       - name: Enable brew and helm-docs
+        # Add brew to the path - see https://github.com/actions/runner-images/issues/6283
         run: |
           echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
           eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"

.github/workflows/superset-applitool-cypress.yml

@@ -58,23 +58,7 @@ jobs:
           submodules: recursive
           ref: master
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
-      - name: OS dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: apt-get-install
-      - name: Install python dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            pip-upgrade
-            pip install -r requirements/testing.txt
-      - name: Setup postgres
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: setup-postgres
+        uses: ./.github/actions/setup-backend/
       - name: Import test data
         uses: ./.github/actions/cached-dependencies
         with:

.github/workflows/superset-cli.yml

@@ -52,22 +52,8 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
       - name: superset init
         if: steps.check.outcome == 'failure'
         run: |

.github/workflows/superset-e2e.yml

@@ -69,9 +69,7 @@ jobs:
         run: ./scripts/ci_check_no_file_changes.sh python frontend
       - name: Setup Python
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
+        uses: ./.github/actions/setup-backend/
       - name: OS dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
@@ -83,7 +81,7 @@ jobs:
         with:
           run: |
             pip-upgrade
-            pip install -r requirements/testing.txt
+            pip install -r requirements/development.txt
       - name: Setup postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

.github/workflows/superset-python-integrationtest.yml

@@ -51,21 +51,13 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup MySQL
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
             setup-mysql
       - name: Run celery
         if: steps.check.outcome == 'failure'
@@ -78,7 +70,6 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           bash .github/workflows/codecov.sh -c -F python -F mysql
-
   test-postgres:
     runs-on: ubuntu-20.04
     strategy:
@@ -117,21 +108,13 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
+      - name: Setup Postgres
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
@@ -177,21 +160,14 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
       - name: Install dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
+            # sqlite needs this working directory
             mkdir ${{ github.workspace }}/.temp
       - name: Run celery
         if: steps.check.outcome == 'failure'

.github/workflows/superset-python-misc.yml

@@ -38,21 +38,8 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
       - name: pylint
         if: steps.check.outcome == 'failure'
         # `-j 0` run Pylint in parallel
@@ -70,18 +57,6 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/base.txt"
-      - name: Install dependencies
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/base.txt
+        uses: ./.github/actions/setup-backend/
       - name: Test babel extraction
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

.github/workflows/superset-python-presto-hive.yml

@@ -67,22 +67,8 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
@@ -144,22 +130,8 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            setup-postgres
       - name: Run celery
         if: steps.check.outcome == 'failure'
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &

.github/workflows/superset-python-unittest.yml

@@ -46,27 +46,15 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        uses: ./.github/actions/setup-backend/
         if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-          cache-dependency-path: "requirements/testing.txt"
-      # TODO: separated requirements.txt file just for unit tests
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: |
-            apt-get-install
-            pip-upgrade
-            pip install wheel
-            pip install -r requirements/testing.txt
-            mkdir ${{ github.workspace }}/.temp
+          requirements-type: 'dev'
       - name: Python unit tests
         if: steps.check.outcome == 'failure'
         env:
           SUPERSET_TESTENV: true
+          SUPERSET_SECRET_KEY: not-a-secret
         run: |
           pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
       - name: Upload code coverage

.github/workflows/superset-translations.yml

@@ -47,9 +47,7 @@ jobs:
           persist-credentials: false
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
+        uses: ./.github/actions/setup-backend/
       - name: Install dependencies
         uses: ./.github/actions/cached-dependencies
         with:

.pre-commit-config.yaml

@@ -56,10 +56,6 @@ repos:
             types-paramiko,
             types-Markdown,
           ]
-  - repo: https://github.com/peterdemin/pip-compile-multi
-    rev: v2.6.2
-    hooks:
-      - id: pip-compile-multi-verify
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks: