Investigation to determine how to improve python module build times

.github/workflows/build2.yml

@@ -0,0 +1,141 @@
+# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Build aissemble 2
+
+on:
+  workflow_dispatch:
+    inputs:
+      buildBranch:
+        description: "Branch you want to build"
+        required: true
+        type: string
+        default: "366-follow-on-to-investigate-sagemaker-poetry-lock-slowness"
+  push:
+    branches: [ "366-follow-on-to-investigate-sagemaker-poetry-lock-slowness" ]
+  schedule:
+    - cron: "0 6 * * *" # every day at 6am UTC
+
+jobs:
+  build:
+    runs-on: arc-runner-set-aissemble
+    env:
+      DOCKER_CONFIG: /home/runner/.docker
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.buildBranch }}
+      - name: clear cache on nightly build
+        if: ${{ github.event.schedule }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            console.log("Clearing cache")
+            const caches = await github.rest.actions.getActionsCacheList({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            })
+            for (const cache of caches.data.actions_caches) {
+              console.log(cache)
+              github.rest.actions.deleteActionsCacheById({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                cache_id: cache.id,
+              })
+            }
+            console.log("Clear completed")
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Install required packages
+        run: |
+          sudo apt-get update
+          sudo apt install -y build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl \
+              git libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev zip unzip \
+              libpython3.11
+      - name: Install Python
+        uses: gabrielfalcao/pyenv-action@v18
+        with:
+          default: 3.11.4
+      # - name: Load m2 repository cache # Manually caching .m2 repo as the setup-java caching isn't falling back to older caches
+      #   id: cached-m2-repo
+      #   uses: actions/cache@v4
+      #   with:
+      #     path: ~/.m2/repository
+      #     key: maven-${{ hashFiles('**/pom.xml') }}
+      #     restore-keys: |
+      #       maven-
+      # - name: Load m2 build cache
+      #   id: cached-m2-build
+      #   uses: actions/cache@v4
+      #   with:
+      #     path: ~/.m2/build-cache
+      #     key: maven-build-cache-${{ hashFiles('**/pom.xml') }}
+      #     restore-keys: |
+      #       maven-build-cache-
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+      # - name: Poetry cache
+      #   id: cached-poetry
+      #   uses: actions/cache@v4
+      #   with:
+      #     path: ~/.cache/pypoetry
+      #     key: poetry-cache-${{ hashFiles('**/pom.xml') }}
+      #     restore-keys: |
+      #       poetry-
+      - name: Install Helm
+        run: |
+          curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+          chmod 700 get_helm.sh
+          ./get_helm.sh
+      - name: Install Helm Unittest Plugin
+        run: |
+          echo "Updating helm unittest plugin to latest version..."
+          helm plugin install https://github.com/helm-unittest/helm-unittest.git
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+      - name: Create Docker Builder Config File
+        run: sudo touch /etc/buildkitd.toml
+      # Generate the settings.xml for ghcr.io, pypi, & dev-pypi server profiles
+      - name: Create settings.xml
+        run: |
+          echo "<settings><servers><server><id>ghcr.io</id><username>${{ secrets.GHCR_IO_USERNAME }}</username><password>${{ secrets.GHCR_IO_TOKEN }}</password></server><server><id>pypi</id><username>${{ secrets.PYPI_USERNAME }}</username><password>${{ secrets.PYPI_TOKEN }}</password></server><server><id>dev-pypi</id><username>${{ secrets.TEST_PYPI_USERNAME }}</username><password>${{ secrets.TEST_PYPI_TOKEN }}</password></server> </servers></settings>" > $HOME/.m2/settings.xml
+      # Run build with the gh-build profile
+      - name: Build aiSSEMBLE
+        run: |
+          ./mvnw -B clean deploy -U -file pom.xml -Pci,gh-build -pl :aissemble-extensions-model-training-api-sagemaker,:aissemble-model-training-api-sagemaker --settings $HOME/.m2/settings.xml
+      # Install Maven which is needed for archetype tests
+      - name: Set up Maven
+        uses: stCarolas/setup-maven@v5
+        with:
+          maven-version: 3.9.9
+      # Execute archetype tests
+      - name: Run Archetype Tests
+        run: |
+          ./mvnw -B clean install -Parchetype-test -pl :foundation-archetype
+      #NB: The following two explicit cache saves are necessary to ensure caches are saved on build failure,
+      # until https://github.com/actions/cache/issues/1315 is resolved
+      # - name: Save m2 repository cache
+      #   id: save-m2-repo
+      #   uses: actions/cache/save@v4
+      #   if: always()
+      #   with:
+      #     path: ~/.m2/repository
+      #     key: maven-${{ hashFiles('**/pom.xml') }}
+      # - name: Save m2 build cache
+      #   id: save-m2-build
+      #   uses: actions/cache/save@v4
+      #   if: always()
+      #   with:
+      #     path: ~/.m2/build-cache
+      #     key: maven-build-cache-${{ hashFiles('**/pom.xml') }}

extensions/aissemble-extensions-model-training-api-sagemaker/pyproject.toml

@@ -17,7 +17,9 @@ kubernetes = ">=26.1.0"
 urllib3 = "^1.26.18"
 krausening = ">=20"
 sagemaker = ">=2.218.0"
-mlflow = "^2.3.1"
+
+# As part of Issue #397, we removed the mlflow dependency as it is included within sagemaker 
+# and specifying it here causes excessive dependency resolution times
 
 [tool.poetry.group.dev.dependencies]
 black = ">=22.1.0"

foundation/aissemble-foundation-versioning-service/pyproject.toml

@@ -17,8 +17,9 @@ python = ">=3.8"
 mlflow = "^2.3.1"
 fastapi = ">=0.95.0"
 uvicorn = {version = "^0.18.0", extras = ["standard"]}
-pydantic = ">=2.8.0"
-starlette = ">=0.25.0"
+
+# Poetry export-without-path-deps fails due to a "Dependency walk failed at urllib3 (>=1.26.0)"
+# Adding to resolve the issue; Poetry changes the value from >= -> ^
 urllib3 = "^1.26.18"
 
 aissemble-foundation-core-python = {path = "../aissemble-foundation-core-python/", develop = true}

test/test-mda-models/test-machine-learning-model/aissemble-machine-learning-sagemaker-training/pyproject.toml

@@ -16,18 +16,15 @@ python = ">=3.8"
 sagemaker-training = ">=4.6.1"
 mlflow = ">=2.16.2"
 
-# Poetry lock was timing out attempting to resolve dependencies
-# I removed the mlflow dependency and ran poetry add mlflow and the dependencies resolved, so I copied
-# over the problematic pacakges' versions from the lock file.
-opentelemetry-api = ">=1.9.0,<3"
-opentelemetry-sdk = ">=1.9.0,<3"
-mlflow-skinny = "2.16.2"
-botocore = ">=1.33.2,<2.0a.0"
+# sagemaker-training (4.8.1) depends on protobuf (>=3.9.2,<=3.20.3)
+# mlflow-skinny (2.16.2) depends on protobuf (>=3.12.0,<6)
+protobuf = ">=3.12.0,<=3.20.3" 
 
 # Poetry export-without-path-deps fails due to a "Dependency walk failed at urllib3 (>=1.26.0)"
 # Adding to resolve the issue; Poetry changes the value from >= -> ^
 urllib3 = "^1.26.18"
 
+
 [tool.poetry.group.dev.dependencies]
 black = ">=22.1.0"
 behave = ">=1.2.6"