Merge branch 'master' into #5319

.github/workflows/monodocs_build.yml

@@ -1,7 +1,7 @@
 name: Monodocs Build
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 on:
@@ -18,8 +18,8 @@ jobs:
     steps:
       - name: Fetch flytekit code
         uses: actions/checkout@v4
-        with:
-          path: "${{ github.workspace }}/flytekit"
+      - name: 'Clear action cache'
+        uses: ./.github/actions/clear-action-cache
       - name: Fetch flyte code
         uses: actions/checkout@v4
         with:
@@ -41,7 +41,6 @@ jobs:
           export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0"
           pip install -e ./flyteidl
       - shell: bash -el {0}
-        working-directory: ${{ github.workspace }}/flytekit
         run: |
           conda activate monodocs-env
           pip install -e .
@@ -54,7 +53,7 @@ jobs:
         working-directory: ${{ github.workspace }}/flyte
         shell: bash -el {0}
         env:
-          FLYTEKIT_LOCAL_PATH: ${{ github.workspace }}/flytekit
+          FLYTEKIT_LOCAL_PATH: ${{ github.workspace }}
         run: |
           conda activate monodocs-env
           make -C docs clean html SPHINXOPTS="-W -vvv"

.github/workflows/pythonbuild.yml

@@ -14,7 +14,7 @@ env:
   FLYTE_SDK_LOGGING_LEVEL: 10 # debug
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 
 jobs:
@@ -28,7 +28,7 @@ jobs:
           if [[ ${{ github.event_name }} == "schedule" ]]; then
             echo "python_versions=[\"3.8\",\"3.9\",\"3.10\",\"3.11\",\"3.12\"]" >> $GITHUB_ENV
           else
-            echo "python_versions=[\"3.12\"]" >> $GITHUB_ENV
+            echo "python_versions=[\"3.9\", \"3.12\"]" >> $GITHUB_ENV
           fi
 
   build:
@@ -42,7 +42,7 @@ jobs:
         python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
     steps:
       - uses: actions/checkout@v4
-      - name: 'Clear action cache'
+      - name: "Clear action cache"
         uses: ./.github/actions/clear-action-cache
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -59,7 +59,7 @@ jobs:
         run: |
           pip install uv
           make setup-global-uv
-          uv pip uninstall --system pandas
+          uv pip uninstall --system pandas pyarrow
           uv pip freeze
       - name: Test with coverage
         run: |
@@ -81,7 +81,7 @@ jobs:
         python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
     steps:
       - uses: actions/checkout@v4
-      - name: 'Clear action cache'
+      - name: "Clear action cache"
         uses: ./.github/actions/clear-action-cache
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -98,7 +98,7 @@ jobs:
         run: |
           pip install uv
           make setup-global-uv
-          uv pip uninstall --system pandas
+          uv pip uninstall --system pandas pyarrow
           uv pip freeze
       - name: Run extras unit tests with coverage
         # Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
@@ -128,10 +128,12 @@ jobs:
             pandas: "pandas<2.0.0"
           - numpy: "numpy<2.0.0"
             pandas: "pandas>=2.0.0"
+          - numpy: "numpy>=2.0.0"
+            python-version: "3.8"
 
     steps:
       - uses: actions/checkout@v4
-      - name: 'Clear action cache'
+      - name: "Clear action cache"
         uses: ./.github/actions/clear-action-cache
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -242,12 +244,16 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
+        makefile-cmd:
+          [integration_test_codecov, integration_test_lftransfers_codecov]
     steps:
       # As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history
       # and tags to work.
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: "Clear action cache"
+        uses: ./.github/actions/clear-action-cache # sandbox has disk pressure, so we need to clear the cache to get more disk space.
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -293,7 +299,7 @@ jobs:
           FLYTEKIT_CI: 1
           PYTEST_OPTS: -n2
         run: |
-          make integration_test_codecov
+          make ${{ matrix.makefile-cmd }}
       - name: Codecov
         uses: codecov/codecov-action@v3.1.0
         with:
@@ -316,6 +322,7 @@ jobs:
           - flytekit-aws-batch
           - flytekit-aws-sagemaker
           - flytekit-bigquery
+          - flytekit-comet-ml
           - flytekit-dask
           - flytekit-data-fsspec
           - flytekit-dbt
@@ -329,18 +336,21 @@ jobs:
           - flytekit-hive
           - flytekit-huggingface
           - flytekit-identity-aware-proxy
+          - flytekit-inference
           - flytekit-k8s-pod
           - flytekit-kf-mpi
           - flytekit-kf-pytorch
           - flytekit-kf-tensorflow
           - flytekit-mlflow
           - flytekit-mmcloud
           - flytekit-modin
+          - flytekit-neptune
           - flytekit-onnx-pytorch
           - flytekit-onnx-scikitlearn
           # onnx-tensorflow needs a version of tensorflow that does not work with protobuf>4.
           # The issue is being tracked on the tensorflow side in https://github.com/tensorflow/tensorflow/issues/53234#issuecomment-1330111693
           # flytekit-onnx-tensorflow
+          - flytekit-omegaconf
           - flytekit-openai
           - flytekit-pandera
           - flytekit-papermill
@@ -353,6 +363,10 @@ jobs:
           - flytekit-vaex
           - flytekit-whylogs
         exclude:
+          - python-version: 3.8
+            plugin-names: "flytekit-aws-sagemaker"
+          - python-version: 3.9
+            plugin-names: "flytekit-aws-sagemaker"
           # flytekit-modin depends on ray which does not have a 3.11 wheel yet.
           # Issue tracked in https://github.com/ray-project/ray/issues/27881
           - python-version: 3.11
@@ -382,9 +396,6 @@ jobs:
           # apache-beam, one of flytekit-airflow dependencies, does not support python 3.12: https://github.com/apache/beam/issues/29149
           - python-version: 3.12
             plugin-names: "flytekit-airflow"
-          # ydata-profiling, a dependency of flytekit-deck-standard, does not support python 3.12: https://github.com/ydataai/ydata-profiling/issues/1510
-          - python-version: 3.12
-            plugin-names: "flytekit-deck-standard"
           # Tensorflow is a dependency of flytekit-mlflow tests and that is not supported yet: https://github.com/tensorflow/tensorflow/issues/62003
           - python-version: 3.12
             plugin-names: "flytekit-mlflow"
@@ -402,7 +413,7 @@ jobs:
             plugin-names: "flytekit-kf-pytorch"
     steps:
       - uses: actions/checkout@v4
-      - name: 'Clear action cache'
+      - name: "Clear action cache"
         uses: ./.github/actions/clear-action-cache
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -425,9 +436,9 @@ jobs:
           uv pip install --system .
           if [ -f dev-requirements.in ]; then uv pip install --system -r dev-requirements.in; fi
           # TODO: move to protobuf>=5. Github issue: https://github.com/flyteorg/flyte/issues/5448
-          uv pip install --system -U $GITHUB_WORKSPACE "protobuf<5"
+          uv pip install --system -U $GITHUB_WORKSPACE "protobuf<5" "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl"
           # TODO: remove this when numpy v2 in onnx has been resolved
-          if [[ ${{ matrix.plugin-names }} == *"onnx"* || ${{ matrix.plugin-names }} == "flytekit-sqlalchemy" ]]; then
+          if [[ ${{ matrix.plugin-names }} == *"onnx"* || ${{ matrix.plugin-names }} == "flytekit-sqlalchemy" || ${{ matrix.plugin-names }} == "flytekit-pandera" ]]; then
             uv pip install --system "numpy<2.0.0"
           fi
           uv pip freeze

.gitignore

@@ -35,6 +35,7 @@ docs/source/_tags/
 .hypothesis
 .npm
 /**/target
+coverage.xml
 
 # Version file is auto-generated by setuptools_scm
 flytekit/_version.py

.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
     hooks:
       - id: check_pdb_hook
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         additional_dependencies:

CODEOWNERS

@@ -1,3 +1,4 @@
 # These owners will be the default owners for everything in
 # the repo. Unless a later match takes precedence.
-*       @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy @samhita-alla
+*       @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy @samhita-alla @thomasjpfan @future-outlier
+plugins/flytekit-kf-pytorch @fg91 @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy @samhita-alla @thomasjpfan @future-outlier

Dockerfile

@@ -1,12 +1,12 @@
-ARG PYTHON_VERSION
+ARG PYTHON_VERSION=3.12
 FROM python:${PYTHON_VERSION}-slim-bookworm
 
-MAINTAINER Flyte Team <users@flyte.org>
+LABEL org.opencontainers.image.authors="Flyte Team <users@flyte.org>"
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit
 
 WORKDIR /root
-ENV PYTHONPATH /root
-ENV FLYTE_SDK_RICH_TRACEBACKS 0
+ENV PYTHONPATH=/root
+ENV FLYTE_SDK_RICH_TRACEBACKS=0
 
 ARG VERSION
 ARG DOCKER_IMAGE
@@ -23,15 +23,16 @@ ARG DOCKER_IMAGE
 RUN apt-get update && apt-get install build-essential -y \
     && pip install uv \
     && uv pip install --system --no-cache-dir -U flytekit==$VERSION \
-        flytekitplugins-deck-standard==$VERSION \
+        kubernetes \
     && apt-get clean autoclean \
     && apt-get autoremove --yes \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/ \
+    && rm -rf /root/.cache/pip \
     && useradd -u 1000 flytekit \
     && chown flytekit: /root \
     && chown flytekit: /home \
     && :
 
 USER flytekit
 
-ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE"
+ENV FLYTE_INTERNAL_IMAGE="$DOCKER_IMAGE"

Dockerfile.agent

@@ -1,6 +1,6 @@
-FROM python:3.10-slim-bookworm as agent-slim
+FROM python:3.10-slim-bookworm AS agent-slim
 
-MAINTAINER Flyte Team <users@flyte.org>
+LABEL org.opencontainers.image.authors="Flyte Team <users@flyte.org>"
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit
 
 ARG VERSION
@@ -19,9 +19,9 @@ RUN pip install --no-cache-dir -U flytekit==$VERSION \
   && rm -rf /var/lib/{apt,dpkg,cache,log}/ \
   && :
 
-CMD pyflyte serve agent --port 8000
+CMD ["pyflyte", "serve", "agent", "--port", "8000"]
 
-FROM agent-slim as agent-all
+FROM agent-slim AS agent-all
 ARG VERSION
 
 RUN pip install --no-cache-dir -U \

Dockerfile.dev

@@ -5,17 +5,17 @@
 # From your test user code
 # $ pyflyte run --image localhost:30000/flytekittest:someversion
 
-ARG PYTHON_VERSION
+ARG PYTHON_VERSION=3.12
 FROM python:${PYTHON_VERSION}-slim-bookworm
 
-MAINTAINER Flyte Team <users@flyte.org>
+LABEL org.opencontainers.image.authors="Flyte Team <users@flyte.org>"
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit
 
 WORKDIR /root
-ENV FLYTE_SDK_RICH_TRACEBACKS 0
+ENV FLYTE_SDK_RICH_TRACEBACKS=0
 
 # Flytekit version of flytekit to be installed in the image
-ARG PSEUDO_VERSION
+ARG PSEUDO_VERSION=1.13.3
 
 
 # Note: Pod tasks should be exposed in the default image
@@ -38,10 +38,15 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \
         uv pip install --system --no-cache-dir -U \
             "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl" \
             -e /flytekit \
-            -e /flytekit/plugins/flytekit-k8s-pod \
             -e /flytekit/plugins/flytekit-deck-standard \
             -e /flytekit/plugins/flytekit-flyteinteractive \
+            markdown \
+            pandas \
+            pillow \
+            plotly \
+            pygments \
             scikit-learn \
+            ydata-profiling \
     && apt-get clean autoclean \
     && apt-get autoremove --yes \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/ \
@@ -51,7 +56,7 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \
     && :
 
 
-ENV PYTHONPATH "/flytekit:/flytekit/plugins/flytekit-k8s-pod:/flytekit/plugins/flytekit-deck-standard:"
+ENV PYTHONPATH="/flytekit:/flytekit/tests/flytekit/integration/remote"
 
 # Switch to the 'flytekit' user for better security.
 USER flytekit

Makefile

@@ -95,7 +95,15 @@ integration_test_codecov:
 
 .PHONY: integration_test
 integration_test:
-	$(PYTEST_AND_OPTS) tests/flytekit/integration ${CODECOV_OPTS}
+	$(PYTEST_AND_OPTS) tests/flytekit/integration ${CODECOV_OPTS} -m "not lftransfers"
+
+.PHONY: integration_test_lftransfers_codecov
+integration_test_lftransfers_codecov:
+	$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" integration_test_lftransfers
+
+.PHONY: integration_test_lftransfers
+integration_test_lftransfers:
+	$(PYTEST) tests/flytekit/integration ${CODECOV_OPTS} -m "lftransfers"
 
 doc-requirements.txt: export CUSTOM_COMPILE_COMMAND := make doc-requirements.txt
 doc-requirements.txt: doc-requirements.in install-piptools
@@ -119,5 +127,6 @@ build-dev: export PLATFORM ?= linux/arm64
 build-dev: export REGISTRY ?= localhost:30000
 build-dev: export PYTHON_VERSION ?= 3.12
 build-dev: export PSEUDO_VERSION ?= $(shell python -m setuptools_scm)
+build-dev: export TAG ?= dev
 build-dev:
 	docker build --platform ${PLATFORM} --push . -f Dockerfile.dev -t ${REGISTRY}/flytekit:${TAG} --build-arg PYTHON_VERSION=${PYTHON_VERSION} --build-arg PSEUDO_VERSION=${PSEUDO_VERSION}

dev-requirements.in

@@ -16,14 +16,14 @@ pre-commit
 codespell
 google-cloud-bigquery
 google-cloud-bigquery-storage
+snowflake-connector-python
 IPython
 keyrings.alt
 setuptools_scm
 pytest-icdiff
-jinja2
 
 # Tensorflow is not available for python 3.12 yet: https://github.com/tensorflow/tensorflow/issues/62003
-tensorflow; python_version<'3.12'
+tensorflow<=2.15.1; python_version<'3.12'
 # Newer versions of torch bring in nvidia dependencies that are not present in windows, so
 # we put this constraint while we do not have per-environment requirements files
 torch<=1.12.1; python_version<'3.11'
@@ -51,6 +51,7 @@ autoflake
 pillow
 numpy
 pandas
+pyarrow
 scikit-learn
 types-requests
 prometheus-client