From a118101aa4a211c326237d861f8b8701b3837ff2 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 19 Jul 2023 10:40:06 -0700 Subject: [PATCH 1/4] Fix typo in flyekitplugins -> flytekitplugins (#1038) Signed-off-by: Haytham Abuelfutuh --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 71b43a7fd..c220bee5b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -70,7 +70,7 @@ Kubernetes cluster so that you can interact with it on your machine. First install [flytekit](https://pypi.org/project/flytekit/), Flyte's Python SDK and [Scikit-learn](https://scikit-learn.org/stable). ```{prompt} bash $ -pip install flytekit flyekitplugins-deck-standard scikit-learn +pip install flytekit flytekitplugins-deck-standard scikit-learn ``` Then install [flytectl](https://docs.flyte.org/projects/flytectl/en/latest/), From e361b7aed36486eddf6104193f730e3f865c02f4 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 19 Jul 2023 12:27:52 -0700 Subject: [PATCH 2/4] Update dockerfile in k8s_spark_plugin (#1039) Signed-off-by: Kevin Su --- examples/k8s_spark_plugin/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/k8s_spark_plugin/Dockerfile b/examples/k8s_spark_plugin/Dockerfile index 86ae6e079..f336776a2 100644 --- a/examples/k8s_spark_plugin/Dockerfile +++ b/examples/k8s_spark_plugin/Dockerfile @@ -44,10 +44,6 @@ COPY . /root/ ARG tag ENV FLYTE_INTERNAL_IMAGE $tag -# Copy over the helper script that the SDK relies on -RUN cp ${VENV}/bin/flytekit_venv /usr/local/bin/ -RUN chmod a+x /usr/local/bin/flytekit_venv - # Set /root user and group RUN chown -R ${spark_uid}:${spark_uid} /root From 4a635af4df8962af0bc5adffd8fbf9b6eecde460 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 19 Jul 2023 16:47:14 -0700 Subject: [PATCH 3/4] Add dockerfile to snowflake example (#1016) * Add dockerfile to snowflake example Signed-off-by: Kevin Su * pip compile with python 3.8 Signed-off-by: Niels Bantilan --------- Signed-off-by: Kevin Su Signed-off-by: Niels Bantilan Co-authored-by: Niels Bantilan Co-authored-by: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> --- examples/snowflake_plugin/Dockerfile | 23 ++ examples/snowflake_plugin/requirements.in | 1 + examples/snowflake_plugin/requirements.txt | 358 +++++++++++++++++++++ 3 files changed, 382 insertions(+) create mode 100644 examples/snowflake_plugin/Dockerfile create mode 100644 examples/snowflake_plugin/requirements.in create mode 100644 examples/snowflake_plugin/requirements.txt diff --git a/examples/snowflake_plugin/Dockerfile b/examples/snowflake_plugin/Dockerfile new file mode 100644 index 000000000..a9ba25521 --- /dev/null +++ b/examples/snowflake_plugin/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.8-slim-buster +LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytesnacks + +WORKDIR /root +ENV VENV /opt/venv +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONPATH /root + + +# Install Python dependencies +COPY requirements.txt /root +RUN pip install -r /root/requirements.txt +RUN pip freeze + +# Copy the actual code +COPY . /root/ + + +# This tag is supplied by the build script and will be used to determine the version +# when registering tasks, workflows, and launch plans +ARG tag +ENV FLYTE_INTERNAL_IMAGE $tag diff --git a/examples/snowflake_plugin/requirements.in b/examples/snowflake_plugin/requirements.in new file mode 100644 index 000000000..029e7ddd5 --- /dev/null +++ b/examples/snowflake_plugin/requirements.in @@ -0,0 +1 @@ +flytekitplugins-snowflake diff --git a/examples/snowflake_plugin/requirements.txt b/examples/snowflake_plugin/requirements.txt new file mode 100644 index 000000000..0afdba87a --- /dev/null +++ b/examples/snowflake_plugin/requirements.txt @@ -0,0 +1,358 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --resolver=backtracking requirements.in +# +adlfs==2023.4.0 + # via flytekit +aiobotocore==2.5.1 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp +arrow==1.2.3 + # via jinja2-time +async-timeout==4.0.2 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.27.1 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs +binaryornot==0.4.4 + # via cookiecutter +botocore==1.29.161 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests +cffi==1.15.1 + # via + # azure-datalake-store + # cryptography +chardet==5.1.0 + # via binaryornot +charset-normalizer==3.1.0 + # via + # aiohttp + # requests +click==8.1.3 + # via + # cookiecutter + # flytekit + # rich-click +cloudpickle==2.2.1 + # via flytekit +cookiecutter==2.1.1 + # via flytekit +croniter==1.4.1 + # via flytekit +cryptography==41.0.1 + # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt + # pyopenssl +dataclasses-json==0.5.9 + # via flytekit +decorator==5.1.1 + # via gcsfs +deprecated==1.2.14 + # via flytekit +diskcache==5.6.1 + # via flytekit +docker==6.1.3 + # via flytekit +docker-image-py==0.1.12 + # via flytekit +docstring-parser==0.15 + # via flytekit +flyteidl==1.5.12 + # via flytekit +flytekit==1.7.1b1 + # via flytekitplugins-snowflake +flytekitplugins-snowflake==1.7.0 + # via -r requirements.in +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.6.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.6.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.31 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 + # via + # flyteidl + # flytekit + # google-api-core + # grpcio-status +grpcio==1.56.0 + # via + # flytekit + # grpcio-status +grpcio-status==1.56.0 + # via flytekit +idna==3.4 + # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # flytekit + # keyring +importlib-resources==5.12.0 + # via keyring +isodate==0.6.1 + # via azure-storage-blob +jaraco-classes==3.2.3 + # via keyring +jinja2==3.1.2 + # via + # cookiecutter + # jinja2-time +jinja2-time==0.2.0 + # via cookiecutter +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +keyring==24.2.0 + # via flytekit +kubernetes==26.1.0 + # via flytekit +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.3 + # via jinja2 +marshmallow==3.19.0 + # via + # dataclasses-json + # marshmallow-enum + # marshmallow-jsonschema +marshmallow-enum==1.5.1 + # via dataclasses-json +marshmallow-jsonschema==0.13.0 + # via flytekit +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 + # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl +mypy-extensions==1.0.0 + # via typing-inspect +natsort==8.4.0 + # via flytekit +numpy==1.24.4 + # via + # flytekit + # pandas + # pyarrow +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 + # via + # docker + # marshmallow +pandas==1.5.3 + # via flytekit +portalocker==2.7.0 + # via msal-extensions +protobuf==4.23.3 + # via + # flyteidl + # google-api-core + # googleapis-common-protos + # grpcio-status + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl +pyarrow==10.0.1 + # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +pycparser==2.21 + # via cffi +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 + # via flytekit +python-dateutil==2.8.2 + # via + # arrow + # botocore + # croniter + # flytekit + # kubernetes + # pandas +python-json-logger==2.0.7 + # via flytekit +python-slugify==8.0.1 + # via cookiecutter +pytimeparse==1.1.8 + # via flytekit +pytz==2023.3 + # via + # flytekit + # pandas +pyyaml==6.0 + # via + # cookiecutter + # flytekit + # kubernetes + # responses +regex==2023.6.3 + # via docker-image-py +requests==2.31.0 + # via + # azure-core + # azure-datalake-store + # cookiecutter + # docker + # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib + # responses +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 + # via flytekit +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.6.0 + # via flytekit +six==1.16.0 + # via + # azure-core + # azure-identity + # google-auth + # isodate + # kubernetes + # python-dateutil +smmap==5.0.0 + # via gitdb +sortedcontainers==2.4.0 + # via flytekit +statsd==3.3.0 + # via flytekit +text-unidecode==1.3 + # via python-slugify +types-pyyaml==6.0.12.10 + # via responses +typing-extensions==4.7.1 + # via + # aioitertools + # azure-core + # azure-storage-blob + # flytekit + # rich + # typing-inspect +typing-inspect==0.9.0 + # via dataclasses-json +urllib3==1.26.16 + # via + # botocore + # docker + # flytekit + # google-auth + # kubernetes + # requests + # responses +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 + # via flytekit +wrapt==1.15.0 + # via + # aiobotocore + # deprecated + # flytekit +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 + # via + # importlib-metadata + # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From 0ea3c6302c62f424fbc6c83cb6d771fe8d0bb8c8 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 21 Jul 2023 14:12:30 -0400 Subject: [PATCH 4/4] execute getting started, future proof flyte decks (#1036) * execute getting started, future proof flyte decks Signed-off-by: Niels Bantilan * bump flytekit Signed-off-by: Niels Bantilan * add deck standard plugin Signed-off-by: Niels Bantilan * debug analytics doc Signed-off-by: Niels Bantilan * debug Signed-off-by: Niels Bantilan * cleanup Signed-off-by: Niels Bantilan * updates Signed-off-by: Niels Bantilan * updates Signed-off-by: Niels Bantilan * use py39 for docs build Signed-off-by: Niels Bantilan * update dbt plugin reqs Signed-off-by: Niels Bantilan * update flytekit dep Signed-off-by: Niels Bantilan --------- Signed-off-by: Niels Bantilan --- .github/workflows/docs_build.yml | 4 + docs-requirements.in | 3 +- docs-requirements.txt | 149 +++--- docs/_static/custom.css | 19 + docs/_static/custom.js | 11 + docs/conf.py | 3 +- docs/getting_started/analytics.md | 13 +- docs/getting_started/visualizing_artifacts.md | 2 +- examples/dbt_plugin/requirements.in | 3 + examples/dbt_plugin/requirements.txt | 436 +++++++++++++++++- 10 files changed, 566 insertions(+), 77 deletions(-) create mode 100644 docs/_static/custom.js create mode 100644 examples/dbt_plugin/requirements.in diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml index e4bca3547..2c87bb6ad 100644 --- a/.github/workflows/docs_build.yml +++ b/.github/workflows/docs_build.yml @@ -15,6 +15,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: "0" + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Report Sphinx Warnings id: sphinx-warnings # NOTE: myst-nb build warnings are logged in docs/_build/reports/*.err.log diff --git a/docs-requirements.in b/docs-requirements.in index 304487079..ac56cea54 100644 --- a/docs-requirements.in +++ b/docs-requirements.in @@ -1,7 +1,8 @@ -flytekit>=1.3.0 +flytekit>=1.8.1 git+https://github.com/flyteorg/furo@main pillow psycopg2-binary +flytekitplugins-deck-standard flytekitplugins-kfpytorch flytekitplugins-sqlalchemy jupytext diff --git a/docs-requirements.txt b/docs-requirements.txt index 094d459fe..6e1974530 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -6,7 +6,7 @@ # adlfs==2023.4.0 # via flytekit -aiobotocore==2.5.0 +aiobotocore==2.5.2 # via s3fs aiohttp==3.8.4 # via @@ -25,8 +25,8 @@ appnope==0.1.3 # ipykernel # ipython arrow==1.2.3 - # via jinja2-time -astroid==2.15.5 + # via cookiecutter +astroid==2.15.6 # via # -r docs-requirements.in # sphinx-autoapi @@ -39,8 +39,9 @@ attrs==23.1.0 # aiohttp # jsonschema # jupyter-cache + # referencing # visions -azure-core==1.27.1 +azure-core==1.28.0 # via # adlfs # azure-identity @@ -49,7 +50,7 @@ azure-datalake-store==0.0.53 # via adlfs azure-identity==1.13.0 # via adlfs -azure-storage-blob==12.16.0 +azure-storage-blob==12.17.0 # via adlfs babel==2.12.1 # via sphinx @@ -61,7 +62,7 @@ beautifulsoup4==4.12.2 # sphinx-code-include binaryornot==0.4.4 # via cookiecutter -botocore==1.29.76 +botocore==1.29.161 # via aiobotocore cachetools==5.3.1 # via google-auth @@ -77,11 +78,11 @@ cfgv==3.3.1 # via pre-commit chardet==5.1.0 # via binaryornot -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests -click==8.1.3 +click==8.1.6 # via # cookiecutter # flytekit @@ -95,11 +96,11 @@ comm==0.1.3 # via ipykernel contourpy==1.1.0 # via matplotlib -cookiecutter==2.1.1 +cookiecutter==2.2.3 # via flytekit croniter==1.4.1 # via flytekit -cryptography==41.0.1 +cryptography==41.0.2 # via # azure-identity # azure-storage-blob @@ -110,7 +111,7 @@ cycler==0.11.0 # via matplotlib dacite==1.8.1 # via ydata-profiling -dataclasses-json==0.5.8 +dataclasses-json==0.5.9 # via flytekit debugpy==1.6.7 # via ipykernel @@ -122,7 +123,7 @@ deprecated==1.2.14 # via flytekit diskcache==5.6.1 # via flytekit -distlib==0.3.6 +distlib==0.3.7 # via virtualenv docker==6.1.3 # via flytekit @@ -145,26 +146,25 @@ filelock==3.12.2 # via # torch # virtualenv -flyteidl==1.5.10 +flyteidl==1.5.13 # via # flytekit # flytekitplugins-kfpytorch -flytekit==1.7.0 +flytekit==1.8.1 # via - # -r ./common/requirements-common.in # -r docs-requirements.in # flytekitplugins-deck-standard # flytekitplugins-kfpytorch # flytekitplugins-sqlalchemy -flytekitplugins-deck-standard==1.7.0 - # via -r ./common/requirements-common.in -flytekitplugins-kfpytorch==1.7.0 +flytekitplugins-deck-standard==1.8.1 + # via -r docs-requirements.in +flytekitplugins-kfpytorch==1.8.1 # via -r docs-requirements.in -flytekitplugins-sqlalchemy==1.7.0 +flytekitplugins-sqlalchemy==1.8.1 # via -r docs-requirements.in -fonttools==4.40.0 +fonttools==4.41.0 # via matplotlib -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal @@ -180,13 +180,13 @@ gcsfs==2023.6.0 # via flytekit gitdb==4.0.10 # via gitpython -gitpython==3.1.31 +gitpython==3.1.32 # via flytekit google-api-core==2.11.1 # via # google-cloud-core # google-cloud-storage -google-auth==2.20.0 +google-auth==2.22.0 # via # gcsfs # google-api-core @@ -196,9 +196,9 @@ google-auth==2.20.0 # kubernetes google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via google-cloud-storage -google-cloud-storage==2.9.0 +google-cloud-storage==2.10.0 # via gcsfs google-crc32c==1.5.0 # via google-resumable-media @@ -212,12 +212,12 @@ googleapis-common-protos==1.59.1 # grpcio-status greenlet==2.0.2 # via sqlalchemy -grpcio==1.54.2 +grpcio==1.56.0 # via # -r docs-requirements.in # flytekit # grpcio-status -grpcio-status==1.54.2 +grpcio-status==1.56.0 # via # -r docs-requirements.in # flytekit @@ -235,7 +235,7 @@ imagehash==4.3.1 # ydata-profiling imagesize==1.4.1 # via sphinx -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via # flytekit # jupyter-cache @@ -244,12 +244,13 @@ importlib-metadata==6.6.0 # markdown # myst-nb # sphinx -importlib-resources==5.12.0 +importlib-resources==6.0.0 # via # jsonschema + # jsonschema-specifications # keyring # matplotlib -ipykernel==6.23.2 +ipykernel==6.24.0 # via # ipywidgets # myst-nb @@ -259,38 +260,37 @@ ipython==8.12.2 # ipykernel # ipywidgets # myst-nb -ipywidgets==8.0.6 +ipywidgets==8.0.7 # via flytekitplugins-deck-standard isodate==0.6.1 # via azure-storage-blob -jaraco-classes==3.2.3 +jaraco-classes==3.3.0 # via keyring jedi==0.18.2 # via ipython jinja2==3.0.3 # via # cookiecutter - # jinja2-time # myst-parser # sphinx # sphinx-autoapi # sphinx-tabs # torch # ydata-profiling -jinja2-time==0.2.0 - # via cookiecutter jmespath==1.0.1 # via botocore -joblib==1.2.0 +joblib==1.3.1 # via # flytekit # phik # scikit-learn -jsonschema==4.17.3 +jsonschema==4.18.4 # via nbformat +jsonschema-specifications==2023.7.1 + # via jsonschema jupyter-cache==0.6.1 # via myst-nb -jupyter-client==8.2.0 +jupyter-client==8.3.0 # via # ipykernel # nbclient @@ -300,15 +300,15 @@ jupyter-core==5.3.1 # jupyter-client # nbclient # nbformat -jupyterlab-widgets==3.0.7 +jupyterlab-widgets==3.0.8 # via ipywidgets -jupytext==1.14.6 +jupytext==1.14.7 # via -r docs-requirements.in -keyring==23.13.1 +keyring==24.2.0 # via flytekit kiwisolver==1.4.4 # via matplotlib -kubernetes==26.1.0 +kubernetes==27.2.0 # via flytekit lazy-object-proxy==1.9.0 # via astroid @@ -328,12 +328,13 @@ marshmallow==3.19.0 # marshmallow-enum # marshmallow-jsonschema marshmallow-enum==1.5.1 - # via dataclasses-json + # via + # dataclasses-json + # flytekit marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib==3.7.1 +matplotlib==3.7.2 # via - # -r ./common/requirements-common.in # phik # seaborn # wordcloud @@ -373,13 +374,13 @@ myst-nb==0.17.2 # via -r docs-requirements.in myst-parser==0.18.1 # via myst-nb -natsort==8.3.1 +natsort==8.4.0 # via flytekit nbclient==0.7.4 # via # jupyter-cache # myst-nb -nbformat==5.9.0 +nbformat==5.9.1 # via # jupyter-cache # jupytext @@ -412,7 +413,9 @@ numpy==1.23.5 # wordcloud # ydata-profiling oauthlib==3.2.2 - # via requests-oauthlib + # via + # kubernetes + # requests-oauthlib packaging==23.1 # via # docker @@ -440,7 +443,7 @@ phik==0.12.3 # via ydata-profiling pickleshare==0.7.5 # via ipython -pillow==9.5.0 +pillow==10.0.0 # via # -r docs-requirements.in # imagehash @@ -449,7 +452,7 @@ pillow==9.5.0 # wordcloud pkgutil-resolve-name==1.3.10 # via jsonschema -platformdirs==3.5.3 +platformdirs==3.9.1 # via # jupyter-core # virtualenv @@ -459,9 +462,9 @@ portalocker==2.7.0 # via msal-extensions pre-commit==3.3.3 # via sphinx-tags -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.39 # via ipython -protobuf==4.23.3 +protobuf==4.23.4 # via # flyteidl # google-api-core @@ -488,7 +491,7 @@ pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pydantic==1.10.9 +pydantic==1.10.11 # via ydata-profiling pygments==2.15.1 # via @@ -498,14 +501,12 @@ pygments==2.15.1 # sphinx # sphinx-prompt # sphinx-tabs -pyjwt[crypto]==2.7.0 +pyjwt[crypto]==2.8.0 # via msal pyopenssl==23.2.0 # via flytekit pyparsing==3.0.9 # via matplotlib -pyrsistent==0.19.3 - # via jsonschema python-dateutil==2.8.2 # via # arrow @@ -529,7 +530,7 @@ pytz==2023.3 # pandas pywavelets==1.4.1 # via imagehash -pyyaml==6.0 +pyyaml==5.4.1 # via # cookiecutter # flytekit @@ -546,6 +547,10 @@ pyzmq==25.1.0 # via # ipykernel # jupyter-client +referencing==0.30.0 + # via + # jsonschema + # jsonschema-specifications regex==2023.6.3 # via docker-image-py requests==2.31.0 @@ -577,11 +582,15 @@ rich==13.4.2 # rich-click rich-click==1.6.1 # via flytekit +rpds-py==0.9.2 + # via + # jsonschema + # referencing rsa==4.9 # via google-auth s3fs==2023.6.0 # via flytekit -scikit-learn==1.2.2 +scikit-learn==1.3.0 # via -r docs-requirements.in scipy==1.10.1 # via @@ -633,7 +642,7 @@ sphinx==4.5.0 # sphinxcontrib-youtube sphinx-autoapi==2.0.1 # via -r docs-requirements.in -sphinx-basic-ng==1.0.0b1 +sphinx-basic-ng==1.0.0b2 # via furo sphinx-code-include==1.1.1 # via -r docs-requirements.in @@ -651,7 +660,7 @@ sphinx-rtd-theme==1.2.2 # via -r docs-requirements.in sphinx-tabs==3.4.0 # via -r docs-requirements.in -sphinx-tags==0.2.0 +sphinx-tags==0.2.1 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.4 # via sphinx @@ -673,7 +682,7 @@ sphinxcontrib-youtube==1.2.0 # via -r docs-requirements.in sphinxext-remoteliteralinclude==0.4.0 # via -r docs-requirements.in -sqlalchemy==2.0.16 +sqlalchemy==2.0.19 # via # flytekitplugins-sqlalchemy # jupyter-cache @@ -693,7 +702,7 @@ tenacity==8.2.2 # via plotly text-unidecode==1.3 # via python-slugify -threadpoolctl==3.1.0 +threadpoolctl==3.2.0 # via scikit-learn toml==0.10.2 # via jupytext @@ -720,7 +729,7 @@ typeguard==2.13.3 # via ydata-profiling types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via # aioitertools # astroid @@ -748,21 +757,19 @@ urllib3==1.26.16 # kubernetes # requests # responses -virtualenv==20.23.0 +virtualenv==20.24.0 # via pre-commit visions[type_image_path]==0.7.5 # via ydata-profiling wcwidth==0.2.6 # via prompt-toolkit -websocket-client==1.5.3 +websocket-client==1.6.1 # via # docker # kubernetes wheel==0.40.0 - # via - # -r ./common/requirements-common.in - # flytekit -widgetsnbextension==4.0.7 + # via flytekit +widgetsnbextension==4.0.8 # via ipywidgets wordcloud==1.9.2 # via ydata-profiling @@ -774,9 +781,9 @@ wrapt==1.15.0 # flytekit yarl==1.9.2 # via aiohttp -ydata-profiling==4.2.0 +ydata-profiling==4.3.1 # via flytekitplugins-deck-standard -zipp==3.15.0 +zipp==3.16.2 # via # importlib-metadata # importlib-resources diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 94be22be8..96f8c2f4b 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -84,3 +84,22 @@ p.tags a { p.tags span:not(:first-child) { display: none; } + +.plotly-graph-div { + width: 100% !important; + opacity: 0; +} + +.plotly-graph-div.show { + opacity: 1; + + -webkit-transition: opacity 0.5s ease-in; + -moz-transition: opacity 0.5s ease-in; + -ms-transition: opacity 0.5s ease-in; + -o-transition: opacity 0.5s ease-in; + transition: opacity 0.5s ease-in; +} + +#flyte-frame-container > div.active { + width: 90% !important; +} diff --git a/docs/_static/custom.js b/docs/_static/custom.js new file mode 100644 index 000000000..4bd3c6bc6 --- /dev/null +++ b/docs/_static/custom.js @@ -0,0 +1,11 @@ +function resizePlotly() { + var plotly = jQuery( ".plotly-graph-div" ) + plotly.attr("style", "height: 100%; width: 100%;") + window.dispatchEvent(new Event('resize')); + plotly.addClass("show"); +} + +jQuery(document).ready(function() { + // make sure that plotly plots only take 100% of the width of the container + resizePlotly() +}) diff --git a/docs/conf.py b/docs/conf.py index 1cb6ef607..c48dfe6aa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -70,6 +70,7 @@ html_static_path = ["_static"] html_css_files = ["sphx_gallery_autogen.css", "custom.css"] +html_js_files = ["custom.js"] suppress_warnings = ["autosectionlabel.*"] @@ -155,7 +156,7 @@ "thumbnail_size": (350, 350), } -nb_execution_mode = "off" +nb_execution_mode = "auto" nb_execution_excludepatterns = [ "auto_examples/**/*", ] diff --git a/docs/getting_started/analytics.md b/docs/getting_started/analytics.md index 080441fc3..598f61e1f 100644 --- a/docs/getting_started/analytics.md +++ b/docs/getting_started/analytics.md @@ -106,6 +106,7 @@ tags: [remove-input] import logging import os import re +from pythonjsonlogger import jsonlogger from IPython.display import HTML @@ -113,13 +114,23 @@ class DeckFilter(logging.Filter): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.formatter = jsonlogger.JsonFormatter( + fmt="%(asctime)s %(name)s %(levelname)s %(message)s" + ) + self.logs = [] self.deck_files = {} def filter(self, record): patt = "(.+) task creates flyte deck html to (.+/deck.html)" - matches = re.match(patt, record.getMessage()) + msg = record.getMessage() + matches = re.match(patt, msg) + + if msg == "Connection error. Skip stats collection.": + return False + if matches: task, filepath = matches.group(1), matches.group(2) + self.logs.append(self.formatter.format(record)) self.deck_files[task] = re.sub("^file://", "", filepath) return False diff --git a/docs/getting_started/visualizing_artifacts.md b/docs/getting_started/visualizing_artifacts.md index 81e81b8cc..65de7e549 100644 --- a/docs/getting_started/visualizing_artifacts.md +++ b/docs/getting_started/visualizing_artifacts.md @@ -94,7 +94,7 @@ import os import re import shutil from pythonjsonlogger import jsonlogger -from IPython.display import HTML, IFrame +from IPython.display import HTML from pathlib import Path diff --git a/examples/dbt_plugin/requirements.in b/examples/dbt_plugin/requirements.in new file mode 100644 index 000000000..d0675062a --- /dev/null +++ b/examples/dbt_plugin/requirements.in @@ -0,0 +1,3 @@ +flytekit>=1.8.1 +flytekitplugins-dbt +dbt-postgres diff --git a/examples/dbt_plugin/requirements.txt b/examples/dbt_plugin/requirements.txt index 52db18fe9..e15fb92e7 100644 --- a/examples/dbt_plugin/requirements.txt +++ b/examples/dbt_plugin/requirements.txt @@ -1,2 +1,434 @@ -flytekitplugins-dbt -dbt-postgres +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --resolver=backtracking requirements.in +# +adlfs==2023.4.0 + # via flytekit +agate==1.6.3 + # via dbt-core +aiobotocore==2.5.2 + # via s3fs +aiohttp==3.8.5 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp +arrow==1.2.3 + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +attrs==23.1.0 + # via + # aiohttp + # jsonschema +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.17.0 + # via adlfs +babel==2.12.1 + # via agate +binaryornot==0.4.4 + # via cookiecutter +botocore==1.29.161 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests +cffi==1.15.1 + # via + # azure-datalake-store + # cryptography + # dbt-core +chardet==5.1.0 + # via binaryornot +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.6 + # via + # cookiecutter + # dbt-core + # flytekit + # rich-click +cloudpickle==2.2.1 + # via flytekit +colorama==0.4.5 + # via dbt-core +cookiecutter==2.2.3 + # via flytekit +croniter==1.4.1 + # via flytekit +cryptography==41.0.2 + # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt + # pyopenssl +dataclasses-json==0.5.9 + # via flytekit +dbt-core==1.2.6 + # via + # dbt-postgres + # flytekitplugins-dbt +dbt-extractor==0.4.1 + # via dbt-core +dbt-postgres==1.2.6 + # via -r requirements.in +decorator==5.1.1 + # via gcsfs +deprecated==1.2.14 + # via flytekit +diskcache==5.6.1 + # via flytekit +docker==6.1.3 + # via flytekit +docker-image-py==0.1.12 + # via flytekit +docstring-parser==0.15 + # via flytekit +flyteidl==1.5.13 + # via flytekit +flytekit==1.8.1 + # via + # -r requirements.in + # flytekitplugins-dbt +flytekitplugins-dbt==1.8.1 + # via -r requirements.in +frozenlist==1.4.0 + # via + # aiohttp + # aiosignal +fsspec==2023.6.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +future==0.18.3 + # via parsedatetime +gcsfs==2023.6.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.22.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 + # via + # flyteidl + # flytekit + # google-api-core + # grpcio-status +grpcio==1.56.0 + # via + # flytekit + # grpcio-status +grpcio-status==1.56.0 + # via flytekit +hologram==0.0.15 + # via dbt-core +idna==3.4 + # via + # dbt-core + # requests + # yarl +importlib-metadata==6.8.0 + # via + # flytekit + # keyring +importlib-resources==6.0.0 + # via keyring +isodate==0.6.1 + # via + # agate + # azure-storage-blob + # dbt-core +jaraco-classes==3.3.0 + # via keyring +jinja2==2.11.3 + # via + # cookiecutter + # dbt-core +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +jsonschema==3.2.0 + # via hologram +keyring==24.2.0 + # via flytekit +kubernetes==27.2.0 + # via flytekit +leather==0.3.4 + # via agate +logbook==1.5.3 + # via dbt-core +markdown-it-py==3.0.0 + # via rich +markupsafe==2.0.1 + # via + # dbt-core + # jinja2 +marshmallow==3.19.0 + # via + # dataclasses-json + # marshmallow-enum + # marshmallow-jsonschema +marshmallow-enum==1.5.1 + # via + # dataclasses-json + # flytekit +marshmallow-jsonschema==0.13.0 + # via flytekit +mashumaro==2.9 + # via dbt-core +mdurl==0.1.2 + # via markdown-it-py +minimal-snowplow-tracker==0.0.2 + # via dbt-core +more-itertools==9.1.0 + # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +msgpack==1.0.5 + # via mashumaro +multidict==6.0.4 + # via + # aiohttp + # yarl +mypy-extensions==1.0.0 + # via typing-inspect +natsort==8.4.0 + # via flytekit +networkx==2.8.8 + # via dbt-core +numpy==1.24.4 + # via + # flytekit + # pandas + # pyarrow +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib +packaging==21.3 + # via + # dbt-core + # docker + # marshmallow +pandas==1.5.3 + # via flytekit +parsedatetime==2.4 + # via agate +portalocker==2.7.0 + # via msal-extensions +protobuf==4.23.4 + # via + # dbt-core + # flyteidl + # google-api-core + # googleapis-common-protos + # grpcio-status + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl +psycopg2-binary==2.9.6 + # via dbt-postgres +pyarrow==10.0.1 + # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +pycparser==2.21 + # via cffi +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.8.0 + # via msal +pyopenssl==23.2.0 + # via flytekit +pyparsing==3.1.0 + # via packaging +pyrsistent==0.19.3 + # via jsonschema +python-dateutil==2.8.2 + # via + # arrow + # botocore + # croniter + # flytekit + # hologram + # kubernetes + # pandas +python-json-logger==2.0.7 + # via flytekit +python-slugify==8.0.1 + # via + # agate + # cookiecutter +pytimeparse==1.1.8 + # via + # agate + # flytekit +pytz==2023.3 + # via + # babel + # dbt-core + # flytekit + # pandas +pyyaml==5.4.1 + # via + # cookiecutter + # flytekit + # kubernetes + # mashumaro + # responses +regex==2023.6.3 + # via docker-image-py +requests==2.31.0 + # via + # azure-core + # azure-datalake-store + # cookiecutter + # dbt-core + # docker + # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # minimal-snowplow-tracker + # msal + # requests-oauthlib + # responses +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 + # via flytekit +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.6.0 + # via flytekit +six==1.16.0 + # via + # agate + # azure-core + # azure-identity + # google-auth + # isodate + # jsonschema + # kubernetes + # leather + # minimal-snowplow-tracker + # python-dateutil +smmap==5.0.0 + # via gitdb +sortedcontainers==2.4.0 + # via flytekit +sqlparse==0.4.3 + # via dbt-core +statsd==3.3.0 + # via flytekit +text-unidecode==1.3 + # via python-slugify +types-pyyaml==6.0.12.10 + # via responses +typing-extensions==4.7.1 + # via + # aioitertools + # azure-core + # azure-storage-blob + # dbt-core + # flytekit + # mashumaro + # rich + # typing-inspect +typing-inspect==0.9.0 + # via dataclasses-json +urllib3==1.26.16 + # via + # botocore + # docker + # flytekit + # google-auth + # kubernetes + # requests + # responses +websocket-client==1.6.1 + # via + # docker + # kubernetes +werkzeug==2.1.2 + # via dbt-core +wheel==0.40.0 + # via flytekit +wrapt==1.15.0 + # via + # aiobotocore + # deprecated + # flytekit +yarl==1.9.2 + # via aiohttp +zipp==3.16.2 + # via + # importlib-metadata + # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools