pyproject.toml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
[tool.poetry]
name = "pyiceberg"
version = "0.7.1"
readme = "README.md"
homepage = "https://py.iceberg.apache.org/"
repository = "https://github.com/apache/iceberg-python"
description = "Apache Iceberg is an open table format for huge analytic datasets"
authors = ["Apache Software Foundation <dev@iceberg.apache.org>"]
license = "Apache License 2.0"
classifiers = [
  "License :: OSI Approved :: Apache Software License",
  "Operating System :: OS Independent",
  "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
]
packages = [
  { include = "pyiceberg" },
  { from = "vendor", include = "fb303" },
  { from = "vendor", include = "hive_metastore" },
  { include = "tests", format = "sdist" },
  { include = "Makefile", format = "sdist" },
  { include = "NOTICE", format = [
    "sdist",
    "wheel",
  ] },
]
include = [
  { path = "dev", format = "sdist" },
  { path = "pyiceberg/**/*.so", format = "wheel" },
  { path = "pyiceberg/**/*.pyd", format = "wheel" },
]

[tool.poetry.dependencies]
python = ">=3.8, <4.0, !=3.9.7"
mmh3 = ">=4.0.0,<5.0.0"
requests = ">=2.20.0,<3.0.0"
click = ">=7.1.1,<9.0.0"
rich = ">=10.11.0,<14.0.0"
strictyaml = ">=1.7.0,<2.0.0" # CVE-2020-14343 was fixed in 5.4.
pydantic = ">=2.0,<3.0,!=2.4.0,!=2.4.1" # 2.4.0, 2.4.1 has a critical bug
sortedcontainers = "2.4.0"
fsspec = ">=2023.1.0,<2025.1.0"
pyparsing = ">=3.1.0,<4.0.0"
zstandard = ">=0.13.0,<1.0.0"
tenacity = ">=8.2.3,<10.0.0"
pyarrow = { version = ">=9.0.0,<18.0.0", optional = true }
pandas = { version = ">=1.0.0,<3.0.0", optional = true }
duckdb = { version = ">=0.5.0,<2.0.0", optional = true }
ray = [
  { version = "==2.10.0", python = "<3.9", optional = true },
  { version = ">=2.10.0,<3.0.0", python = ">=3.9", optional = true },
]
python-snappy = { version = ">=0.6.0,<1.0.0", optional = true }
thrift = { version = ">=0.13.0,<1.0.0", optional = true }
mypy-boto3-glue = { version = ">=1.28.18", optional = true }
boto3 = { version = ">=1.24.59", optional = true }
s3fs = { version = ">=2023.1.0,<2024.1.0", optional = true }
adlfs = { version = ">=2023.1.0,<2024.8.0", optional = true }
gcsfs = { version = ">=2023.1.0,<2024.1.0", optional = true }
psycopg2-binary = { version = ">=2.9.6", optional = true }
sqlalchemy = { version = "^2.0.18", optional = true }
getdaft = { version = ">=0.2.12", optional = true }
numpy = { version = "^1.22.4", optional = true }

[tool.poetry.group.dev.dependencies]
pytest = "7.4.4"
pytest-checkdocs = "2.10.1"
pytest-lazy-fixture = "0.6.3"
pre-commit = "3.5.0"
fastavro = "1.9.5"
coverage = { version = "^7.4.2", extras = ["toml"] }
requests-mock = "1.12.1"
moto = { version = "^5.0.2", extras = ["server"] }
typing-extensions = "4.12.2"
pytest-mock = "3.14.0"
pyspark = "3.5.1"
cython = "3.0.11"
deptry = ">=0.14,<0.20"
docutils = "!=0.21"

[[tool.mypy.overrides]]
module = "pytest_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "numpy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "numpy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "numpy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[tool.poetry.scripts]
pyiceberg = "pyiceberg.cli.console:run"

[build-system]
requires = ["poetry-core>=1.0.0", "wheel", "Cython>=3.0.0", "setuptools"]
build-backend = "poetry.core.masonry.api"

[tool.poetry.build]
generate-setup-file = false
script = "build-module.py"

[tool.poetry.extras]
pyarrow = ["pyarrow", "numpy"]
pandas = ["pandas", "pyarrow", "numpy"]
duckdb = ["duckdb", "pyarrow", "numpy"]
ray = ["ray", "pyarrow", "pandas", "numpy"]
daft = ["getdaft"]
snappy = ["python-snappy"]
hive = ["thrift"]
s3fs = ["s3fs"]
glue = ["boto3", "mypy-boto3-glue"]
adlfs = ["adlfs"]
dynamodb = ["boto3"]
zstandard = ["zstandard"]
sql-postgres = ["sqlalchemy", "psycopg2-binary"]
sql-sqlite = ["sqlalchemy"]
gcsfs = ["gcsfs"]

[tool.pytest.ini_options]
markers = [
  "unmarked: marks a test as a unittest",
  "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint args)",
  "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)",
  "integration: marks integration tests against Apache Spark",
  "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint)",
]

# Turns a warning into an error
filterwarnings = [
  "error",
  "ignore:A plugin raised an exception during an old-style hookwrapper teardown.",
  "ignore:unclosed <socket.socket",
  # Remove this in a future release of PySpark.
  "ignore:distutils Version classes are deprecated. Use packaging.version instead.",
]

[tool.black]
line-length = 130
target-version = ['py38']

[tool.pycln]
all = true

[tool.mypy]
mypy_path = "python"
no_implicit_optional = true
namespace_packages = false
warn_redundant_casts = true
warn_unreachable = true
warn_unused_ignores = true
disallow_any_generics = true
disallow_untyped_defs = true
implicit_reexport = false

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "numpy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[tool.coverage.run]
source = ['pyiceberg/']