From c2e38cb97d2001c5563f2707c28ebca753f04e85 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Wed, 11 Oct 2023 13:32:39 -0700 Subject: [PATCH 1/6] Report coverage to codecov (#5660) Use the codecov uploader in azure pipelines to report coverage to codecov --------- Signed-off-by: Justin Chu --- .azure-pipelines/Linux-CI.yml | 10 +++++++++- .reuse/dep5 | 3 +++ codecov.yml | 8 ++++++++ requirements-release.txt | 1 + 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 codecov.yml diff --git a/.azure-pipelines/Linux-CI.yml b/.azure-pipelines/Linux-CI.yml index ff83ea2ad8f..7a5bc1477e8 100644 --- a/.azure-pipelines/Linux-CI.yml +++ b/.azure-pipelines/Linux-CI.yml @@ -81,7 +81,7 @@ jobs: - script: | source venv/bin/activate - pytest -sv + pytest -sv --cov=onnx --cov-report=xml --cov-append --cov-branch --junit-xml pytest.xml if [ $? -ne 0 ]; then echo "pytest failed" exit 1 @@ -97,6 +97,14 @@ jobs: displayName: 'Run ONNX tests' + - script: | + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + ./codecov + + continueOnError: true + displayName: 'Upload to codecov' + - script: | source venv/bin/activate python onnx/backend/test/cmd_tools.py generate-data --clean diff --git a/.reuse/dep5 b/.reuse/dep5 index 37cb4daad74..5deb7d9248a 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -13,6 +13,9 @@ Files: .azure-pipelines/*.yml COPYRIGHT: Copyright (c) ONNX Project Contributors License: Apache-2.0 +Files: codecov.yml +COPYRIGHT: Copyright (c) ONNX Project Contributors +License: Apache-2.0 Files: .github/**/*.md .github/pull_request_template.md COPYRIGHT: Copyright (c) ONNX Project Contributors diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000000..bfdc9877d9a --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true diff --git a/requirements-release.txt b/requirements-release.txt index 3c61641246e..4cada27e188 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -5,6 +5,7 @@ numpy==1.24.3 parameterized protobuf==4.21.12 pytest +pytest-cov setuptools twine wheel From b61d2a8ab7a7e60eb386a387c73e449dd3461b3e Mon Sep 17 00:00:00 2001 From: Chun-Wei Chen Date: Thu, 12 Oct 2023 16:21:15 -0700 Subject: [PATCH 2/6] Use absolute link in README.md entirely (#5663) ### Description Use absolute link in README.md entirely to correctly show links on PyPI: https://pypi.org/project/onnx/. ### Motivation and Context Fixes https://github.com/onnx/onnx/issues/5616. There are still few relative links in README.md. Signed-off-by: jcwchen --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c5d42605b74..2c1af7c19af 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,12 @@ ONNX is [widely supported](http://onnx.ai/supported-tools) and can be found in m # Contribute -ONNX is a community project and the open governance model is described [here](community/readme.md). We encourage you to join the effort and contribute feedback, ideas, and code. You can participate in the [Special Interest Groups](community/sigs.md) and [Working Groups](community/working-groups.md) to shape the future of ONNX. +ONNX is a community project and the open governance model is described [here](https://github.com/onnx/onnx/blob/main/community/readme.md). We encourage you to join the effort and contribute feedback, ideas, and code. You can participate in the [Special Interest Groups](https://github.com/onnx/onnx/blob/main/community/sigs.md) and [Working Groups](https://github.com/onnx/onnx/blob/main/community/working-groups.md) to shape the future of ONNX. -Check out our [contribution guide](/CONTRIBUTING.md) to get started. +Check out our [contribution guide](https://github.com/onnx/onnx/blob/main/CONTRIBUTING.md) to get started. If you think some operator should be added to ONNX specification, please read -[this document](docs/AddNewOp.md). +[this document](https://github.com/onnx/onnx/blob/main/docs/AddNewOp.md). # Community meetings @@ -280,7 +280,7 @@ For full list refer to CMakeLists.txt * `USE_MSVC_STATIC_RUNTIME` should be 1 or 0, not ON or OFF. When set to 1 onnx links statically to runtime library. **Default**: `USE_MSVC_STATIC_RUNTIME=0` -* `DEBUG` should be 0 or 1. When set to 1 onnx is built in debug mode. or debug versions of the dependencies, you need to open the [CMakeLists file](CMakeLists.txt) and append a letter `d` at the end of the package name lines. For example, `NAMES protobuf-lite` would become `NAMES protobuf-lited`. +* `DEBUG` should be 0 or 1. When set to 1 onnx is built in debug mode. or debug versions of the dependencies, you need to open the [CMakeLists file](https://github.com/onnx/onnx/blob/main/CMakeLists.txt) and append a letter `d` at the end of the package name lines. For example, `NAMES protobuf-lite` would become `NAMES protobuf-lited`. **Default**: `Debug=0` ### CMake variables @@ -321,7 +321,7 @@ pytest # Development -Check out the [contributor guide](/CONTRIBUTING.md) for instructions. +Check out the [contributor guide](https://github.com/onnx/onnx/blob/main/CONTRIBUTING.md) for instructions. # License From 34cc49dc5a2b16d1c7ffd0b3be4ee74c5ce50a6d Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Sat, 14 Oct 2023 01:06:15 -0700 Subject: [PATCH 3/6] Delete onnx/test/reference_evaluator_backend_test.py (#5667) Removed because `reference_evaluator_backend_test` is a duplicating test for the reference evaluator. Fix https://github.com/onnx/onnx/issues/5662 Signed-off-by: Justin Chu --- onnx/test/reference_evaluator_backend_test.py | 876 ------------------ 1 file changed, 876 deletions(-) delete mode 100644 onnx/test/reference_evaluator_backend_test.py diff --git a/onnx/test/reference_evaluator_backend_test.py b/onnx/test/reference_evaluator_backend_test.py deleted file mode 100644 index 1df173b4eea..00000000000 --- a/onnx/test/reference_evaluator_backend_test.py +++ /dev/null @@ -1,876 +0,0 @@ -# Copyright (c) ONNX Project Contributors - -# SPDX-License-Identifier: Apache-2.0 -# type: ignore - -""" -These test evaluates the python runtime (class ReferenceEvaluator) against -all the backend tests (in onnx/backend/test/case/node) and checks -the runtime produces the expected outputs. - -You may run one specific test with following command line: - -:: - - python onnx/test/reference_evaluator_backend_test.py TestOnnxBackEndWithReferenceEvaluator.test_group_normalization_example - -You may bypass a test newly added by adding to the global variable `SKIP_TESTS`. -You may refine the absolute or relative tolerance for a test by -adding an item in method `setUpClass` and attributes -`atol` or `rtol`. -""" - -import os -import pprint -import sys -import unittest -from os import getenv - -import numpy as np -import version_utils -from numpy import object_ as dtype_object -from numpy.testing import assert_allclose # type: ignore - -from onnx import ONNX_ML, OptionalProto, SequenceProto, TensorProto, load -from onnx.backend.test import __file__ as backend_folder -from onnx.helper import __file__ as onnx_file -from onnx.numpy_helper import bfloat16_to_float32, to_list, to_optional -from onnx.reference import ReferenceEvaluator -from onnx.reference.op_run import to_array_extended -from onnx.reference.ops.op_cast import cast_to - -# TODO (https://github.com/microsoft/onnxruntime/issues/14932): Get max supported version from onnxruntime directly -# For now, bump the version in CIs whenever there is a new onnxruntime release -ORT_MAX_IR_SUPPORTED_VERSION = int(getenv("ORT_MAX_IR_SUPPORTED_VERSION", "8")) -ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION = int( - getenv("ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION", "18") -) - -# Number of tests expected to pass without raising an exception. -MIN_PASSING_TESTS = 1235 - -# Update this list if one new operator does not have any implementation. -SKIP_TESTS = { - # mismatches - # shapes (10, 9, 3), (10, 8, 3) shape mismatch unexpected as the operator is inlined - "test_center_crop_pad_crop_axes_hwc_expanded", - # deprecated - "test_scan_sum", # deprecated, opset 8 -> not implemented - "test_scatter_with_axis", # deprecated, scatter is removed - "test_scatter_without_axis", # deprecated, scatter is removed - # not implemented - "test__simple_gradient_of_add", # gradient not implemented - "test__simple_gradient_of_add_and_mul", # gradient not implemented -} - -if version_utils.numpy_older_than("1.21.5"): - SKIP_TESTS |= { - "test_cast_FLOAT_to_BFLOAT16", - "test_castlike_FLOAT_to_BFLOAT16", - "test_castlike_FLOAT_to_BFLOAT16_expanded", - } -if sys.platform == "win32": - SKIP_TESTS |= { - "test_regex_full_match_basic", - "test_regex_full_match_email_domain", - "test_regex_full_match_empty", - "test_image_decoder_decode_jpeg_rgb", - "test_image_decoder_decode_jpeg_grayscale", - "test_image_decoder_decode_jpeg_bgr", - "test_image_decoder_decode_jpeg2k_rgb", - "test_image_decoder_decode_bmp_rgb", - "test_image_decoder_decode_png_rgb", - "test_image_decoder_decode_tiff_rgb", - "test_image_decoder_decode_webp_rgb", - "test_image_decoder_decode_pnm_rgb", - } - -if version_utils.numpy_older_than("1.21.5"): - # op_dft requires numpy >= 1.21.5 - # op_stft depends on op_dft - SKIP_TESTS |= { - "test_stft", - "test_stft_with_window", - "test_stft_cpu", - "test_dft", - "test_dft_axis", - "test_dft_inverse", - "test_dft_opset19", - "test_dft_axis_opset19", - "test_dft_inverse_opset19", - } - - -def assert_allclose_string(expected, value): - """ - Compares two arrays knowing they contain strings. - Raises an exception if the test fails. - - :param expected: expected array - :param value: value - """ - - def is_float(x): - try: - float(x) - return True - except ValueError: - return False - - if all(map(is_float, expected.ravel())): - expected_float = expected.astype(np.float32) - value_float = value.astype(np.float32) - assert_allclose(expected_float, value_float) - else: # noqa: PLR5501 - if expected.tolist() != value.tolist(): - raise AssertionError(f"Mismatches {expected} != {value}.") - - -class OnnxBackendTest: - """ - Definition of a backend test. It starts with a folder, - in this folder, one onnx file must be there, then a subfolder - for each test to run with this model. - - :param folder: test folder - :param onnx_path: onnx file - :param onnx_model: loaded onnx file - :param tests: list of test - """ - - @staticmethod - def _sort(filenames): - temp = [] - for f in filenames: - name = os.path.splitext(f)[0] - i = name.split("_")[-1] - temp.append((int(i), f)) - temp.sort() - return [_[1] for _ in temp] - - @staticmethod - def _read_proto_from_file(full): - if not os.path.exists(full): - raise FileNotFoundError(f"File not found: {full!r}.") - with open(full, "rb") as f: - serialized = f.read() - return OnnxBackendTest._read_proto_from_serialized(serialized, full) - - @staticmethod - def _read_proto_from_serialized(serialized, full): - if not os.path.exists(full): - raise FileNotFoundError(f"File not found: {full!r}.") - with open(full, "rb") as f: - serialized = f.read() - proto_types = [ - (TensorProto, to_array_extended), - (SequenceProto, to_list), - (OptionalProto, to_optional), - ] - exc = None - for pt, cvt in proto_types: - obj = pt() - try: - obj.ParseFromString(serialized) - try: - return cvt(obj) - except ValueError as e: - exc = e - continue - except Exception as e: - exc = e - raise RuntimeError( - f"Unable to read {full!r}, error is {exc}, " - f"content is {serialized[:100]!r}." - ) from exc - - @staticmethod - def _load(folder, names): - res = [] - for name in names: - full = os.path.join(folder, name) - obj = OnnxBackendTest._read_proto_from_file(full) - res.append(obj) - return res - - def __repr__(self): - "usual" - return f"{self.__class__.__name__}({self.folder!r})" - - def __init__(self, folder): - if not os.path.exists(folder): - raise FileNotFoundError(f"Unable to find folder {folder!r}.") - content = os.listdir(folder) - onx = [c for c in content if os.path.splitext(c)[-1] in {".onnx"}] - if len(onx) != 1: - raise ValueError( - f"There is more than one onnx file in {folder!r} ({onx!r})." - ) - self.folder = folder - self.onnx_path = os.path.join(folder, onx[0]) - self.onnx_model = load(self.onnx_path) - - self.tests = [] - for sub in content: - full = os.path.join(folder, sub) - if os.path.isdir(full): - pb = [c for c in os.listdir(full) if os.path.splitext(c)[-1] in {".pb"}] - inputs = OnnxBackendTest._sort(c for c in pb if c.startswith("input_")) - outputs = OnnxBackendTest._sort( - c for c in pb if c.startswith("output_") - ) - self.tests.append( - { - "inputs": OnnxBackendTest._load(full, inputs), - "outputs": OnnxBackendTest._load(full, outputs), - } - ) - - @property - def name(self): - "Returns the test name." - return os.path.split(self.folder)[-1] - - @property - def fname(self): - folder = self.folder.replace("\\", "/").split("/")[-2] - if folder.endswith("node"): - fname = self.name - else: - fname = f"test__{folder.replace('-', '_')}_{self.name[5:]}" - if "/" in fname or fname == "test__test_AvgPool1d_AvgPool1d": - raise AssertionError( - f"name={self.name!r}, folder={folder!r}, self.folder={self.folder}." - ) - return fname - - def __len__(self): - "Returns the number of tests." - return len(self.tests) - - def _compare_results( - self, index, i_output, desired, output, rtol=0, atol=0, comment="", inputs=None - ): - """ - Compares the expected output and the output produced - by the runtime. Raises an exception if not equal. - - :param index: test index - :param i_output: output index - :param desired: expected output - :param output: output - :param rtol: relative tolerance - :param atol: absolute tolerance - :param comment: addition text to give more insights to the user - :param inputs: inputs to the model - """ - if comment == "": - raise RuntimeError("Argument comment should be filled.") - if atol is None: - atol = 0 - if rtol is None: - rtol = 0 - if isinstance(desired, np.ndarray): - if isinstance(output, np.ndarray): - if rtol == 0: - if desired.dtype == np.float32: - rtl = 1e-5 - elif desired.dtype == np.float64: - rtl = 1e-12 - else: - rtl = rtol - else: - rtl = rtol - if desired.dtype == dtype_object: - try: - assert_allclose_string(desired, output) - except AssertionError as ex: - raise AssertionError( - f"Output {i_output} of test {index} in folder {self.folder!r} failed, comment={comment}." - ) from ex - else: - equal_nan = desired.dtype in (np.float16, np.float32, np.float64) - if equal_nan: - try: - assert_allclose( - desired, - output, - atol=atol, - rtol=rtl, - equal_nan=equal_nan, - ) - except AssertionError as ex: - try: - diff = output - desired - except ValueError: - diff = None - raise AssertionError( - f"Output {i_output} of test {index} in folder {self.folder!r} failed " - f"(rtol={rtl}, atol={atol}), comment={comment}\n---\n{desired}\n----" - f"\n{output}\n-----\n{diff}\n------INPUTS----\n{pprint.pformat(inputs)}." - ) from ex - else: - # float 8 types - if desired.dtype != output.dtype: - raise AssertionError( - f"Output {i_output} of test {index} in folder {self.folder!r} " - f"has unexpected type {output.dtype} (expecting {desired.dtype}.)" - ) - if desired.tolist() != output.tolist(): - raise AssertionError( - f"Output {i_output} of test {index} in folder {self.folder!r} " - f"has unexpected values {output} (expecting {desired}.)" - ) - - if desired.shape != output.shape: - raise AssertionError( - f"Output {i_output} of test {index} in folder {self.folder!r} failed " - f"(expected shape={desired.shape} but shape={output.shape}), " - f"comment={comment}\n---\n{desired}\n----" - f"\n{output}\n------INPUTS----\n{pprint.pformat(inputs)}." - ) - elif hasattr(output, "is_compatible"): - # A shape - if desired.dtype != output.dtype: - raise AssertionError( - f"Output {i_output} of test {index} in folder {self.folder!r} failed " - f"(desired.dtype={desired.dtype!r}, output={output!r}), comment={comment}." - ) - if not output.is_compatible(desired.shape): - raise AssertionError( - f"Output {i_output} of test {index} in folder {self.folder!r} failed " - f"(desired.shape={desired.shape}, output={output!r}), comment={comment}." - ) - elif isinstance(desired, list): - if not isinstance(output, list): - raise AssertionError( - f"Expected result is 'list' but output type is {type(output)} for output {i_output}" - f", comment={comment}\n--EXPECTED--\n{desired}\n--GOT--\n{output}." - ) - if len(desired) != len(output): - raise AssertionError( - f"Expected has {len(desired)} but output has {len(output)} for output {i_output}" - f", comment={comment}\n--EXPECTED--\n{desired}\n--GOT--\n{output}." - ) - for a, b in zip(desired, output): - self._compare_results( - index, i_output, a, b, rtol=rtol, atol=atol, comment=comment - ) - else: - raise NotImplementedError( - f"Comparison not implemented for type {type(desired)} and output {i_output}, comment={comment}." - ) - - def is_random(self): - "Tells if a test is random or not." - if "bernoulli" in self.folder: - return True - return False - - def run( - self, - load_fct, - run_fct, - index=None, - rtol=1e-07, - atol=0, - comment="", - print_io=False, - ): - """ - Executes a tests or all tests if index is None. - The function crashes if the tests fails. - - :param load_fct: loading function, takes a loaded onnx graph, - and returns an object - :param run_fct: running function, takes the result of previous - function, the inputs, and returns the outputs - :param index: index of the test to run or all. - :param rtol: relative tolerance - :param atol: absolute tolerance - :param comment: additional information for the user - :param print_io: prints out the input and output - """ - if index is None: - res = [] - for i in range(len(self)): - res.append( - self.run( - load_fct, - run_fct, - index=i, - atol=atol, - rtol=rtol, - comment=comment, - print_io=print_io, - ) - ) - return res - - if print_io: - print("------ INPUTS") - for k, v in enumerate(self.tests[index]["inputs"]): - print(f"input {k!r}, shape={v.shape}, dtype={v.dtype}") - print("------ EXPECTED OUTPUTS") - for k, v in enumerate(self.tests[index]["outputs"]): - print(f"output {k!r}, shape={v.shape}, dtype={v.dtype}") - - obj = load_fct(self.onnx_model) - - got = run_fct(obj, *self.tests[index]["inputs"]) - expected = self.tests[index]["outputs"] - if len(got) != len(expected): - raise AssertionError( - f"Unexpected number of output (test {index}, folder {self.folder!r}), " - f"got {len(got)}, expected {len(expected)}." - ) - res = { - "inputs": self.tests[index]["inputs"], - "expected": self.tests[index]["outputs"], - "results": got, - } - for i, (e, o) in enumerate(zip(expected, got)): - if self.is_random(): - if e.dtype != o.dtype: - raise AssertionError( - f"Output {i} of test {index} in folder {self.folder!r} failed " - f"(type mismatch {e.dtype} != {o.dtype!r})." - ) - if e.shape != o.shape: - raise AssertionError( - f"Output {i} of test {index} in folder {self.folder!r} failed " - f"(shape mismatch {e.shape} != {o.shape})." - ) - else: - self._compare_results( - index, - i, - e, - o, - atol=atol, - rtol=rtol, - comment=comment + "\n" + str(self.onnx_model), - inputs=self.tests[index]["inputs"], - ) - return res - - -def enumerate_onnx_tests(series, fct_filter=None): - """ - Collects test from a sub folder of `onnx/backend/test`. - Works as an enumerator to start processing them - without waiting or storing too much of them. - - :param series: which subfolder to load, possible values: - (`'node'`, ...) - :param fct_filter: function `lambda testname: boolean` - to load or skip the test, None for all - :return: list of @see cl OnnxBackendTest - """ - root = os.path.dirname(backend_folder) - sub = os.path.join(root, "data", series) - if not os.path.exists(sub): - content = "\n".join(os.listdir(root)) - raise FileNotFoundError( - f"Unable to find series of tests in {root!r}, subfolders:\n{content}" - ) - tests = os.listdir(sub) - for t in tests: - if fct_filter is not None and not fct_filter(t): - continue - folder = os.path.join(sub, t) - if not ONNX_ML and "ai_onnx_ml" in folder: - continue - content = os.listdir(folder) - onx = [c for c in content if os.path.splitext(c)[-1] in {".onnx"}] - if len(onx) == 1: - yield OnnxBackendTest(folder) - - -class TestOnnxBackEndWithReferenceEvaluator(unittest.TestCase): - folder = os.path.join( - os.path.abspath(os.path.dirname(__file__)), "onnx_backend_test_code" - ) - - @classmethod - def add_test_methods(cls): - for folder in ["node", "pytorch-converted", "pytorch-operator", "simple"]: - for te in enumerate_onnx_tests(folder): - - def _test_( - self, te=te, check_other_runtime=None, verbose=0, print_io=False - ): - if te.fname in getattr(cls, "skip_test", set()): - cls.skipped.append((te, None)) - return - rtol = getattr(cls, "rtol", {}) - atol = getattr(cls, "atol", {}) - if len(rtol) == 0 or len(atol) == 0: - raise AssertionError("rtol or atol is empty.") - self.common_test_onnx_test_run( - te, - getattr(cls, "successes", []), - getattr(cls, "missed", []), - getattr(cls, "skipped", []), - getattr(cls, "load_failed", []), - getattr(cls, "exec_failed", []), - getattr(cls, "mismatch", []), - verbose=verbose, - rtol=rtol, - atol=atol, - check_other_runtime=check_other_runtime, - print_io=print_io, - ) - - setattr(TestOnnxBackEndWithReferenceEvaluator, te.fname, _test_) - - def test_onnx_backend_test_abs(self): - name = "test_abs" - code = [] - for te in enumerate_onnx_tests("node", lambda folder: folder == name): - code.append(te) - self.assertEqual(len(code), 1) - - def test_onnx_backend_test_expand_shape_model1(self): - name = "test_expand_shape_model1" - code = [] - for te in enumerate_onnx_tests("simple", lambda folder: folder == name): - code.append(te) - self.assertEqual(len(code), 1) - - @staticmethod - def load_fct(obj, verbose=0): - return ReferenceEvaluator(obj, verbose=verbose) - - @staticmethod - def run_fct(obj, *inputs, verbose=0): - if hasattr(obj, "input_names"): - input_names = obj.input_names - elif hasattr(obj, "get_inputs"): - input_names = [_.name for _ in obj.get_inputs()] - else: - raise AttributeError( - f"Unable to extract the number to guess the number of inputs for type {type(obj)}." - ) - if len(input_names) < len(inputs): - raise AssertionError( - f"Got {len(inputs)} inputs but expecting {len(obj.input_names)}." - ) - rewrite = False - for i in range(len(inputs)): - if ( - isinstance(inputs[i], np.ndarray) - and inputs[i].dtype == np.uint16 - and obj.input_types[i].tensor_type.elem_type != TensorProto.UINT16 - ): - rewrite = True - if rewrite: - # bfloat16 does not exist for numpy. - inputs = list(inputs) - for i in range(len(inputs)): - if ( - isinstance(inputs[i], np.ndarray) - and inputs[i].dtype == np.uint16 - and obj.input_types[i].tensor_type.elem_type != TensorProto.UINT16 - ): - xr = inputs[i].ravel() - xf = np.empty(xr.shape[0], dtype=np.float32) - for ie in range(xr.shape[0]): - el = bfloat16_to_float32(xr[ie]) - xf[ie] = el - inputs[i] = cast_to( - xf.astype(np.float32).reshape(inputs[i].shape), - TensorProto.BFLOAT16, - True, - ) - feeds = {input_names[i]: inputs[i] for i in range(len(inputs))} - got = obj.run(None, feeds) - return got - - def common_test_onnx_test_run( - self, - te, - successes, - missed, - skipped, - load_failed, - exec_failed, - mismatch, - verbose=0, - rtol=None, - atol=None, - check_other_runtime=None, - print_io=False, - ): - if verbose > 6: - print("TEST:", te.name) - if verbose > 7: - print(" check runtime") - self.assertIn(te.name, repr(te)) - self.assertGreater(len(te), 0) - try: - if verbose > 7: - print(" run") - if verbose > 5: - te.run( - lambda *args, verbose=verbose: TestOnnxBackEndWithReferenceEvaluator.load_fct( - *args, verbose - ), - TestOnnxBackEndWithReferenceEvaluator.run_fct, - atol=atol.get(te.name, None), - rtol=rtol.get(te.name, None), - comment=f"[runtime=ReferenceEvaluator, verbose={verbose}]", - print_io=print_io, - ) - else: - te.run( - TestOnnxBackEndWithReferenceEvaluator.load_fct, - TestOnnxBackEndWithReferenceEvaluator.run_fct, - atol=atol.get(te.fname, atol.get(te.name, None)), - rtol=rtol.get(te.fname, rtol.get(te.name, None)), - comment="[runtime=ReferenceEvaluator]", - print_io=print_io, - ) - if verbose > 7: - print(" end run") - if verbose > 8: - print(te.onnx_model) - except NotImplementedError as e: - if verbose > 7: - print(" ", e, type(e)) - missed.append((te, e)) - with open(f"missed_{te.name}.onnx", "wb") as f: - f.write(te.onnx_model.SerializeToString()) - raise e - except (AssertionError, ValueError) as e: - if verbose > 7: - print(" ", e, type(e)) - mismatch.append((te, e)) - with open(f"mismatch_{te.name}.onnx", "wb") as f: - f.write(te.onnx_model.SerializeToString()) - if check_other_runtime is None: - raise e - if "onnxruntime" in check_other_runtime: - print("CHECK RUNTIME onnxruntime") - from onnxruntime import InferenceSession - - onnx_domain_opset = ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION - for opset in te.onnx_model.opset_import: - if opset.domain in ("", "ai.onnx"): - onnx_domain_opset = opset.version - break - - # The new IR or opset version is not supported by onnxruntime yet - if ( - te.onnx_model.ir_version > ORT_MAX_IR_SUPPORTED_VERSION - or onnx_domain_opset > ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION - ): - print( - "Skip test because of IR or opset version is not supported by onnxruntime yet" - ) - return - - te.run( - lambda obj: InferenceSession( - obj.SerializeToString(), providers=["CPUExecutionProvider"] - ), - lambda *a, **b: TestOnnxBackEndWithReferenceEvaluator.run_fct( - *a, verbose=1, **b - ), - atol=1e-5, - rtol=1e-3, - comment="[runtime=onnxruntime]", - ) - print("done") - raise e - except Exception as e: - if verbose > 7: - print(" ", e, type(e)) - with open(f"issue_{te.name}.onnx", "wb") as f: - f.write(te.onnx_model.SerializeToString()) - raise AssertionError( - f"Unable to run test {te.name!r} due to {e}\n{te.onnx_model}" - ) from e - successes.append((te, atol.get(te.fname, None), rtol.get(te.fname, None))) - if verbose > 7: - print(" end example.") - - @staticmethod - def _postprocess( - successes, missed, skipped, load_failed, exec_failed, mismatch, verbose - ): - success = len(successes) - failed = [ - len(missed), - len(skipped), - len(load_failed), - len(exec_failed), - len(mismatch), - ] - coverage = success / (success + sum(failed)) - - if verbose: - path = os.path.dirname(onnx_file) - print("-----------") - print( - f"success={success}, skipped={len(skipped)}, missed={len(missed)}, load_failed={len(load_failed)}, " - f"exec_failed={len(exec_failed)}, mismatch={len(mismatch)}" - ) - print( - f"coverage {coverage * 100:.1f}% out of {success + sum(failed)} tests" - ) - - if verbose > 3: - - def _print(s, path): - return ( - str(s) - .replace("\\\\", "\\") - .replace(path, "onnx") - .replace("\\", "/") - ) - - print("-----------") - for t in sorted(load_failed, key=lambda m: m[0].fname): - print("loading failed", t[0].fname, "---", _print(t[0], path)) - for t in sorted(exec_failed, key=lambda m: m[0].fname): - print("execution failed", t[0].fname, "---", _print(t[0], path)) - for t in sorted(mismatch, key=lambda m: m[0].fname): - print("mismatch", t[0].fname, "---", _print(t[0], path)) - for t in sorted(missed, key=lambda m: m[0].fname): - print("missed ", t[0].fname, "---", _print(t[0], path)) - for t in sorted(skipped, key=lambda m: m[0].fname): - print("skipped", t[0].fname, "---", _print(t[0], path)) - - if success > 30: - print("-----------") - print( - f"success={success}, skipped={len(skipped)}, missed={len(missed)}, load_failed={len(load_failed)}, " - f"exec_failed={len(exec_failed)}, mismatch={len(mismatch)}" - ) - print( - f"coverage {coverage * 100:.1f}% out of {success + sum(failed)} tests" - ) - print("-----------") - - if len(mismatch) > 0: - te, e = mismatch[0] - raise AssertionError( - f"Mismatch in test {te.name!r}\n{te.onnx_model}." - ) from e - - if sum(failed) > len(SKIP_TESTS): - raise AssertionError( - f"Unexpected failures. {sum(failed)}/{success + sum(failed)} tests have failed." - f"The coverage is {coverage * 100:.1f}%. " - f"New operators were added with no corresponding runtime." - ) - - @classmethod - def setUpClass(cls, all_tests=False): - # test not supported yet - # not supported yet - # see https://onnx.ai/backend-scoreboard/onnxruntime_details_stable.html - # to compare with onnxruntime - - cls.rtol = { - "test_adam_multiple": 1e-2, - "test_blackmanwindow_expanded": 0, - "test_blackmanwindow_symmetric_expanded": 0, - "test_simple_rnn_batchwise": 0, - "test__pytorch_converted_Conv1d_pad1": 1e-4, - "test__pytorch_converted_Conv2d": 1e-5, - "test__pytorch_converted_Conv2d_no_bias": 1e-3, - "test__pytorch_converted_Conv2d_strided": 1e-4, - "test_layer_normalization_4d_axis1_expanded_ver18": 1e-4, - "test_layer_normalization_4d_axis_negative_1_expanded_ver18": 1e-4, - "test_layer_normalization_4d_axis_negative_3_expanded_ver18": 1e-4, - "test_ConvTranspose2d": 1e-4, - "test__pytorch_converted_ConvTranspose2d": 1e-4, - } - - cls.atol = { - "test_blackmanwindow": 1e-7, - "test_blackmanwindow_expanded": 1e-4, - "test_blackmanwindow_symmetric": 1e-7, - "test_blackmanwindow_symmetric_expanded": 1e-4, - "test_Conv1d": 1e-6, - "test_Conv2d_depthwise_padded": 1e-7, - "test_Conv3d_dilated": 1e-6, - "test_gridsample_bicubic": 1e-4, - "test_gru_seq_length": 1e-7, - "test_hammingwindow_expanded": 1e-4, - "test_hammingwindow_symmetric_expanded": 1e-4, - "test_hannwindow_expanded": 1e-4, - "test_hannwindow_symmetric": 1e-7, - "test_hannwindow_symmetric_expanded": 1e-4, - "test_layer_normalization_4d_axis_negative_1_expanded": 1e-6, - "test_layer_normalization_4d_axis1_expanded": 1e-6, - "test_layer_normalization_4d_axis_negative_3_expanded": 1e-6, - "test_mish": 1e-6, - "test_mish_expanded": 1e-6, - "test_roialign_aligned_false": 1e-4, - "test_roialign_aligned_true": 1e-4, - # extended list - "test__pytorch_converted_ConvTranspose2d_no_bias": 1e-4, - "test__pytorch_converted_Linear_no_bias": 1e-5, - "test_Linear_no_bias": 1e-5, - "test__pytorch_converted_Conv1d_pad1": 1e-6, - "test__pytorch_converted_Conv2d": 1e-5, - "test__pytorch_converted_Conv2d_depthwise": 1e-4, - "test__pytorch_converted_Conv2d_depthwise_strided": 1e-4, - "test__pytorch_converted_Conv2d_depthwise_with_multiplier": 1e-4, - "test__pytorch_converted_Conv2d_depthwise_padded": 1e-4, - "test__pytorch_converted_Conv2d_groups": 1e-4, - "test__pytorch_converted_Conv2d_groups_thnn": 1e-4, - "test__pytorch_converted_Conv2d_no_bias": 1e-5, - "test__pytorch_converted_Conv2d_strided": 1e-4, - "test__pytorch_operator_operator_symbolic_override": 1e-5, - "test_operator_symbolic_override": 1e-4, - "test__pytorch_converted_Conv3d_dilated_strided": 1e-4, - "test__pytorch_converted_Conv3d_groups": 1e-4, - "test_affine_grid_2d": 1e-4, - "test_affine_grid_2d_expanded": 1e-4, - "test_affine_grid_2d_align_corners": 1e-4, - "test_affine_grid_2d_align_corners_expanded": 1e-4, - "test_affine_grid_3d": 1e-4, - "test_affine_grid_3d_expanded": 1e-4, - "test_affine_grid_3d_align_corners": 1e-4, - "test_affine_grid_3d_align_corners_expanded": 1e-4, - } - - cls.skip_test = SKIP_TESTS - if all_tests: - cls.skip_test = set() - cls.successes = [] - cls.missed = [] - cls.skipped = [] - cls.load_failed = [] - cls.exec_failed = [] - cls.mismatch = [] - - @classmethod - def tearDownClass(cls): - if len(cls.successes) == 0: - failed = cls.mismatch + cls.missed + cls.load_failed + cls.exec_failed - if len(failed) > 0: - raise RuntimeError( - f"No test was successful, {len(failed)} failed." - ) from failed[0][1] - raise RuntimeError("No test was successful.") - cls._postprocess( - cls.successes, - cls.missed, - cls.skipped, - cls.load_failed, - cls.exec_failed, - cls.mismatch, - 10, - ) - - -TestOnnxBackEndWithReferenceEvaluator.add_test_methods() - - -if __name__ == "__main__": - unittest.main(verbosity=2) From f556cffc40dc769a2c8e3bb5fd9a90f1e7fbb7eb Mon Sep 17 00:00:00 2001 From: Leso_KN Date: Sun, 15 Oct 2023 06:37:20 +0200 Subject: [PATCH 4/6] Do not use LFS64 on non-glibc linux (#5668) (#5669) ### Description This PR closes #5668 and fixes building onnx on non-glibc linux systems (e.g. Alpine Linux / musl). ### Motivation and Context The latest version of onnx (1.14.1) currently fails to compile on non-glibc systems (e.g. musl) due to an unsupported call to `stat64()` in checker.cc. See the latest [pipeline log](https://gitlab.alpinelinux.org/leso-kn/aports/-/jobs/1143766#L1155) of [aports!52138](https://gitlab.alpinelinux.org/alpine/aports/-/merge_requests/52138) for an example case. Signed-off-by: leso-kn --- onnx/checker.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onnx/checker.cc b/onnx/checker.cc index 1beb1b882a6..fac56f5655f 100644 --- a/onnx/checker.cc +++ b/onnx/checker.cc @@ -190,11 +190,11 @@ void check_tensor(const TensorProto& tensor, const CheckerContext& ctx) { } std::string data_path = path_join(ctx.get_model_dir(), relative_path); // use stat64 to check whether the file exists -#if defined(__APPLE__) || defined(__wasm__) - struct stat buffer; // APPLE does not have stat64 +#if defined(__APPLE__) || defined(__wasm__) || !defined(__GLIBC__) + struct stat buffer; // APPLE, wasm and non-glic stdlibs do not have stat64 if (stat((data_path).c_str(), &buffer) != 0) { #else - struct stat64 buffer; // All POSIX except APPLE have stat64 + struct stat64 buffer; // All POSIX under glibc except APPLE and wasm have stat64 if (stat64((data_path).c_str(), &buffer) != 0) { #endif fail_check( From a93ec80919a4370a4b160eef811488d569487cfc Mon Sep 17 00:00:00 2001 From: liqun Fu Date: Mon, 16 Oct 2023 03:10:54 -0700 Subject: [PATCH 5/6] Update reducesum test name - test name was duplicated (#5664) ### Description fix test name for reducesum op ### Motivation and Context fix test data Signed-off-by: Liqun Fu --- docs/Operators.md | 2 +- docs/TestCoverage.md | 2 +- onnx/backend/test/case/node/reducesum.py | 2 +- .../node/test_reduce_sum_empty_set/model.onnx | Bin 182 -> 182 bytes .../test_data_set_0/input_1.pb | Bin 20 -> 20 bytes .../test_data_set_0/output_0.pb | Bin 19 -> 51 bytes .../model.onnx | Bin 0 -> 204 bytes .../test_data_set_0/input_0.pb | Bin 0 -> 16 bytes .../test_data_set_0/input_1.pb | Bin 0 -> 20 bytes .../test_data_set_0/output_0.pb | Bin 0 -> 19 bytes 10 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/model.onnx create mode 100644 onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/input_0.pb create mode 100644 onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/input_1.pb create mode 100644 onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/output_0.pb diff --git a/docs/Operators.md b/docs/Operators.md index bd21b28e028..56ac3118452 100644 --- a/docs/Operators.md +++ b/docs/Operators.md @@ -23094,7 +23094,7 @@ expect( node, inputs=[data, axes], outputs=[reduced], - name="test_reduce_sum_empty_set", + name="test_reduce_sum_empty_set_non_reduced_axis_zero", ) ``` diff --git a/docs/TestCoverage.md b/docs/TestCoverage.md index b8cf5520ef9..6140f51845c 100644 --- a/docs/TestCoverage.md +++ b/docs/TestCoverage.md @@ -15681,7 +15681,7 @@ expect( node, inputs=[data, axes], outputs=[reduced], - name="test_reduce_sum_empty_set", + name="test_reduce_sum_empty_set_non_reduced_axis_zero", ) ``` diff --git a/onnx/backend/test/case/node/reducesum.py b/onnx/backend/test/case/node/reducesum.py index 04e7d6e0256..98f9afbbe79 100644 --- a/onnx/backend/test/case/node/reducesum.py +++ b/onnx/backend/test/case/node/reducesum.py @@ -242,5 +242,5 @@ def export_non_reduced_axis_zero() -> None: node, inputs=[data, axes], outputs=[reduced], - name="test_reduce_sum_empty_set", + name="test_reduce_sum_empty_set_non_reduced_axis_zero", ) diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_set/model.onnx b/onnx/backend/test/data/node/test_reduce_sum_empty_set/model.onnx index 816bed5902c58c1b490e7f50b188bff74448b3a7..0ef5cabe84d3ea59ad5fc3ddebb29f69a48f26d5 100644 GIT binary patch delta 18 ZcmdnSxQ%hbYHmg@CJq)S7A^(>UH~jZ13Lf! delta 18 ZcmdnSxQ%hbYHkKDCJsg?7A^(>UH~j312+Hw diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_set/test_data_set_0/input_1.pb b/onnx/backend/test/data/node/test_reduce_sum_empty_set/test_data_set_0/input_1.pb index e8256ea4dee5836050a3f38e60689bcd2ff89ba2..31799432830506cff63208274e24952586e34bda 100644 GIT binary patch literal 20 Wcmd;J6kvB^NvudM_TpeFG@`*O-@bmVgLXayaSv7 diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/model.onnx b/onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/model.onnx new file mode 100644 index 0000000000000000000000000000000000000000..4decf892cc2bd8b5e9166b499c4497f1eea3f67a GIT binary patch literal 204 zcmd;J7vf1uOwLZtOVKS!EiSRz%*bWP#gdX(lE}r9Sdm&R#9ow|QktBaqQn^lW(1e! zYVmV%WT&PUq-5q6OE4~AWD?Q`>5Ye|j4v+DjZe)jD5;DuPA!Sg%g=*Kq{JsyWERI) zr55E!NkQxo65 FUI5pZGARH6 literal 0 HcmV?d00001 diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/input_0.pb b/onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/input_0.pb new file mode 100644 index 0000000000000000000000000000000000000000..99a6775f25536560f7ceea7b986d781c662212b3 GIT binary patch literal 16 Xcmd;J;$Yxl5nyy;Nl7e8^kM)23_}81 literal 0 HcmV?d00001 diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/input_1.pb b/onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/input_1.pb new file mode 100644 index 0000000000000000000000000000000000000000..e8256ea4dee5836050a3f38e60689bcd2ff89ba2 GIT binary patch literal 20 Wcmd;J6kvB^NvudM_Tpe-fB*m=2?A^Y literal 0 HcmV?d00001 diff --git a/onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/output_0.pb b/onnx/backend/test/data/node/test_reduce_sum_empty_set_non_reduced_axis_zero/test_data_set_0/output_0.pb new file mode 100644 index 0000000000000000000000000000000000000000..2ebb1a27b06d5e00ac4b15aab9ab6cea63bcdec2 GIT binary patch literal 19 acmd;J;$Yxl6kv2>FG@`*O-@bmVgLXayaSv7 literal 0 HcmV?d00001 From 5f908a9c433d86c4f1dbf6b963f51daf68daa86e Mon Sep 17 00:00:00 2001 From: Aditya Goel <48102515+adityagoel4512@users.noreply.github.com> Date: Mon, 16 Oct 2023 19:09:15 +0100 Subject: [PATCH 6/6] Drop "one of" default attribute check in LabelEncoder (#5673) ### Description Drop "one of" checking on the default value in the `LabelEncoder`. ### Motivation and Context When implementing `LabelEncoder` in onnxruntime (for the upcoming ai.onnx.ml opset 4), it became clear that the implemented type inference for LabelEncoder would not work well in practice - specifically, the type inference pass ensures that only one of the default_* attributes is set. Since some of these are optional with default values, this is not very workable in practice. This PR addresses this by dropping this check. Notice that this was [not checked](https://github.com/onnx/onnx/blob/f556cffc40dc769a2c8e3bb5fd9a90f1e7fbb7eb/onnx/defs/traditionalml/old.cc#L277-L338) in previous opsets anyway. --------- Signed-off-by: Aditya Goel --- onnx/defs/traditionalml/defs.cc | 38 ++++++++++++++------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/onnx/defs/traditionalml/defs.cc b/onnx/defs/traditionalml/defs.cc index 4355b1b31ee..50e1c5d3f0b 100644 --- a/onnx/defs/traditionalml/defs.cc +++ b/onnx/defs/traditionalml/defs.cc @@ -382,28 +382,6 @@ ONNX_ML_OPERATOR_SET_SCHEMA( fail_shape_inference( "At least one of values_tensor, values_strings, values_int64s, values_floats must be set."); } - - int default_length, default_type; - std::tie(default_type, default_length) = getAttributeElementTypeAndLength( - ctx, {"default_tensor", "default_string", "default_int64", "default_float"}); - if (default_type != TensorProto::UNDEFINED) { - if (value_type != default_type) { - fail_shape_inference( - "The value type ", - value_type, - " and the default type ", - default_type, - " are different, which is not permitted for LabelEncoders."); - } - - // Ensure default_tensor is a singleton if set - const AttributeProto* default_tensor = ctx.getAttribute("default_tensor"); - if (default_tensor != nullptr && - (default_tensor->t().dims_size() != 1 || default_tensor->t().dims(0) != 1)) { - fail_shape_inference("default_tensor must be a singleton if set."); - } - } - if (value_length != key_length) { fail_shape_inference( "The number of keys ", @@ -413,6 +391,22 @@ ONNX_ML_OPERATOR_SET_SCHEMA( " must be the same in the LabelEncoder."); } + auto default_attr = ctx.getAttribute("default_tensor"); + if (nullptr != default_attr && default_attr->has_t() && default_attr->t().has_data_type() && + default_attr->t().data_type() != TensorProto_DataType_UNDEFINED) { + auto default_tensor = default_attr->t(); + if (default_tensor.data_type() != value_type) { + fail_shape_inference( + "The default tensor type ", + default_tensor.data_type(), + " and the value type ", + value_type, + " must be the same in the LabelEncoder."); + } + if (1 != default_tensor.dims_size() || 1 != default_tensor.dims(0)) { + fail_shape_inference("The default tensor must be a singleton 1D tensor."); + } + } // Propagate shape from input type and assign output type based on value type ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(value_type); propagateShapeFromInputToOutput(ctx, 0, 0);