Refactor manifest writing.

Signed-off-by: dblock <dblock@dblock.org>

bundle-workflow/src/assemble_workflow/bundle_recorder.py

@@ -7,8 +7,6 @@
 import os
 from urllib.parse import urljoin
 
-import yaml
-
 from manifests.bundle_manifest import BundleManifest
 
 
@@ -67,10 +65,8 @@ def get_manifest(self):
         return self.bundle_manifest.to_manifest()
 
     def write_manifest(self, folder):
-        output_manifest = self.get_manifest()
         manifest_path = os.path.join(folder, "manifest.yml")
-        with open(manifest_path, "w") as file:
-            yaml.dump(output_manifest.to_dict(), file)
+        self.get_manifest().to_file(manifest_path)
 
     class BundleManifestBuilder:
         def __init__(self, build_id, name, version, arch, location):

bundle-workflow/src/build_workflow/build_recorder.py

@@ -9,8 +9,6 @@
 import shutil
 from zipfile import ZipFile
 
-import yaml
-
 from manifests.build_manifest import BuildManifest
 from system.properties_file import PropertiesFile
 
@@ -57,10 +55,8 @@ def get_manifest(self):
         return self.build_manifest.to_manifest()
 
     def write_manifest(self):
-        output_manifest = self.get_manifest()
         manifest_path = os.path.join(self.target.output_dir, "manifest.yml")
-        with open(manifest_path, "w") as file:
-            yaml.dump(output_manifest.to_dict(), file)
+        self.get_manifest().to_file(manifest_path)
         logging.info(f'Created build manifest {manifest_path}')
 
     def __check_artifact(self, artifact_type, artifact_file):

bundle-workflow/src/manifests/build_manifest.py

@@ -12,7 +12,7 @@
 and the components that made up the build in the `components` section.
 
 The format for schema version 1.0 is:
-schema-version: 1.0
+schema-version: "1.0"
 build:
   name: string
   version: string
@@ -45,12 +45,12 @@ def __init__(self, data):
             map(lambda entry: self.Component(entry), data["components"])
         )
 
-    def to_dict(self):
+    def __to_dict__(self):
         return {
             "schema-version": "1.0",
-            "build": self.build.to_dict(),
+            "build": self.build.__to_dict__(),
             "components": list(
-                map(lambda component: component.to_dict(), self.components)
+                map(lambda component: component.__to_dict__(), self.components)
             ),
         }
 
@@ -61,7 +61,7 @@ def __init__(self, data):
             self.architecture = data["architecture"]
             self.id = data["id"]
 
-        def to_dict(self):
+        def __to_dict__(self):
             return {
                 "name": self.name,
                 "version": self.version,
@@ -78,7 +78,7 @@ def __init__(self, data):
             self.artifacts = data["artifacts"]
             self.version = data["version"]
 
-        def to_dict(self):
+        def __to_dict__(self):
             return {
                 "name": self.name,
                 "repository": self.repository,

bundle-workflow/src/manifests/bundle_manifest.py

@@ -14,7 +14,7 @@ class BundleManifest(Manifest):
     and the components that made up the bundle in the `components` section.
 
     The format for schema version 1.0 is:
-        schema-version: 1.0
+        schema-version: "1.0"
         build:
           name: string
           version: string
@@ -36,12 +36,12 @@ def __init__(self, data):
             map(lambda entry: self.Component(entry), data["components"])
         )
 
-    def to_dict(self):
+    def __to_dict__(self):
         return {
             "schema-version": "1.0",
-            "build": self.build.to_dict(),
+            "build": self.build.__to_dict__(),
             "components": list(
-                map(lambda component: component.to_dict(), self.components)
+                map(lambda component: component.__to_dict__(), self.components)
             ),
         }
 
@@ -53,7 +53,7 @@ def __init__(self, data):
             self.location = data["location"]
             self.id = data["id"]
 
-        def to_dict(self):
+        def __to_dict__(self):
             return {
                 "name": self.name,
                 "version": self.version,
@@ -70,7 +70,7 @@ def __init__(self, data):
             self.commit_id = data["commit_id"]
             self.location = data["location"]
 
-        def to_dict(self):
+        def __to_dict__(self):
             return {
                 "name": self.name,
                 "repository": self.repository,

bundle-workflow/src/manifests/input_manifest.py

@@ -10,7 +10,7 @@
 and the components that make up the product in the `components` section.
 
 The format for schema version 1.0 is:
-schema-version: 1.0
+schema-version: "1.0"
 build:
   name: string
   version: string
@@ -37,15 +37,38 @@ def __init__(self, data):
             map(lambda entry: self.Component(entry), data["components"])
         )
 
+    def __to_dict__(self):
+        return {
+            "schema-version": "1.0",
+            "build": self.build.__to_dict__(),
+            "components": list(
+                map(lambda component: component.__to_dict__(), self.components)
+            ),
+        }
+
     class Build:
         def __init__(self, data):
             self.name = data["name"]
             self.version = data["version"]
 
+        def __to_dict__(self):
+            return {"name": self.name, "version": self.version}
+
     class Component:
         def __init__(self, data):
             self.name = data["name"]
             self.repository = data["repository"]
             self.ref = data["ref"]
             self.working_directory = data.get("working_directory", None)
             self.checks = data.get("checks", [])
+
+        def __to_dict__(self):
+            return Manifest.compact(
+                {
+                    "name": self.name,
+                    "repository": self.repository,
+                    "ref": self.ref,
+                    "working_directory": self.working_directory,
+                    "checks": self.checks,
+                }
+            )

bundle-workflow/src/manifests/manifest.py

@@ -19,6 +19,28 @@ def from_path(cls, path):
         with open(path, "r") as f:
             return cls.from_file(f)
 
+    @classmethod
+    def compact(cls, d):
+        clean = {}
+        for k, v in d.items():
+            if isinstance(v, dict):
+                nested = cls.compact(v)
+                if len(nested.keys()) > 0:
+                    clean[k] = nested
+            elif v is not None and v != []:
+                clean[k] = v
+        return clean
+
+    def __to_dict(self):
+        return {}
+
+    def to_dict(self):
+        return Manifest.compact(self.__to_dict__())
+
+    def to_file(self, path):
+        with open(path, "w") as file:
+            yaml.dump(self.to_dict(), file)
+
     @abstractmethod
     def __init__(self, data):
         self.version = str(data["schema-version"])

bundle-workflow/src/manifests/test_manifest.py

@@ -42,11 +42,11 @@ def __init__(self, data):
             map(lambda entry: self.Component(entry), data["components"])
         )
 
-    def to_dict(self):
+    def __to_dict__(self):
         return {
             "schema-version": "1.0",
             "components": list(
-                map(lambda component: component.to_dict(), self.components)
+                map(lambda component: component.__to_dict__(), self.components)
             ),
         }
 
@@ -56,7 +56,7 @@ def __init__(self, data):
             self.integ_test = data["integ-test"]
             self.bwc_test = data["bwc-test"]
 
-        def to_dict(self):
+        def __to_dict__(self):
             return {
                 "name": self.name,
                 "integ-test": self.integ_test,

bundle-workflow/tests/tests_assemble_workflow/test_bundle_recorder.py

@@ -72,7 +72,6 @@ def test_get_manifest(self):
                     "name": "OpenSearch",
                     "version": "1.1.0",
                 },
-                "components": [],
                 "schema-version": "1.0",
             },
         )

bundle-workflow/tests/tests_build_workflow/test_build_recorder.py

@@ -235,7 +235,6 @@ def test_get_manifest(self):
                     "name": "OpenSearch",
                     "version": "1.1.0",
                 },
-                "components": [],
                 "schema-version": "1.0",
             },
         )

bundle-workflow/tests/tests_manifests/test_input_manifest.py

@@ -7,45 +7,55 @@
 import os
 import unittest
 
+import yaml
+
 from manifests.input_manifest import InputManifest
 
 
 class TestInputManifest(unittest.TestCase):
     def setUp(self):
+        self.maxDiff = None
         self.manifests_path = os.path.realpath(
             os.path.join(os.path.dirname(__file__), "../../../manifests")
         )
 
-    def test_from_file_1_0(self):
-        with open(os.path.join(self.manifests_path, "opensearch-1.0.0.yml")) as f:
-            manifest = InputManifest.from_file(f)
-            self.assertEqual(manifest.version, "1.0")
-            self.assertEqual(manifest.build.name, "OpenSearch")
-            self.assertEqual(manifest.build.version, "1.0.0")
-            self.assertEqual(len(manifest.components), 12)
-            opensearch_component = manifest.components[0]
-            self.assertEqual(opensearch_component.name, "OpenSearch")
-            self.assertEqual(
-                opensearch_component.repository,
-                "https://github.com/opensearch-project/OpenSearch.git",
-            )
-            self.assertEqual(opensearch_component.ref, "1.0")
-            for component in manifest.components:
-                self.assertIsInstance(component.ref, str)
-
-    def test_from_file_1_1(self):
-        with open(os.path.join(self.manifests_path, "opensearch-1.1.0.yml")) as f:
-            manifest = InputManifest.from_file(f)
-            self.assertEqual(manifest.version, "1.0")
-            self.assertEqual(manifest.build.name, "OpenSearch")
-            self.assertEqual(manifest.build.version, "1.1.0")
-            self.assertEqual(len(manifest.components), 14)
-            opensearch_component = manifest.components[0]
-            self.assertEqual(opensearch_component.name, "OpenSearch")
-            self.assertEqual(
-                opensearch_component.repository,
-                "https://github.com/opensearch-project/OpenSearch.git",
-            )
-            self.assertEqual(opensearch_component.ref, "1.x")
-            for component in manifest.components:
-                self.assertIsInstance(component.ref, str)
+    def test_1_0(self):
+        path = os.path.join(self.manifests_path, "opensearch-1.0.0.yml")
+        manifest = InputManifest.from_path(path)
+        self.assertEqual(manifest.version, "1.0")
+        self.assertEqual(manifest.build.name, "OpenSearch")
+        self.assertEqual(manifest.build.version, "1.0.0")
+        self.assertEqual(len(manifest.components), 12)
+        opensearch_component = manifest.components[0]
+        self.assertEqual(opensearch_component.name, "OpenSearch")
+        self.assertEqual(
+            opensearch_component.repository,
+            "https://github.com/opensearch-project/OpenSearch.git",
+        )
+        self.assertEqual(opensearch_component.ref, "1.0")
+        for component in manifest.components:
+            self.assertIsInstance(component.ref, str)
+
+    def test_1_1(self):
+        path = os.path.join(self.manifests_path, "opensearch-1.1.0.yml")
+        manifest = InputManifest.from_path(path)
+        self.assertEqual(manifest.version, "1.0")
+        self.assertEqual(manifest.build.name, "OpenSearch")
+        self.assertEqual(manifest.build.version, "1.1.0")
+        self.assertEqual(len(manifest.components), 14)
+        opensearch_component = manifest.components[0]
+        self.assertEqual(opensearch_component.name, "OpenSearch")
+        self.assertEqual(
+            opensearch_component.repository,
+            "https://github.com/opensearch-project/OpenSearch.git",
+        )
+        self.assertEqual(opensearch_component.ref, "1.x")
+        for component in manifest.components:
+            self.assertIsInstance(component.ref, str)
+
+    def test_to_dict(self):
+        path = os.path.join(self.manifests_path, "opensearch-1.1.0.yml")
+        manifest = InputManifest.from_path(path)
+        data = manifest.to_dict()
+        with open(path) as f:
+            self.assertEqual(yaml.safe_load(f), data)

bundle-workflow/tests/tests_manifests/test_manifest.py

@@ -5,12 +5,23 @@
 # compatible open source license.
 
 import os
+import tempfile
 import unittest
 
+import yaml
+
 from manifests.manifest import Manifest
 
 
 class TestManifest(unittest.TestCase):
+    class SampleManifest(Manifest):
+        def __init__(self, data):
+            super().__init__(data)
+            self.data = data
+
+        def __to_dict__(self):
+            return self.data
+
     def setUp(self):
         self.data_path = os.path.join(os.path.dirname(__file__), "data")
 
@@ -23,14 +34,28 @@ def test_manifest_is_abstract(self):
         )
 
     def test_invalid_version(self):
-        class TestManifest(Manifest):
-            def __init__(self, data):
-                super().__init__(data)
-
         manifest_path = os.path.join(self.data_path, "invalid-schema-version.yml")
 
         with self.assertRaises(ValueError) as context:
-            TestManifest.from_path(manifest_path)
+            TestManifest.SampleManifest.from_path(manifest_path)
         self.assertEqual(
             "Unsupported schema version: invalid", context.exception.__str__()
         )
+
+    def test_compact(self):
+        self.assertEqual(Manifest.compact({}), {})
+        self.assertEqual(Manifest.compact({"x": "y"}), {"x": "y"})
+        self.assertEqual(Manifest.compact({"x": "y", "z": []}), {"x": "y"})
+        self.assertEqual(Manifest.compact({"x": "y", "z": None}), {"x": "y"})
+        self.assertEqual(Manifest.compact({"x": "y", "z": {"t": None}}), {"x": "y"})
+
+    def test_to_file(self):
+        manifest_path = os.path.join(self.data_path, "opensearch-build-1.1.0.yml")
+        manifest = TestManifest.SampleManifest.from_path(manifest_path)
+
+        with tempfile.TemporaryDirectory() as path:
+            output_path = os.path.join(path, "manifest.yml")
+            manifest.to_file(output_path)
+            self.assertTrue(os.path.isfile(manifest_path))
+            with open(output_path) as f:
+                self.assertEqual(yaml.safe_load(f), manifest.to_dict())

manifests/1.0.1/manifest.linux.arm64.1.0.1.yml

@@ -1,6 +1,6 @@
 ---
 manifest:
-  schema-version: 1.0
+  schema-version: "1.0"
 build:
   version: 1.0.1
   platform: linux