JSON and YAML export_set fixes (#51)

* json and yaml export_set fixes
* bump version
* add tests
* linting

.github/workflows/validate.yml

@@ -30,8 +30,8 @@ jobs:
           poetry install
       - name: Linting
         run: |
-          poetry run isort .
-          poetry run black .
+          poetry run black --check .
+          poetry run isort --check-only .
           poetry run flake8 . --extend-ignore=D,E501,W601 --extend-exclude=docs/ --statistics --count
       - name: Security
         run: poetry run bandit -c pyproject.toml -r .

multi_import/fields.py

@@ -1,5 +1,9 @@
-from django.core.exceptions import (FieldError, MultipleObjectsReturned,
-                                    ObjectDoesNotExist, ValidationError)
+from django.core.exceptions import (
+    FieldError,
+    MultipleObjectsReturned,
+    ObjectDoesNotExist,
+    ValidationError,
+)
 from django.utils.translation import gettext_lazy as _
 from rest_framework import relations
 

multi_import/formats.py

@@ -1,7 +1,9 @@
+import json as pyjson
 from csv import Error as NullError
 from io import BytesIO, StringIO
 
 import chardet
+import yaml as pyyaml
 from django.utils.translation import gettext_lazy as _
 from tablib.core import Dataset, InvalidDimensions, UnsupportedFormat
 from tablib.formats import registry
@@ -148,9 +150,8 @@ def __init__(self):
         )
 
     def export_set(self, dataset):
-        return _json.json.dumps(
+        return pyjson.dumps(
             dataset.dict,
-            default=_json.date_handler,
             ensure_ascii=False,
             sort_keys=False,
             indent=2,
@@ -167,16 +168,16 @@ def export_set(self, dataset):
         # By default use the C-based CSafeDumper,
         # otherwise fallback to pure Python SafeDumper.
         if CSafeDumper:
-            return _yaml.yaml.dump(
-                dataset._package(ordered=False),
+            return pyyaml.dump(
+                dataset._package(),
+                Dumper=CSafeDumper,
                 allow_unicode=True,
                 default_flow_style=False,
                 sort_keys=False,
-                Dumper=CSafeDumper,
             )
         else:
-            return _yaml.yaml.safe_dump(
-                dataset._package(ordered=False),
+            return pyyaml.safe_dump(
+                dataset._package(),
                 allow_unicode=True,
                 default_flow_style=False,
                 sort_keys=False,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "django-multi_import"
-version = "2.0.2"
+version = "2.0.3"
 homepage = "https://github.com/sdelements/django-multi-import"
 description = "Import/export multi Django resources together atomically."
 authors = ["Security Compass <contact@securitycompass.com>"]
@@ -67,3 +67,6 @@ build-backend = "poetry.core.masonry.api"
 exclude_dirs = [
   './tests/',
 ]
+
+[tool.isort]
+profile = "black"

tests/test_csv_format.py

@@ -2,7 +2,8 @@
 
 from multi_import.exceptions import InvalidFileError
 from multi_import.formats import csv
-from multi_import.helpers.files import decode_contents, read as multi_import_read
+from multi_import.helpers.files import decode_contents
+from multi_import.helpers.files import read as multi_import_read
 
 
 class CSVFormatTest(TestCase):
@@ -22,7 +23,7 @@ def test_detect_invalid_csv_files(self):
         invalid_files = [
             "tests/fixtures/test_file.yaml",
             "tests/fixtures/test_file.json",
-            "tests/fixtures/test_file.xlsx"
+            "tests/fixtures/test_file.xlsx",
         ]
         for invalid_file in invalid_files:
             with open(invalid_file, "rb") as file:
@@ -34,25 +35,45 @@ def test_detect_invalid_csv_files(self):
                     file_handler=file, file_contents=decoded_contents
                 )
 
-                self.assertFalse(file_detection_result, "Expected %s to be an invalid file" % invalid_file)
+                self.assertFalse(
+                    file_detection_result,
+                    "Expected %s to be an invalid file" % invalid_file,
+                )
 
     def test_read_csv_file(self):
         with open("tests/fixtures/test_file.csv", "rb") as file:
             output = multi_import_read([csv], file)
 
-        expected_data = ('1', 'c0eb1608-7a75-11ee-b962-0242ac120002', 'Jedi', 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', 'lorem', '1', '00c5755c-7a76-11ee-b962-0242ac120002', '1900-11-20T13:47:10-04:00', '1900-11-20T13:47:10-04:00', 'mocha', 'True', 'mocha', '00c5755c-7a76-11ee-b962-0242ac120002', 'lorem', '', '', '', '')
+        expected_data = (
+            "1",
+            "c0eb1608-7a75-11ee-b962-0242ac120002",
+            "Jedi",
+            "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
+            "lorem",
+            "1",
+            "00c5755c-7a76-11ee-b962-0242ac120002",
+            "1900-11-20T13:47:10-04:00",
+            "1900-11-20T13:47:10-04:00",
+            "mocha",
+            "True",
+            "mocha",
+            "00c5755c-7a76-11ee-b962-0242ac120002",
+            "lorem",
+            "",
+            "",
+            "",
+            "",
+        )
 
         self.assertEqual(expected_data, output[0])
 
     def test_read_invalid_csv_files(self):
         invalid_files = [
             "tests/fixtures/test_file.yaml",
             "tests/fixtures/test_file.json",
-            "tests/fixtures/test_file.xlsx"
+            "tests/fixtures/test_file.xlsx",
         ]
 
         for invalid_file in invalid_files:
-            with open(invalid_file, "rb") as file, self.assertRaises(
-                InvalidFileError
-            ):
+            with open(invalid_file, "rb") as file, self.assertRaises(InvalidFileError):
                 multi_import_read([csv], file)

tests/test_json_format.py

@@ -1,8 +1,13 @@
+from io import StringIO
+from json import dumps as json_dumps
+
 from django.test import TestCase
+from tablib import Dataset
 
 from multi_import.exceptions import InvalidFileError
 from multi_import.formats import json
-from multi_import.helpers.files import decode_contents, read as multi_import_read
+from multi_import.helpers.files import decode_contents
+from multi_import.helpers.files import read as multi_import_read
 
 
 class JSONFormatTest(TestCase):
@@ -22,7 +27,7 @@ def test_detect_invalid_files(self):
         invalid_files = [
             "tests/fixtures/test_file.yaml",
             "tests/fixtures/test_file.csv",
-            "tests/fixtures/test_file.xlsx"
+            "tests/fixtures/test_file.xlsx",
         ]
         for invalid_file in invalid_files:
             with open(invalid_file, "rb") as file:
@@ -34,25 +39,67 @@ def test_detect_invalid_files(self):
                     file_handler=file, file_contents=decoded_contents
                 )
 
-                self.assertFalse(file_detection_result, "Expected %s to be an invalid file" % invalid_file)
+                self.assertFalse(
+                    file_detection_result,
+                    "Expected %s to be an invalid file" % invalid_file,
+                )
 
     def test_read_valid_file(self):
         with open("tests/fixtures/test_file.json", "rb") as file:
             output = multi_import_read([json], file)
 
-        expected_data = ('1', 'c0eb1608-7a75-11ee-b962-0242ac120002', 'Luke', 'Lorem ipsum dolor sit', 'Lorem ipsum dolor sit', '7', 'c0eb1608-7a75-11ee-b962-0242ac120002', '1900-10-20T13:47:10-04:00', '1901-03-10T04:03:03.622000-05:00', 'tea', 'True', 'tea', 'c0eb1608-7a75-11ee-b962-0242ac120002', '1')
+        expected_data = (
+            "1",
+            "c0eb1608-7a75-11ee-b962-0242ac120002",
+            "Luke",
+            "Lorem ipsum dolor sit",
+            "Lorem ipsum dolor sit",
+            "7",
+            "c0eb1608-7a75-11ee-b962-0242ac120002",
+            "1900-10-20T13:47:10-04:00",
+            "1901-03-10T04:03:03.622000-05:00",
+            "tea",
+            "True",
+            "tea",
+            "c0eb1608-7a75-11ee-b962-0242ac120002",
+            "1",
+        )
 
         self.assertEqual(expected_data, output[0])
 
     def test_read_invalid_files(self):
         invalid_files = [
             "tests/fixtures/test_file.yaml",
             "tests/fixtures/test_file.csv",
-            "tests/fixtures/test_file.xlsx"
+            "tests/fixtures/test_file.xlsx",
         ]
 
         for invalid_file in invalid_files:
-            with open(invalid_file, "rb") as file, self.assertRaises(
-                InvalidFileError
-            ):
+            with open(invalid_file, "rb") as file, self.assertRaises(InvalidFileError):
                 multi_import_read([json], file)
+
+    def test_key_property(self):
+        self.assertEqual(json.key, "json")
+
+    def test_extension_property(self):
+        self.assertEqual(json.extension, "json")
+
+    def test_pre_read(self):
+        file_object = StringIO(json_dumps([{"name": "John", "age": 30}]))
+
+        self.assertEqual(json.pre_read(file_object), file_object)
+
+    def test_export_set(self):
+        sample_data = [{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]
+
+        dataset = Dataset()
+        dataset.dict = sample_data
+        result = json.export_set(dataset)
+        expected = json_dumps(
+            sample_data,
+            ensure_ascii=False,
+            sort_keys=False,
+            indent=2,
+        )
+
+        self.assertEqual(result, expected)

tests/test_xlsx_format.py

@@ -2,7 +2,8 @@
 
 from multi_import.exceptions import InvalidFileError
 from multi_import.formats import xlsx
-from multi_import.helpers.files import decode_contents, read as multi_import_read
+from multi_import.helpers.files import decode_contents
+from multi_import.helpers.files import read as multi_import_read
 
 
 class XLSXFormatTest(TestCase):
@@ -22,7 +23,7 @@ def test_detect_invalid_files(self):
         invalid_files = [
             "tests/fixtures/test_file.yaml",
             "tests/fixtures/test_file.csv",
-            "tests/fixtures/test_file.json"
+            "tests/fixtures/test_file.json",
         ]
         for invalid_file in invalid_files:
             with open(invalid_file, "rb") as file:
@@ -34,25 +35,41 @@ def test_detect_invalid_files(self):
                     file_handler=file, file_contents=decoded_contents
                 )
 
-                self.assertFalse(file_detection_result, "Expected %s to be an invalid file" % invalid_file)
+                self.assertFalse(
+                    file_detection_result,
+                    "Expected %s to be an invalid file" % invalid_file,
+                )
 
     def test_read_valid_file(self):
         with open("tests/fixtures/test_file.xlsx", "rb") as file:
             output = multi_import_read([xlsx], file)
 
-        expected_data = (1, 'c0eb1608-7a75-11ee-b962-0242ac120002', 'Lorem', 'Lorem', 'ipsum', 2, 'i[sum', '1900-10-20T13:47:10-04:00', '1901-03-10T04:03:03.622000-05:00', 'ice tea', 'True', 'ice tea', 'c0eb1608-7a75-11ee-b962-0242ac120002', 1)
+        expected_data = (
+            1,
+            "c0eb1608-7a75-11ee-b962-0242ac120002",
+            "Lorem",
+            "Lorem",
+            "ipsum",
+            2,
+            "i[sum",
+            "1900-10-20T13:47:10-04:00",
+            "1901-03-10T04:03:03.622000-05:00",
+            "ice tea",
+            "True",
+            "ice tea",
+            "c0eb1608-7a75-11ee-b962-0242ac120002",
+            1,
+        )
 
         self.assertEqual(expected_data, output[0])
 
     def test_read_invalid_files(self):
         invalid_files = [
             "tests/fixtures/test_file.yaml",
             "tests/fixtures/test_file.csv",
-            "tests/fixtures/test_file.json"
+            "tests/fixtures/test_file.json",
         ]
 
         for invalid_file in invalid_files:
-            with open(invalid_file, "rb") as file, self.assertRaises(
-                InvalidFileError
-            ):
+            with open(invalid_file, "rb") as file, self.assertRaises(InvalidFileError):
                 multi_import_read([xlsx], file)