Update more tests to ignore headers

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

src/scancode/cli_test_utils.py

@@ -289,6 +289,7 @@ def check_jsonlines_scan(
     result_file,
     regen=False,
     remove_file_date=False,
+    ignore_headers=False,
 ):
     """
     Check the scan result_file JSON Lines results against the expected_file
@@ -311,6 +312,10 @@ def check_jsonlines_scan(
         expected = json.load(res)
 
     streamline_jsonlines_scan(expected, remove_file_date)
+
+    if ignore_headers:
+        results[0].pop('headers', None)
+        expected[0].pop('headers', None)
 
     expected = json.dumps(expected, indent=2, separators=(',', ': '))
     results = json.dumps(results, indent=2, separators=(',', ': '))

tests/licensedcode/test_plugin_license.py

@@ -78,7 +78,7 @@ def test_license_match_reference():
     args = ['--license', '--license-text', '--license-text-diagnostics', '--strip-root', test_dir, '--json', result_file, '--verbose']
     run_scan_click(args)
     test_loc = test_env.get_test_loc('plugin_license/license_reference/scan-ref.expected.json')
-    check_json_scan(test_loc, result_file, regen=False)
+    check_json_scan(test_loc, result_file, regen=False, ignore_headers=True)
 
 
 def test_license_match_without_reference():
@@ -87,7 +87,7 @@ def test_license_match_without_reference():
     args = ['--license', '--license-text', '--license-text-diagnostics', '--strip-root', test_dir, '--json', result_file, '--verbose']
     run_scan_click(args)
     test_loc = test_env.get_test_loc('plugin_license/license_reference/scan-wref.expected.json')
-    check_json_scan(test_loc, result_file, regen=False)
+    check_json_scan(test_loc, result_file, regen=False, ignore_headers=True)
 
 
 def test_get_referenced_filenames():

tests/scancode/test_plugin_mark_source.py

@@ -42,4 +42,4 @@ def test_scan_mark_source_with_info(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_mark_source/with_info.expected.json')
         run_scan_click(['--info', '--mark-source', test_dir, '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False)
+        check_json_scan(expected_file, result_file, regen=False, ignore_headers=True)

tests/scancode/test_plugin_only_findings.py

@@ -27,19 +27,19 @@ def test_scan_only_findings(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_only_findings/basic.expected.json')
         run_scan_click(['-clip', '--only-findings', '--json', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_scan_only_findings_with_errors(self):
         test_file = self.get_test_loc('plugin_only_findings/errors.json')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_only_findings/errors.expected.json')
         run_scan_click(['--from-json', test_file, '--only-findings',
                         '--json-pp', result_file])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_scan_only_findings_with_only_info(self):
         test_dir = self.extract_test_tar('plugin_only_findings/basic.tgz')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_only_findings/info.expected.json')
         run_scan_click(['--info', '--only-findings', '--json', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)

tests/summarycode/test_classify.py

@@ -67,4 +67,4 @@ def test_classify_cli_option(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('classify/cli.expected.json')
         run_scan_click(['--info', '--classify', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)

tests/summarycode/test_facet.py

@@ -29,4 +29,4 @@ def test_facet_cli_option(self):
             '--facet', 'data=*.json',
             '--facet', 'docs=*/docs/*',
             '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)

tests/summarycode/test_generated.py

@@ -86,5 +86,5 @@ def test_generated_cli_option(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('generated/cli.expected.json')
         run_scan_click(['--generated', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 

tests/summarycode/test_plugin_consolidate.py

@@ -28,7 +28,7 @@ def test_consolidate_package(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_consolidate/package-fileset-expected.json')
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
 
     def test_consolidate_package_files_should_not_be_considered_in_license_holder_consolidated_component(self):
         scan_loc = self.get_test_loc('plugin_consolidate/package-files-not-counted-in-license-holders')
@@ -38,7 +38,7 @@ def test_consolidate_package_files_should_not_be_considered_in_license_holder_co
         # expression and holder
         expected_file = self.get_test_loc('plugin_consolidate/package-files-not-counted-in-license-holders-expected.json')
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
 
     def test_consolidate_component_package_from_json_can_run_twice(self):
         scan_file = self.get_scan('plugin_consolidate/component-package', cli_options='-clip')
@@ -65,7 +65,7 @@ def test_consolidate_package_always_include_own_manifest_file(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_consolidate/package-manifest-expected.json')
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
 
     def test_get_package_resources_on_nested_packages_should_include_manifest(self):
         from packagedcode import get_package_instance
@@ -92,7 +92,7 @@ def test_consolidate_origin_summary_license_holder_rollup(self):
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
         # There should be two consolidated components for things under
         # no-majority and one consolidated component for clear-majority
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
 
     def test_consolidate_return_nested_local_majority(self):
         scan_loc = self.get_test_loc('plugin_consolidate/return-nested-local-majority')
@@ -101,7 +101,7 @@ def test_consolidate_return_nested_local_majority(self):
         run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file])
         # The nested majority is just 1 file, but has a different origin than the rest of the files above it
         # and should be reported as a separate consolidated component
-        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True)
+        check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True)
 
     def test_consolidate_component_package_build_from_live_scan(self):
         scan_loc = self.get_test_loc('plugin_consolidate/component-package-build')

tests/summarycode/test_score.py

@@ -52,7 +52,9 @@ def closure_test_function(*args, **kwargs):
             test_env.get_test_loc(expected_file),
             result_file,
             remove_file_date=True,
-            regen=regen)
+            regen=regen,
+            ignore_headers=True,
+        )
 
     test_name = 'test_license_clarity_score_%(test_name)s' % locals()
     test_name = python_safe_name(test_name)

tests/summarycode/test_summarizer.py

@@ -29,42 +29,42 @@ def test_copyright_summary_base(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('copyright_summary/summary.expected.json')
         run_scan_click(['-c', '--summary', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_copyright_summary_with_details(self):
         test_dir = self.get_test_loc('copyright_summary/scan')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('copyright_summary/summary_details.expected.json')
         run_scan_click(['-c', '--summary-with-details', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_copyright_summary_with_details_plain_json(self):
         test_dir = self.get_test_loc('copyright_summary/scan')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('copyright_summary/summary_details.expected2.json')
         run_scan_click(['-c', '--summary-with-details', '--json', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_copyright_summary_does_not_crash(self):
         test_dir = self.get_test_loc('copyright_summary/scan2')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('copyright_summary/summary2.expected.json')
         run_scan_click(['-c', '--summary', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_full_summary_base(self):
         test_dir = self.get_test_loc('full_summary/scan')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('full_summary/summary.expected.json')
         run_scan_click(['-clip', '--summary', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_full_summary_with_details(self):
         test_dir = self.get_test_loc('full_summary/scan')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('full_summary/summary_details.expected.json')
         run_scan_click(['-clip', '--summary-with-details', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_copyright_summary_key_files(self):
         test_dir = self.get_test_loc('copyright_summary/scan')
@@ -74,7 +74,7 @@ def test_copyright_summary_key_files(self):
             ['-c', '-i', '--classify', '--summary', '--summary-key-files',
              '--json-pp', result_file, test_dir])
 
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_full_summary_key_files(self):
         test_dir = self.get_test_loc('full_summary/scan')
@@ -83,7 +83,7 @@ def test_full_summary_key_files(self):
         run_scan_click(
             ['-cli', '--classify', '--summary', '--summary-key-files',
              '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_full_summary_key_files_json_lines(self):
         test_dir = self.get_test_loc('full_summary/scan')
@@ -92,7 +92,7 @@ def test_full_summary_key_files_json_lines(self):
         run_scan_click(
             ['-cli', '--classify', '--summary', '--summary-key-files',
              '--json-lines', result_file, test_dir])
-        check_jsonlines_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_jsonlines_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_full_summary_by_facet(self):
         test_dir = self.get_test_loc('full_summary/scan')
@@ -110,7 +110,7 @@ def test_full_summary_by_facet(self):
             '--summary-by-facet',
             '--json-pp', result_file, test_dir
         ])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_end2end_summary_and_classify_works_with_empty_dir_and_empty_values(self):
         test_dir = self.extract_test_tar('end-2-end/bug-1141.tar.gz')
@@ -124,7 +124,7 @@ def test_end2end_summary_and_classify_works_with_empty_dir_and_empty_values(self
             '--summary-key-files',
             '--json-pp', result_file, test_dir
         ])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)
 
     def test_summary_with_packages_reports_packages_with_files(self):
         test_dir = self.get_test_loc('packages/scan')
@@ -135,4 +135,4 @@ def test_summary_with_packages_reports_packages_with_files(self):
             '--summary',
             '--json-pp', result_file, test_dir
         ])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False, ignore_headers=True)