Create pipeline step to tag ignorable files #252

Signed-off-by: Jono Yang <jyang@nexb.com>

scanpipe/pipes/rootfs.py

@@ -20,6 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
+import fnmatch
 import logging
 import os
 from functools import partial
@@ -28,6 +29,7 @@
 from django.db.models import Q
 
 import attr
+from commoncode.ignore import default_ignores
 from container_inspector.distro import Distro
 
 from scanpipe import pipes
@@ -338,3 +340,21 @@ def tag_uninteresting_codebase_resources(project):
 
     qs = project.codebaseresources.no_status()
     qs.filter(lookups).update(status="ignored-not-interesting")
+
+
+def tag_ignorable_codebase_resources(project):
+    """
+    Using the glob patterns from commoncode.ignore of ignorable files/directories,
+    tag codebase resources from `project` if their paths match an ignorable pattern.
+    """
+    lookups = Q()
+    for pattern in default_ignores.keys():
+        # Translate glob pattern to regex
+        translated_pattern = fnmatch.translate(pattern)
+        # postgresql does not like parts of Python regex
+        if translated_pattern.startswith("(?s"):
+            translated_pattern = translated_pattern.replace("(?s", "(?")
+        lookups |= Q(rootfs_path__icontains=pattern)
+        lookups |= Q(rootfs_path__iregex=translated_pattern)
+    qs = project.codebaseresources.no_status()
+    qs.filter(lookups).update(status='ignored-default-ignores')

scanpipe/tests/test_pipes.py

@@ -872,6 +872,38 @@ def test_scanpipe_pipes_windows_tag_program_files(self):
         self.assertEqual("", resource5.status)
         self.assertEqual("", resource6.status)
 
+    def test_scanpipe_pipes_rootfs_tag_ignorable_codebase_resources(self):
+        p1 = Project.objects.create(name="Analysis")
+        resource1 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/user/cmake_install.cmake",
+            rootfs_path="/user/cmake_install.cmake"
+        )
+        resource2 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/user/example.pot",
+            rootfs_path="/user/example.pot"
+        )
+        resource3 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/user/__pycache__/foo.pyc",
+            rootfs_path="/user/__pycache__/foo.pyc"
+        )
+        resource4 = CodebaseResource.objects.create(
+            project=p1,
+            path="root/user/foo.css.map",
+            rootfs_path="/user/foo.css.map"
+        )
+        rootfs.tag_ignorable_codebase_resources(p1)
+        resource1.refresh_from_db()
+        resource2.refresh_from_db()
+        resource3.refresh_from_db()
+        resource4.refresh_from_db()
+        self.assertEqual("ignored-default-ignores", resource1.status)
+        self.assertEqual("ignored-default-ignores", resource2.status)
+        self.assertEqual("ignored-default-ignores", resource3.status)
+        self.assertEqual("ignored-default-ignores", resource4.status)
+
 
 class ScanPipePipesTransactionTest(TransactionTestCase):
     """