fix: add script to find python dependencies

edx_repo_tools/audit_gh_users/extra.txt

@@ -6,7 +6,7 @@
 #
 click==8.1.7
     # via -r edx_repo_tools/audit_gh_users/extra.in
-fastcore==1.5.38
+fastcore==1.5.44
     # via ghapi
 ghapi==1.0.5
     # via -r edx_repo_tools/audit_gh_users/extra.in

edx_repo_tools/conventional_commits/extra-py312.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --output-file=edx_repo_tools/conventional_commits/extra-py312.txt edx_repo_tools/conventional_commits/extra-py312.in
+#    make upgrade
 #
 alembic==1.13.1
     # via dataset
@@ -13,8 +13,8 @@ contourpy==1.2.1
 cycler==0.12.1
     # via matplotlib
 dataset==1.6.2
-    # via -r edx_repo_tools/conventional_commits/extra-py312.in
-fonttools==4.51.0
+    # via -r extra-py312.in
+fonttools==4.52.1
     # via matplotlib
 kiwisolver==1.4.5
     # via matplotlib
@@ -23,7 +23,7 @@ mako==1.3.5
 markupsafe==2.1.5
     # via mako
 matplotlib==3.9.0
-    # via -r edx_repo_tools/conventional_commits/extra-py312.in
+    # via -r extra-py312.in
 numpy==1.26.4
     # via
     #   contourpy
@@ -32,7 +32,7 @@ numpy==1.26.4
 packaging==24.0
     # via matplotlib
 pandas==2.2.2
-    # via -r edx_repo_tools/conventional_commits/extra-py312.in
+    # via -r extra-py312.in
 pillow==10.3.0
     # via matplotlib
 pyparsing==3.1.2

edx_repo_tools/conventional_commits/extra.txt

@@ -14,12 +14,8 @@ cycler==0.12.1
     # via matplotlib
 dataset==1.6.2
     # via -r edx_repo_tools/conventional_commits/extra.in
-fonttools==4.51.0
+fonttools==4.53.0
     # via matplotlib
-greenlet==3.0.3
-    # via
-    #   -c edx_repo_tools/conventional_commits/../../requirements/constraints.txt
-    #   sqlalchemy
 importlib-metadata==6.11.0
     # via
     #   -c edx_repo_tools/conventional_commits/../../requirements/common_constraints.txt
@@ -61,11 +57,11 @@ sqlalchemy==1.4.52
     # via
     #   alembic
     #   dataset
-typing-extensions==4.11.0
+typing-extensions==4.12.1
     # via alembic
 tzdata==2024.1
     # via pandas
-zipp==3.18.2
+zipp==3.19.2
     # via
     #   importlib-metadata
     #   importlib-resources

edx_repo_tools/find_dependencies/extra.in

@@ -4,3 +4,4 @@
 
 rich
 requests
+requirements-parser

edx_repo_tools/find_dependencies/extra.txt

@@ -4,7 +4,7 @@
 #
 #    make upgrade
 #
-certifi==2024.2.2
+certifi==2024.6.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
@@ -16,11 +16,15 @@ mdurl==0.1.2
     # via markdown-it-py
 pygments==2.18.0
     # via rich
-requests==2.32.2
+requests==2.32.3
+    # via -r edx_repo_tools/find_dependencies/extra.in
+requirements-parser==0.9.0
     # via -r edx_repo_tools/find_dependencies/extra.in
 rich==13.7.1
     # via -r edx_repo_tools/find_dependencies/extra.in
-typing-extensions==4.11.0
+types-setuptools==70.0.0.20240524
+    # via requirements-parser
+typing-extensions==4.12.1
     # via rich
 urllib3==2.2.1
     # via requests

edx_repo_tools/find_dependencies/find_python_dependencies.py

@@ -3,6 +3,7 @@
 $ python find_python_dependencies.py $FILE_PATH
 """
 
+import click
 import json
 import os
 import requirements
@@ -12,12 +13,6 @@
 
 
 # The first of these we find is the requirements file we'll examine:
-PY_REQS = [
-    "requirements/edx/base.txt",
-    "requirements/base.txt",
-    "requirements.txt",
-]
-
 def request_package_info_url(package):
         base_url = "https://pypi.org/pypi/"
         url = f"{base_url}{package}/json"
@@ -47,29 +42,44 @@ def urls_in_orgs(urls, orgs):
         if any(f"/{org}/" in url for org in orgs)
     )
 
-def main(dirs=None, org=None):
+@click.command()
+@click.option(
+    '--req-file', 'directories',
+    multiple=True,
+    required=True,
+    help="The absolute file paths to locate Python dependencies" 
+        "within a particular repository. You can provide this "
+        "option multiple times to include multiple requirement files.",
+)
+@click.option(
+    '--ignore', 'ignore_paths',
+    multiple=True,
+    help="Dependency Repo URL to ignore even if it's"
+            "outside of your organization's approved list",
+)
+
+def main(directories=None, ignore_paths=None):
     """
     Analyze the requirements in input directory mentioned on the command line.    
     """
-    packages_url = [] 
-    if dirs is None:
-         dirs = sys.argv[1:]
 
-    for i_dir, repo_dir in enumerate(dirs, start=1):
-        with open(repo_dir) as fbase:
-            # Read each line (package name) in the file
+    home_page = set()
+    for directory in directories:
+        with open(directory) as fbase:
             for req in requirements.parse(fbase):
-                home_page = request_package_info_url(req.name)
-                if home_page is not None:
-                    if match := urls_in_orgs([home_page], SECOND_PARTY_ORGS):
-                        packages_url.append(home_page)
-
-    print("== DONE ==============")
-    print("Second party packages:")
-    print("\n".join(packages_url))
+                url = request_package_info_url(req.name)
+                if url is not None:
+                    home_page.add(url)
+
+    packages_urls = set(urls_in_orgs(home_page, SECOND_PARTY_ORGS))
 
-    if packages_url:
-        sys.exit(1)
+    if diff:= packages_urls.symmetric_difference(set(ignore_paths)):
+            print("third party packages:")
+            print("\n".join(diff))
+
+            print("ignore paths:")
+            print("\n".join(ignore_paths))
+            sys.exit(1)
 
 if __name__ == "__main__":
     main()

edx_repo_tools/repo_access_scraper/extra.txt

@@ -12,5 +12,5 @@ playwright==1.44.0
     # via -r edx_repo_tools/repo_access_scraper/extra.in
 pyee==11.1.0
     # via playwright
-typing-extensions==4.11.0
+typing-extensions==4.12.1
     # via pyee

edx_repo_tools/repo_checks/extra.txt

@@ -6,13 +6,13 @@
 #
 cache-to-disk==2.0.0
     # via -r edx_repo_tools/repo_checks/extra.in
-certifi==2024.2.2
+certifi==2024.6.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via -r edx_repo_tools/repo_checks/extra.in
-fastcore==1.5.38
+fastcore==1.5.44
     # via ghapi
 ghapi==1.0.5
     # via -r edx_repo_tools/repo_checks/extra.in
@@ -24,7 +24,7 @@ packaging==24.0
     #   ghapi
 pyyaml==6.0.1
     # via -r edx_repo_tools/repo_checks/extra.in
-requests==2.32.2
+requests==2.32.3
     # via -r edx_repo_tools/repo_checks/extra.in
 urllib3==2.2.1
     # via requests

requirements/base.txt

@@ -18,7 +18,7 @@ cachecontrol==0.14.0
     # via -r requirements/base.in
 cachetools==5.3.3
     # via tox
-certifi==2024.2.2
+certifi==2024.6.2
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -33,7 +33,7 @@ click==8.1.7
     #   moreorless
 colorama==0.4.6
     # via tox
-cryptography==42.0.7
+cryptography==42.0.8
     # via pyjwt
 distlib==0.3.8
     # via virtualenv
@@ -92,7 +92,7 @@ pyjwt[crypto]==2.8.0
     # via github3-py
 pyproject-api==1.6.1
     # via tox
-pytest==8.2.1
+pytest==8.2.2
     # via
     #   -r requirements/base.in
     #   pytest-logging
@@ -109,7 +109,7 @@ python-dotenv==1.0.1
     # via -r requirements/base.in
 pyyaml==6.0.1
     # via -r requirements/base.in
-requests==2.32.2
+requests==2.32.3
     # via
     #   -r requirements/base.in
     #   cachecontrol

requirements/development.txt

@@ -28,7 +28,7 @@ cachetools==5.3.3
     # via
     #   -r requirements/base.txt
     #   tox
-certifi==2024.2.2
+certifi==2024.6.2
     # via
     #   -r requirements/base.txt
     #   requests
@@ -61,7 +61,7 @@ colorama==0.4.6
     # via
     #   -r requirements/base.txt
     #   tox
-cryptography==42.0.7
+cryptography==42.0.8
     # via
     #   -r requirements/base.txt
     #   pyjwt
@@ -194,7 +194,7 @@ pyproject-hooks==1.1.0
     # via
     #   build
     #   pip-tools
-pytest==8.2.1
+pytest==8.2.2
     # via
     #   -r requirements/base.txt
     #   -r requirements/development.in
@@ -220,7 +220,7 @@ pyyaml==6.0.1
     #   -r requirements/base.txt
     #   code-annotations
     #   responses
-requests==2.32.2
+requests==2.32.3
     # via
     #   -r requirements/base.txt
     #   cachecontrol
@@ -264,7 +264,7 @@ tox==4.15.0
     # via -r requirements/base.txt
 tqdm==4.66.4
     # via -r requirements/base.txt
-typing-extensions==4.11.0
+typing-extensions==4.12.1
     # via
     #   astroid
     #   pylint
@@ -289,7 +289,7 @@ volatile==2.1.0
     #   bowler
 wheel==0.43.0
     # via pip-tools
-zipp==3.18.2
+zipp==3.19.2
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/pip-tools.txt

@@ -26,7 +26,7 @@ tomli==2.0.1
     #   pip-tools
 wheel==0.43.0
     # via pip-tools
-zipp==3.18.2
+zipp==3.19.2
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:

setup.py

@@ -96,6 +96,7 @@ def is_requirement(line):
             'clone_org = edx_repo_tools.dev.clone_org:main',
             'conventional_commits = edx_repo_tools.conventional_commits.commitstats:main',
             'find_dependencies = edx_repo_tools.find_dependencies.find_dependencies:main',
+            'find_python_dependencies = edx_repo_tools.find_dependencies.find_python_dependencies:main',
             'get_org_repo_urls = edx_repo_tools.dev.get_org_repo_urls:main',
             'modernize_github_actions = edx_repo_tools.codemods.django3.github_actions_modernizer:main',
             'modernize_github_actions_django = edx_repo_tools.codemods.django3.github_actions_modernizer_django:main',