Replaced imp module with importlib.util

weecology · Nov 7, 2023 · 6bad6fa · 6bad6fa
1 parent 3798257
commit 6bad6fa
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 23 deletions.
diff --git a/retriever/lib/repository.py b/retriever/lib/repository.py
@@ -1,7 +1,7 @@
 """Checks the repository for updates."""
 import os
 import requests
-import imp
+import importlib.util
 from tqdm import tqdm
 from pkg_resources import parse_version
 from retriever.lib.defaults import (REPOSITORY, RETRIEVER_REPOSITORY, SCRIPT_WRITE_PATH,
@@ -59,9 +59,11 @@ def check_for_updates(repo=REPOSITORY):
                 _download_from_repository("scripts/" + script_name, download_path, repo)
             need_to_download = False
             try:
-                file_object, pathname, desc = imp.find_module(
-                    ''.join(script_name.split('.')[:-1]), [SCRIPT_WRITE_PATH])
-                new_module = imp.load_module(script_name, file_object, pathname, desc)
+                _script_name = ''.join(script_name.split('.')[:-1])
+                script_path = f"{SCRIPT_WRITE_PATH}/{_script_name}.py"
+                spec = importlib.util.spec_from_file_location(_script_name, script_path)
+                new_module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(new_module)
                 m = str(new_module.SCRIPT.version)
                 need_to_download = parse_version(str(script_version)) > parse_version(m)
             except:

diff --git a/retriever/lib/scripts.py b/retriever/lib/scripts.py
@@ -1,5 +1,5 @@
 import csv
-import imp
+import importlib.util
 import io
 import os
 import re
@@ -29,7 +29,7 @@ def check_retriever_minimum_version(module):
     m = module.name
 
     if hasattr(module, "retriever_minimum_version"):
-        if not parse_version(VERSION) >= parse_version("{}".format(mod_ver)):
+        if mod_ver != '' and not parse_version(VERSION) >= parse_version("{}".format(mod_ver)):
             print("{} is supported by Retriever version "
                   "{}".format(m, mod_ver))
             print("Current version is {}".format(VERSION))
@@ -80,9 +80,11 @@ def reload_scripts():
             script_name = ".".join(script.split(".")[:-1])
             if script_name not in loaded_files:
                 loaded_files.append(script_name)
-                file, pathname, desc = imp.find_module(script_name, [search_path])
+                script_path = f"{search_path}/{script_name}.py"
+                spec = importlib.util.spec_from_file_location(script_name, script_path)
                 try:
-                    new_module = imp.load_module(script_name, file, pathname, desc)
+                    new_module = importlib.util.module_from_spec(spec)
+                    spec.loader.exec_module(new_module)
                     if not hasattr(new_module, "SCRIPT"):
                         continue
                     if hasattr(new_module.SCRIPT, "retriever_minimum_version"):
@@ -274,8 +276,10 @@ def get_script_upstream(dataset, repo=REPOSITORY):
         setattr(read_script, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name))
         setattr(read_script, "_name", script)
         return read_script
-    file, pathname, desc = imp.find_module(script, [SCRIPT_WRITE_PATH])
-    new_module = imp.load_module(script, file, pathname, desc)
+    script_path = f"{SCRIPT_WRITE_PATH}/{script_name}.py"
+    spec = importlib.util.spec_from_file_location(script_name, script_path)
+    new_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(new_module)
     setattr(new_module.SCRIPT, "_file", os.path.join(SCRIPT_WRITE_PATH, script_name))
     setattr(new_module.SCRIPT, "_name", script)
     return new_module.SCRIPT
@@ -461,9 +465,11 @@ def read_json_version(json_file):
 
 def read_py_version(script_name, search_path):
     """Read the version of a script from a python file"""
-    file, pathname, desc = imp.find_module(script_name, [search_path])
+    script_path = f"{search_path}/{script_name}.py"
+    spec = importlib.util.spec_from_file_location(script_name, script_path)
     try:
-        new_module = imp.load_module(script_name, file, pathname, desc)
+        new_module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(new_module)
         if hasattr(new_module.SCRIPT, "version"):
             return new_module.SCRIPT.version
     except:

diff --git a/test/test_regression.py b/test/test_regression.py
@@ -1,14 +1,13 @@
 from __future__ import absolute_import
 
-import imp
+import importlib
 import os
 import shlex
 import shutil
 import subprocess
 import sys
 import time
 from distutils.dir_util import copy_tree
-from imp import reload
 
 import pytest
 import retriever as rt
@@ -35,7 +34,7 @@
     mysqldb_host = "mysqldb_retriever"
 
 mysql_engine, postgres_engine, sqlite_engine, msaccess_engine, \
-csv_engine, download_engine, json_engine, xml_engine, _ = engine_list
+    csv_engine, download_engine, json_engine, xml_engine, _ = engine_list
 file_location = os.path.dirname(os.path.realpath(__file__))
 retriever_root_dir = os.path.abspath(os.path.join(file_location, os.pardir))
 working_script_dir = os.path.abspath(os.path.join(retriever_root_dir, "scripts"))
@@ -58,11 +57,11 @@
 ]
 
 spatial_db_md5 = [
-    ("test-eco-level-four", ["gid", "us_l3code", "na_l3code", "na_l2code"], 'd1c01d8046143e9700f5cf92cbd6be3d',[]),
+    ("test-eco-level-four", ["gid", "us_l3code", "na_l3code", "na_l2code"], 'd1c01d8046143e9700f5cf92cbd6be3d', []),
     # ("test-raster-bio1clip", ["rid","filename"], '29f702992ae42cc221d6ca9149635024',[60, 50, 100, 0]),
-    ("test-raster-bio1", ["rid", "filename"], '27e0472ddc2da9fe807bfb48b786a251',[]),
-    ("test-raster-bio2", ["rid", "filename"], '2983a9f7e099355db2ce2fa312a94cc6',[]),
-    ("test-us-eco", ["gid", "us_l3code", "na_l3code", "na_l2code"], 'eaab9fa30c745557ff6ba7c116910b45',[]),
+    ("test-raster-bio1", ["rid", "filename"], '27e0472ddc2da9fe807bfb48b786a251', []),
+    ("test-raster-bio2", ["rid", "filename"], '2983a9f7e099355db2ce2fa312a94cc6', []),
+    ("test-us-eco", ["gid", "us_l3code", "na_l3code", "na_l2code"], 'eaab9fa30c745557ff6ba7c116910b45', []),
     # h5py has compatibility issues in linux-Travis
     # Tests pass locally
     # ("sample-hdf", ["*"], '31e61867e9990138788a946542c4b1bf')
@@ -99,7 +98,7 @@
               'consumerstageid', 'resourcestageid', 'linknumber', 'linktype', 'linkevidence',
               'linkevidencenotes', 'linkfrequency', 'linkn',
               'dietfraction', 'consumptionrate', 'vectorfrom', 'preyfrom']
-         ]
+     ]
      }])
 ]
 
@@ -143,9 +142,11 @@ def teardown_module():
 def get_script_module(script_name):
     """Load a script module"""
     if script_name in python_files:
-        file, pathname, desc = imp.find_module(script_name,
-                                               [working_script_dir])
-        return imp.load_module(script_name + '.py', file, pathname, desc)
+        script_path = f"{working_script_dir}/{script_name}.py"
+        spec = importlib.util.spec_from_file_location(script_name, script_path)
+        new_module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(new_module)
+        return new_module
     return read_json(os.path.join(retriever_root_dir, 'scripts', script_name))