Merge pull request #1482 from york-stsci/update_readnoise

Update readnoise monitor to use django database models

docs/source/database.rst

@@ -12,4 +12,4 @@ reset_database.py
 -----------------
 .. automodule:: jwql.database.reset_database
     :members:
-    :undoc-members:
+    :undoc-members:

docs/source/website.rst

@@ -44,6 +44,12 @@ monitor_views.py
     :members:
     :undoc-members:
 
+monitor_models
+--------------
+.. automodule:: jwql.website.apps.jwql.monitor_models.common
+    :members:
+    :undoc-members:
+
 settings.py
 -----------
 .. automodule:: jwql.website.jwql_proj.settings
@@ -60,4 +66,4 @@ views.py
 --------
 .. automodule:: jwql.website.apps.jwql.views
     :members:
-    :undoc-members:
+    :undoc-members:

jwql/instrument_monitors/common_monitors/readnoise_monitor.py

@@ -47,14 +47,26 @@
 import matplotlib.pyplot as plt  # noqa: E348 (comparison to true)
 import numpy as np  # noqa: E348 (comparison to true)
 from pysiaf import Siaf  # noqa: E348 (comparison to true)
-from sqlalchemy.sql.expression import and_  # noqa: E348 (comparison to true)
-
-from jwql.database.database_interface import FGSReadnoiseQueryHistory, FGSReadnoiseStats  # noqa: E348 (comparison to true)
-from jwql.database.database_interface import MIRIReadnoiseQueryHistory, MIRIReadnoiseStats  # noqa: E348 (comparison to true)
-from jwql.database.database_interface import NIRCamReadnoiseQueryHistory, NIRCamReadnoiseStats  # noqa: E348 (comparison to true)
-from jwql.database.database_interface import NIRISSReadnoiseQueryHistory, NIRISSReadnoiseStats  # noqa: E348 (comparison to true)
-from jwql.database.database_interface import NIRSpecReadnoiseQueryHistory, NIRSpecReadnoiseStats  # noqa: E348 (comparison to true)
-from jwql.database.database_interface import session, engine  # noqa: E348 (comparison to true)
+
+# astroquery.mast import that depends on value of auth_mast
+# this import has to be made before any other import of astroquery.mast
+ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
+
+# Determine if the code is being run as part of a Readthedocs build
+ON_READTHEDOCS = False
+if 'READTHEDOCS' in os.environ:  # pragma: no cover
+    ON_READTHEDOCS = os.environ['READTHEDOCS']
+
+if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    # Need to set up django apps before we can access the models
+    import django  # noqa: E402 (module level import not at top of file)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    django.setup()
+
+    # Import * is okay here because this module specifically only contains database models
+    # for this monitor
+    from jwql.website.apps.jwql.monitor_models.readnoise import *  # noqa: E402 (module level import not at top of file)
+
 from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline  # noqa: E348 (comparison to true)
 from jwql.instrument_monitors import pipeline_tools  # noqa: E348 (comparison to true)
 from jwql.utils import instrument_properties, monitor_utils  # noqa: E348 (comparison to true)
@@ -149,17 +161,8 @@ def file_exists_in_database(self, filename):
         file_exists : bool
             ``True`` if filename exists in the readnoise stats database.
         """
-
-        query = session.query(self.stats_table)
-        results = query.filter(self.stats_table.uncal_filename == filename).all()
-
-        if len(results) != 0:
-            file_exists = True
-        else:
-            file_exists = False
-
-        session.close()
-        return file_exists
+        results = self.stats_table.objects.filter(uncal_filename__iexact=filename).values()
+        return (len(results) != 0)
 
     def get_amp_stats(self, image, amps):
         """Calculates the sigma-clipped mean and stddev, as well as the
@@ -385,17 +388,18 @@ def most_recent_search(self):
             Date (in MJD) of the ending range of the previous MAST query
             where the readnoise monitor was run.
         """
-
-        query = session.query(self.query_table).filter(and_(self.query_table.aperture == self.aperture,
-                self.query_table.run_monitor == True)).order_by(self.query_table.end_time_mjd).all()  # noqa: E712 (comparison to True)
+        filter_kwargs = {
+            'aperture__iexact': self.aperture,
+            'run_monitor__exact': True
+        }
+        query = self.query_table.objects.filter(**filter_kwargs).order_by("-end_time_mjd").all()
 
         if len(query) == 0:
             query_result = 59607.0  # a.k.a. Jan 28, 2022 == First JWST images (MIRI)
             logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'.format(self.aperture, query_result)))
         else:
-            query_result = query[-1].end_time_mjd
+            query_result = query[0].end_time_mjd
 
-        session.close()
         return query_result
 
     def process(self, file_list):
@@ -512,24 +516,24 @@ def process(self, file_list):
                                   'readnoise_filename': os.path.basename(readnoise_outfile),
                                   'full_image_mean': float(full_image_mean),
                                   'full_image_stddev': float(full_image_stddev),
-                                  'full_image_n': full_image_n.astype(float),
-                                  'full_image_bin_centers': full_image_bin_centers.astype(float),
+                                  'full_image_n': list(full_image_n.astype(float)),
+                                  'full_image_bin_centers': list(full_image_bin_centers.astype(float)),
                                   'readnoise_diff_image': os.path.basename(readnoise_diff_png),
                                   'diff_image_mean': float(diff_image_mean),
                                   'diff_image_stddev': float(diff_image_stddev),
-                                  'diff_image_n': diff_image_n.astype(float),
-                                  'diff_image_bin_centers': diff_image_bin_centers.astype(float),
+                                  'diff_image_n': list(diff_image_n.astype(float)),
+                                  'diff_image_bin_centers': list(diff_image_bin_centers.astype(float)),
                                   'entry_date': datetime.datetime.now()
                                   }
             for key in amp_stats.keys():
                 if isinstance(amp_stats[key], (int, float)):
                     readnoise_db_entry[key] = float(amp_stats[key])
                 else:
-                    readnoise_db_entry[key] = amp_stats[key].astype(float)
+                    readnoise_db_entry[key] = list(amp_stats[key].astype(float))
 
             # Add this new entry to the readnoise database table
-            with engine.begin() as connection:
-                connection.execute(self.stats_table.__table__.insert(), readnoise_db_entry)
+            entry = self.stats_table(**readnoise_db_entry)
+            entry.save()
             logging.info('\tNew entry added to readnoise database table')
 
             # Remove the raw and calibrated files to save memory space
@@ -658,8 +662,8 @@ def run(self):
                              'files_found': len(new_files),
                              'run_monitor': monitor_run,
                              'entry_date': datetime.datetime.now()}
-                with engine.begin() as connection:
-                    connection.execute(self.query_table.__table__.insert(), new_entry)
+                stats_entry = self.query_table(**new_entry)
+                stats_entry.save()
                 logging.info('\tUpdated the query history table')
 
         logging.info('Readnoise Monitor completed successfully.')

jwql/tests/test_bad_pixel_monitor.py

@@ -163,6 +163,7 @@ def test_get_possible_apertures(instrument, expected_list):
     assert ap_list == expected_list
 
 
+@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_identify_tables():
     """Be sure the correct database tables are identified
     """

jwql/tests/test_bias_monitor.py

@@ -65,7 +65,7 @@ def test_extract_zeroth_group():
     # Extract the zeroth group using the bias monitor
     # nosec comment added to ignore bandit security check
     output_filename = monitor.extract_zeroth_group(filename)
-    os.chmod(output_filename, 508) # nosec
+    os.chmod(output_filename, 508)  # nosec
     data = fits.getdata(output_filename, 'SCI')[0, 0, :, :]
 
     # Remove the copied test file and its zeroth group file so this test can be properly repeated
@@ -96,6 +96,7 @@ def test_get_amp_medians():
     assert amp_medians == amp_medians_truth
 
 
+@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_identify_tables():
     """Be sure the correct database tables are identified"""
 

jwql/tests/test_readnoise_monitor.py

@@ -92,6 +92,7 @@ def test_get_metadata():
     assert monitor.expstart == '2016-01-18T04:35:14.523'
 
 
+@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_identify_tables():
     """Be sure the correct database tables are identified"""
 

jwql/website/apps/jwql/monitor_models/bad_pixel.py

@@ -27,9 +27,10 @@
 # This is an auto-generated Django model module.
 # Feel free to rename the models, but don't rename db_table values or field names.
 from django.db import models
+from django.contrib.postgres.fields import ArrayField
 
 
-class FgsBadPixelQueryHistory(models.Model):
+class FGSBadPixelQueryHistory(models.Model):
     entry_date = models.DateTimeField(unique=True)
     instrument = models.CharField(blank=True, null=True)
     aperture = models.CharField(blank=True, null=True)
@@ -50,11 +51,11 @@ class Meta:
         app_label = 'monitors'
 
 
-class FgsBadPixelStats(models.Model):
+class FGSBadPixelStats(models.Model):
     entry_date = models.DateTimeField(unique=True)
     detector = models.CharField(blank=True, null=True)
-    x_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
-    y_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
+    x_coord = ArrayField(models.IntegerField())
+    y_coord = ArrayField(models.IntegerField())
     type = models.CharField(blank=True, null=True)
     source_files = models.TextField(blank=True, null=True)  # This field type is a guess.
     obs_start_time = models.DateTimeField(blank=True, null=True)
@@ -69,7 +70,7 @@ class Meta:
         app_label = 'monitors'
 
 
-class MiriBadPixelQueryHistory(models.Model):
+class MIRIBadPixelQueryHistory(models.Model):
     entry_date = models.DateTimeField(unique=True)
     instrument = models.CharField(blank=True, null=True)
     aperture = models.CharField(blank=True, null=True)
@@ -90,11 +91,11 @@ class Meta:
         app_label = 'monitors'
 
 
-class MiriBadPixelStats(models.Model):
+class MIRIBadPixelStats(models.Model):
     entry_date = models.DateTimeField(unique=True)
     detector = models.CharField(blank=True, null=True)
-    x_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
-    y_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
+    x_coord = ArrayField(models.IntegerField())
+    y_coord = ArrayField(models.IntegerField())
     type = models.CharField(blank=True, null=True)
     source_files = models.TextField(blank=True, null=True)  # This field type is a guess.
     obs_start_time = models.DateTimeField(blank=True, null=True)
@@ -109,7 +110,7 @@ class Meta:
         app_label = 'monitors'
 
 
-class NircamBadPixelQueryHistory(models.Model):
+class NIRCamBadPixelQueryHistory(models.Model):
     entry_date = models.DateTimeField(unique=True)
     instrument = models.CharField(blank=True, null=True)
     aperture = models.CharField(blank=True, null=True)
@@ -130,11 +131,11 @@ class Meta:
         app_label = 'monitors'
 
 
-class NircamBadPixelStats(models.Model):
+class NIRCamBadPixelStats(models.Model):
     entry_date = models.DateTimeField(unique=True)
     detector = models.CharField(blank=True, null=True)
-    x_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
-    y_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
+    x_coord = ArrayField(models.IntegerField())
+    y_coord = ArrayField(models.IntegerField())
     type = models.CharField(blank=True, null=True)
     source_files = models.TextField(blank=True, null=True)  # This field type is a guess.
     obs_start_time = models.DateTimeField(blank=True, null=True)
@@ -149,7 +150,7 @@ class Meta:
         app_label = 'monitors'
 
 
-class NirissBadPixelQueryHistory(models.Model):
+class NIRISSBadPixelQueryHistory(models.Model):
     entry_date = models.DateTimeField(unique=True)
     instrument = models.CharField(blank=True, null=True)
     aperture = models.CharField(blank=True, null=True)
@@ -170,11 +171,11 @@ class Meta:
         app_label = 'monitors'
 
 
-class NirissBadPixelStats(models.Model):
+class NIRISSBadPixelStats(models.Model):
     entry_date = models.DateTimeField(unique=True)
     detector = models.CharField(blank=True, null=True)
-    x_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
-    y_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
+    x_coord = ArrayField(models.IntegerField())
+    y_coord = ArrayField(models.IntegerField())
     type = models.CharField(blank=True, null=True)
     source_files = models.TextField(blank=True, null=True)  # This field type is a guess.
     obs_start_time = models.DateTimeField(blank=True, null=True)
@@ -189,7 +190,7 @@ class Meta:
         app_label = 'monitors'
 
 
-class NirspecBadPixelQueryHistory(models.Model):
+class NIRSpecBadPixelQueryHistory(models.Model):
     entry_date = models.DateTimeField(unique=True)
     instrument = models.CharField(blank=True, null=True)
     aperture = models.CharField(blank=True, null=True)
@@ -210,11 +211,11 @@ class Meta:
         app_label = 'monitors'
 
 
-class NirspecBadPixelStats(models.Model):
+class NIRSpecBadPixelStats(models.Model):
     entry_date = models.DateTimeField(unique=True)
     detector = models.CharField(blank=True, null=True)
-    x_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
-    y_coord = models.TextField(blank=True, null=True)  # This field type is a guess.
+    x_coord = ArrayField(models.IntegerField())
+    y_coord = ArrayField(models.IntegerField())
     type = models.CharField(blank=True, null=True)
     source_files = models.TextField(blank=True, null=True)  # This field type is a guess.
     obs_start_time = models.DateTimeField(blank=True, null=True)