Merge branch 'main' into get_hdu_cache

CHANGES.rst

@@ -1,14 +1,31 @@
 1.10.1 (unreleased)
 ===================
 
+- Added ALL_MRS to allowed values for keyword MRSPRCHN in core
+  schema. [#285]
+
 - Provide existing ``AsdfFile`` instance to ``validate`` to
-  speed up assignment validation ``check_value``. [#276] 
+  speed up assignment validation ``check_value``. [#276]
 
 - Deprecate ``deprecate_class`` unused by downstream. [#274] 
 
 - Add cache to hdu accesses during ``_load_from_schema``
   to speed up file opening. [#278]
 
+- Remove ``TEXPTIME`` keyword from the JWST core datamodel schema
+  because it duplicates the information of ``XPOSURE``. [#277]
+
+- Deprecate ``check_memory_allocation``. This function did not
+  work as intended. [#273]
+
+- Decrease size of ``SPECTYP`` and ``TARGET`` columns in
+  ``OI_TARGET`` table of oifits schema to 16 characters. [#281]
+
+- Change ``integration_number`` from int16 to int32 in ``group``
+  schema. [#283]
+
+- Fix datamodel schema ids for abvegaoffset, keyword_lampmode, nrsfs_apcorr [#258]
+
 
 1.10.0 (2024-02-29)
 ===================

pyproject.toml

@@ -13,6 +13,7 @@ classifiers = [
 ]
 dependencies = [
     "asdf>=2.15.0",
+    "asdf-transform-schemas>=0.5.0",
     "asdf-astropy>=0.3.0",
     "psutil>=5.7.2",
     "numpy>=1.18",
@@ -50,6 +51,7 @@ test = [
     "pytest-doctestplus",
     "crds>=11.16.14",
     "scipy>=1.5",
+    "importlib_resources; python_version < '3.10'"
 ]
 docs = [
     "sphinx",

src/stdatamodels/fits_support.py

@@ -35,7 +35,10 @@
 
 _ASDF_EXTENSION_NAME = "ASDF"
 _FITS_SOURCE_PREFIX = "fits:"
-_NDARRAY_TAG = "tag:stsci.edu:asdf/core/ndarray-1.0.0"
+if asdf.versioning.default_version > "1.5.0":
+    _NDARRAY_TAG = "tag:stsci.edu:asdf/core/ndarray-1.1.0"
+else:
+    _NDARRAY_TAG = "tag:stsci.edu:asdf/core/ndarray-1.0.0"
 
 _builtin_regexes = [
     '', 'NAXIS[0-9]{0,3}', 'BITPIX', 'XTENSION', 'PCOUNT', 'GCOUNT',

src/stdatamodels/jwst/datamodels/schemas/abvegaoffset.schema.yaml

@@ -1,7 +1,7 @@
 %YAML 1.1
 ---
 $schema: "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0"
-id: "http://stsci.edu/schemas/jwst_datamodel/abvegamag_offset.schema"
+id: "http://stsci.edu/schemas/jwst_datamodel/abvegaoffset.schema"
 title: AB to Vega magnitude offset reference file model
 allOf:
 - $ref: referencefile.schema

src/stdatamodels/jwst/datamodels/schemas/core.schema.yaml

@@ -1332,7 +1332,7 @@ properties:
           primary_channel:
             title: MRS primary channel
             type: string
-            enum: [ALL, CHANNEL1, CHANNEL2, CHANNEL3, CHANNEL4, IMAGER]
+            enum: [ALL, ALL_MRS, CHANNEL1, CHANNEL2, CHANNEL3, CHANNEL4, IMAGER]
             fits_keyword: MRSPRCHN
           dithered_ra:
             title: RA of dithered pointing location
@@ -2597,10 +2597,6 @@ properties:
             title: "Number of groups/pointings included in resampled product"
             type: integer
             fits_keyword: NDRIZ
-          product_exposure_time:
-            title: "[s] Total exposure time for product"
-            type: number
-            fits_keyword: TEXPTIME
           weight_type:
             title: Type of drizzle weighting to use in resampling input
             type: string

src/stdatamodels/jwst/datamodels/schemas/group.schema.yaml

@@ -9,7 +9,7 @@ properties:
     fits_hdu: GROUP
     datatype:
     - name: integration_number
-      datatype: int16
+      datatype: int32
     - name: group_number
       datatype: int16
     - name: end_day

src/stdatamodels/jwst/datamodels/schemas/ifuslicer.schema.yaml

@@ -14,7 +14,7 @@ allOf:
         An instance of astropy.modeling.Model.
       type: object
     data:
-      $ref: http://stsci.edu/schemas/asdf/core/ndarray-1.0.0
+      tag: tag:stsci.edu:asdf/core/ndarray-1.*
     meta:
       type: object
       properties:

src/stdatamodels/jwst/datamodels/schemas/keyword_lampstate.schema.yaml

@@ -1,7 +1,7 @@
 %YAML 1.1
 ---
 $schema: "http://stsci.edu/schemas/fits-schema/fits-schema"
-id: "http://stsci.edu/schemas/jwst_datamodel/keyword_lampmode.schema"
+id: "http://stsci.edu/schemas/jwst_datamodel/keyword_lampstate.schema"
 type: object
 properties:
   meta:

src/stdatamodels/jwst/datamodels/schemas/msa.schema.yaml

@@ -18,7 +18,7 @@ allOf:
             An instance of astropy.modeling.Model.
           type: object
         data:
-          $ref: http://stsci.edu/schemas/asdf/core/ndarray-1.0.0
+          tag: tag:stsci.edu:asdf/core/ndarray-1.*
     Q2:
       type: object
       properties:
@@ -27,7 +27,7 @@ allOf:
             An instance of astropy.modeling.Model.
           type: object
         data:
-          $ref: http://stsci.edu/schemas/asdf/core/ndarray-1.0.0
+          tag: tag:stsci.edu:asdf/core/ndarray-1.*
     Q3:
       type: object
       properties:
@@ -36,7 +36,7 @@ allOf:
             An instance of astropy.modeling.Model.
           type: object
         data:
-          $ref: http://stsci.edu/schemas/asdf/core/ndarray-1.0.0
+          tag: tag:stsci.edu:asdf/core/ndarray-1.*
     Q4:
       type: object
       properties:
@@ -45,7 +45,7 @@ allOf:
             An instance of astropy.modeling.Model.
           type: object
         data:
-          $ref: http://stsci.edu/schemas/asdf/core/ndarray-1.0.0
+          tag: tag:stsci.edu:asdf/core/ndarray-1.*
     Q5:
       type: object
       properties:
@@ -54,7 +54,7 @@ allOf:
             An instance of astropy.modeling.Model.
           type: object
         data:
-          $ref: http://stsci.edu/schemas/asdf/core/ndarray-1.0.0
+          tag: tag:stsci.edu:asdf/core/ndarray-1.*
     meta:
       type: object
       properties:

src/stdatamodels/jwst/datamodels/schemas/nrsfs_apcorr.schema.yaml

@@ -1,7 +1,7 @@
 %YAML 1.1
 ---
 $schema: "http://stsci.edu/schemas/fits-schema/fits-schema"
-id: "http://stsci.edu/schemas/jwst_datamodel/nrsmos_apcorr.schema"
+id: "http://stsci.edu/schemas/jwst_datamodel/nrsfs_apcorr.schema"
 title: NIRSpec Fixed Slit aperture correction data model
 allOf:
 - $ref: referencefile.schema
@@ -41,4 +41,4 @@ allOf:
     sizeunit:
       title: Units for the SIZE or RADIUS column in the APCORR table
       fits_hdu: APCORR
-      fits_keyword: SIZEUNIT
+      fits_keyword: SIZEUNIT

src/stdatamodels/jwst/datamodels/schemas/oifits.schema.yaml

@@ -220,7 +220,7 @@ allOf:
       - name: TARGET_ID
         datatype: int16
       - name: TARGET
-        datatype: [ascii, 32]
+        datatype: [ascii, 16]
       - name: RAEP0
         datatype: float64
       - name: DECEP0
@@ -250,7 +250,7 @@ allOf:
       - name: PARA_ERR
         datatype: float32
       - name: SPECTYP
-        datatype: [ascii, 32]
+        datatype: [ascii, 16]
     t3:
       title: OIFITS OI_T3 table
       fits_hdu: OI_T3

src/stdatamodels/jwst/datamodels/schemas/regions.schema.yaml

@@ -11,4 +11,4 @@ allOf:
 - type: object
   properties:
     regions:
-      $ref: http://stsci.edu/schemas/asdf/core/ndarray-1.0.0
+      tag: tag:stsci.edu:asdf/core/ndarray-1.*

src/stdatamodels/jwst/datamodels/schemas/wavecorr.schema.yaml

@@ -18,7 +18,7 @@ definitions:
           [SLIT] or pitch [MOS]).  An instance of astropy.modeling.Model.
         type: object
       variance:
-        $ref: http://stsci.edu/schemas/asdf/core/ndarray-1.0.0
+        tag: tag:stsci.edu:asdf/core/ndarray-1.*
         title: Variance of the zero-point offset
         description: |
           Estimated variance on the zero-point offset (in units of detector pixel)

src/stdatamodels/jwst/datamodels/tests/test_integration.py

@@ -0,0 +1,59 @@
+import sys
+
+if sys.version_info < (3, 10):
+    import importlib_resources
+else:
+    import importlib.resources as importlib_resources
+
+import asdf
+import pytest
+import yaml
+
+
+METASCHEMAS = list(
+    importlib_resources.files("stdatamodels.jwst.datamodels.metaschema").glob("*.yaml")
+)
+
+DATAMODEL_SCHEMAS = list(
+    importlib_resources.files("stdatamodels.jwst.datamodels.schemas").glob("*.yaml")
+)
+# transform schemas are nested in a directory with a '.'
+TRANSFORM_SCHEMAS = list(
+    next(
+        importlib_resources.files(
+            "stdatamodels.jwst.transforms.resources.schemas"
+        ).iterdir()
+    ).glob("**/*.yaml")
+)
+
+SCHEMAS = METASCHEMAS + DATAMODEL_SCHEMAS + TRANSFORM_SCHEMAS
+
+TRANSFORM_MANIFESTS = list(
+    importlib_resources.files("stdatamodels.jwst.transforms.resources.manifests").glob(
+        "*.yaml"
+    )
+)
+
+RESOURCES = SCHEMAS + TRANSFORM_MANIFESTS
+
+
+@pytest.mark.parametrize("resource", RESOURCES)
+def test_resource_id(resource):
+    """
+    Test that all "resources" (schemas, metaschemas and manifests) are
+    registered with asdf using the "id" in the resource.
+    """
+    with open(resource, "rb") as f:
+        contents = f.read()
+    schema = yaml.safe_load(contents.decode("ascii"))
+    resource_manager = asdf.get_config().resource_manager
+
+    # check that asdf is aware of the "id"
+    assert (
+        schema["id"] in resource_manager
+    ), f"id[{schema['id']}] for resource[{resource}] was not registered with asdf"
+
+    # and that using the "id" to fetch the resource returns the contents of the file
+    assert (
+        resource_manager[schema["id"]] == contents
+    ), f"id[{schema['id']}] for resource[{resource}] did not return the contents of the resource"

src/stdatamodels/jwst/datamodels/tests/test_open.py

@@ -89,9 +89,10 @@ def test_check_memory_allocation_env(monkeypatch, mock_get_available_memory,
         monkeypatch.setenv('DMODEL_ALLOWED_MEMORY', str(allowed_env))
 
     # Allocate amount that would fit at 100% + swap.
-    can_allocate, required = util.check_memory_allocation(
-        (MEMORY // 2, 1), allowed=allowed_explicit,
-    )
+    with pytest.warns(DeprecationWarning, match="check_memory_allocation is deprecated"):
+        can_allocate, required = util.check_memory_allocation(
+            (MEMORY // 2, 1), allowed=allowed_explicit,
+        )
     assert can_allocate is result
 
 
@@ -107,9 +108,10 @@ def test_check_memory_allocation_env(monkeypatch, mock_get_available_memory,
 )
 def test_check_memory_allocation(mock_get_available_memory, dim, allowed, include_swap, result):
     """Check general operation of check_memory_allocation"""
-    can_allocate, required = util.check_memory_allocation(
-        (dim, 1), allowed=allowed, include_swap=include_swap
-    )
+    with pytest.warns(DeprecationWarning, match="check_memory_allocation is deprecated"):
+        can_allocate, required = util.check_memory_allocation(
+            (dim, 1), allowed=allowed, include_swap=include_swap
+        )
     assert can_allocate is result
 
 

src/stdatamodels/jwst/datamodels/tests/test_util.py

@@ -0,0 +1,20 @@
+import platform
+
+from stdatamodels.jwst.datamodels import util
+
+import pytest
+
+
+@pytest.mark.parametrize("func_name", [
+    "get_available_memory",
+    "get_available_memory_linux",
+])
+def test_deprecated(func_name):
+    with pytest.warns(DeprecationWarning, match=f"{func_name} is deprecated"):
+        getattr(util, func_name)()
+
+
+@pytest.mark.skipif(platform.system() != 'Darwin', reason="only runs on darwin")
+def test_deprecated_get_available_memory_darwin():
+    with pytest.warns(DeprecationWarning, match="get_available_memory_darwin is deprecated"):
+        util.get_available_memory_darwin()

src/stdatamodels/jwst/datamodels/util.py

@@ -415,6 +415,10 @@ def check_memory_allocation(shape, allowed=None, model_type=None, include_swap=T
     can_instantiate, required_memory : bool, number
         True if the model can be instantiated and the predicted memory footprint.
     """
+    warnings.warn(
+        "check_memory_allocation is deprecated and did not work as intended",
+        DeprecationWarning
+    )
     # Determine desired allowed amount.
     if allowed is None:
         allowed = os.environ.get('DMODEL_ALLOWED_MEMORY', None)
@@ -438,7 +442,9 @@ def check_memory_allocation(shape, allowed=None, model_type=None, include_swap=T
         size *= dimension
 
     # Get available memory
-    available = get_available_memory(include_swap=include_swap)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=DeprecationWarning)
+        available = get_available_memory(include_swap=include_swap)
     log.debug(f'Model size {bytes2human(size)} available system memory {bytes2human(available)}')
 
     if size > available:
@@ -469,15 +475,22 @@ def get_available_memory(include_swap=True):
     available : number
         The amount available.
     """
+    warnings.warn(
+        "get_available_memory is deprecated. Please use psutil",
+        DeprecationWarning
+    )
     system = platform_system()
 
     # Apple MacOS
     log.debug(f'Running OS is "{system}"')
     if system in ['Darwin']:
-        return get_available_memory_darwin(include_swap=include_swap)
+        get_available_memory_os = get_available_memory_darwin
+    else:
+        get_available_memory_os = get_available_memory_linux
 
-    # Default to Linux-like:
-    return get_available_memory_linux(include_swap=include_swap)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=DeprecationWarning)
+        return get_available_memory_os(include_swap=include_swap)
 
 
 def get_available_memory_linux(include_swap=True):
@@ -496,6 +509,10 @@ def get_available_memory_linux(include_swap=True):
     available : number
         The amount available.
     """
+    warnings.warn(
+        "get_available_memory_linux is deprecated. Please use psutil",
+        DeprecationWarning
+    )
     vm_stats = psutil.virtual_memory()
     available = vm_stats.available
     if include_swap:
@@ -523,6 +540,10 @@ def get_available_memory_darwin(include_swap=True):
     available : number
         The amount available.
     """
+    warnings.warn(
+        "get_available_memory_darwin is deprecated. Please use psutil",
+        DeprecationWarning
+    )
     vm_stats = psutil.virtual_memory()
     available = vm_stats.available
     if include_swap: