SSP Section 2 Parsing & Validation

tdrs-backend/tdpservice/parsers/parse.py

@@ -240,7 +240,7 @@ def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False)
         records = schema_manager.parse_and_validate(line, generate_error)
         return records
     except AttributeError as e:
-        logging.error(e)
+        logger.error(e)
         return [(None, False, [
             generate_error(
                 schema=None,
@@ -251,50 +251,6 @@ def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False)
             )
         ])]
 
-
-def get_schema_manager_options(program_type):
-    """Return the allowed schema options."""
-    match program_type:
-        case 'TAN':
-            return {
-                'A': {
-                    'T1': schema_defs.tanf.t1,
-                    'T2': schema_defs.tanf.t2,
-                    'T3': schema_defs.tanf.t3,
-                },
-                'C': {
-                    'T4': schema_defs.tanf.t4,
-                    'T5': schema_defs.tanf.t5,
-                },
-                'G': {
-                    'T6': schema_defs.tanf.t6,
-                },
-                'S': {
-                    'T7': schema_defs.tanf.t7,
-                },
-            }
-        case 'SSP':
-            return {
-                'A': {
-                    'M1': schema_defs.ssp.m1,
-                    'M2': schema_defs.ssp.m2,
-                    'M3': schema_defs.ssp.m3,
-                },
-                'C': {
-                    # 'M4': schema_options.m4,
-                    # 'M5': schema_options.m5,
-                },
-                'G': {
-                    # 'M6': schema_options.m6,
-                },
-                'S': {
-                    # 'M7': schema_options.m7,
-                },
-            }
-        # case tribal?
-    return None
-
-
 def get_schema_manager(line, section, program_type):
     """Return the appropriate schema for the line."""
     line_type = line[0:2]

tdrs-backend/tdpservice/parsers/schema_defs/ssp/__init__.py

@@ -1,7 +1,11 @@
 from .m1 import m1
 from .m2 import m2
 from .m3 import m3
+from .m4 import m4
+from .m5 import m5
 
 m1 = m1
 m2 = m2
 m3 = m3
+m4 = m4
+m5 = m5

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py

@@ -1,10 +1,10 @@
 """Schema for SSP M1 record type."""
 
 
-from ...util import SchemaManager
-from ...fields import Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.fields import Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M1
 
 m1 = SchemaManager(

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py

@@ -1,11 +1,11 @@
 """Schema for SSP M1 record type."""
 
 
-from ...util import SchemaManager
-from ...transforms import ssp_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.transforms import ssp_ssn_decryption_func
+from tdpservice.parsers.fields import TransformField, Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M2
 
 

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py

@@ -1,11 +1,11 @@
 """Schema for SSP M1 record type."""
 
 
-from ...util import SchemaManager
-from ...transforms import ssp_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.transforms import ssp_ssn_decryption_func
+from tdpservice.parsers.fields import TransformField, Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.ssp import SSP_M3
 
 first_part_schema = RowSchema(

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py

@@ -0,0 +1,50 @@
+"""Schema for SSP M1 record type."""
+
+
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.fields import Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
+from tdpservice.search_indexes.models.ssp import SSP_M4
+
+m4 = SchemaManager(
+      schemas=[
+        RowSchema(
+          model=SSP_M4,
+          preparsing_validators=[
+              validators.hasLength(66),
+          ],
+          postparsing_validators=[],
+          fields=[
+              Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2,
+                    required=True, validators=[]),
+              Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8,
+                    required=True, validators=[validators.dateYearIsLargerThan(1998),
+                                               validators.dateMonthIsValid(),]),
+              Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19,
+                    required=True, validators=[validators.isAlphaNumeric()]),
+              Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22,
+                    required=True, validators=[validators.isInStringRange(0, 999)]),
+              Field(item="4", name='STRATUM', type='string', startIndex=22, endIndex=24,
+                    required=False, validators=[validators.isInStringRange(0, 99)]),
+              Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29,
+                    required=True, validators=[validators.isInStringRange(0, 99999)]),
+              Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30,
+                    required=True, validators=[validators.isInLimits(1, 2)]),
+              Field(item="8", name='CLOSURE_REASON', type='string', startIndex=30, endIndex=32,
+                    required=True, validators=[validators.or_validators(validators.isInStringRange(1, 19),
+                                                                        validators.matches('19'))]),
+              Field(item="9", name='REC_SUB_HOUSING', type='number', startIndex=32, endIndex=33,
+                    required=True, validators=[validators.isInLimits(1, 3)]),
+              Field(item="10`", name='REC_MED_ASSIST', type='number', startIndex=33, endIndex=34,
+                    required=True, validators=[validators.isInLimits(1, 2)]),
+              Field(item="11", name='REC_FOOD_STAMPS', type='number', startIndex=34, endIndex=35,
+                    required=True, validators=[validators.isInLimits(1, 2)]),
+              Field(item="12", name='REC_SUB_CC', type='number', startIndex=35, endIndex=36,
+                    required=True, validators=[validators.isInLimits(1, 3)]),
+              Field(item="-1", name='BLANK', type='string', startIndex=36, endIndex=66, required=False,
+                    validators=[]),
+          ]
+        )
+      ]
+  )

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py

@@ -0,0 +1,138 @@
+"""Schema for SSP M1 record type."""
+
+
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.transforms import ssp_ssn_decryption_func
+from tdpservice.parsers.fields import TransformField, Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
+from tdpservice.search_indexes.models.ssp import SSP_M5
+
+
+m5 = SchemaManager(
+      schemas=[
+        RowSchema(
+          model=SSP_M5,
+          preparsing_validators=[
+              validators.hasLength(66),
+          ],
+          postparsing_validators=[
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1),
+                        result_field='SSN', result_function=validators.validateSSN(),
+                  ),
+              validators.validate__FAM_AFF__SSN(),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3),
+                        result_field='RACE_HISPANIC', result_function=validators.isInLimits(1, 2),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3),
+                        result_field='RACE_AMER_INDIAN', result_function=validators.isInLimits(1, 2),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3),
+                        result_field='RACE_ASIAN', result_function=validators.isInLimits(1, 2),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3),
+                        result_field='RACE_BLACK', result_function=validators.isInLimits(1, 2),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3),
+                        result_field='RACE_HAWAIIAN', result_function=validators.isInLimits(1, 2),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3),
+                        result_field='RACE_WHITE', result_function=validators.isInLimits(1, 2),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3),
+                        result_field='MARITAL_STATUS', result_function=validators.isInLimits(1, 5),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 2),
+                        result_field='PARENT_MINOR_CHILD', result_function=validators.isInLimits(1, 3),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.isInLimits(1, 3),
+                        result_field='EDUCATION_LEVEL', result_function=validators.or_validators(
+                            validators.isInStringRange(1, 16), validators.isInStringRange(98, 99)),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1),
+                        result_field='CITIZENSHIP_STATUS', result_function=validators.isInLimits(1, 3),
+                  ),
+              validators.if_then_validator(
+                        condition_field='DATE_OF_BIRTH', condition_function=validators.olderThan(18),
+                        result_field='REC_OASDI_INSURANCE', result_function=validators.isInLimits(1, 2),
+                  ),
+              validators.if_then_validator(
+                        condition_field='FAMILY_AFFILIATION', condition_function=validators.matches(1),
+                        result_field='REC_FEDERAL_DISABILITY', result_function=validators.isInLimits(1, 2),
+                  ),
+          ],
+          fields=[
+              Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2,
+                    required=True, validators=[]),
+              Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8,
+                    required=True, validators=[validators.dateYearIsLargerThan(1998),
+                                               validators.dateMonthIsValid(),]),
+              Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19,
+                    required=True, validators=[validators.isAlphaNumeric()]),
+              Field(item="13", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20,
+                    required=True, validators=[validators.isInLimits(1, 5)]),
+              Field(item="14", name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28,
+                    required=True, validators=[validators.dateYearIsLargerThan(1900),
+                                               validators.dateMonthIsValid(),]),
+              TransformField(transform_func=ssp_ssn_decryption_func, item="15", name='SSN', type='string',
+                             startIndex=28, endIndex=37, required=True, validators=[validators.validateSSN()],
+                             is_encrypted=False),
+              Field(item="16A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=False,
+                    validators=[validators.validateRace()]),
+              Field(item="16B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39,
+                    required=False, validators=[validators.validateRace()]),
+              Field(item="16C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40,
+                    required=False, validators=[validators.validateRace()]),
+              Field(item="16D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41,
+                    required=False, validators=[validators.validateRace()]),
+              Field(item="16E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42,
+                    required=False, validators=[validators.validateRace()]),
+              Field(item="16F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43,
+                    required=False, validators=[validators.validateRace()]),
+              Field(item="17", name='GENDER', type='number', startIndex=43, endIndex=44,
+                    required=True, validators=[validators.isInLimits(0, 9)]),
+              Field(item="18A", name='REC_OASDI_INSURANCE', type='number', startIndex=44, endIndex=45,
+                    required=True, validators=[validators.isInLimits(0, 2)]),
+              Field(item="18B", name='REC_FEDERAL_DISABILITY', type='number', startIndex=45, endIndex=46,
+                    required=False, validators=[validators.isInLimits(0, 2)]),
+              Field(item="18C", name='REC_AID_TOTALLY_DISABLED', type='number', startIndex=46, endIndex=47,
+                    required=False, validators=[validators.isInLimits(0, 2)]),
+              Field(item="18D", name='REC_AID_AGED_BLIND', type='number', startIndex=47, endIndex=48,
+                    required=False, validators=[validators.isInLimits(0, 2)]),
+              Field(item="18E", name='REC_SSI', type='number', startIndex=48, endIndex=49,
+                    required=True, validators=[validators.isInLimits(1, 2)]),
+              Field(item="19", name='MARITAL_STATUS', type='number', startIndex=49, endIndex=50,
+                    required=False, validators=[validators.isInLimits(0, 5)]),
+              Field(item="20", name='RELATIONSHIP_HOH', type='string', startIndex=50, endIndex=52,
+                    required=True, validators=[validators.isInStringRange(1, 10)]),
+              Field(item="21", name='PARENT_MINOR_CHILD', type='number', startIndex=52, endIndex=53,
+                    required=False, validators=[validators.isInLimits(0, 2)]),
+              Field(item="22", name='NEEDS_OF_PREGNANT_WOMAN', type='number', startIndex=53, endIndex=54,
+                    required=False, validators=[validators.isInLimits(0, 9)]),
+              Field(item="23", name='EDUCATION_LEVEL', type='string', startIndex=54, endIndex=56,
+                    required=False, validators=[validators.or_validators(validators.isInStringRange(0, 16),
+                                                                         validators.isInStringRange(98, 99))]),
+              Field(item="24", name='CITIZENSHIP_STATUS', type='number', startIndex=56, endIndex=57,
+                    required=False, validators=[validators.or_validators(validators.isInLimits(0, 3),
+                                                                         validators.matches(9))]),
+              Field(item="25", name='EMPLOYMENT_STATUS', type='number', startIndex=57, endIndex=58,
+                    required=False, validators=[validators.isInLimits(0, 3)]),
+              Field(item="26", name='AMOUNT_EARNED_INCOME', type='string', startIndex=58, endIndex=62,
+                    required=True, validators=[validators.isInStringRange(0, 9999)]),
+              Field(item="27", name='AMOUNT_UNEARNED_INCOME', type='string', startIndex=62, endIndex=66,
+                    required=True, validators=[validators.isInStringRange(0, 9999)]),
+          ],
+        )
+      ]
+  )

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py

@@ -1,9 +1,9 @@
 """Schema for t1 record types."""
 
-from ...util import SchemaManager
-from ...fields import Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.fields import Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T1
 
 

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py

@@ -1,11 +1,11 @@
 """Schema for HEADER row of all submission types."""
 
 
-from ...util import SchemaManager
-from ...transforms import tanf_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.transforms import tanf_ssn_decryption_func
+from tdpservice.parsers.fields import TransformField, Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T2
 
 
@@ -105,7 +105,7 @@
             Field(item="31", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True,
                   validators=[validators.oneOf([1, 2])]),
             Field(item="32", name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, required=True,
-                  validators=[validators.isLargerThan(0),]),
+                  validators=[validators.dateYearIsLargerThan(1900), validators.dateMonthIsValid(),]),
             TransformField(transform_func=tanf_ssn_decryption_func, item="33", name='SSN', type='string', startIndex=29,
                            endIndex=38, required=True,
                            validators=[validators.validateSSN()], is_encrypted=False),

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py

@@ -1,11 +1,11 @@
 """Schema for HEADER row of all submission types."""
 
 
-from ...util import SchemaManager
-from ...transforms import tanf_ssn_decryption_func
-from ...fields import TransformField, Field
-from ...row_schema import RowSchema
-from ... import validators
+from tdpservice.parsers.util import SchemaManager
+from tdpservice.parsers.transforms import tanf_ssn_decryption_func
+from tdpservice.parsers.fields import TransformField, Field
+from tdpservice.parsers.row_schema import RowSchema
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.models.tanf import TANF_T3
 
 
@@ -77,7 +77,7 @@
               ]),
         Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28,
               required=True, validators=[
-                  validators.dateYearIsLargerThan(1998),
+                  validators.dateYearIsLargerThan(1950),
                   validators.dateMonthIsValid(),
               ]),
         TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=28,
@@ -206,7 +206,7 @@
               ]),
         Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69,
               required=True, validators=[
-                  validators.dateYearIsLargerThan(1998),
+                  validators.dateYearIsLargerThan(1950),
                   validators.dateMonthIsValid(),
               ]),
         TransformField(transform_func=tanf_ssn_decryption_func, item="69", name='SSN', type='string', startIndex=69,