Tribal Section 1 Parsing & Validation

tdrs-backend/Pipfile

@@ -58,7 +58,7 @@ django-elasticsearch-dsl = "==7.3"
 django-elasticsearch-dsl-drf = "==0.22.5"
 requests-aws4auth = "==1.1.2"
 cerberus = "==1.3.4"
-xlsxwriter = "==3.0.1"
+xlsxwriter = "==3.1.9"
 sendgrid = "==6.10.0"
 
 [requires]

tdrs-backend/tdpservice/parsers/fields.py

@@ -4,12 +4,13 @@
 
 logger = logging.getLogger(__name__)
 
+
 def value_is_empty(value, length):
     """Handle 'empty' values as field inputs."""
     empty_values = [
-        ' '*length,  # '     '
-        '#'*length,  # '#####'
-        '_'*length,  # '_____'
+        " " * length,  # '     '
+        "#" * length,  # '#####'
+        "_" * length,  # '_____'
     ]
 
     return value is None or value in empty_values
@@ -18,9 +19,20 @@ def value_is_empty(value, length):
 class Field:
     """Provides a mapping between a field name and its position."""
 
-    def __init__(self, item, name, type, startIndex, endIndex, required=True, validators=[]):
+    def __init__(
+        self,
+        item,
+        name,
+        friendly_name,
+        type,
+        startIndex,
+        endIndex,
+        required=True,
+        validators=[],
+    ):
         self.item = item
         self.name = name
+        self.friendly_name = friendly_name
         self.type = type
         self.startIndex = startIndex
         self.endIndex = endIndex
@@ -37,32 +49,43 @@ def __repr__(self):
 
     def parse_value(self, line):
         """Parse the value for a field given a line, startIndex, endIndex, and field type."""
-        value = line[self.startIndex:self.endIndex]
+        value = line[self.startIndex: self.endIndex]
 
-        if value_is_empty(value, self.endIndex-self.startIndex):
-            logger.debug(f"Field: '{self.name}' at position: [{self.startIndex}, {self.endIndex}) is empty.")
+        if value_is_empty(value, self.endIndex - self.startIndex):
+            logger.debug(
+                f"Field: '{self.name}' at position: [{self.startIndex}, {self.endIndex}) is empty."
+            )
             return None
 
         match self.type:
-            case 'number':
+            case "number":
                 try:
                     value = int(value)
                     return value
                 except ValueError:
                     logger.error(f"Error parsing field value: {value} to integer.")
                     return None
-            case 'string':
+            case "string":
                 return value
             case _:
                 logger.warn(f"Unknown field type: {self.type}.")
                 return None
 
+
 class TransformField(Field):
     """Represents a field that requires some transformation before serializing."""
 
-    def __init__(self, transform_func, item, name, type, startIndex, endIndex, required=True,
+    def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True,
                  validators=[], **kwargs):
-        super().__init__(item, name, type, startIndex, endIndex, required, validators)
+        super().__init__(
+            item=item,
+            name=name,
+            type=type,
+            friendly_name=friendly_name,
+            startIndex=startIndex,
+            endIndex=endIndex,
+            required=required,
+            validators=validators)
         self.transform_func = transform_func
         self.kwargs = kwargs
 

tdrs-backend/tdpservice/parsers/parse.py

@@ -28,11 +28,31 @@ def parse_datafile(datafile):
         bulk_create_errors({1: header_errors}, 1, flush=True)
         return errors
 
-    is_encrypted = util.contains_encrypted_indicator(header_line, schema_defs.header.get_field_by_name("encryption"))
+    field_values = schema_defs.header.get_field_values_by_names(header_line,
+                                                                {"encryption", "tribe_code", "state_fips"})
+
+    # Validate tribe code in submission across program type and fips code
+    generate_error = util.make_generate_parser_error(datafile, 1)
+    tribe_is_valid, tribe_error = validators.validate_tribe_fips_program_agree(header['program_type'],
+                                                                               field_values["tribe_code"],
+                                                                               field_values["state_fips"],
+                                                                               generate_error)
+
+    if not tribe_is_valid:
+        logger.info(f"Tribe Code ({field_values['tribe_code']}) inconsistency with Program Type " +
+                    f"({header['program_type']}) and FIPS Code ({field_values['state_fips']}).",)
+        errors['header'] = [tribe_error]
+        bulk_create_errors({1: [tribe_error]}, 1, flush=True)
+        return errors
+
+    is_encrypted = field_values["encryption"] == "E"
+    is_tribal = util.is_string_field_valid(field_values["tribe_code"], 3)
+
     logger.debug(f"Datafile has encrypted fields: {is_encrypted}.")
+    logger.debug(f"Datafile: {datafile.__repr__()}, is Tribal: {is_tribal}.")
 
     # ensure file section matches upload section
-    program_type = header['program_type']
+    program_type = f"Tribal {header['program_type']}" if is_tribal else header['program_type']
     section = header['type']
     logger.debug(f"Program type: {program_type}, Section: {section}.")
 

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -150,21 +150,31 @@ def run_postparsing_validators(self, instance, generate_error):
         errors = []
 
         for validator in self.postparsing_validators:
-            validator_is_valid, validator_error = validator(instance)
+            validator_is_valid, validator_error, field_names = validator(instance)
             is_valid = False if not validator_is_valid else is_valid
             if validator_error:
+                # get field from field name
+                fields = [self.get_field_by_name(name) for name in field_names]
                 errors.append(
                     generate_error(
                         schema=self,
                         error_category=ParserErrorCategoryChoices.VALUE_CONSISTENCY,
                         error_message=validator_error,
                         record=instance,
-                        field=None
+                        field=fields,
                     )
                 )
 
         return is_valid, errors
 
+    def get_field_values_by_names(self, line, names={}):
+        """Return dictionary of field values keyed on their name."""
+        field_values = {}
+        for field in self.fields:
+            if field.name in names:
+                field_values[field.name] = field.parse_value(line)
+        return field_values
+
     def get_field_by_name(self, name):
         """Get field by it's name."""
         for field in self.fields:

tdrs-backend/tdpservice/parsers/schema_defs/__init__.py

@@ -1,9 +1,11 @@
 from .header import header
 from .trailer import trailer
 from . import tanf
+from . import tribal_tanf
 from . import ssp
 
 header = header
 trailer = trailer
 tanf = tanf
+tribal_tanf = tribal_tanf
 ssp = ssp

tdrs-backend/tdpservice/parsers/schema_defs/header.py

@@ -20,6 +20,7 @@
         Field(
             item="2",
             name="title",
+            friendly_name="title",
             type="string",
             startIndex=0,
             endIndex=6,
@@ -31,6 +32,7 @@
         Field(
             item="4",
             name="year",
+            friendly_name="year",
             type="number",
             startIndex=6,
             endIndex=10,
@@ -40,6 +42,7 @@
         Field(
             item="5",
             name="quarter",
+            friendly_name="quarter",
             type="string",
             startIndex=10,
             endIndex=11,
@@ -49,6 +52,7 @@
         Field(
             item="6",
             name="type",
+            friendly_name="type",
             type="string",
             startIndex=11,
             endIndex=12,
@@ -58,74 +62,27 @@
         Field(
             item="1",
             name="state_fips",
+            friendly_name="state fips",
             type="string",
             startIndex=12,
             endIndex=14,
             required=True,
             validators=[
-                validators.oneOf(
-                    (
-                        "01",
-                        "02",
-                        "04",
-                        "05",
-                        "06",
-                        "08",
-                        "09",
-                        "10",
-                        "11",
-                        "12",
-                        "13",
-                        "15",
-                        "16",
-                        "17",
-                        "18",
-                        "19",
-                        "20",
-                        "21",
-                        "22",
-                        "23",
-                        "24",
-                        "25",
-                        "26",
-                        "27",
-                        "28",
-                        "29",
-                        "30",
-                        "31",
-                        "32",
-                        "33",
-                        "34",
-                        "35",
-                        "36",
-                        "37",
-                        "38",
-                        "39",
-                        "40",
-                        "41",
-                        "42",
-                        "44",
-                        "45",
-                        "46",
-                        "47",
-                        "48",
-                        "49",
-                        "50",
-                        "51",
-                        "53",
-                        "54",
-                        "55",
-                        "56",
-                        "66",
-                        "72",
-                        "78",
-                    )
+                validators.or_validators(
+                    validators.isInStringRange(0, 2),
+                    validators.isInStringRange(4, 6),
+                    validators.isInStringRange(8, 13),
+                    validators.isInStringRange(15, 42),
+                    validators.isInStringRange(44, 51),
+                    validators.isInStringRange(53, 56),
+                    validators.oneOf(["66", "72", "78"]),
                 )
             ],
         ),
         Field(
             item="3",
             name="tribe_code",
+            friendly_name="tribe code",
             type="string",
             startIndex=14,
             endIndex=17,
@@ -135,6 +92,7 @@
         Field(
             item="7",
             name="program_type",
+            friendly_name="program type",
             type="string",
             startIndex=17,
             endIndex=20,
@@ -144,6 +102,7 @@
         Field(
             item="8",
             name="edit",
+            friendly_name="edit",
             type="string",
             startIndex=20,
             endIndex=21,
@@ -153,6 +112,7 @@
         Field(
             item="9",
             name="encryption",
+            friendly_name="encryption",
             type="string",
             startIndex=21,
             endIndex=22,
@@ -162,6 +122,7 @@
         Field(
             item="10",
             name="update",
+            friendly_name="update",
             type="string",
             startIndex=22,
             endIndex=23,