Merge branch 'develop' of https://github.com/raft-tech/TANF-app into …

…2664-ext-msg-2

.circleci/build-and-test/jobs.yml

@@ -5,14 +5,14 @@
       - checkout
       - docker-compose-check
       - docker-compose-up-backend
-      - run:
-          name: Execute Python Linting Test
-          command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ."
       - run:
           name: Run Unit Tests And Create Code Coverage Report
           command: |
             cd tdrs-backend;
             docker-compose run --rm web bash -c "./wait_for_services.sh && pytest --cov-report=xml"
+      - run:
+          name: Execute Python Linting Test
+          command: cd tdrs-backend; docker-compose run --rm web bash -c "flake8 ."
       - upload-codecov:
           component: backend
           coverage-report: ./tdrs-backend/coverage.xml

.circleci/config.yml

@@ -82,5 +82,5 @@ workflows:
                 - develop
                 - main
                 - master
-                - /^release.*/      
-      
+                - /^release.*/
+

scripts/zap-scanner.sh

@@ -139,7 +139,6 @@ ZAP_CLI_OPTIONS="\
   -config globalexcludeurl.url_list.url\(21\).regex='^https:\/\/.*\.identitysandbox.gov\/.*$' \
   -config globalexcludeurl.url_list.url\(21\).description='Site - IdentitySandbox.gov' \
   -config globalexcludeurl.url_list.url\(21\).enabled=true \
-
   -config spider.postform=true"
 
 # How long ZAP will crawl the app with the spider process

tdrs-backend/docker-compose.local.yml

@@ -80,7 +80,7 @@ services:
     build: .
     command: >
       bash -c "./wait_for_services.sh &&
-               ./gunicorn_start.sh && 
+               ./gunicorn_start.sh &&
                celery -A tdpservice.settings worker -l info"
     ports:
       - "5555:5555"
@@ -106,5 +106,5 @@ volumes:
 
 networks:
   default:
-    external:
-      name: external-net
+    name: external-net
+    external: true

tdrs-backend/docker-compose.yml

@@ -124,5 +124,5 @@ volumes:
 
 networks:
   default:
-    external:
       name: external-net
+      external: true

tdrs-backend/tdpservice/data_files/test/factories.py

@@ -18,7 +18,7 @@ class Meta:
     extension = "txt"
     section = "Active Case Data"
     quarter = "Q1"
-    year = "2020"
+    year = 2020
     version = 1
     user = factory.SubFactory(UserFactory)
     stt = factory.SubFactory(STTFactory)

tdrs-backend/tdpservice/parsers/admin.py

@@ -15,4 +15,11 @@ class ParserErrorAdmin(admin.ModelAdmin):
     ]
 
 
+class DataFileSummaryAdmin(admin.ModelAdmin):
+    """ModelAdmin class for DataFileSummary objects generated in parsing."""
+
+    list_display = ['status', 'case_aggregates', 'datafile']
+
+
 admin.site.register(models.ParserError, ParserErrorAdmin)
+admin.site.register(models.DataFileSummary, DataFileSummaryAdmin)

tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py

@@ -14,5 +14,5 @@ class Migration(migrations.Migration):
             model_name='parsererror',
             name='error_type',
             field=models.TextField(choices=[('1', 'File pre-check'), ('2', 'Record value invalid'), ('3', 'Record value consistency'), ('4', 'Case consistency'), ('5', 'Section consistency'), ('6', 'Historical consistency')], max_length=128),
-        ),
+        )
     ]

tdrs-backend/tdpservice/parsers/migrations/0007_datafilesummary.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.15 on 2023-09-20 15:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_files', '0012_datafile_s3_versioning_id'),
+        ('parsers', '0006_auto_20230810_1500'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DataFileSummary',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('Pending', 'Pending'), ('Accepted', 'Accepted'), ('Accepted with Errors', 'Accepted With Errors'), ('Rejected', 'Rejected')], default='Pending', max_length=50)),
+                ('case_aggregates', models.JSONField(null=True)),
+                ('datafile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='data_files.datafile')),
+            ],
+        ),
+    ]

tdrs-backend/tdpservice/parsers/models.py

@@ -5,7 +5,7 @@
 from django.utils.translation import gettext_lazy as _
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
-
+from tdpservice.data_files.models import DataFile
 
 class ParserErrorCategoryChoices(models.TextChoices):
     """Enum of ParserError error_type."""
@@ -62,8 +62,49 @@ def __repr__(self):
 
     def __str__(self):
         """Return a string representation of the model."""
-        return f"error_message: {self.error_message}"
+        return f"ParserError {self.__dict__}"
 
     def _get_error_message(self):
         """Return the error message."""
         return self.error_message
+
+class DataFileSummary(models.Model):
+    """Aggregates information about a parsed file."""
+
+    class Status(models.TextChoices):
+        """Enum for status of parsed file."""
+
+        PENDING = "Pending"  # file has been uploaded, but not validated
+        ACCEPTED = "Accepted"
+        ACCEPTED_WITH_ERRORS = "Accepted with Errors"
+        REJECTED = "Rejected"
+
+    status = models.CharField(
+        max_length=50,
+        choices=Status.choices,
+        default=Status.PENDING,
+    )
+
+    datafile = models.ForeignKey(DataFile, on_delete=models.CASCADE)
+
+    case_aggregates = models.JSONField(null=True, blank=False)
+
+    def get_status(self):
+        """Set and return the status field based on errors and models associated with datafile."""
+        errors = ParserError.objects.filter(file=self.datafile)
+        [print(error) for error in errors]
+
+        # excluding row-level pre-checks and trailer pre-checks.
+        precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\
+                                .exclude(field_name="Record_Type")\
+                                .exclude(error_message__icontains="trailer")\
+                                .exclude(error_message__icontains="Unknown Record_Type was found.")
+
+        if errors is None:
+            return DataFileSummary.Status.PENDING
+        elif errors.count() == 0:
+            return DataFileSummary.Status.ACCEPTED
+        elif precheck_errors.count() > 0:
+            return DataFileSummary.Status.REJECTED
+        else:
+            return DataFileSummary.Status.ACCEPTED_WITH_ERRORS

tdrs-backend/tdpservice/parsers/parse.py

@@ -38,8 +38,8 @@ def parse_datafile(datafile):
 
     section_is_valid, section_error = validators.validate_header_section_matches_submission(
         datafile,
-        program_type,
-        section,
+        util.get_section_reference(program_type, section),
+        util.make_generate_parser_error(datafile, 1)
     )
 
     if not section_is_valid:
@@ -123,7 +123,6 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
     errors = {}
 
     line_number = 0
-    schema_manager_options = get_schema_manager_options(program_type)
 
     unsaved_records = {}
     unsaved_parser_errors = {}
@@ -180,11 +179,9 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
             prev_sum = header_count + trailer_count
             continue
 
-        schema_manager = get_schema_manager(line, section, schema_manager_options)
-
-        schema_manager.update_encrypted_fields(is_encrypted)
+        schema_manager = get_schema_manager(line, section, program_type)
 
-        records = manager_parse_line(line, schema_manager, generate_error)
+        records = manager_parse_line(line, schema_manager, generate_error, is_encrypted)
 
         record_number = 0
         for i in range(len(records)):
@@ -236,68 +233,25 @@ def parse_datafile_lines(datafile, program_type, section, is_encrypted):
     return errors
 
 
-def manager_parse_line(line, schema_manager, generate_error):
+def manager_parse_line(line, schema_manager, generate_error, is_encrypted=False):
     """Parse and validate a datafile line using SchemaManager."""
-    if schema_manager.schemas:
+    try:
+        schema_manager.update_encrypted_fields(is_encrypted)
         records = schema_manager.parse_and_validate(line, generate_error)
         return records
+    except AttributeError as e:
+        logging.error(e)
+        return [(None, False, [
+            generate_error(
+                schema=None,
+                error_category=ParserErrorCategoryChoices.PRE_CHECK,
+                error_message="Unknown Record_Type was found.",
+                record=None,
+                field="Record_Type",
+            )
+        ])]
 
-    logger.debug("Record Type is missing from record.")
-    return [(None, False, [
-        generate_error(
-            schema=None,
-            error_category=ParserErrorCategoryChoices.PRE_CHECK,
-            error_message="Record Type is missing from record.",
-            record=None,
-            field=None
-        )
-    ])]
-
-
-def get_schema_manager_options(program_type):
-    """Return the allowed schema options."""
-    match program_type:
-        case 'TAN':
-            return {
-                'A': {
-                    'T1': schema_defs.tanf.t1,
-                    'T2': schema_defs.tanf.t2,
-                    'T3': schema_defs.tanf.t3,
-                },
-                'C': {
-                    'T4': schema_defs.tanf.t4,
-                    'T5': schema_defs.tanf.t5,
-                },
-                'G': {
-                    'T6': schema_defs.tanf.t6,
-                },
-                'S': {
-                    # 'T7': schema_options.t7,
-                },
-            }
-        case 'SSP':
-            return {
-                'A': {
-                    'M1': schema_defs.ssp.m1,
-                    'M2': schema_defs.ssp.m2,
-                    'M3': schema_defs.ssp.m3,
-                },
-                'C': {
-                    # 'M4': schema_options.m4,
-                    # 'M5': schema_options.m5,
-                },
-                'G': {
-                    # 'M6': schema_options.m6,
-                },
-                'S': {
-                    # 'M7': schema_options.m7,
-                },
-            }
-        # case tribal?
-    return None
-
-
-def get_schema_manager(line, section, schema_options):
+def get_schema_manager(line, section, program_type):
     """Return the appropriate schema for the line."""
     line_type = line[0:2]
-    return schema_options.get(section, {}).get(line_type, util.SchemaManager([]))
+    return util.get_program_model(program_type, section, line_type)

tdrs-backend/tdpservice/parsers/row_schema.py

@@ -81,7 +81,7 @@ def run_preparsing_validators(self, line, generate_error):
                         error_category=ParserErrorCategoryChoices.PRE_CHECK,
                         error_message=validator_error,
                         record=None,
-                        field=None
+                        field="Record_Type"
                     )
                 )
 

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py

@@ -1,4 +1,4 @@
-"""Schema for HEADER row of all submission types."""
+"""Schema for t1 record types."""
 
 from ...util import SchemaManager
 from ...fields import Field

tdrs-backend/tdpservice/parsers/serializers.py

@@ -1,7 +1,7 @@
 """Serializers for parsing errors."""
 
 from rest_framework import serializers
-from .models import ParserError
+from .models import ParserError, DataFileSummary
 
 
 class ParsingErrorSerializer(serializers.ModelSerializer):
@@ -23,3 +23,13 @@ class Meta:
 
         model = ParserError
         fields = '__all__'
+
+
+class DataFileSummarySerializer(serializers.ModelSerializer):
+    """Serializer for Parsing Errors."""
+
+    class Meta:
+        """Metadata."""
+
+        model = DataFileSummary
+        fields = ['status', 'case_aggregates', 'datafile']

tdrs-backend/tdpservice/parsers/test/data/small_tanf_section1.txt

@@ -1,12 +1,12 @@
 HEADER20204A06   TAN1EN
 T12020101111111111223003403361110213120000300000000000008730010000000000000000000000000000000000222222000000002229012                                       
-T2202010111111111121219740114WTTTTTY@W2221222222221012212110014722011400000000000000000000000000000000000000000000000000000000000000000000000000000000000291
+T2202010111111111121219740114WTTTTTY@W2221222222221012212110014722011500000000000000000000000000000000000000000000000000000000000000000000000000000000000291
 T320201011111111112120190127WTTTT90W022212222204398100000000                                                                                                
 T12020101111111111524503401311110233110374300000000000005450320000000000000000000000000000000000222222000000002229021                                       
 T2202010111111111152219730113WTTTT@#Z@2221222122211012210110630023080700000000000000000000000000000000000000000000000000000000000000000000000551019700000000
 T320201011111111115120160401WTTTT@BTB22212212204398100000000                                                                                                
 T12020101111111114023001401101120213110336300000000000002910410000000000000000000000000000000000222222000000002229012                                       
-T2202010111111111401219910501WTTTT@9#T2221222222221012212210421322011400000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+T2202010111111111401219910501WTTTT@9#T2221222222221012212210421322011500000000000000000000000000000000000000000000000000000000000000000000000000000000000000
 T320201011111111140120170423WTTTT@@T#22212222204398100000000                                                                                                
 T12020101111111114721801401711120212110374300000000000003820060000000000000000000000000000000000222222000000002229012                                       
 T2202010111111111471219800223WTTTT@TTW2222212222221012212110065423010700000000000000000000000000000000000000000000000000000000000000000000000000000000000000