diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index 3e34bd4212..4977f2ebd1 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -12,6 +12,7 @@ from tdpservice.security.models import ClamAVFileScan from tdpservice.stts.models import STT from tdpservice.users.models import User +from tdpservice.parsers.serializers import DataFileSummarySerializer logger = logging.getLogger(__name__) class DataFileSerializer(serializers.ModelSerializer): @@ -22,6 +23,7 @@ class DataFileSerializer(serializers.ModelSerializer): user = serializers.PrimaryKeyRelatedField(queryset=User.objects.all()) ssp = serializers.BooleanField(write_only=True) has_error = serializers.SerializerMethodField() + summary = DataFileSummarySerializer(many=False, read_only=True) class Meta: """Metadata.""" @@ -45,6 +47,7 @@ class Meta: 's3_location', 's3_versioning_id', 'has_error', + 'summary' ] read_only_fields = ("version",) diff --git a/tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py b/tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py index e55c856ce1..5236b5c297 100644 --- a/tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py +++ b/tdrs-backend/tdpservice/parsers/migrations/0002_alter_parsererror_error_type.py @@ -14,5 +14,5 @@ class Migration(migrations.Migration): model_name='parsererror', name='error_type', field=models.TextField(choices=[('1', 'File pre-check'), ('2', 'Record value invalid'), ('3', 'Record value consistency'), ('4', 'Case consistency'), ('5', 'Section consistency'), ('6', 'Historical consistency')], max_length=128), - ) + ), ] diff --git a/tdrs-backend/tdpservice/parsers/migrations/0008_alter_datafilesummary_datafile.py b/tdrs-backend/tdpservice/parsers/migrations/0008_alter_datafilesummary_datafile.py new file mode 100644 index 0000000000..dc9f0b2f64 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/migrations/0008_alter_datafilesummary_datafile.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.15 on 2023-07-20 20:50 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_files', '0012_datafile_s3_versioning_id'), + ('parsers', '0007_datafilesummary'), + ] + + operations = [ + migrations.AlterField( + model_name='datafilesummary', + name='datafile', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='summary', to='data_files.datafile'), + ), + ] diff --git a/tdrs-backend/tdpservice/parsers/migrations/0009_alter_datafilesummary_status.py b/tdrs-backend/tdpservice/parsers/migrations/0009_alter_datafilesummary_status.py new file mode 100644 index 0000000000..dd05c0f4fd --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/migrations/0009_alter_datafilesummary_status.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-08-23 12:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0008_alter_datafilesummary_datafile'), + ] + + operations = [ + migrations.AlterField( + model_name='datafilesummary', + name='status', + field=models.CharField(choices=[('Pending', 'Pending'), ('Accepted', 'Accepted'), ('Accepted with Errors', 'Accepted With Errors'), ('Partially Accepted with Errors', 'Partially Accepted'), ('Rejected', 'Rejected')], default='Pending', max_length=50), + ), + ] diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 0c0ccdc505..12c70a2c07 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -77,6 +77,7 @@ class Status(models.TextChoices): PENDING = "Pending" # file has been uploaded, but not validated ACCEPTED = "Accepted" ACCEPTED_WITH_ERRORS = "Accepted with Errors" + PARTIALLY_ACCEPTED = "Partially Accepted with Errors" REJECTED = "Rejected" status = models.CharField( @@ -85,7 +86,7 @@ class Status(models.TextChoices): default=Status.PENDING, ) - datafile = models.ForeignKey(DataFile, on_delete=models.CASCADE) + datafile = models.OneToOneField(DataFile, on_delete=models.CASCADE, related_name="summary") case_aggregates = models.JSONField(null=True, blank=False) @@ -100,11 +101,17 @@ def get_status(self): .exclude(error_message__icontains="trailer")\ .exclude(error_message__icontains="Unknown Record_Type was found.") + row_precheck_errors = errors.filter(error_type=ParserErrorCategoryChoices.PRE_CHECK)\ + .filter(field_name="Record_Type")\ + .exclude(error_message__icontains="trailer") + if errors is None: return DataFileSummary.Status.PENDING elif errors.count() == 0: return DataFileSummary.Status.ACCEPTED elif precheck_errors.count() > 0: return DataFileSummary.Status.REJECTED + elif row_precheck_errors.count() > 0: + return DataFileSummary.Status.PARTIALLY_ACCEPTED else: return DataFileSummary.Status.ACCEPTED_WITH_ERRORS diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 409d239b8e..05c3da3ca8 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -20,7 +20,7 @@ def parse_datafile(datafile): header_line = rawfile.readline().decode().strip() header, header_is_valid, header_errors = schema_defs.header.parse_and_validate( header_line, - util.make_generate_parser_error(datafile, 1) + util.make_generate_file_precheck_parser_error(datafile, 1) ) if not header_is_valid: logger.info(f"Preparser Error: {len(header_errors)} header errors encountered.") diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index a92cc49466..f9ca02fb67 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -608,7 +608,7 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field parse.parse_datafile(bad_tanf_s1__row_missing_required_field) - assert dfs.get_status() == DataFileSummary.Status.ACCEPTED_WITH_ERRORS + assert dfs.get_status() == DataFileSummary.Status.PARTIALLY_ACCEPTED parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 diff --git a/tdrs-backend/tdpservice/parsers/util.py b/tdrs-backend/tdpservice/parsers/util.py index 073b7b8d8a..e3e13538c4 100644 --- a/tdrs-backend/tdpservice/parsers/util.py +++ b/tdrs-backend/tdpservice/parsers/util.py @@ -35,7 +35,7 @@ def generate_parser_error(datafile, line_number, schema, error_category, error_m row_number=line_number, column_number=getattr(field, 'item', None), item_number=getattr(field, 'item', None), - field_name=getattr(field, 'name', None), + field_name=getattr(field, 'name', None) if hasattr(field, 'name') else field, rpt_month_year=getattr(record, 'RPT_MONTH_YEAR', None), case_number=getattr(record, 'CASE_NUMBER', None), error_message=error_message, @@ -64,6 +64,22 @@ def generate(schema, error_category, error_message, record=None, field=None): return generate +def make_generate_file_precheck_parser_error(datafile, line_number): + """Configure a generate_parser_error that acts as a file pre-check error.""" + def generate(schema, error_category, error_message, record=None, field=None): + return generate_parser_error( + datafile=datafile, + line_number=line_number, + schema=schema, + error_category=error_category, + error_message=error_message, + record=record, + field=None, # purposely overridden to force a "Rejected" status for certain file precheck errors + ) + + return generate + + class SchemaManager: """Manages one or more RowSchema's and runs all parsers and validators.""" diff --git a/tdrs-backend/tdpservice/scheduling/parser_task.py b/tdrs-backend/tdpservice/scheduling/parser_task.py index f9fab7f6f6..be47703c5e 100644 --- a/tdrs-backend/tdpservice/scheduling/parser_task.py +++ b/tdrs-backend/tdpservice/scheduling/parser_task.py @@ -19,7 +19,8 @@ def parse(data_file_id): # for undetermined amount of time. data_file = DataFile.objects.get(id=data_file_id) - logger.info(f"DataFile parsing started for file -> {repr(data_file)}") + logger.info(f"DataFile parsing started for file {data_file.filename}") + dfs = DataFileSummary.objects.create(datafile=data_file, status=DataFileSummary.Status.PENDING) errors = parse_datafile(data_file) dfs.status = dfs.get_status() diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index 844b0c7449..d7045e15d0 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -3,6 +3,13 @@ import axios from 'axios' import PropTypes from 'prop-types' import { useDispatch, useSelector } from 'react-redux' import { fileUploadSections } from '../../reducers/reports' +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome' +import { + faCheckCircle, + faExclamationCircle, + faXmarkCircle, + faClock, +} from '@fortawesome/free-solid-svg-icons' import Paginator from '../Paginator' import { getAvailableFileList, download } from '../../actions/reports' import { useEffect } from 'react' @@ -11,6 +18,39 @@ import { getParseErrors } from '../../actions/createXLSReport' const formatDate = (dateStr) => new Date(dateStr).toLocaleString() +const SubmissionSummaryStatusIcon = ({ status }) => { + let icon = null + let color = null + + switch (status) { + case 'Pending': + icon = faClock + color = '#005EA2' + break + case 'Accepted': + icon = faCheckCircle + color = '#40bb45' + break + case 'Partially Accepted with Errors': + icon = faExclamationCircle + color = '#ec4e11' + break + case 'Accepted with Errors': + icon = faExclamationCircle + color = '#ec4e11' + break + case 'Rejected': + icon = faXmarkCircle + color = '#bb0000' + break + default: + break + } + return ( + + ) +} + const SubmissionHistoryRow = ({ file }) => { const dispatch = useDispatch() @@ -41,14 +81,24 @@ const SubmissionHistoryRow = ({ file }) => { {file.fileName} + + {file.summary ? ( + <> + + {file.summary && file.summary.status + ? file.summary.status + : 'Pending'} + + ) : ( + 'N/A' + )} + {file.hasError > 0 ? ( - ) : ( - 'Currently Unavailable' - )} + ) : null} ) @@ -78,6 +128,7 @@ const SectionSubmissionHistory = ({ section, label, files }) => { Submitted On Submitted By File Name + Acceptance Status Error Reports (In development) diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js index 72f3e65f80..1926cce32a 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js @@ -244,8 +244,6 @@ describe('SubmissionHistory', () => { expect( screen.queryByText('Error Reports (In development)') ).toBeInTheDocument() - - expect(screen.queryByText('Currently Unavailable')).toBeInTheDocument() }) it('Shows SSP results when SSP-MOE file type selected', () => { @@ -339,4 +337,63 @@ describe('SubmissionHistory', () => { expect(screen.queryByText('test5.txt')).toBeInTheDocument() expect(screen.queryByText('test6.txt')).not.toBeInTheDocument() }) + + it.each([ + 'Pending', + 'Accepted', + 'Accepted with Errors', + 'Partially Accepted with Errors', + 'Rejected', + null, + ])('Shows the submission acceptance status', (status) => { + const state = { + reports: { + files: [ + { + id: '123', + fileName: 'test1.txt', + fileType: 'TANF', + quarter: 'Q1', + section: 'Active Case Data', + uuid: '123-4-4-321', + year: '2023', + s3_version_id: '321-0-0-123', + createdAt: '12/12/2012 12:12', + submittedBy: 'test@teamraft.com', + summary: { + datafile: '123', + status: status, + case_aggregates: { + Oct: { + total: 0, + accepted: 0, + rejected: 0, + }, + Nov: { + total: 0, + accepted: 0, + rejected: 0, + }, + Dec: { + total: 0, + accepted: 0, + rejected: 0, + }, + }, + }, + }, + ], + }, + } + + const store = appConfigureStore(state) + const dispatch = jest.fn(store.dispatch) + store.dispatch = dispatch + + setup(store) + + expect(screen.queryByText('Acceptance Status')).toBeInTheDocument() + expect(screen.queryByText('test1.txt')).toBeInTheDocument() + expect(screen.queryByText(status || 'Pending')).toBeInTheDocument() + }) }) diff --git a/tdrs-frontend/src/reducers/reports.js b/tdrs-frontend/src/reducers/reports.js index ac16c6ac64..8c085bd994 100644 --- a/tdrs-frontend/src/reducers/reports.js +++ b/tdrs-frontend/src/reducers/reports.js @@ -67,6 +67,7 @@ export const serializeApiDataFile = (dataFile) => ({ createdAt: dataFile.created_at, submittedBy: dataFile.submitted_by, hasError: dataFile.has_error, + summary: dataFile.summary, }) const initialState = { @@ -132,6 +133,7 @@ const reports = (state = initialState, action) => { created_at: null, submitted_by: null, has_error: null, + summary: null, }) }), }