Skip to content

Commit

Permalink
Merging in Jan's 1610 code for parserError useful-ness
Browse files Browse the repository at this point in the history
  • Loading branch information
andrew-jameson committed Apr 26, 2023
1 parent b948a5b commit c5796da
Show file tree
Hide file tree
Showing 11 changed files with 942 additions and 106 deletions.
16 changes: 16 additions & 0 deletions tdrs-backend/tdpservice/parsers/admin.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
"""Django admin customizations for the parser models."""
from django.contrib import admin
from . import models


# Register your models here.
class ParserErrorAdmin(admin.ModelAdmin):
"""ModelAdmin class for parsed M1 data files."""

list_display = [
'row_number',
'field_name',
'category',
'error_type',
'error_message',
]


admin.site.register(models.ParserError, ParserErrorAdmin)
97 changes: 92 additions & 5 deletions tdrs-backend/tdpservice/parsers/parse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,13 @@


import os
<<<<<<< HEAD
from . import schema_defs
from . import validators
=======
from . import schema_defs, validators, util
# from .models import ParserError
>>>>>>> 6772b8e6 (wip parser error generator)
from tdpservice.data_files.models import DataFile
from .models import DataFileSummary

Expand All @@ -13,7 +18,10 @@ def parse_datafile(datafile):
rawfile = datafile.file
errors = {}

document_is_valid, document_error = validators.validate_single_header_trailer(rawfile)
document_is_valid, document_error = validators.validate_single_header_trailer(
rawfile,
util.make_generate_parser_error(datafile, 1)
)
if not document_is_valid:
errors['document'] = [document_error]
return errors
Expand All @@ -31,12 +39,18 @@ def parse_datafile(datafile):
trailer_line = rawfile.readline().decode().strip('\n')

# parse header, trailer
header, header_is_valid, header_errors = schema_defs.header.parse_and_validate(header_line)
header, header_is_valid, header_errors = schema_defs.header.parse_and_validate(
header_line,
util.make_generate_parser_error(datafile, 1)
)
if not header_is_valid:
errors['header'] = header_errors
return errors

trailer, trailer_is_valid, trailer_errors = schema_defs.trailer.parse_and_validate(trailer_line)
trailer, trailer_is_valid, trailer_errors = schema_defs.trailer.parse_and_validate(
trailer_line,
util.make_generate_parser_error(datafile, -1)
)
if not trailer_is_valid:
errors['trailer'] = trailer_errors

Expand All @@ -60,6 +74,7 @@ def parse_datafile(datafile):
section = header['type']

if datafile.section != section_names.get(program_type, {}).get(section):
# error_func call
errors['document'] = ['Section does not match.']
return errors

Expand All @@ -78,6 +93,7 @@ def parse_datafile(datafile):
schema = get_schema(line, section, schema_options)
record_is_valid, record_errors = parse_datafile_line(line, schema)

<<<<<<< HEAD
if not record_is_valid:
errors[line_number] = record_errors

Expand All @@ -86,17 +102,88 @@ def parse_datafile(datafile):
summary.save()

return summary, errors
=======
if isinstance(schema, util.MultiRecordRowSchema):
records = parse_multi_record_line(
line,
schema,
util.make_generate_parser_error(datafile, line_number)
)

n = 0
for r in records:
n += 1
record, record_is_valid, record_errors = r
if not record_is_valid:
line_errors = errors.get(line_number, {})
line_errors[n] = record_errors
errors[line_number] = line_errors
else:
record_is_valid, record_errors = parse_datafile_line(
line,
schema,
util.make_generate_parser_error(datafile, line_number)
)
if not record_is_valid:
errors[line_number] = record_errors

return errors

def parse_multi_record_line(line, schema, error_func):
if schema:
records = schema.parse_and_validate(line, error_func)

for r in records:
record, record_is_valid, record_errors = r

if record:
record.save()

# for error_msg in record_errors:
# error_obj = ParserError.objects.create(
# file=None,
# row_number=None,
# column_number=None,
# field_name=None,
# category=None,
# case_number=getattr(record, 'CASE_NUMBER', None),
# error_message=error_msg,
# error_type=None,
# content_type=schema.model,
# object_id=record.pk,
# fields_json=None
# )

return records

def parse_datafile_line(line, schema):
return [(None, False, ['No schema selected.'])]
>>>>>>> 6772b8e6 (wip parser error generator)


def parse_datafile_line(line, schema, error_func):
"""Parse and validate a datafile line and save any errors to the model."""
if schema:
record, record_is_valid, record_errors = schema.parse_and_validate(line)
record, record_is_valid, record_errors = schema.parse_and_validate(line, error_func)

if record:
record.errors = record_errors
record.save()

# for error_msg in record_errors:
# error_obj = ParserError.objects.create(
# file=None,
# row_number=None,
# column_number=None,
# field_name=None,
# category=None,
# case_number=None,
# error_message=error_msg,
# error_type=None,
# content_type=schema.model,
# object_id=record.pk,
# fields_json=None
# )

return record_is_valid, record_errors

return (False, ['No schema selected.'])
Expand Down
20 changes: 10 additions & 10 deletions tdrs-backend/tdpservice/parsers/schema_defs/header.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,34 +13,34 @@
],
postparsing_validators=[],
fields=[
Field(name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[
Field(item=1, name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[
validators.matches('HEADER'),
]),
Field(name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[
Field(item=2, name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[
validators.between(2000, 2099)
]),
Field(name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[
Field(item=3, name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[
validators.oneOf(['1', '2', '3', '4'])
]),
Field(name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[
Field(item=4, name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[
validators.oneOf(['A', 'C', 'G', 'S'])
]),
Field(name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[
Field(item=5, name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[
validators.between(0, 99)
]),
Field(name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[
Field(item=6, name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[
validators.between(0, 999)
]),
Field(name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[
Field(item=7, name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[
validators.oneOf(['TAN', 'SSP'])
]),
Field(name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[
Field(item=8, name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[
validators.oneOf(['1', '2'])
]),
Field(name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[
Field(item=9, name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[
validators.matches('E')
]),
Field(name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[
Field(item=10, name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[
validators.oneOf(['N', 'D', 'U'])
]),
],
Expand Down
108 changes: 108 additions & 0 deletions tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
"""Schema for SSP M1 record type."""


from ...util import RowSchema, Field
from ... import validators
from tdpservice.search_indexes.models.ssp import SSP_M1


m1 = RowSchema(
model=SSP_M1,
preparsing_validators=[
validators.hasLength(150),
],
postparsing_validators=[],
fields=[
Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[
]),
Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[
]),
Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[
]),
Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[
]),
Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[
]),
Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[
]),
Field(item=7, name='DISPOSITION', type='number', startIndex=29, endIndex=30, required=True, validators=[
]),
# Field(itemNumbe=1, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[
# ]),
Field(item=8, name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, required=True, validators=[
]),
Field(item=9, name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33, required=True, validators=[
]),
Field(item=10, name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, required=True, validators=[
]),
Field(item=11, name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, required=True, validators=[
]),
Field(item=12, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, required=True, validators=[
]),
Field(item=13, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, required=True, validators=[
]),
Field(item=14, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, required=True, validators=[
]),
Field(item=15, name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, required=True, validators=[
]),
Field(item=16, name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, required=True, validators=[
]),
Field(item=17, name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, required=True, validators=[
]),
Field(item=18, name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, required=True, validators=[
]),
Field(item=19, name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, required=True, validators=[
]),
Field(item=20, name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, required=True, validators=[
]),
Field(item=21, name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, required=True, validators=[
]),
Field(item=22, name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, required=True, validators=[
]),
Field(item=23, name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, required=True, validators=[
]),
Field(item=24, name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, required=True, validators=[
]),
Field(item=25, name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, required=True, validators=[
]),
Field(item=26, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, required=True, validators=[
]),
Field(item=27, name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, required=True, validators=[
]),
Field(item=28, name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, required=True, validators=[
]),
Field(item=29, name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, required=True, validators=[
]),
Field(item=30, name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, required=True, validators=[
]),
Field(item=31, name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, required=True, validators=[
]),
Field(item=32, name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, required=True, validators=[
]),
Field(item=33, name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, required=True, validators=[
]),
Field(item=34, name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, required=True, validators=[
]),
Field(item=35, name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, required=True, validators=[
]),
Field(item=36, name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, required=True, validators=[
]),
Field(item=37, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, required=True, validators=[
]),
Field(item=38, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, required=True, validators=[
]),
Field(item=39, name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, required=True, validators=[
]),
Field(item=40, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, required=True, validators=[
]),
Field(item=41, name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, required=True, validators=[
]),
Field(item=42, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, required=True, validators=[
]),
# Field(item=1, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[
# ]),
# Field(item=1, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[
# ]),
Field(item=43, name='BLANK', type='string', startIndex=113, endIndex=150, required=False, validators=[]),
],
)
Loading

0 comments on commit c5796da

Please sign in to comment.