From f7dcaf659ef58a99b51e8364bb67112d697a4709 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Fri, 23 Jun 2023 14:24:42 -0600 Subject: [PATCH 01/35] Item Number Mismatch (#2578) * - Updated schemas and models to reflect correct item numbers of fields * - Revert migration * - Updated header/trailer item numbers * - Fixed item numbers off by one errors --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../0006_alter_parsererror_item_number.py | 18 +++ tdrs-backend/tdpservice/parsers/models.py | 2 +- .../tdpservice/parsers/schema_defs/header.py | 20 +-- .../tdpservice/parsers/schema_defs/ssp/m1.py | 86 +++++------ .../tdpservice/parsers/schema_defs/ssp/m2.py | 132 ++++++++--------- .../tdpservice/parsers/schema_defs/ssp/m3.py | 84 +++++------ .../tdpservice/parsers/schema_defs/tanf/t1.py | 92 ++++++------ .../tdpservice/parsers/schema_defs/tanf/t2.py | 138 +++++++++--------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 84 +++++------ .../tdpservice/parsers/schema_defs/trailer.py | 6 +- .../tdpservice/parsers/test/factories.py | 2 +- 11 files changed, 342 insertions(+), 322 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py diff --git a/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py b/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py new file mode 100644 index 0000000000..79607377ed --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-06-15 18:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0005_auto_20230601_1510'), + ] + + operations = [ + migrations.AlterField( + model_name='parsererror', + name='item_number', + field=models.CharField(max_length=8, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index f6775021fb..4356cfb68f 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -35,7 +35,7 @@ class Meta: ) row_number = models.IntegerField(null=False) column_number = models.IntegerField(null=True) - item_number = models.IntegerField(null=True) + item_number = models.CharField(null=True, max_length=8) field_name = models.TextField(null=True, max_length=128) rpt_month_year = models.IntegerField(null=True, blank=False) case_number = models.TextField(null=True, max_length=128) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index b51c433ede..3183142521 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -16,34 +16,34 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ + Field(item="2", name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ validators.matches('HEADER'), ]), - Field(item=2, name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ + Field(item="4", name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ validators.between(2000, 2099) ]), - Field(item=3, name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ + Field(item="5", name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ validators.oneOf(['1', '2', '3', '4']) ]), - Field(item=4, name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ + Field(item="6", name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ validators.oneOf(['A', 'C', 'G', 'S']) ]), - Field(item=5, name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ + Field(item="1", name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ validators.between(0, 99) ]), - Field(item=6, name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ + Field(item="3", name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ validators.between(0, 999) ]), - Field(item=7, name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ + Field(item="7", name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ validators.oneOf(['TAN', 'SSP']) ]), - Field(item=8, name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ + Field(item="8", name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ validators.oneOf(['1', '2']) ]), - Field(item=9, name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ + Field(item="9", name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ validators.matches('E') ]), - Field(item=10, name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ + Field(item="10", name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ validators.oneOf(['N', 'D', 'U']) ]), ], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 79b9c8bc83..590ca4ea1e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -13,89 +13,91 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, + Field(item="4", name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[]), - Field(item=7, name='DISPOSITION', type='number', startIndex=29, endIndex=30, + Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30, required=True, validators=[]), - Field(item=8, name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, + Field(item="8", name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, required=True, validators=[]), - Field(item=10, name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, + Field(item="9", name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33, required=True, validators=[]), - Field(item=11, name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, + Field(item="10", name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, required=True, validators=[]), - Field(item=12, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, + Field(item="11", name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, required=True, validators=[]), - Field(item=13, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, + Field(item="12", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, required=True, validators=[]), - Field(item=14, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, + Field(item="13", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, required=True, validators=[]), - Field(item=15, name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, + Field(item="14", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, required=True, validators=[]), - Field(item=16, name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, + Field(item="15", name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=17, name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, + Field(item="16", name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, required=True, validators=[]), - Field(item=18, name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, + Field(item="17", name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, required=True, validators=[]), - Field(item=19, name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, + Field(item="18", name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, required=True, validators=[]), - Field(item=20, name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, + Field(item="19A", name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, required=True, validators=[]), - Field(item=21, name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, + Field(item="19B", name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, required=True, validators=[]), - Field(item=22, name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, + Field(item="20A", name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, required=True, validators=[]), - Field(item=23, name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, + Field(item="20B", name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, required=True, validators=[]), - Field(item=24, name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, + Field(item="20C", name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, required=True, validators=[]), - Field(item=25, name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, + Field(item="21A", name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, required=True, validators=[]), - Field(item=26, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, + Field(item="21B", name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, required=True, validators=[]), - Field(item=27, name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, + Field(item="22A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, required=True, validators=[]), - Field(item=28, name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, + Field(item="22B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, required=True, validators=[]), - Field(item=29, name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, + Field(item="23A", name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, required=True, validators=[]), - Field(item=30, name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, + Field(item="23B", name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, required=True, validators=[]), - Field(item=31, name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, + Field(item="24AI", name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, required=True, validators=[]), - Field(item=32, name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, + Field(item="24AII", name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, required=True, validators=[]), - Field(item=33, name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, + Field(item="24AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, required=True, validators=[]), - Field(item=34, name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, + Field(item="24AIV", name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, required=True, validators=[]), - Field(item=35, name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, + Field(item="24AV", name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, required=True, validators=[]), - Field(item=36, name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, + Field(item="24AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, required=True, validators=[]), - Field(item=37, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, + Field(item="24AVII", name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, required=True, validators=[]), - Field(item=38, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, + Field(item="24B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, required=True, validators=[]), - Field(item=39, name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, + Field(item="24CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, required=True, validators=[]), - Field(item=40, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, + Field(item="24CII", name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, required=True, validators=[]), - Field(item=41, name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, + Field(item="24CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, required=True, validators=[]), - Field(item=42, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, + Field(item="24CIV", name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, required=True, validators=[]), - Field(item=43, name='BLANK', type='string', startIndex=113, endIndex=150, + Field(item="25", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, + required=True, validators=[]), + Field(item="-1", name='BLANK', type='string', startIndex=113, endIndex=150, required=False, validators=[]), ] ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index fd855181be..cc80e3431c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -13,137 +13,137 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="26", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, + Field(item="27", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[]), - Field(item=6, name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, + Field(item="28", name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, required=True, validators=[]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, + Field(item="29", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, + Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, + Field(item="30B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, + Field(item="30C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_BLACK', type='number', startIndex=41, endIndex=42, + Field(item="30D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, + Field(item="30E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='RACE_WHITE', type='number', startIndex=43, endIndex=44, + Field(item="30F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, + Field(item="31", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, + Field(item="32A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, + Field(item="32B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, + Field(item="32C", name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, required=True, validators=[]), - Field(item=18, name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, + Field(item="32D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=19, name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, + Field(item="32E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[]), - Field(item=20, name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, + Field(item="33", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, validators=[]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, + Field(item="34", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[]), - Field(item=22, name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, + Field(item="35", name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, validators=[]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, + Field(item="36", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, validators=[]), - Field(item=24, name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, + Field(item="37", name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, required=True, validators=[]), - Field(item=25, name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, + Field(item="38", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, validators=[]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, + Field(item="39", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, validators=[]), - Field(item=27, name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, + Field(item="40", name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, required=True, validators=[]), - Field(item=28, name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, + Field(item="41", name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, required=True, validators=[]), - Field(item=29, name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, + Field(item="42", name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, required=True, validators=[]), - Field(item=30, name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, + Field(item="43", name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, required=True, validators=[]), - Field(item=31, name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, + Field(item="44", name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=32, name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, + Field(item="45", name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, required=True, validators=[]), - Field(item=33, name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, + Field(item="46A", name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, required=True, validators=[]), - Field(item=34, name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, + Field(item="46B", name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, required=True, validators=[]), - Field(item=35, name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, + Field(item="46C", name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, required=True, validators=[]), - Field(item=36, name='OJT', type='number', startIndex=76, endIndex=78, + Field(item="47", name='OJT', type='number', startIndex=76, endIndex=78, required=True, validators=[]), - Field(item=37, name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, + Field(item="48A", name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, required=True, validators=[]), - Field(item=38, name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, + Field(item="48B", name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, required=True, validators=[]), - Field(item=39, name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, + Field(item="48C", name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, required=True, validators=[]), - Field(item=40, name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, + Field(item="49A", name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, required=True, validators=[]), - Field(item=41, name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, + Field(item="49B", name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, required=True, validators=[]), - Field(item=42, name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, + Field(item="49C", name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, required=True, validators=[]), - Field(item=43, name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, + Field(item="50A", name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=44, name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, + Field(item="50B", name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, required=True, validators=[]), - Field(item=45, name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, + Field(item="50C", name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, required=True, validators=[]), - Field(item=46, name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, + Field(item="51A", name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, required=True, validators=[]), - Field(item=47, name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, + Field(item="51B", name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, required=True, validators=[]), - Field(item=48, name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, + Field(item="51C", name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, required=True, validators=[]), - Field(item=49, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, + Field(item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, required=True, validators=[]), - Field(item=50, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, + Field(item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, required=True, validators=[]), - Field(item=51, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, + Field(item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, required=True, validators=[]), - Field(item=52, name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, + Field(item="53A", name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, required=True, validators=[]), - Field(item=53, name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, + Field(item="53B", name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, required=True, validators=[]), - Field(item=54, name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, + Field(item="53C", name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, required=True, validators=[]), - Field(item=55, name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, + Field(item="54A", name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, required=True, validators=[]), - Field(item=56, name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, + Field(item="54B", name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, required=True, validators=[]), - Field(item=57, name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, + Field(item="54C", name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, required=True, validators=[]), - Field(item=58, name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, + Field(item="55", name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, required=True, validators=[]), - Field(item=59, name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, + Field(item="56", name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, required=True, validators=[]), - Field(item=60, name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, + Field(item="57", name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, required=True, validators=[]), - Field(item=61, name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, + Field(item="58", name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, required=True, validators=[]), - Field(item=62, name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, + Field(item="59A", name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, required=True, validators=[]), - Field(item=63, name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, + Field(item="59B", name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, required=True, validators=[]), - Field(item=64, name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, + Field(item="59C", name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, required=True, validators=[]), - Field(item=65, name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, + Field(item="59D", name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, required=True, validators=[]), - Field(item=66, name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, + Field(item="59E", name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, required=True, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index c269861ff7..c2f1551cc6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -13,47 +13,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, + Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, required=True, validators=[]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, + Field(item="62", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[]), - Field(item=7, name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, + Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, + Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, + Field(item="63C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_BLACK', type='number', startIndex=40, endIndex=41, + Field(item="63D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, + Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_WHITE', type='number', startIndex=42, endIndex=43, + Field(item="63F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, + Field(item="64", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, + Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, + Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, + Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[]), - Field(item=17, name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, + Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=18, name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, + Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, required=True, validators=[]), - Field(item=19, name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, + Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[]), - Field(item=20, name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, + Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, required=True, validators=[]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, + Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, required=True, validators=[]) ] ) @@ -67,47 +67,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, + Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[]), - Field(item=23, name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, + Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, required=True, validators=[]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, + Field(item="62", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[]), - Field(item=25, name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, + Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[]), - Field(item=26, name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, + Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[]), - Field(item=27, name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, + Field(item="63C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[]), - Field(item=28, name='RACE_BLACK', type='number', startIndex=81, endIndex=82, + Field(item="63D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[]), - Field(item=29, name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, + Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[]), - Field(item=30, name='RACE_WHITE', type='number', startIndex=83, endIndex=84, + Field(item="63F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, + Field(item="64", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[]), - Field(item=32, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, + Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, required=True, validators=[]), - Field(item=33, name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, + Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, required=True, validators=[]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, + Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[]), - Field(item=35, name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, + Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[]), - Field(item=36, name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, + Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=37, name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, + Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[]), - Field(item=38, name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, + Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, required=True, validators=[]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, + Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, required=True, validators=[]) ] ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index ceb59db10e..7626278aa2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -13,96 +13,96 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, + Field(item="5", name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[]), - Field(item=7, name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, + Field(item="8", name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[]), - Field(item=8, name='DISPOSITION', type='number', startIndex=30, endIndex=31, + Field(item="9", name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[]), - Field(item=9, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, + Field(item="10", name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[]), - Field(item=10, name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, + Field(item="11", name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[]), - Field(item=11, name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, + Field(item="12", name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[]), - Field(item=12, name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, + Field(item="13", name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[]), - Field(item=13, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, + Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[]), - Field(item=14, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, + Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=15, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, + Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[]), - Field(item=16, name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, + Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=17, name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, + Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[]), - Field(item=18, name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, + Field(item="19", name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[]), - Field(item=19, name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, + Field(item="20", name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[]), - Field(item=20, name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, + Field(item="21A", name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[]), - Field(item=21, name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, + Field(item="21B", name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[]), - Field(item=22, name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, + Field(item="22A", name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[]), - Field(item=23, name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, + Field(item="22B", name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=24, name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, + Field(item="22C", name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[]), - Field(item=25, name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, + Field(item="23A", name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[]), - Field(item=26, name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, + Field(item="23B", name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[]), - Field(item=27, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, + Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[]), - Field(item=28, name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, + Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[]), - Field(item=29, name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, + Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[]), - Field(item=30, name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, + Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[]), - Field(item=31, name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, + Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[]), - Field(item=32, name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, + Field(item="26AII", name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[]), - Field(item=33, name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, + Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[]), - Field(item=34, name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, + Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[]), - Field(item=35, name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, + Field(item="26AV", name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[]), - Field(item=36, name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, + Field(item="26AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[]), - Field(item=37, name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, + Field(item="26AVII", name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[]), - Field(item=38, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, + Field(item="26B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[]), - Field(item=39, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, + Field(item="26CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[]), - Field(item=40, name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, + Field(item="26CII", name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[]), - Field(item=41, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, + Field(item="26CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[]), - Field(item=42, name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, + Field(item="26CIV", name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[]), - Field(item=43, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, + Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[]), - Field(item=44, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, + Field(item="28", name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[]), - Field(item=45, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, + Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[]), - Field(item=46, name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), + Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index dc94263d45..49ec6eff1e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -13,143 +13,143 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="30", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, + Field(item="31", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[]), - Field(item=6, name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, + Field(item="32", name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, required=True, validators=[]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, + Field(item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_HISPANIC', type='string', startIndex=38, endIndex=39, + Field(item="34A", name='RACE_HISPANIC', type='string', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_AMER_INDIAN', type='string', startIndex=39, endIndex=40, + Field(item="34B", name='RACE_AMER_INDIAN', type='string', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_ASIAN', type='string', startIndex=40, endIndex=41, + Field(item="34C", name='RACE_ASIAN', type='string', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_BLACK', type='string', startIndex=41, endIndex=42, + Field(item="34D", name='RACE_BLACK', type='string', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_HAWAIIAN', type='string', startIndex=42, endIndex=43, + Field(item="34E", name='RACE_HAWAIIAN', type='string', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='RACE_WHITE', type='string', startIndex=43, endIndex=44, + Field(item="34F", name='RACE_WHITE', type='string', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, + Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='FED_OASDI_PROGRAM', type='string', startIndex=45, endIndex=46, + Field(item="36A", name='FED_OASDI_PROGRAM', type='string', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='FED_DISABILITY_STATUS', type='string', startIndex=46, endIndex=47, + Field(item="36B", name='FED_DISABILITY_STATUS', type='string', startIndex=46, endIndex=47, required=True, validators=[]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, validators=[]), - Field(item=18, name='AID_AGED_BLIND', type='string', startIndex=48, endIndex=49, + Field(item="36D", name='AID_AGED_BLIND', type='string', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=19, name='RECEIVE_SSI', type='string', startIndex=49, endIndex=50, + Field(item="36E", name='RECEIVE_SSI', type='string', startIndex=49, endIndex=50, required=True, validators=[]), - Field(item=20, name='MARITAL_STATUS', type='string', startIndex=50, endIndex=51, + Field(item="37", name='MARITAL_STATUS', type='string', startIndex=50, endIndex=51, required=True, validators=[]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, + Field(item="38", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[]), - Field(item=22, name='PARENT_WITH_MINOR_CHILD', type='string', startIndex=53, endIndex=54, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='string', startIndex=53, endIndex=54, required=True, validators=[]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='string', startIndex=54, endIndex=55, + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='string', startIndex=54, endIndex=55, required=True, validators=[]), - Field(item=24, name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, validators=[]), - Field(item=25, name='CITIZENSHIP_STATUS', type='string', startIndex=57, endIndex=58, + Field(item="42", name='CITIZENSHIP_STATUS', type='string', startIndex=57, endIndex=58, required=True, validators=[]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='string', startIndex=58, endIndex=59, + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='string', startIndex=58, endIndex=59, required=True, validators=[]), - Field(item=27, name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, validators=[]), - Field(item=28, name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, validators=[]), - Field(item=29, name='CURRENT_MONTH_STATE_EXEMPT', type='string', startIndex=64, endIndex=65, + Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='string', startIndex=64, endIndex=65, required=True, validators=[]), - Field(item=30, name='EMPLOYMENT_STATUS', type='string', startIndex=65, endIndex=66, + Field(item="47", name='EMPLOYMENT_STATUS', type='string', startIndex=65, endIndex=66, required=True, validators=[]), - Field(item=31, name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, + Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=32, name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, validators=[]), - Field(item=33, name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, validators=[]), - Field(item=34, name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, validators=[]), - Field(item=35, name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, validators=[]), - Field(item=36, name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, validators=[]), - Field(item=37, name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, validators=[]), - Field(item=38, name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, validators=[]), - Field(item=39, name='OJT', type='string', startIndex=82, endIndex=84, + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[]), - Field(item=40, name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, validators=[]), - Field(item=41, name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, validators=[]), - Field(item=42, name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, validators=[]), - Field(item=43, name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=44, name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, validators=[]), - Field(item=45, name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, validators=[]), - Field(item=46, name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, + Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, required=False, validators=[]), - Field(item=47, name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, + Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, required=False, validators=[]), - Field(item=48, name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, + Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, required=False, validators=[]), - Field(item=49, name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, + Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, required=False, validators=[]), - Field(item=50, name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, + Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, required=False, validators=[]), - Field(item=51, name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, + Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, required=False, validators=[]), - Field(item=52, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, + Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, required=False, validators=[]), - Field(item=53, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, + Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, required=False, validators=[]), - Field(item=54, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, + Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, required=False, validators=[]), - Field(item=55, name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, + Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, required=False, validators=[]), - Field(item=56, name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, + Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, required=False, validators=[]), - Field(item=57, name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, + Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, required=False, validators=[]), - Field(item=58, name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, + Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, required=False, validators=[]), - Field(item=59, name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, + Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, required=False, validators=[]), - Field(item=60, name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, + Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, required=False, validators=[]), - Field(item=61, name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, + Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, required=False, validators=[]), - Field(item=62, name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, + Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, required=False, validators=[]), - Field(item=63, name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, + Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, required=False, validators=[]), - Field(item=64, name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, + Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=False, validators=[]), - Field(item=65, name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, + Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, required=False, validators=[]), - Field(item=67, name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, + Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=False, validators=[]), - Field(item=68, name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, + Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=False, validators=[]), - Field(item=69, name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, + Field(item="66D", name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, required=False, validators=[]), - Field(item=70, name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, + Field(item="66E", name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index c22dddc7a7..4da311c988 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -13,47 +13,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="67", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, + Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, required=True, validators=[]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, + Field(item="69", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[]), - Field(item=7, name='RACE_HISPANIC', type='string', startIndex=37, endIndex=38, + Field(item="70A", name='RACE_HISPANIC', type='string', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_AMER_INDIAN', type='string', startIndex=38, endIndex=39, + Field(item="70B", name='RACE_AMER_INDIAN', type='string', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_ASIAN', type='string', startIndex=39, endIndex=40, + Field(item="70C", name='RACE_ASIAN', type='string', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_BLACK', type='string', startIndex=40, endIndex=41, + Field(item="70D", name='RACE_BLACK', type='string', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_HAWAIIAN', type='string', startIndex=41, endIndex=42, + Field(item="70E", name='RACE_HAWAIIAN', type='string', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_WHITE', type='string', startIndex=42, endIndex=43, + Field(item="70F", name='RACE_WHITE', type='string', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, + Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=44, endIndex=45, + Field(item="72A", name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='RECEIVE_SSI', type='string', startIndex=45, endIndex=46, + Field(item="72B", name='RECEIVE_SSI', type='string', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, + Field(item="73", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[]), - Field(item=17, name='PARENT_MINOR_CHILD', type='string', startIndex=48, endIndex=49, + Field(item="74", name='PARENT_MINOR_CHILD', type='string', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=18, name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, + Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[]), - Field(item=19, name='CITIZENSHIP_STATUS', type='string', startIndex=51, endIndex=52, + Field(item="76", name='CITIZENSHIP_STATUS', type='string', startIndex=51, endIndex=52, required=True, validators=[]), - Field(item=20, name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, + Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, required=False, validators=[]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, + Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, required=False, validators=[]), ], ) @@ -66,47 +66,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, + Field(item="67", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[]), - Field(item=23, name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, + Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, required=True, validators=[]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, + Field(item="69", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[]), - Field(item=25, name='RACE_HISPANIC', type='string', startIndex=78, endIndex=79, + Field(item="70A", name='RACE_HISPANIC', type='string', startIndex=78, endIndex=79, required=True, validators=[]), - Field(item=26, name='RACE_AMER_INDIAN', type='string', startIndex=79, endIndex=80, + Field(item="70B", name='RACE_AMER_INDIAN', type='string', startIndex=79, endIndex=80, required=True, validators=[]), - Field(item=27, name='RACE_ASIAN', type='string', startIndex=80, endIndex=81, + Field(item="70C", name='RACE_ASIAN', type='string', startIndex=80, endIndex=81, required=True, validators=[]), - Field(item=28, name='RACE_BLACK', type='string', startIndex=81, endIndex=82, + Field(item="70D", name='RACE_BLACK', type='string', startIndex=81, endIndex=82, required=True, validators=[]), - Field(item=29, name='RACE_HAWAIIAN', type='string', startIndex=82, endIndex=83, + Field(item="70E", name='RACE_HAWAIIAN', type='string', startIndex=82, endIndex=83, required=True, validators=[]), - Field(item=30, name='RACE_WHITE', type='string', startIndex=83, endIndex=84, + Field(item="70F", name='RACE_WHITE', type='string', startIndex=83, endIndex=84, required=True, validators=[]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, + Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[]), - Field(item=32, name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=85, endIndex=86, + Field(item="72A", name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=85, endIndex=86, required=True, validators=[]), - Field(item=33, name='RECEIVE_SSI', type='string', startIndex=86, endIndex=87, + Field(item="72B", name='RECEIVE_SSI', type='string', startIndex=86, endIndex=87, required=True, validators=[]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, + Field(item="73", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[]), - Field(item=35, name='PARENT_MINOR_CHILD', type='string', startIndex=89, endIndex=90, + Field(item="74", name='PARENT_MINOR_CHILD', type='string', startIndex=89, endIndex=90, required=True, validators=[]), - Field(item=36, name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, + Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=37, name='CITIZENSHIP_STATUS', type='string', startIndex=92, endIndex=93, + Field(item="76", name='CITIZENSHIP_STATUS', type='string', startIndex=92, endIndex=93, required=True, validators=[]), - Field(item=38, name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, + Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, required=False, validators=[]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, + Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 3657cfc515..086c83e85a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -16,13 +16,13 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ + Field(item="1", name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ validators.matches('TRAILER') ]), - Field(item=2, name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ + Field(item="2", name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ validators.between(0, 9999999) ]), - Field(item=3, name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ + Field(item="-1", name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ validators.matches(' ') ]), ], diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index 74bf925297..8d0e709505 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -13,7 +13,7 @@ class Meta: file = factory.SubFactory(DataFileFactory) row_number = 1 column_number = 1 - item_number = 1 + item_number = "1" field_name = "test field name" case_number = '1' rpt_month_year = 202001 From ab0966005a875f1ccb2c007d6d905d570998a1a7 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 27 Jun 2023 11:40:22 -0600 Subject: [PATCH 02/35] pipeline filtering (#2538) * pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md --------- Co-authored-by: George Hudson Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .circleci/README.md | 4 +- .circleci/base_config.yml | 18 ++++++ .circleci/build-and-test/workflows.yml | 62 ++++++++++++++++--- .circleci/config.yml | 39 +++++++++++- .circleci/infrastructure/workflows.yml | 8 ++- .github/workflows/build-all.yml | 58 +++++++++++++++++ .github/workflows/build-backend.yml | 44 +++++++++++++ .github/workflows/build-frontend.yml | 44 +++++++++++++ .github/workflows/build-pr.yml | 54 ++++++++++++++++ .github/workflows/deploy-develop-on-merge.yml | 2 +- .github/workflows/deploy-infrastructure.yml | 46 ++++++++++++++ .../020-pipeline-build-flow.md | 44 +++++++++++++ .../Technical-Documentation/github-actions.md | 5 ++ 13 files changed, 414 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/build-all.yml create mode 100644 .github/workflows/build-backend.yml create mode 100644 .github/workflows/build-frontend.yml create mode 100644 .github/workflows/build-pr.yml create mode 100644 .github/workflows/deploy-infrastructure.yml create mode 100644 docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md create mode 100644 docs/Technical-Documentation/github-actions.md diff --git a/.circleci/README.md b/.circleci/README.md index 39e484871c..1b0c4c0311 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -14,10 +14,10 @@ This script will generate a complete config for building, testing, and deploying ### Directory structure #### build-and-test -Contains workflows, jobs, and commands for building and testing the application. +Contains workflows, jobs, and commands for building and testing the application. For all development side builds, these are now triggered by GitHub Actions that serve as a filter so only the code that's changed is tested. See [build-all](../.github/workflows/build-all.yml), [build-backend](../.github/workflows/build-backend.yml), and [build-frontend](../.github/workflows/build-frontend.yml) #### infrastructure -Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. +Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. For all development side builds, this is now triggered by GitHub Actions that serve as a filter so only runs when infrastructure code is changed. See [deploy-infrastructure](../.github/workflows/deploy-infrastructure.yml) #### deployment Contains workflows, jobs, and commands for deploying the application on Cloud gov. Note: merges to develop now automatically trigger a develop deploy using [deploy-develop-on-merge](../.github/workflows/deploy-develop-on-merge.yml) and deploys to dev environments happen when a label is created on the PR using [deploy-on-label](../.github/workflows/deploy-on-label.yml) diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml index eb4b9af15c..d722abecf1 100644 --- a/.circleci/base_config.yml +++ b/.circleci/base_config.yml @@ -21,6 +21,18 @@ executors: resource_class: large parameters: + build_and_test_all: + type: boolean + default: false + build_and_test_backend: + type: boolean + default: false + build_and_test_frontend: + type: boolean + default: false + deploy_infrastructure: + type: boolean + default: false develop_branch_deploy: type: boolean default: false @@ -36,3 +48,9 @@ parameters: target_env: type: string default: '' + triggered: + type: boolean + default: false + util_make_erd: + type: boolean + default: false diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml index 9122ab1666..7c0e559b0f 100644 --- a/.circleci/build-and-test/workflows.yml +++ b/.circleci/build-and-test/workflows.yml @@ -1,19 +1,67 @@ # workflows: - build-and-test: - unless: - or: - - << pipeline.parameters.run_dev_deployment >> - - << pipeline.parameters.develop_branch_deploy >> - - << pipeline.parameters.run_owasp_scan >> - - << pipeline.parameters.run_nightly_owasp_scan >> + build-and-test-all: + when: << pipeline.parameters.build_and_test_all >> jobs: - secrets-check + - test-backend: + requires: + - secrets-check - test-frontend: requires: - secrets-check + - test-e2e: + requires: + - secrets-check + + ci-build-and-test-all: + jobs: + - secrets-check: + filters: + branches: + only: + - main + - master + - /^release.*/ - test-backend: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - test-frontend: + filters: + branches: + only: + - main + - master + - /^release.*/ requires: - secrets-check - test-e2e: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + + build-and-test-backend: + when: << pipeline.parameters.build_and_test_backend >> + jobs: + - secrets-check + - test-backend: + requires: + - secrets-check + + build-and-test-frontend: + when: << pipeline.parameters.build_and_test_frontend >> + jobs: + - secrets-check + - test-frontend: requires: - secrets-check diff --git a/.circleci/config.yml b/.circleci/config.yml index ea0ddd1c49..5d0be7af65 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,18 @@ orbs: # parameters from github actions parameters: + build_and_test_all: + type: boolean + default: false + build_and_test_backend: + type: boolean + default: false + build_and_test_frontend: + type: boolean + default: false + deploy_infrastructure: + type: boolean + default: false develop_branch_deploy: type: boolean default: false @@ -28,6 +40,12 @@ parameters: target_env: type: string default: '' + triggered: + type: boolean + default: false + util_make_erd: + type: boolean + default: false jobs: setup: @@ -45,6 +63,23 @@ jobs: # our single workflow, that triggers the setup job defined above workflows: - setup: + github-triggered-setup: + when: << pipeline.parameters.triggered >> + jobs: + - setup: + filters: + branches: + ignore: + - main + - master + - /^release.*/ + circle-ci-setup: jobs: - - setup + - setup: + filters: + branches: + only: + - main + - master + - /^release.*/ + diff --git a/.circleci/infrastructure/workflows.yml b/.circleci/infrastructure/workflows.yml index 46f29aec7a..9cb85e8ed8 100644 --- a/.circleci/infrastructure/workflows.yml +++ b/.circleci/infrastructure/workflows.yml @@ -1,6 +1,10 @@ -#workflows: +#workflows: enable-versioning-for-s3-buckets: - unless: << pipeline.parameters.run_nightly_owasp_scan >> + when: + or: + - << pipeline.parameters.deploy_infrastructure >> + - equal: [ 'main', << pipeline.git.branch >> ] + - equal: [ 'master', << pipeline.git.branch >> ] jobs: - enable-versioning: filters: diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml new file mode 100644 index 0000000000..c03d344abb --- /dev/null +++ b/.github/workflows/build-all.yml @@ -0,0 +1,58 @@ +########################################################################### +# GitHub Action Workflow +# On changes to scripts or changes to the pipeline code to any branch +# besides develop, staging and master triggers the full build and test +# pipeline. +# +# NOTE: develop, staging(main) and master are skipped on the push because this +# would be redundant after running the full set of tests from the PR. +# See build-pr.yml for builds that run on code being merged into develop. +# See deploy-develop-on-merge.yml and make_erd for the workflow +# pipelines that run on merge to develop, staging, and master branches. +# HHS (main and master and release/**) branches build all only +# and are managed in CircleCI +# +# Step 0: Make changes on your branch to files in scripts/ .circleci or .github +# and push changes to your remote branch. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend and +# build_and_test_frontend to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build and test All on push when scripts/commands change +on: + push: + branches-ignore: + - develop + - main + - master + - 'release/**' + paths: + - 'scripts/**' + - '.circleci/**' + - '.github/**' +jobs: + build_and_test_all: + runs-on: ubuntu-latest + name: Initiate deploy job in CircleCI + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }} + payload: '{ + "build_and_test_all": true, + "triggered": true + }' diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..26ef5c03eb --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,44 @@ +########################################################################### +# GitHub Action Workflow +# On push to any branch, triggers the back end build and test pipeline +# if the tdrs-backend has changed. +# +# Step 0: make changes on your branch to non-documentation files in +# tdrs-backend and push changes to your remote branch +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend +# to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build Only Backend When tdrs-backend/ Files Change +on: + push: + paths: 'tdrs-backend/**' + branches-ignore: + - develop + - main + - master +jobs: + build_and_test_backend: + runs-on: ubuntu-latest + name: Build and Test Backend + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "build_and_test_backend": true, + "triggered": true + }' diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml new file mode 100644 index 0000000000..b9b60a9141 --- /dev/null +++ b/.github/workflows/build-frontend.yml @@ -0,0 +1,44 @@ +########################################################################### +# GitHub Action Workflow +# On push to any branch, triggers the front end build and test pipeline +# if the tdrs-frontend has changed. +# +# Step 0: make changes on your branch to non-documentation files in +# tdrs-frontend and push changes to your remote branch +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_frontend +# to run the workflow/jobs listed here: +# build-and-test:[ +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build Only Frontend When tdrs-frontend Files Change +on: + push: + paths: 'tdrs-frontend/**' + branches-ignore: + - develop + - main + - master +jobs: + build_and_test_frontend: + runs-on: ubuntu-latest + name: Build and Test Frontend + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "build_and_test_frontend": true, + "triggered": true + }' diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml new file mode 100644 index 0000000000..fa54a2a097 --- /dev/null +++ b/.github/workflows/build-pr.yml @@ -0,0 +1,54 @@ +########################################################################### +# GitHub Action Workflow +# On pull requests requesting review from individuals, besides staging, +# master, and release branches triggers the full build and test pipeline. +# +# NOTE: release branches, staging(main) and master are skipped because +# these branch builds are managed in CircleCI +# +# Step 0: make PR from your branch into develop, then select reviewers. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend and +# build_and_test_frontend to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build and test All for PRs +on: + pull_request: + branches-ignore: #handled in circleci + - main + - master + - 'release/**' + types: [review_requested, ready_for_review, synchronize] + paths-ignore: + - 'docs/**' + - '**.md' + - '**.txt' + - '.gitattributes' + - '.gitignore' + - 'LICENSE' +jobs: + build_and_test_pr: + runs-on: ubuntu-latest + name: Initiate deploy job in CircleCI + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }} + payload: '{ + "build_and_test_all": true, + "triggered": true + }' \ No newline at end of file diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml index 80ad9b3088..a9df826455 100644 --- a/.github/workflows/deploy-develop-on-merge.yml +++ b/.github/workflows/deploy-develop-on-merge.yml @@ -24,7 +24,7 @@ on: push: branches: - develop - paths_ignore: + paths-ignore: - 'docs/**' - '**.md' - '**.txt' diff --git a/.github/workflows/deploy-infrastructure.yml b/.github/workflows/deploy-infrastructure.yml new file mode 100644 index 0000000000..e5eeeb611a --- /dev/null +++ b/.github/workflows/deploy-infrastructure.yml @@ -0,0 +1,46 @@ +########################################################################### +# GitHub Action Workflow +# On push changing terraform files or infrastructure pipelines, triggers the +# terraform deploy pipeline for the appropriate cf space. +# +# Step 0: make changes to non-documentation files in terraform/ or +# .circleci/infrastructure/ and push/merge changes. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon terraform: true flag +# to run the workflow/jobs listed here: +# build-and-test:[ +# enable-versioning-for-s3-buckets +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Run Infrastructure Pipeline When Terraform or Infrastructure Files Change +on: + push: + branches-ignore: #handled in CircleCI + - main + - master + - 'release/**' + paths: + - 'terraform/**' + - '.circleci/infrastructure/**' +jobs: + run_infrastructure_deployment: + runs-on: ubuntu-latest + name: Deploy Infrastructure + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "deploy_infrastructure": true, + "triggered": true + }' + \ No newline at end of file diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md new file mode 100644 index 0000000000..0e5f9b17ce --- /dev/null +++ b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md @@ -0,0 +1,44 @@ +# 20. Pipeline Build Flow + +Date: 2023-06-07 (_Updated 2023-06-13_) + +## Status + +Pending + +## Context + +We use [CircleCI](https://app.circleci.com/pipelines/github/raft-tech/TANF-app)] as our primary pipeline build tool. On the HHS side, the standard pipeline, build all and deploy ran by CircleCI is sufficiently granular to meet all our build needs. However, on the dev Raft-TANF side, where work can often be held up by waiting for build and test pipelines, it's useful to reduce build times by filtering which workflows run based upon which code is changing. In order to do this, GitHub Actions is leveraged to kick off different CircleCI build workflows since Actions has more granular control over what paths are changed. + +These pipelines are closely tied to the [Git workflow](./009-git-workflow.md) and should run build and test as expected based on what code changed, and still deploy to the correct CF spaces based upon the Decision tree in the [Deployment Flow](./008-deployment-flow.md) + +## Build Logic + +For all release branches, the main, and the master branch, CircleCI should run the full build, infrastructure deploy, and app deployment. + +for feature/development branches, only the build and test pertainint to the code changed should be built. +Front end tests should be run if /tdrs-frontent changes +Back end tests should be run if /tdrs-backend changes +the entire build and test all should run if anything pertaining to the pipeline changes +infrastructure deploy should run if /terraform or infrastructure deploy pipeline changes + +Once a pull request is flagged as ready for review and/or has reviewers assigned, a full build and test all should be run (and tests must pass before merge to develop) + +Develop merges trigger a deploy to the develop CF space, and then a special integration end-2-end test that tests against the real development environment instead of a simulated environment on the CircleCI build servers. + +## Consequences + +**Pros** +* reduce time of build and tests by only running the appropriate workflows. +* only run infrastructure deploys when changes are made to infrastructure code. +* only run end-2-end tests when code is ready to be reviewed. +* only run full integration end-2-end testing when the develop environment assets are updated. + +**Risks** +* Increased pipeline logic complexity + +## Notes + +- For the nuanced build triggers, CircleCI documentation recommends using Actions to trigger CircleCI builds and send different flags to tell which workflows should run. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0 + +- This could be an argument for future complete pipeline switchover to GitHub Actions in order to reduce pipeline complexity, while maintaining the desired build granularity. \ No newline at end of file diff --git a/docs/Technical-Documentation/github-actions.md b/docs/Technical-Documentation/github-actions.md new file mode 100644 index 0000000000..05839498a4 --- /dev/null +++ b/docs/Technical-Documentation/github-actions.md @@ -0,0 +1,5 @@ +# How We Use GitHub Actions +For now, the only use case we have for GitHub Actions is to help up trigger CircleCI builds the way we want to. This is actually the preferred method CircleCI advises for branch, path, pull-request, and labelled filtering and job triggering. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0 + +## Path Filtering +We use Actions to filter which workflows are getting run by CircleCI by sending different flags to CircleCI through the promiseofcake CircleCI API trigger. See the individual files in [.github](../../.github/) for detailed instructions for how to use each. From 487adfc35e91ce3c0d62a6dc69acbdca32f198b7 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 29 Jun 2023 08:54:40 -0600 Subject: [PATCH 03/35] Hotfix Devops/2457 path filtering for documentation (#2597) * pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md * build and test all on PRs even for documentation --------- Co-authored-by: George Hudson --- .github/workflows/build-pr.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index fa54a2a097..5aef710990 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -28,13 +28,6 @@ on: - master - 'release/**' types: [review_requested, ready_for_review, synchronize] - paths-ignore: - - 'docs/**' - - '**.md' - - '**.txt' - - '.gitattributes' - - '.gitignore' - - 'LICENSE' jobs: build_and_test_pr: runs-on: ubuntu-latest From b4032ae9466f6289067cf6bfa8bd62005679ea45 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Thu, 29 Jun 2023 08:05:08 -0700 Subject: [PATCH 04/35] Create sprint-74-summary.md (#2596) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-74-summary.md | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/Sprint-Review/sprint-74-summary.md diff --git a/docs/Sprint-Review/sprint-74-summary.md b/docs/Sprint-Review/sprint-74-summary.md new file mode 100644 index 0000000000..9d64430d39 --- /dev/null +++ b/docs/Sprint-Review/sprint-74-summary.md @@ -0,0 +1,54 @@ +# Sprint 74 Summary +05/24/23 - 06/06/23 + +Velocity: 21 + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX to continue STT onboarding (focusing on onboarding CyberFusion users), errors research sessions, provide copy for 2509 (e-mail notification for data submission and errors/transmission report) +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + +## Tickets + +#### Completed/Merged +* [#2439 - [Design] Video Release — Embed videos in knowledge center +](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2439) +* [#2503 [Design] May Knowledge Center Enhancements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2503) +* [#2531 Bug/Max file size](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2531) +* [#2424 Complete TANF section 1 parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) +* [#2368 errors reporting research](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2368) + + + +#### Submitted (QASP Review, OCIO Review) +* N/A + + +#### Closed (not merged) +* N/A + +### Moved to Next Sprint (Blocked, Raft Review, In Progress) + +#### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +#### Raft Review +* [2486 [Spike - Parser Performance]](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2521 - Update to CFlinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2550 - Deactivation warning emails are missing hyperlinks in staging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2550) + +#### In Progress +* [#2116 [DevOps] Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + + + +### Sprint 74 Demo +* Internal: + * UX walkthrough of Knowledge Center enhancements + * [#2424 Complete TANF section 1 parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) +* External: + * N/A From ce4bcbe29fae9566e56059a3e2895fb97a081172 Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Wed, 5 Jul 2023 09:52:35 -0400 Subject: [PATCH 05/35] added URL filters (#2580) * added URL filters * allow github to trigger owasp and label deploys (#2601) Co-authored-by: George Hudson --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> Co-authored-by: George Hudson Co-authored-by: George Hudson --- .github/workflows/deploy-develop-on-merge.yml | 6 +- .github/workflows/deploy-on-label.yml | 6 +- .github/workflows/qasp-owasp-scan.yml | 3 +- scripts/zap-scanner.sh | 80 ++++++++++++++++++- 4 files changed, 90 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml index a9df826455..ee4eee0577 100644 --- a/.github/workflows/deploy-develop-on-merge.yml +++ b/.github/workflows/deploy-develop-on-merge.yml @@ -45,4 +45,8 @@ jobs: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} project-slug: ${{ github.repository }} branch: ${{ github.ref_name }} - payload: '{"develop_branch_deploy": true, "target_env": "develop"}' + payload: '{ + "develop_branch_deploy": true, + "target_env": "develop", + "triggered": true + }' diff --git a/.github/workflows/deploy-on-label.yml b/.github/workflows/deploy-on-label.yml index 857dd3a35d..20fbfd8b33 100644 --- a/.github/workflows/deploy-on-label.yml +++ b/.github/workflows/deploy-on-label.yml @@ -66,4 +66,8 @@ jobs: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} project-slug: ${{ github.repository }} branch: ${{ github.head_ref }} - payload: '{"run_dev_deployment": true, "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}"}' + payload: '{ + "run_dev_deployment": true, + "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}", + "triggered": true + }' diff --git a/.github/workflows/qasp-owasp-scan.yml b/.github/workflows/qasp-owasp-scan.yml index 3a4dea99bd..963bf430c7 100644 --- a/.github/workflows/qasp-owasp-scan.yml +++ b/.github/workflows/qasp-owasp-scan.yml @@ -34,5 +34,6 @@ jobs: branch: ${{ github.head_ref }} payload: | { - "run_owasp_scan": ${{ env.HAS_QASP_LABEL }} + "run_owasp_scan": ${{ env.HAS_QASP_LABEL }}, + "triggered": true } diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh index 4e017eb7e2..dbce417a84 100755 --- a/scripts/zap-scanner.sh +++ b/scripts/zap-scanner.sh @@ -59,15 +59,91 @@ ZAP_CLI_OPTIONS="\ -config globalexcludeurl.url_list.url\(0\).regex='.*/robots\.txt.*' \ -config globalexcludeurl.url_list.url\(0\).description='Exclude robots.txt' \ -config globalexcludeurl.url_list.url\(0\).enabled=true \ + -config globalexcludeurl.url_list.url\(1\).regex='^https?://.*\.cdn\.mozilla\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(1\).description='Site - Mozilla CDN (requests such as getpocket)' \ -config globalexcludeurl.url_list.url\(1\).enabled=true \ + -config globalexcludeurl.url_list.url\(2\).regex='^https?://.*\.amazonaws\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(2\).description='TDP S3 buckets' \ -config globalexcludeurl.url_list.url\(2\).enabled=true \ - -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.acf.hhs.gov\/v1\/login\/.*$' \ - -config globalexcludeurl.url_list.url\(3\).description='Site - identity pages' \ + + -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.hhs.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(3\).description='Site - acf.hhs.gov' \ -config globalexcludeurl.url_list.url\(3\).enabled=true \ + + -config globalexcludeurl.url_list.url\(4\).regex='^https:\/\/.*\.google.com\/.*$' \ + -config globalexcludeurl.url_list.url\(4\).description='Site - Google.com' \ + -config globalexcludeurl.url_list.url\(4\).enabled=true \ + + -config globalexcludeurl.url_list.url\(5\).regex='^https:\/\/.*\.youtube.com\/.*$' \ + -config globalexcludeurl.url_list.url\(5\).description='Site - youtube.com' \ + -config globalexcludeurl.url_list.url\(5\).enabled=true \ + + -config globalexcludeurl.url_list.url\(6\).regex='^https:\/\/.*\.monsido.com\/.*$' \ + -config globalexcludeurl.url_list.url\(6\).description='Site - monsido.com' \ + -config globalexcludeurl.url_list.url\(6\).enabled=true \ + + -config globalexcludeurl.url_list.url\(7\).regex='^https:\/\/.*\.crazyegg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(7\).description='Site - crazyegg.com' \ + -config globalexcludeurl.url_list.url\(7\).enabled=true \ + + -config globalexcludeurl.url_list.url\(8\).regex='^https:\/\/.*\.gstatic.com\/.*$' \ + -config globalexcludeurl.url_list.url\(8\).description='Site - gstatic.com' \ + -config globalexcludeurl.url_list.url\(8\).enabled=true \ + + -config globalexcludeurl.url_list.url\(9\).regex='^https:\/\/.*\.googleapis.com\/.*$' \ + -config globalexcludeurl.url_list.url\(9\).description='Site - GoogleAPIs.com' \ + -config globalexcludeurl.url_list.url\(9\).enabled=true \ + + -config globalexcludeurl.url_list.url\(10\).regex='^https:\/\/.*\.crazyegg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(10\).description='Site - CrazyEgg.com' \ + -config globalexcludeurl.url_list.url\(10\).enabled=true \ + + -config globalexcludeurl.url_list.url\(11\).regex='^https:\/\/.*\.doubleclick.net\/.*$' \ + -config globalexcludeurl.url_list.url\(11\).description='Site - DoubleClick.net' \ + -config globalexcludeurl.url_list.url\(11\).enabled=true \ + + -config globalexcludeurl.url_list.url\(12\).regex='^https:\/\/.*unpkg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(12\).description='Site - Unpkg.com' \ + -config globalexcludeurl.url_list.url\(12\).enabled=true \ + + -config globalexcludeurl.url_list.url\(13\).regex='^https:\/\/.*\.readspeaker.com\/.*$' \ + -config globalexcludeurl.url_list.url\(13\).description='Site - ReadSpeaker.com' \ + -config globalexcludeurl.url_list.url\(13\).enabled=true \ + + -config globalexcludeurl.url_list.url\(14\).regex='^https:\/\/.*\.fontawesome.com\/.*$' \ + -config globalexcludeurl.url_list.url\(14\).description='Site - FontAwesome.com' \ + -config globalexcludeurl.url_list.url\(14\).enabled=true \ + + -config globalexcludeurl.url_list.url\(15\).regex='^https:\/\/.*\.cloud.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(15\).description='Site - Cloud.gov' \ + -config globalexcludeurl.url_list.url\(15\).enabled=true \ + + -config globalexcludeurl.url_list.url\(16\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \ + -config globalexcludeurl.url_list.url\(16\).description='Site - googletagmanager.com' \ + -config globalexcludeurl.url_list.url\(16\).enabled=true \ + + -config globalexcludeurl.url_list.url\(17\).regex='^https:\/\/.*\.cloudflare.com\/.*$' \ + -config globalexcludeurl.url_list.url\(17\).description='Site - CloudFlare.com' \ + -config globalexcludeurl.url_list.url\(17\).enabled=true \ + + -config globalexcludeurl.url_list.url\(18\).regex='^https:\/\/.*\.google-analytics.com\/.*$' \ + -config globalexcludeurl.url_list.url\(18\).description='Site - Google-Analytics.com' \ + -config globalexcludeurl.url_list.url\(18\).enabled=true \ + + -config globalexcludeurl.url_list.url\(19\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \ + -config globalexcludeurl.url_list.url\(19\).description='Site - googletagmanager.com' \ + -config globalexcludeurl.url_list.url\(19\).enabled=true \ + + -config globalexcludeurl.url_list.url\(20\).regex='^https:\/\/.*\.digitalgov.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(20\).description='Site - DigitalGov.gov' \ + -config globalexcludeurl.url_list.url\(20\).enabled=true \ + + -config globalexcludeurl.url_list.url\(21\).regex='^https:\/\/.*\.identitysandbox.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(21\).description='Site - IdentitySandbox.gov' \ + -config globalexcludeurl.url_list.url\(21\).enabled=true \ + -config spider.postform=true" # How long ZAP will crawl the app with the spider process From e96e634e165ca1718f2c6da9252fb9acb085c803 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Mon, 10 Jul 2023 06:43:09 -0700 Subject: [PATCH 06/35] Create sprint-75-summary.md (#2608) --- docs/Sprint-Review/sprint-75-summary.md | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/Sprint-Review/sprint-75-summary.md diff --git a/docs/Sprint-Review/sprint-75-summary.md b/docs/Sprint-Review/sprint-75-summary.md new file mode 100644 index 0000000000..967b0381b8 --- /dev/null +++ b/docs/Sprint-Review/sprint-75-summary.md @@ -0,0 +1,49 @@ +# Sprint 75 Summary + +06/06/23 - 06/20/23 + +Velocity: Dev (1) + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX to continue STT onboarding (focusing on onboarding CyberFusion users), errors research synthesis, copy for e-mail notification of data submission and errors/transmission reports - 2559 +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [2550 Deactivation warning emails are missing e-mail links in staging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2550) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (In Progress, Blocked, Raft Review) + +### In Progress +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) + +### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +### Raft Review +* [#2563 Assess OWASP scan accuracy and URL filter](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2563) +* [#2551 [Bug] - All users are not returned in API response](https://](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2551)) +* [#2564 Adjust TANF and SSP Section 1 item numbers](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2564) +* [#2457 Path filtering for CI Builds](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2457) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2516 Errors Research Synthesis ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2516) + + From 48734a9339a2ee02a06dd394e2953d7fc42631aa Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Mon, 10 Jul 2023 06:54:17 -0700 Subject: [PATCH 07/35] Create sprint-76-summary.md (#2609) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-76-summary.md | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/Sprint-Review/sprint-76-summary.md diff --git a/docs/Sprint-Review/sprint-76-summary.md b/docs/Sprint-Review/sprint-76-summary.md new file mode 100644 index 0000000000..0d35a73143 --- /dev/null +++ b/docs/Sprint-Review/sprint-76-summary.md @@ -0,0 +1,54 @@ +# Sprint 76 Summary + +06/21/23 - 07/04/23 + +Velocity: Dev (17) + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX errors template, follow-on research, onboarding +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [#2563 Assess OWASP scan accuracy and URL filter](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2563) +* [#2564 Adjust TANF and SSP Section 1 item numbers](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2564) +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2551 [Bug] - All users are not returned in API response](https://](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2551)) +* [#2457 Path filtering for CI Builds](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2457) +* [#2516 Errors Research Synthesis ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2516) +* [#2527 Error Research informed excel prototype](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2527) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1784 - Email Relay](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1784) +* [#2347 decouple backend apps spike](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2347) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2598 HHS Staging Deployment Failure](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2598) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) + + +### Blocked +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) + + +### Raft Review +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + + + From 45f296d9646aeb2e1ad6e4d2c15a5d63df01ece9 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 09:15:23 -0600 Subject: [PATCH 08/35] - Fixed failing test --- tdrs-backend/tdpservice/parsers/test/test_summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 814fa25969..4f92ab8651 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -44,7 +44,7 @@ def test_dfs_set_status(dfs): parser_errors = [] for i in range(2, 4): - parser_errors.append(ParserErrorFactory(row_number=i, category=str(i))) + parser_errors.append(ParserErrorFactory(row_number=i, error_type=str(i))) dfs.status = dfs.get_status(errors={'document': parser_errors}) From c0bce698754b102ae33dd6e74de769a2c230a8d7 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Fri, 5 May 2023 11:21:53 -0400 Subject: [PATCH 09/35] add adminUrl to deployment cypress overrides --- .circleci/deployment/jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index 078fb3172b..dc1ee1d5b7 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -43,7 +43,7 @@ app-dir: tdrs-frontend - run: name: Run Cypress e2e tests - command: cd tdrs-frontend; npm run test:e2e-ci -- --config baseUrl="https://tdp-frontend-develop.acf.hhs.gov" --env cypressToken=$CYPRESS_TOKEN,apiUrl="https://tdp-frontend-develop.acf.hhs.gov/v1" + command: cd tdrs-frontend; npm run test:e2e-ci -- --config baseUrl="https://tdp-frontend-develop.acf.hhs.gov" --env cypressToken=$CYPRESS_TOKEN,apiUrl="https://tdp-frontend-develop.acf.hhs.gov/v1",adminUrl="https://tdp-frontend-develop.acf.hhs.gov/admin" - store_artifacts: path: tdrs-frontend/cypress/screenshots/ - store_artifacts: From 57c179db5c89c30aa4a3846202def6a1b4f1b99f Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Fri, 2 Jun 2023 13:43:27 -0400 Subject: [PATCH 10/35] Adding "beta" banners to relevant error report sections (#2522) * Update views.py * Update views.py * Update SubmissionHistory.jsx * Update SubmissionHistory.test.js * Apply suggestions from code review Co-authored-by: Miles Reiter * lint fixes --------- Co-authored-by: Miles Reiter Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: andrew-jameson --- tdrs-backend/tdpservice/parsers/views.py | 6 ++++++ .../src/components/SubmissionHistory/SubmissionHistory.jsx | 2 +- .../components/SubmissionHistory/SubmissionHistory.test.js | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/views.py b/tdrs-backend/tdpservice/parsers/views.py index 90023597bb..d39965ee3a 100644 --- a/tdrs-backend/tdpservice/parsers/views.py +++ b/tdrs-backend/tdpservice/parsers/views.py @@ -52,6 +52,12 @@ def _get_xls_serialized_file(self, data): 'row_number', 'column_number'] + # write beta banner + worksheet.write(row, col, + "Error reporting in TDP is still in development." + + "We'll be in touch when it's ready to use!" + + "For now please refer to the reports you receive via email") + row, col = 2, 0 # write csv header [worksheet.write(row, col, key) for col, key in enumerate(report_columns)] diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index 7174f3999f..844b0c7449 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -78,7 +78,7 @@ const SectionSubmissionHistory = ({ section, label, files }) => { Submitted On Submitted By File Name - Error Reports + Error Reports (In development) diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js index 23d538df39..72f3e65f80 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js @@ -241,7 +241,10 @@ describe('SubmissionHistory', () => { expect(screen.queryByText('test5.txt')).not.toBeInTheDocument() expect(screen.queryByText('test6.txt')).toBeInTheDocument() - expect(screen.queryByText('Error Reports')).toBeInTheDocument() + expect( + screen.queryByText('Error Reports (In development)') + ).toBeInTheDocument() + expect(screen.queryByText('Currently Unavailable')).toBeInTheDocument() }) From 8e7d0e3917e37b107e0fcc8dd2773b0df0adf591 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Thu, 8 Jun 2023 07:30:21 -0700 Subject: [PATCH 11/35] Create sprint-73-summary.md (#2565) --- docs/Sprint-Review/sprint-73-summary.md | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/Sprint-Review/sprint-73-summary.md diff --git a/docs/Sprint-Review/sprint-73-summary.md b/docs/Sprint-Review/sprint-73-summary.md new file mode 100644 index 0000000000..9d9c0c9b98 --- /dev/null +++ b/docs/Sprint-Review/sprint-73-summary.md @@ -0,0 +1,50 @@ +# Sprint 73 Summary + +05/10/23 - 5/23/23 + +Velocity: 10 + +## Sprint Goal +* Release v3.1.0. Continue parsing engine development and close out integration test epic (310). +* UX will maintain STT onboarding velocity, errors research sessions, provide copy for 2509 (e-mail notification for data submission and errors/transmission report) and iterate on communication method ahead of release v1/beta of downloadable errors report from 1610.1. +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + +## Tickets + +#### Completed/Merged +* [#2466 As sys admin, I need the API viewsets updated for the user, stt, and data file apps](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2466) +* [#1113 SSP Active Data (01)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1113) + + +#### Submitted (QASP Review, OCIO Review) +* [#2439 [Design] Embed YouTube videos in Knowledge Center](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2439) +* [#2424 Complete TANF section parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) + + +#### Closed (not merged) +* N/A + +### Moved to Next Sprint (Blocked, Raft Review, In Progress) + +#### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +#### Raft Review +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2503 [Design] May Knowledge Center Enhancements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2503) + +#### In Progress +* [#2116 [DevOps] Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2368 .csv errors reporting](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2368) + + +### Sprint 73 Demo +* Internal: + * [#2466 As sys admin, I need the API viewsets updated for the user, stt, and data file apps](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2466) + * [#1113 SSP Active Data (01)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1113) +* External: + * N/A From bd3558ff3b625a29ccf50b632082e0b7d25e4c83 Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Thu, 8 Jun 2023 11:09:07 -0400 Subject: [PATCH 12/35] hotfix for large file sizes (#2542) * hotfix for large file sizes * apply timeouts/req limits to dev * filter identity pages from scan * IGNORE sql injection --------- Co-authored-by: Jan Timpe Co-authored-by: mo sohani Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> --- scripts/zap-scanner.sh | 3 +++ tdrs-backend/gunicorn_dev_cfg.py | 2 +- tdrs-backend/gunicorn_prod_cfg.py | 2 +- tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf | 2 +- tdrs-frontend/nginx/local/default.conf.template | 4 ++-- tdrs-frontend/reports/zap.conf | 10 +++++----- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh index 2864746c37..4e017eb7e2 100755 --- a/scripts/zap-scanner.sh +++ b/scripts/zap-scanner.sh @@ -65,6 +65,9 @@ ZAP_CLI_OPTIONS="\ -config globalexcludeurl.url_list.url\(2\).regex='^https?://.*\.amazonaws\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(2\).description='TDP S3 buckets' \ -config globalexcludeurl.url_list.url\(2\).enabled=true \ + -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.acf.hhs.gov\/v1\/login\/.*$' \ + -config globalexcludeurl.url_list.url\(3\).description='Site - identity pages' \ + -config globalexcludeurl.url_list.url\(3\).enabled=true \ -config spider.postform=true" # How long ZAP will crawl the app with the spider process diff --git a/tdrs-backend/gunicorn_dev_cfg.py b/tdrs-backend/gunicorn_dev_cfg.py index 6c4cd7254f..04c04e6ee7 100644 --- a/tdrs-backend/gunicorn_dev_cfg.py +++ b/tdrs-backend/gunicorn_dev_cfg.py @@ -21,4 +21,4 @@ # Daemonize the Gunicorn process (detach & enter background) # daemon = True -timeout = 10 +timeout = 100 diff --git a/tdrs-backend/gunicorn_prod_cfg.py b/tdrs-backend/gunicorn_prod_cfg.py index 3174beb2fc..b6a16164ce 100644 --- a/tdrs-backend/gunicorn_prod_cfg.py +++ b/tdrs-backend/gunicorn_prod_cfg.py @@ -21,4 +21,4 @@ # Daemonize the Gunicorn process (detach & enter background) # daemon = True -timeout = 10 +timeout = 100 diff --git a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf index a07fdcaaa0..27f9122452 100644 --- a/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf +++ b/tdrs-frontend/nginx/cloud.gov/buildpack.nginx.conf @@ -40,7 +40,7 @@ http { access_log /home/vcap/app/nginx_access.log compression; include locations.conf; - client_max_body_size 50m; + client_max_body_size 100m; # Block all requests except ones listed in whitelist; disabled for local # First have to correct the source IP address using real_ip_header, otherwise diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template index c40cf188d9..4e6e7e4083 100644 --- a/tdrs-frontend/nginx/local/default.conf.template +++ b/tdrs-frontend/nginx/local/default.conf.template @@ -10,7 +10,7 @@ http { server { listen 80; - client_max_body_size 50m; + client_max_body_size 100m; root /usr/share/nginx/html; include locations.conf; @@ -84,7 +84,7 @@ http { add_header Content-Security-Policy $CSP; } - limit_req_zone $binary_remote_addr zone=limitreqsbyaddr:200m rate=100r/s; + limit_req_zone $binary_remote_addr zone=limitreqsbyaddr:20m rate=1000r/s; limit_req_status 444; # Logging format diff --git a/tdrs-frontend/reports/zap.conf b/tdrs-frontend/reports/zap.conf index 88bc3b7a33..854c0ea39f 100644 --- a/tdrs-frontend/reports/zap.conf +++ b/tdrs-frontend/reports/zap.conf @@ -79,13 +79,13 @@ 40014 FAIL (Cross Site Scripting (Persistent) - Active/release) 40016 FAIL (Cross Site Scripting (Persistent) - Prime - Active/release) 40017 FAIL (Cross Site Scripting (Persistent) - Spider - Active/release) -40018 FAIL (SQL Injection - Active/release) -40019 FAIL (SQL Injection - MySQL - Active/beta) -40020 FAIL (SQL Injection - Hypersonic SQL - Active/beta) -40021 FAIL (SQL Injection - Oracle - Active/beta) +40018 IGNORE (SQL Injection - Active/release) +40019 IGNORE (SQL Injection - MySQL - Active/beta) +40020 IGNORE (SQL Injection - Hypersonic SQL - Active/beta) +40021 IGNORE (SQL Injection - Oracle - Active/beta) 40022 FAIL (SQL Injection - PostgreSQL - Active/beta) 40023 FAIL (Possible Username Enumeration - Active/beta) -40024 FAIL (SQL Injection - SQLite - Active/beta) +40024 IGNORE (SQL Injection - SQLite - Active/beta) 40025 IGNORE (Proxy Disclosure - Active/beta) 40026 FAIL (Cross Site Scripting (DOM Based) - Active/beta) 40027 FAIL (SQL Injection - MsSQL - Active/beta) From 2946dad1f04b4cca107e7681f0b9e0c26e3bf06f Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 12 Jun 2023 09:29:41 -0400 Subject: [PATCH 13/35] updating validation error language --- tdrs-backend/tdpservice/parsers/parse.py | 31 +++----- .../tdpservice/parsers/schema_defs/header.py | 13 +++- .../tdpservice/parsers/schema_defs/trailer.py | 6 +- .../tdpservice/parsers/test/test_parse.py | 16 ++-- tdrs-backend/tdpservice/parsers/validators.py | 26 +++++-- tdrs-backend/tdpservice/test/test_backends.py | 78 +++++++++++++++++++ 6 files changed, 135 insertions(+), 35 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 3ee7dffadd..66a18a2dca 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -3,7 +3,6 @@ import os from . import schema_defs, validators, util from .models import ParserErrorCategoryChoices -from tdpservice.data_files.models import DataFile def parse_datafile(datafile): @@ -45,30 +44,13 @@ def parse_datafile(datafile): errors['trailer'] = trailer_errors # ensure file section matches upload section - section_names = { - 'TAN': { - 'A': DataFile.Section.ACTIVE_CASE_DATA, - 'C': DataFile.Section.CLOSED_CASE_DATA, - 'G': DataFile.Section.AGGREGATE_DATA, - 'S': DataFile.Section.STRATUM_DATA, - }, - 'SSP': { - 'A': DataFile.Section.SSP_ACTIVE_CASE_DATA, - 'C': DataFile.Section.SSP_CLOSED_CASE_DATA, - 'G': DataFile.Section.SSP_AGGREGATE_DATA, - 'S': DataFile.Section.SSP_STRATUM_DATA, - }, - } - - # TODO: utility transformations between text to schemas and back - # text > prog > sections > schemas - program_type = header['program_type'] section = header['type'] section_is_valid, section_error = validators.validate_header_section_matches_submission( datafile, - section_names.get(program_type, {}).get(section) + program_type, + section, ) if not section_is_valid: @@ -189,6 +171,15 @@ def parse_datafile_line(line, schema, generate_error): record.save() return record_is_valid, record_errors + return (False, [ + generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Record Type is missing from record.", + record=None, + field=None + ) + ]) def get_schema_options(program_type): diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index c2b0b55c0c..e57eccd63d 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -5,10 +5,21 @@ from .. import validators +# def header_length_validator(length): +# return validators.make_validator( +# lambda value: len(value) == length, +# lambda value: f'Header length is {len(value)} but must be {length} characters.' +# ) + + header = RowSchema( model=dict, preparsing_validators=[ - validators.hasLength(23), + # header_length_validator(23), + validators.hasLength( + 23, + lambda value, length: f'Header length is {len(value)} but must be {length} characters.' + ), validators.startsWith('HEADER'), ], postparsing_validators=[], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 625b2042e9..48bf4ccca4 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -8,7 +8,11 @@ trailer = RowSchema( model=dict, preparsing_validators=[ - validators.hasLength(23), + # validators.hasLength(23), + validators.hasLength( + 23, + lambda value, length: f'Trailer length is {len(value)} but must be {length} characters.' + ), validators.startsWith('TRAILER') ], postparsing_validators=[], diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 00f2c90eeb..dbab071656 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -78,7 +78,7 @@ def test_parse_section_mismatch(test_datafile, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Section does not match.' + assert err.error_message == 'Data does not match the expected layout for Closed Case Data.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -102,7 +102,7 @@ def test_parse_wrong_program_type(test_datafile, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Section does not match.' + assert err.error_message == 'Data does not match the expected layout for SSP Active Case Data.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -162,7 +162,7 @@ def test_parse_bad_test_file(bad_test_file, dfs): assert err.row_number == 1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Value length 24 does not match 23.' + assert err.error_message == 'Header length is 24 but must be 23 characters.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -265,7 +265,7 @@ def test_parse_bad_trailer_file(bad_trailer_file): trailer_error = parser_errors.get(row_number=-1) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'Value length 11 does not match 23.' + assert trailer_error.error_message == 'Trailer length is 11 but must be 23 characters.' assert trailer_error.content_type is None assert trailer_error.object_id is None @@ -299,7 +299,7 @@ def test_parse_bad_trailer_file2(bad_trailer_file_2): trailer_error_1 = trailer_errors.first() assert trailer_error_1.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error_1.error_message == 'Value length 7 does not match 23.' + assert trailer_error_1.error_message == 'Trailer length is 7 but must be 23 characters.' assert trailer_error_1.content_type is None assert trailer_error_1.object_id is None @@ -379,7 +379,7 @@ def test_parse_small_ssp_section1_datafile(small_ssp_section1_datafile): assert err.row_number == -1 assert err.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert err.error_message == 'Value length 15 does not match 23.' + assert err.error_message == 'Trailer length is 15 but must be 23 characters.' assert err.content_type is None assert err.object_id is None assert errors == { @@ -410,7 +410,7 @@ def ssp_section1_datafile(stt_user, stt): # trailer_error = parser_errors.get(row_number=-1) # assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK -# assert trailer_error.error_message == 'Value length 14 does not match 23.' +# assert trailer_error.error_message == 'Trailer length is 14 but must be 23 characters.' # row_12430_error = parser_errors.get(row_number=12430) # assert row_12430_error.error_type == ParserErrorCategoryChoices.PRE_CHECK @@ -591,7 +591,7 @@ def test_parse_bad_ssp_s1_missing_required(bad_ssp_s1__row_missing_required_fiel trailer_error = parser_errors.get(row_number=-1) assert trailer_error.error_type == ParserErrorCategoryChoices.PRE_CHECK - assert trailer_error.error_message == 'Value length 15 does not match 23.' + assert trailer_error.error_message == 'Trailer length is 15 but must be 23 characters.' assert trailer_error.content_type is None assert trailer_error.object_id is None diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index bd7603c22f..c3b58b8d10 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -2,6 +2,7 @@ from .util import generate_parser_error from .models import ParserErrorCategoryChoices +from tdpservice.data_files.models import DataFile # higher order validator func @@ -36,11 +37,11 @@ def between(min, max): ) -def hasLength(length): +def hasLength(length, error_func=None): """Validate that value (string or array) has a length matching length param.""" return make_validator( lambda value: len(value) == length, - lambda value: f'Value length {len(value)} does not match {length}.' + lambda value: error_func(value, length) if error_func else f'Value length {len(value)} does not match {length}.' ) @@ -116,9 +117,24 @@ def validate_single_header_trailer(datafile): return is_valid, error -def validate_header_section_matches_submission(datafile, section): +def validate_header_section_matches_submission(datafile, program_type, section): """Validate header section matches submission section.""" - is_valid = datafile.section == section + section_names = { + 'TAN': { + 'A': DataFile.Section.ACTIVE_CASE_DATA, + 'C': DataFile.Section.CLOSED_CASE_DATA, + 'G': DataFile.Section.AGGREGATE_DATA, + 'S': DataFile.Section.STRATUM_DATA, + }, + 'SSP': { + 'A': DataFile.Section.SSP_ACTIVE_CASE_DATA, + 'C': DataFile.Section.SSP_CLOSED_CASE_DATA, + 'G': DataFile.Section.SSP_AGGREGATE_DATA, + 'S': DataFile.Section.SSP_STRATUM_DATA, + }, + } + + is_valid = datafile.section == section_names.get(program_type, {}).get(section) error = None if not is_valid: @@ -127,7 +143,7 @@ def validate_header_section_matches_submission(datafile, section): line_number=1, schema=None, error_category=ParserErrorCategoryChoices.PRE_CHECK, - error_message="Section does not match.", + error_message=f"Data does not match the expected layout for {datafile.section}.", record=None, field=None ) diff --git a/tdrs-backend/tdpservice/test/test_backends.py b/tdrs-backend/tdpservice/test/test_backends.py index 6e6a0b25dd..ef7842c759 100644 --- a/tdrs-backend/tdpservice/test/test_backends.py +++ b/tdrs-backend/tdpservice/test/test_backends.py @@ -1,5 +1,6 @@ """Tests for storage backends available for use within tdpservice.""" import pytest +import math from tdpservice.backends import DataFilesS3Storage, StaticFilesS3Storage @@ -43,3 +44,80 @@ def test_datafiles_and_staticfiles_storages_have_distinct_credentials( """Test that the credentials used differ between backends.""" assert datafiles_backend.access_key != staticfiles_backend.access_key assert datafiles_backend.secret_key != staticfiles_backend.secret_key + + +char_map = [ + (1, 'I'), + (4, 'IV'), + (5, 'V'), + (9, 'IX'), + (10, 'X'), + (40, 'XL'), + (50, 'L'), + (90, 'XC'), + (100, 'C'), + (400, 'CD'), + (500, 'D'), + (900, 'CM'), + (1000, 'M'), +] + + +def arabic_to_roman(arabic): + for (a, r) in reversed(char_map): + if arabic == a: + return r + if arabic > a: + return f"{r}{arabic_to_roman(arabic - a)}" + + +@pytest.mark.parametrize('arabic,roman', [ + (1, 'I'), + (2, 'II'), + (3, 'III'), + (4, 'IV'), + (5, 'V'), + (6, 'VI'), + (7, 'VII'), + (8, 'VIII'), + (9, 'IX'), + (10, 'X'), + (11, 'XI'), + (20, 'XX'), + (23, 'XXIII'), + (24, 'XXIV'), + (25, 'XXV'), + (27, 'XXVII'), + (29, 'XXIX'), + (30, 'XXX'), + (39, 'XXXIX'), + (40, 'XL'), + (50, 'L'), + (60, 'LX'), + (70, 'LXX'), + (73, 'LXXIII'), + (80, 'LXXX'), + (90, 'XC'), + (100, 'C'), + (110, 'CX'), + (190, 'CXC'), + (200, 'CC'), + (300, 'CCC'), + (400, 'CD'), + (492, 'CDXCII'), + (500, 'D'), + (600, 'DC'), + (700, 'DCC'), + (800, 'DCCC'), + (900, 'CM'), + (1000, 'M'), + (1048, 'MXLVIII'), + (1100, 'MC'), + (2000, 'MM'), + (2001, 'MMI'), + (3000, 'MMM'), + (3549, 'MMMDXLIX'), + (3999, 'MMMCMXCIX'), +]) +def test_arabic_to_roman(arabic, roman): + assert arabic_to_roman(arabic) == roman From d9d6be90334fe1d22078df1c9df80e0057e73993 Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 12 Jun 2023 09:37:24 -0400 Subject: [PATCH 14/35] accidentally included coding challenge --- tdrs-backend/tdpservice/test/test_backends.py | 78 ------------------- 1 file changed, 78 deletions(-) diff --git a/tdrs-backend/tdpservice/test/test_backends.py b/tdrs-backend/tdpservice/test/test_backends.py index ef7842c759..6e6a0b25dd 100644 --- a/tdrs-backend/tdpservice/test/test_backends.py +++ b/tdrs-backend/tdpservice/test/test_backends.py @@ -1,6 +1,5 @@ """Tests for storage backends available for use within tdpservice.""" import pytest -import math from tdpservice.backends import DataFilesS3Storage, StaticFilesS3Storage @@ -44,80 +43,3 @@ def test_datafiles_and_staticfiles_storages_have_distinct_credentials( """Test that the credentials used differ between backends.""" assert datafiles_backend.access_key != staticfiles_backend.access_key assert datafiles_backend.secret_key != staticfiles_backend.secret_key - - -char_map = [ - (1, 'I'), - (4, 'IV'), - (5, 'V'), - (9, 'IX'), - (10, 'X'), - (40, 'XL'), - (50, 'L'), - (90, 'XC'), - (100, 'C'), - (400, 'CD'), - (500, 'D'), - (900, 'CM'), - (1000, 'M'), -] - - -def arabic_to_roman(arabic): - for (a, r) in reversed(char_map): - if arabic == a: - return r - if arabic > a: - return f"{r}{arabic_to_roman(arabic - a)}" - - -@pytest.mark.parametrize('arabic,roman', [ - (1, 'I'), - (2, 'II'), - (3, 'III'), - (4, 'IV'), - (5, 'V'), - (6, 'VI'), - (7, 'VII'), - (8, 'VIII'), - (9, 'IX'), - (10, 'X'), - (11, 'XI'), - (20, 'XX'), - (23, 'XXIII'), - (24, 'XXIV'), - (25, 'XXV'), - (27, 'XXVII'), - (29, 'XXIX'), - (30, 'XXX'), - (39, 'XXXIX'), - (40, 'XL'), - (50, 'L'), - (60, 'LX'), - (70, 'LXX'), - (73, 'LXXIII'), - (80, 'LXXX'), - (90, 'XC'), - (100, 'C'), - (110, 'CX'), - (190, 'CXC'), - (200, 'CC'), - (300, 'CCC'), - (400, 'CD'), - (492, 'CDXCII'), - (500, 'D'), - (600, 'DC'), - (700, 'DCC'), - (800, 'DCCC'), - (900, 'CM'), - (1000, 'M'), - (1048, 'MXLVIII'), - (1100, 'MC'), - (2000, 'MM'), - (2001, 'MMI'), - (3000, 'MMM'), - (3549, 'MMMDXLIX'), - (3999, 'MMMCMXCIX'), -]) -def test_arabic_to_roman(arabic, roman): - assert arabic_to_roman(arabic) == roman From 5cdb8869a721df31e5cdf7caf9933db9965d7f9c Mon Sep 17 00:00:00 2001 From: Jan Timpe Date: Mon, 12 Jun 2023 11:53:31 -0400 Subject: [PATCH 15/35] rm comments --- tdrs-backend/tdpservice/parsers/schema_defs/header.py | 8 -------- tdrs-backend/tdpservice/parsers/schema_defs/trailer.py | 1 - 2 files changed, 9 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index e57eccd63d..b51c433ede 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -5,17 +5,9 @@ from .. import validators -# def header_length_validator(length): -# return validators.make_validator( -# lambda value: len(value) == length, -# lambda value: f'Header length is {len(value)} but must be {length} characters.' -# ) - - header = RowSchema( model=dict, preparsing_validators=[ - # header_length_validator(23), validators.hasLength( 23, lambda value, length: f'Header length is {len(value)} but must be {length} characters.' diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 48bf4ccca4..3657cfc515 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -8,7 +8,6 @@ trailer = RowSchema( model=dict, preparsing_validators=[ - # validators.hasLength(23), validators.hasLength( 23, lambda value, length: f'Trailer length is {len(value)} but must be {length} characters.' From fe32cdcebff6238b91389974ede290249799714a Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Tue, 20 Jun 2023 07:55:04 -0600 Subject: [PATCH 16/35] 2550 deactivation email link (#2557) * - updated nginx buildpack * - specifying different nginx version * - Updating changelog * - added script to update certain apps in cf - added workflow for each environment in circi * - fixed base config * - fixing jobs * - Updated based on feedback in OH * - Updating defaults * - Removing defaults * - Fixing indent * - Adding params to config * test * test * - updating work dir * - Adding checkout * - adding cf check * - logging into cf * - update cf check to install required binary * - removing unnecessary switch * - Forcing plugin installation * - test installing plugin from script also * - Adding url to email * - test code for sandbox * - using my email * Revert "Merge branch 'update-cf-os' into 2551-deactivation-email-link" This reverts commit e963b9df48dd1f72ca0c5b192c979bac11851d11, reversing changes made to cc9cf81e9d76c42f51ffd5e102f6027d3eb5e645. * Revert "- using my email" This reverts commit cc9cf81e9d76c42f51ffd5e102f6027d3eb5e645. * Revert "- test code for sandbox" This reverts commit 06037747197d17ed8e63b086fcfcf048ecb50dc4. --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../tdpservice/email/helpers/account_deactivation_warning.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py b/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py index 53b67bbc55..2a184e93ae 100644 --- a/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py +++ b/tdrs-backend/tdpservice/email/helpers/account_deactivation_warning.py @@ -2,6 +2,7 @@ from tdpservice.email.email_enums import EmailType from tdpservice.email.email import automated_email from datetime import datetime, timedelta, timezone +from django.conf import settings def send_deactivation_warning_email(users, days): @@ -16,7 +17,8 @@ def send_deactivation_warning_email(users, days): context = { 'first_name': user.first_name, 'days': days, - 'deactivation_date': deactivation_date + 'deactivation_date': deactivation_date, + 'url': f'{settings.FRONTEND_BASE_URL}/login/' } logger_context = { From c26703ea836d2fdd17739e5dd5d779fa4710a514 Mon Sep 17 00:00:00 2001 From: Lauren Frohlich <61251539+lfrohlich@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:05:42 -0400 Subject: [PATCH 17/35] Update README.md (#2577) Add ATO Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Security-Compliance/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Security-Compliance/README.md b/docs/Security-Compliance/README.md index 9cb4c4f4e6..c55f72986c 100644 --- a/docs/Security-Compliance/README.md +++ b/docs/Security-Compliance/README.md @@ -2,6 +2,8 @@ This subdirectory contains documentation that describes our practices as it relates to keeping the TDP system in compliance with security requirements for [FISMA moderate systems](https://csrc.nist.gov/CSRC/media/Projects/risk-management/documents/02-Categorize%20Step/NIST%20RMF%20Categorize%20Step-FAQs.pdf) (*Note: see section 18 for definition*) and other related federal system regulations. +The TANF Data Portal received its Authority to Operate (ATO) from the ACF Chief Information Officer on May 18, 2021. The ATO was extended on March 29, 2023 through May 24, 2024. + ## Table of Contents * **[Security Controls](./Security-Controls)** - Herein you will find information about TDP's security controls (security requirements laid out by the [National Institute of Standards and Technology (NIST)](https://www.nist.gov/)), documented as part of the authority to operate (ATO) process in coordination with ACF's Office of the Chief Information Officer (OCIO). * **[White House Cybersecurity Executive Order 14208](./WH_CybersecurityEO.md)** - includes information about the status of TANF Data Portal's compliance with White House Cybersecurity Executive Order 14208 issued on May 12, 2021. From 075ff86e5af9b55089291204e92cbdd27d289579 Mon Sep 17 00:00:00 2001 From: Miles Reiter Date: Wed, 21 Jun 2023 02:30:38 -0400 Subject: [PATCH 18/35] Create 2023, Spring - Testing CSV & Excel-based error reports.md --- ...Testing CSV & Excel-based error reports.md | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 docs/User-Experience/Research-Syntheses/2023, Spring - Testing CSV & Excel-based error reports.md diff --git a/docs/User-Experience/Research-Syntheses/2023, Spring - Testing CSV & Excel-based error reports.md b/docs/User-Experience/Research-Syntheses/2023, Spring - Testing CSV & Excel-based error reports.md new file mode 100644 index 0000000000..c6ca830664 --- /dev/null +++ b/docs/User-Experience/Research-Syntheses/2023, Spring - Testing CSV & Excel-based error reports.md @@ -0,0 +1,209 @@ +# Spring 2023 - Testing CSV & Excel-based error reports + +## Contents: +* [Key insights](#Key-insights) +* [Who we talked to](#Who-we-talked-to) +* [What we did](#What-we-did) +* [What we tested](#What-we-tested) +* [What we learned](#What-we-learned) +* [What's next](#Whats-next) +___ + +### Key insights +As research conducted in [January 2023 Errors Research](https://hackmd.io/WxtMOPCUQiO-JHjahVMPDQ) focused on error messaging and specific internal workflows occuring after data submission, our research in this cycle explored the end-to-end error report experience and simulated it in the direct context of the file formats that the parsing engine will ultimately produce. + + + +- All participants were able to automatically open all variations of the error report prototypes that we tested in Excel. Participants were also able to expand collapsed columns themselves, but we found that it decreased time and effort on the part of the grantee when we provided versions of the prototype with columns that were pre-expanded to fit their content. + + - **Impact**: As a consequence of this finding and conversations with the development team we shifted from CSV format to XLSX to take advantage of AutoFit column widths to enhance the immediate readability of reports. + + +- We identified which columns should be included—and in what order—to maximize the actionable : extraneous data ratio of reports. + - **Impact**: Case Number, Month, and Error Message columns proved most critical for participants with Item Number and Field Name frequently assessed as a "nice to have". A final prototype reflecting our complete findings in this area will be produced for [#2527](https://github.com/raft-tech/TANF-app/issues/2527) and include the following columns: + + | Case Number | Month | Error Message | Item Number | Field Name | + | ----------- | ----- | ------------- | ----------- | ---------- | + +- Item number and field name represent an area where the language used in TANF documentation has shifted over time. Participants frequently appeared more familiar with the term "Data Element" than "Item" or "Field" but were ultimately all able to make the correct association with the relevant content of the data reporting instructions. + - **Impact** After discussions with ACF regarding upcoming revisions to the instructions "Item number" does reflect the preferred terminology going forward. This will be captured in [#2527](https://github.com/raft-tech/TANF-app/issues/2527) and additionally justify future knowledge center content to address any resulting confusion for TDP users. + +**Overall the feedback on the post data file submission was positive and encouraging to continue developing the email communication notice and Error Report.** + +> “This [Excel file] is helpful too [...] after further review, I’m able to understand the spreadsheet more. I’m excited for this project!" - Tribe + +> "I think this excel sheet is more user friendly than the [legacy system report] table. I think this has more information than compared to what I get now" - State + +> "[...]I think this is perfect. Once I knew what I was looking at it’s very easy to follow." - Tribe + +> “I’m excited about [having this in] an online portal. One of the things that was an issue prior was they would ask for proof that we submitted and we didn’t have it.” - Tribe + +> "I like the words and not just the [item] numbers. This saves me a step." - State +___ + + + +### Who we talked to + +We recruited based primarily on the following criteria: +* Participants from past error message research sessions +* TANF programs which had previously encountered errors on their Section 1 data files +* Programs and past participants who had expressed specific interest in participating in research sessions + +Research sessions were conducted with: +* 5 States +* 4 Tribes +___ + +### What we did + +Our research was conducted in order to: + +* Test the usability of an Excel-based error report located in Submission History to evaluate whether participants would be successful in downloading, opening, and ultimately acting on the reports. +* Explore optimal column structures and data needed to help guide error correction and cut down on the time required to understand the report itself. + +___ + +### What we tested + +During these sessions, we wanted to simulate the future experience of the error reports with our participants. We asked participants to: + +* Speak about their expectations at each step of the process including the error report email, the submission history page (including acceptance status and the link to the error report), and the downloadable error report prototype itself. +* Open the error report prototype once they downloaded it. +* Identify the meaning in their own words of key columns. +* Rank the columns in order of their priority in regard to error correction. +* Use the Error Report as if they were using it to resolve the errors listed and expand on how it might fit into their current workflows. + +**Supporting Documentation:** +- [A variation of our HTML/XLSX prototype](https://reitermb.github.io/HTML-Table-example/xlsx-login) +___ + +### What we learned + +Jump to: +* [Collapsed columns in the Error Report created extra steps for participants when first opening the CSV-based prototype](#Collapsed-columns-in-the-error-report-created-extra-steps-for-participants-when-first-opening-the-csv-based-prototype) +* [We identified which columns should be included—and in what order—to maximize the actionable : extraneous data ratio of reports](#We-identified-which-columns-should-be-included-and-in-what-order-to-maximize-the-actionable-extraneous-data-ratio-of-reports) +* [Participants use the coding instructions in conjunction with the Error Report for error definition, but it can be challenging to know whether all information is available](#Participants-use-the-coding-instructions-in-conjunction-with-the-error-report-for-error-definition-but-it-can-be-challenging-to-know-whether-all-information-is-available) +* [Item number and field name represent an area where the language used in TANF documentation has shifted over time](#Item-number-and-field-name-represent-an-area-where-the-language-used-in-TANF-documentation-has-shifted-over-time) +* [We further validated the utility of submission history and notification emails as ways for TANF programs to show proof of transmission for audit purposes](#We-further-validated-the-utility-of-submission-history-and-notification-emails-as-ways-for-tanf-programs-to-show-proof-of-transmission-for-audit-purposes) + +--- + +#### Collapsed columns in the Error Report created extra steps for participants when first opening the CSV-based prototype + +The following observations were made of the first few participants when opening the initial file format of .csv (comma-separated values) + +- Participant expanded the columns one-by-one manually +- Participant began to expand the first few columns, but then stopped as the screen size hid the final column from view +- One participant did not expand any of the columns and moved from cell to cell to read the content in the formula bar +- The Case Number field was truncated by Excel and could not be easily read (e.g. 1.111+08) + +After this initial round of review we determined (via conversations with the product and development teams) that we could move forward with a .xlsx file format which could take advantage of column AutoFit. All further research sessions were conducted with the .xlsx-based prototypes, which eliminated the need for participants to adjust columns and allowed more time to be spent dedicated to the content of the error report. + +--- + +#### We identified which columns should be included—and in what order—to maximize the actionable : extraneous data ratio of reports. + +We started the first few research sessions with a version of the column headers and iterated as we went on to further sessions based on early participant feedback. + +Our first version used the following columns/orderings: + + +| Case Number | Month | Item Number | Field Name | Error Message | Error Type | +| ----------- | ----- | ----------- | ---------- | ------------- | ---------- | + + + +> “I’m not sure about the item number - what does this represent?” - Tribe + +> “I don’t know what item number and field name are.” Initial reaction was that the Error Report did not provide the information they needed - State + +> “I’d suggest removing item number and field name but leave the rest” - Tribe + +> “I would prefer the Error Message come before the Item Number and Field Name” - Tribe + +After the first three research sessions we reevaluated our hypothesis that that Item Number and Field Name near the beginning was the most digestible way to present an error report. Those columns coming before the Error Message seemed to be at odds with participant's priorities and expectations. + +Our subsequent version—used for the remaining research sessions—had the following columns/orderings: + + +| Case Number | Month | Error Message | Item Number | Field Name | Error Type | +| ----------- | ----- | ------------- | ----------- | ---------- | ---------- | + + +> “I like the flow of the columns - Tribe + +> “I think the sequence is good for me” - State + +> “I am strictly looking at Case Numbers, Month and Error Message here. The rest is noise to me” - State + +> “I think these columns are pretty good at showing me where things are at” - Tribe + +We received a wide range of feedback around the utility of Item Number and Field Name. Participants who had many years of experience with TANF data tended to feel they were redundant to the Error Message—but might be useful for those new to correcting TANF data. Others seemed to appreciate the degree to which those columns reinforced key parts of the Error Message. + +While this research did not directly content test Error Types, some participants were able to correctly interpret their meanings. However, it seemed that the primary way Data Analysts categorize different types of errors among themselves is by classification under their Item/Field names. For example, a Data Analyst receiving the error, "If Item 49 (Work Participation Status) = 99 Or Blank, Item 30 (Family Affiliation - Adult) Must = 2-5" would consider its type to be a "Work Participation Status Error". + + +--- + +#### Participants use the coding instructions in conjunction with the Error Report for error definition, but it can be challenging to know whether all information is available + +When we asked participants to use the error report prototype to resolve the errors they consistently understood how to pair it with the coding instructions. + +> “I want to cheat here” (Participant referring to the Tribal Coding Instructions as a cheat sheet while responding to a particular error) - Tribe + +> “I will look at this line and then go to the guide to find more information for Item 7” - State + +> “The way the error report is… it is kind of like a maze where we have to go here and then back” - Tribe + +While all had access to at least some of the coding instructions, it wasn't always clear whether the participants had the latest or even complete versions of them. We observed one participant who had printed a few pages to keep in their workspace to help handle common errors. + +> “Is there any updated code instructions book so that we have the latest?” - State + +> “Can you get to the Instructions from [the excel error report] or no?” - Tribe + +Linking to the coding instructions; and particularly linking directly to relevant items may help reduce lookup times and make the whole process easier to navigate for those new to TANF data coding. + +> “A link to the coding instructions would not be helpful because it is 97 pages long”. - State + +--- + +#### Item number and field name represent an area where the language used in TANF documentation has shifted over time. + +The term “Item Number” originated from the fTANF system where each field is labeled as “Item [x]”. However, in the current coding instructions, it is referred to as a "Data Element". + +> “Are Item Numbers related to Data Elements?” - State + +> “What is it called… it’s not item number…” (Participant flips through a printed excerpt of the Coding Instructions) - Tribe + +After discussions with OFA regarding upcoming revisions to the coding instructions "Item number" does reflect the preferred terminology going forward. All participants ultimately managed to connect the two terms, but this is still a likely area for future knowledge center content such as an FAQ item to reassure TDP users that they're making the correct connection of terms. + +--- + +#### We further validated the utility of submission history and notification emails as ways for TANF programs to show proof of transmission for audit purposes + +The error report email and submission history page would serve to help answer two common questions that Data Analysts—and/or their auditors—have after data file submission. "When did we transmit the data files and has ACF received them?"" + +> “Referenced the transmission file as what they still gets for confirmation on when files are transmitted to ACF.” - State + +> “Transmission report is what the auditors get to show that I’ve met the timeline” - State + +> “The date and time that I submitted is critical for the auditors” - State + +> “Auditors want proof” - State + +Participants also spoke to the usefulness of TDP's notification emails for fulfilling elements of these audits. + +A common subsequent question is "Are my files without errors or do they need correction?". While evaluating TDP's acceptance statuses were not a primary goal of this research, multiple participants did respond to its presence in our prototype—commentating on acceptance status and its lack of clarity in TDRS. + +> “[After I submit] has it been accepted or is the data okay? Is it acceptable data? Do I need correct anything?” - Tribe + +> “[I] wasn’t too clear whether the current transmission report provided [all] errors or files were transmitted successfully” - State + + + +--- + +### What's next +- We plan to capture final prototype encompassing all changes informed by this research in [#2527](https://github.com/raft-tech/TANF-app/issues/2527). +- Further errors research with participants from this round who submitted files with errors for the May 15th deadline to repeat many elements of this test but directly in the context of their real data (rather than example data). \ No newline at end of file From 2ff7d2ec2f6160634e02063cf9f133483461f4f7 Mon Sep 17 00:00:00 2001 From: Miles Reiter Date: Wed, 21 Jun 2023 12:13:39 -0400 Subject: [PATCH 19/35] Update README.md --- docs/User-Experience/Research-Syntheses/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/User-Experience/Research-Syntheses/README.md b/docs/User-Experience/Research-Syntheses/README.md index 261a30fe24..43496f75a8 100644 --- a/docs/User-Experience/Research-Syntheses/README.md +++ b/docs/User-Experience/Research-Syntheses/README.md @@ -5,9 +5,15 @@ With a few exceptions, we've tended to publish markdown research syntheses to su The syntheses included herein are organized reverse-chronologically from newest to oldest: +### [2023, Sprint - TDP 3.0 Pilot Program](https://github.com/raft-tech/TANF-app/blob/develop/docs/User-Experience/Research-Syntheses/2023%2C%20Spring%20-%20Testing%20CSV%20%26%20Excel-based%20error%20reports.md#spring-2023---testing-csv--excel-based-error-reports) + +- Research sessions conducted with 5 states and 4 Tribes with a focus on programs that had errors on their Section 1 Data Files. +- Tested the usability of an Excel-based error report located in Submission History to evaluate whether participants would be successful in downloading, opening, and ultimately acting on the reports. +* Explored optimal column structures and data needed to help guide error correction and cut down on the time required to understand the report itself. + ### [2023, Winter - TDP 3.0 Pilot Program](https://github.com/raft-tech/TANF-app/blob/develop/docs/User-Experience/Research-Syntheses/2023%2C%20Winter%20-%20TDP%203.0%20Pilot%20Program.md) -- A total of 17 states, 17 tribes and two territories participated in the pilot expansion using the production version of TDP to submit their FY2023 Q1 data files. +- A total of 17 states, 17 Tribes and two territories participated in the pilot expansion using the production version of TDP to submit their FY2023 Q1 data files. - Conducted a retrospective on the usability of Django (system admin) as the TDP user base continues to expand and what might the System Admins need to keep up with onboarding, and managing users and logs at scale. ### [2022, Winter - Understanding How STTs Would Use the Transmission Report](https://github.com/raft-tech/TANF-app/blob/develop/docs/User-Experience/Research-Syntheses/2022%2C%20Winter%20-%20Understanding%20How%20STTs%20Would%20Use%20the%20Transmission%20Report.md) From 093f67f220204020630b5c822bc4b8410af495bb Mon Sep 17 00:00:00 2001 From: Andrew <84722778+andrew-jameson@users.noreply.github.com> Date: Wed, 21 Jun 2023 13:54:57 -0400 Subject: [PATCH 20/35] Updating deliverable links (#2584) --- .github/pull_request_template.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 895d9cd92a..625fe7f9c0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,15 +16,15 @@ cd tdrs-backend && docker-compose -f docker-compose.yml -f docker-compose.local. > *Demo GIF(s) and screenshots for testing procedure* ## Deliverables -_More details on how deliverables herein are assessed included [here](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverables)._ +_More details on how deliverables herein are assessed included [here](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverables)._ -### [Deliverable 1: Accepted Features](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-1-Accepted-Features) +### [Deliverable 1: Accepted Features](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-1-Accepted-Features) Checklist of ACs: + [ ] [**_insert ACs here_**] + [ ] **`lfrohlich`** and/or **`adpennington`** confirmed that ACs are met. -### [Deliverable 2: Tested Code](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-2-Tested-Code) +### [Deliverable 2: Tested Code](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-2-Tested-Code) + Are all areas of code introduced in this PR meaningfully tested? + [ ] If this PR introduces backend code changes, are they meaningfully tested? @@ -33,13 +33,13 @@ Checklist of ACs: + [ ] Frontend coverage: [_insert coverage %_] (see `CodeCov Report` comment in PR) + [ ] Backend coverage: [_insert coverage %_] (see `CodeCov Report` comment in PR) -### [Deliverable 3: Properly Styled Code](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-3-Properly-Styled-Code) +### [Deliverable 3: Properly Styled Code](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-3-Properly-Styled-Code) + [ ] Are backend code style checks passing on CircleCI? + [ ] Are frontend code style checks passing on CircleCI? + [ ] Are code maintainability principles being followed? -### [Deliverable 4: Accessible](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-4-Accessibility) +### [Deliverable 4: Accessible](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-4-Accessibility) + [ ] Does this PR complete the epic? + [ ] Are links included to any other gov-approved PRs associated with epic? @@ -47,11 +47,11 @@ Checklist of ACs: + [ ] Did automated and manual testing with `iamjolly` and `ttran-hub` using Accessibility Insights reveal any errors introduced in this PR? -### [Deliverable 5: Deployed](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-5-Deployed) +### [Deliverable 5: Deployed](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-5-Deployed) + [ ] Was the code successfully deployed via automated CircleCI process to development on Cloud.gov? -### [Deliverable 6: Documented](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-6-Code-documentation) +### [Deliverable 6: Documented](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-6-Code-documentation) + [ ] Does this PR provide background for why coding decisions were made? + [ ] If this PR introduces backend code, is that code easy to understand and sufficiently documented, both inline and overall? @@ -59,13 +59,13 @@ Checklist of ACs: + [ ] If this PR introduces dependencies, are their licenses documented? + [ ] Can reviewer explain and take ownership of these elements presented in this code review? -### [Deliverable 7: Secure](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-7-Secure) +### [Deliverable 7: Secure](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-7-Secure) + [ ] Does the OWASP Scan pass on CircleCI? + [ ] Do manual code review and manual testing detect any new security issues? + [ ] If new issues detected, is investigation and/or remediation plan documented? -### [Deliverable 8: User Research](../docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-8-User-Research) +### [Deliverable 8: User Research](https://github.com/raft-tech/TANF-app/blob/develop/docs/How-We-Work/our-priorities-values-expectations.md#Deliverable-8-User-Research) Research product(s) clearly articulate(s): + [ ] the purpose of the research From 9534d255d071719c5a42ba13aeb99a6b8133b2d7 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Wed, 21 Jun 2023 12:16:21 -0600 Subject: [PATCH 21/35] User viewset not returning/duplicating users (#2573) * - Fixed issue not allowing pagination to work locally with nginx - Added ordering to user field to fix duplicates issue * - fix lint error * - Removing ID check since we cannot guarantee that the uuid that is generated per test run will be lexigraphically consistent --------- Co-authored-by: Alex P <63075587+ADPennington@users.noreply.github.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- tdrs-backend/tdpservice/users/models.py | 5 +++++ .../tdpservice/users/test/test_api/test_endpoints.py | 1 - tdrs-frontend/nginx/local/locations.conf | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/users/models.py b/tdrs-backend/tdpservice/users/models.py index 7d9a9507c4..d0a9c924dc 100644 --- a/tdrs-backend/tdpservice/users/models.py +++ b/tdrs-backend/tdpservice/users/models.py @@ -33,6 +33,11 @@ class AccountApprovalStatusChoices(models.TextChoices): class User(AbstractUser): """Define user fields and methods.""" + class Meta: + """Define meta user model attributes.""" + + ordering = ['pk'] + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) stt = models.ForeignKey( diff --git a/tdrs-backend/tdpservice/users/test/test_api/test_endpoints.py b/tdrs-backend/tdpservice/users/test/test_api/test_endpoints.py index a9265b74fb..a20244538e 100644 --- a/tdrs-backend/tdpservice/users/test/test_api/test_endpoints.py +++ b/tdrs-backend/tdpservice/users/test/test_api/test_endpoints.py @@ -231,7 +231,6 @@ def test_list_users(self, api_client, user): results = response.data['results'] assert response.status_code == status.HTTP_200_OK assert len(results) == 2 - assert results[1]['id'] == user.id def test_get_user(self, api_client, user): """Get a specific user, expect 200 and ability to get any user.""" diff --git a/tdrs-frontend/nginx/local/locations.conf b/tdrs-frontend/nginx/local/locations.conf index 4f53dd27b7..9859f5fd19 100644 --- a/tdrs-frontend/nginx/local/locations.conf +++ b/tdrs-frontend/nginx/local/locations.conf @@ -7,7 +7,7 @@ location = /nginx_status { location ~ ^/(v1|admin|static/admin|swagger|redocs) { limit_req zone=limitreqsbyaddr delay=5; proxy_pass http://${BACK_END}:8080$request_uri; - proxy_set_header Host $host; + proxy_set_header Host $host:3000; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; From 9a392ba49b8033942273bac39628b7441dd4b363 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Thu, 22 Jun 2023 09:17:36 -0600 Subject: [PATCH 22/35] Update cf os (#2523) * - updated nginx buildpack * - specifying different nginx version * - Updating changelog * - added script to update certain apps in cf - added workflow for each environment in circi * - fixed base config * - fixing jobs * - Updated based on feedback in OH * - Updating defaults * - Removing defaults * - Fixing indent * - Adding params to config * test * test * - updating work dir * - Adding checkout * - adding cf check * - logging into cf * - update cf check to install required binary * - removing unnecessary switch * - Forcing plugin installation * - test installing plugin from script also * - Adding new dependencies * - adding package * - fixing broken install * - fixing libs * - using correct command * - gettign correct version of libc * - trying to upgrade libs * - testing * - Updated README and script * Revert "- Updated README and script" This reverts commit 92697b3e53d1fd87b8d3e7995abb9093aa26e307. * - Removed unnecessary circi stuff - Removed script - Updated docs to callout updating secondary apps * - Correct spelling error --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .circleci/README.md | 14 ++++++++++++++ .circleci/deployment/commands.yml | 4 ++-- .circleci/deployment/jobs.yml | 2 +- .circleci/deployment/workflows.yml | 3 ++- .../Technical-Documentation/buildpack-changelog.md | 4 +++- scripts/cf-check.sh | 1 + tdrs-frontend/manifest.buildpack.yml | 2 +- 7 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.circleci/README.md b/.circleci/README.md index f08a5f0e0a..39e484871c 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -71,3 +71,17 @@ You want to set the branch to be the branch you want this scan to be run on. You want to add a Pipeline Parameter with `run_nightly_owasp_scan`to be a boolean and set to `true`. You want Attribution set to Scheduled Actor (Scheduling System) + +## Updating Cloud Foundry App OS +Cloud Foundry (CF) occasionally releases OS updates. In doing so they deprecate the previous OS and after a short time +do not allow any apps to run/deploy on anything but the latest OS. The steps below describe how the main TDP apps are +updated along with the secondary apps running in CF. + +### Frontend/Backend + - Before updating, make sure the current buildpacks that these apps use are supported by the latest OS. If they aren't you can update the manifest to point them to the correct buildpacks. + - To update the apps you can either deploy each of the environments (sandbox, raft, qasp, etc) from CircleCi or you can use the `tdrs-deploy ` command from `commands.sh`. Assuming the buildpacks are up to date, that is all you need to do. + +### Secondary apps + - Before you can make the update, you need to ensure you have the CF plugin that allows you to do so. Download the binary for your respective OS [HERE](https://github.com/cloudfoundry/stack-auditor/releases) and follow the installation instructions [HERE](https://docs.cloudfoundry.org/adminguide/stack-auditor.html#install). + - Verify the installation succeeded by running `cf audit-stack`. Note you need to be logged in and have targeted a space via `cf target -o hhs-acf-ofa -s ` + - To update the remaining apps you need to run `cf change-stack ` against every app that is not a frontend/backend app. diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index 65cab6a939..b259fb57f1 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -110,10 +110,10 @@ default: tdp-frontend type: string # So the frontend knows what space its in for the banner. -# I am unclear if the domain is a reliable metric to make this function +# I am unclear if the domain is a reliable metric to make this function # It seems like it might not be working cf-space: - default: dev + default: dev type: string steps: - install-nodejs: diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index dc1ee1d5b7..a24baef461 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -20,7 +20,7 @@ cf-password: CF_PASSWORD_STAGING cf-space: tanf-staging cf-username: CF_USERNAME_STAGING - + deploy-develop: executor: docker-executor working_directory: ~/tdp-deploy diff --git a/.circleci/deployment/workflows.yml b/.circleci/deployment/workflows.yml index 730a87373f..91e5ea56a9 100644 --- a/.circleci/deployment/workflows.yml +++ b/.circleci/deployment/workflows.yml @@ -11,7 +11,7 @@ - deploy-infrastructure-dev staging-deployment: - unless: + unless: or: - << pipeline.parameters.run_dev_deployment >> - << pipeline.parameters.run_nightly_owasp_scan >> @@ -71,3 +71,4 @@ branches: only: - master + diff --git a/docs/Technical-Documentation/buildpack-changelog.md b/docs/Technical-Documentation/buildpack-changelog.md index 6208437faf..a53fdcf6af 100644 --- a/docs/Technical-Documentation/buildpack-changelog.md +++ b/docs/Technical-Documentation/buildpack-changelog.md @@ -1,4 +1,6 @@ ## Buildpacks Changelog +- 05/15/2023 [nginx-buildpack v1.2.2](https://github.com/cloudfoundry/nginx-buildpack.git#v1.2.2) + - 11/09/2022 [python-buildpack v1.8.3](https://github.com/cloudfoundry/python-buildpack/releases/tag/v1.8.3) - 11/09/2022 [nginx-buildpack v1.1.45](https://github.com/cloudfoundry/nginx-buildpack/releases/tag/v1.1.45) @@ -10,7 +12,7 @@ - 05/11/2022 [python-buildpack v1.7.53:](https://github.com/cloudfoundry/python-buildpack/releases/tag/v1.7.53) Forced python version to 3.10.4 - 03/10/2022 [python-buildpack v1.7.51:](https://github.com/cloudfoundry/python-buildpack/releases/tag/v1.7.51) - + - 03/10/2022 [nginx-buildpack v1.1.36:](https://github.com/cloudfoundry/nginx-buildpack/releases/tag/v1.1.36) - 01/03/2022 [python-buildpack v1.7.48:](https://github.com/cloudfoundry/python-buildpack/releases/tag/v1.7.48) diff --git a/scripts/cf-check.sh b/scripts/cf-check.sh index c5f513ed7d..5eacbd9523 100755 --- a/scripts/cf-check.sh +++ b/scripts/cf-check.sh @@ -13,4 +13,5 @@ else apt-get update apt-get install cf7-cli + fi diff --git a/tdrs-frontend/manifest.buildpack.yml b/tdrs-frontend/manifest.buildpack.yml index 097d89323a..870321b775 100755 --- a/tdrs-frontend/manifest.buildpack.yml +++ b/tdrs-frontend/manifest.buildpack.yml @@ -3,7 +3,7 @@ version: 1 applications: - name: tdp-frontend buildpacks: - - https://github.com/cloudfoundry/nginx-buildpack.git#v1.1.45 + - https://github.com/cloudfoundry/nginx-buildpack.git#v1.2.2 memory: 128M instances: 1 disk_quota: 256M From f8d75cc32e6459b594098e0083328cef49bd5692 Mon Sep 17 00:00:00 2001 From: Eric Lipe <125676261+elipe17@users.noreply.github.com> Date: Fri, 23 Jun 2023 14:24:42 -0600 Subject: [PATCH 23/35] Item Number Mismatch (#2578) * - Updated schemas and models to reflect correct item numbers of fields * - Revert migration * - Updated header/trailer item numbers * - Fixed item numbers off by one errors --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .../0006_alter_parsererror_item_number.py | 18 +++ tdrs-backend/tdpservice/parsers/models.py | 2 +- .../tdpservice/parsers/schema_defs/header.py | 20 +-- .../tdpservice/parsers/schema_defs/ssp/m1.py | 86 +++++------ .../tdpservice/parsers/schema_defs/ssp/m2.py | 132 ++++++++--------- .../tdpservice/parsers/schema_defs/ssp/m3.py | 84 +++++------ .../tdpservice/parsers/schema_defs/tanf/t1.py | 92 ++++++------ .../tdpservice/parsers/schema_defs/tanf/t2.py | 138 +++++++++--------- .../tdpservice/parsers/schema_defs/tanf/t3.py | 84 +++++------ .../tdpservice/parsers/schema_defs/trailer.py | 6 +- .../tdpservice/parsers/test/factories.py | 2 +- 11 files changed, 342 insertions(+), 322 deletions(-) create mode 100644 tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py diff --git a/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py b/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py new file mode 100644 index 0000000000..79607377ed --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-06-15 18:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0005_auto_20230601_1510'), + ] + + operations = [ + migrations.AlterField( + model_name='parsererror', + name='item_number', + field=models.CharField(max_length=8, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 78b12a163c..9ffa8306bc 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -35,7 +35,7 @@ class Meta: ) row_number = models.IntegerField(null=False) column_number = models.IntegerField(null=True) - item_number = models.IntegerField(null=True) + item_number = models.CharField(null=True, max_length=8) field_name = models.TextField(null=True, max_length=128) rpt_month_year = models.IntegerField(null=True, blank=False) case_number = models.TextField(null=True, max_length=128) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index b51c433ede..3183142521 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -16,34 +16,34 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ + Field(item="2", name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ validators.matches('HEADER'), ]), - Field(item=2, name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ + Field(item="4", name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ validators.between(2000, 2099) ]), - Field(item=3, name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ + Field(item="5", name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ validators.oneOf(['1', '2', '3', '4']) ]), - Field(item=4, name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ + Field(item="6", name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ validators.oneOf(['A', 'C', 'G', 'S']) ]), - Field(item=5, name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ + Field(item="1", name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ validators.between(0, 99) ]), - Field(item=6, name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ + Field(item="3", name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ validators.between(0, 999) ]), - Field(item=7, name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ + Field(item="7", name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ validators.oneOf(['TAN', 'SSP']) ]), - Field(item=8, name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ + Field(item="8", name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ validators.oneOf(['1', '2']) ]), - Field(item=9, name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ + Field(item="9", name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ validators.matches('E') ]), - Field(item=10, name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ + Field(item="10", name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ validators.oneOf(['N', 'D', 'U']) ]), ], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 79b9c8bc83..590ca4ea1e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -13,89 +13,91 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, + Field(item="4", name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[]), - Field(item=7, name='DISPOSITION', type='number', startIndex=29, endIndex=30, + Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30, required=True, validators=[]), - Field(item=8, name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, + Field(item="8", name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, required=True, validators=[]), - Field(item=10, name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, + Field(item="9", name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33, required=True, validators=[]), - Field(item=11, name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, + Field(item="10", name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, required=True, validators=[]), - Field(item=12, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, + Field(item="11", name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, required=True, validators=[]), - Field(item=13, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, + Field(item="12", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, required=True, validators=[]), - Field(item=14, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, + Field(item="13", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, required=True, validators=[]), - Field(item=15, name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, + Field(item="14", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, required=True, validators=[]), - Field(item=16, name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, + Field(item="15", name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=17, name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, + Field(item="16", name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, required=True, validators=[]), - Field(item=18, name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, + Field(item="17", name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, required=True, validators=[]), - Field(item=19, name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, + Field(item="18", name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, required=True, validators=[]), - Field(item=20, name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, + Field(item="19A", name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, required=True, validators=[]), - Field(item=21, name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, + Field(item="19B", name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, required=True, validators=[]), - Field(item=22, name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, + Field(item="20A", name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, required=True, validators=[]), - Field(item=23, name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, + Field(item="20B", name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, required=True, validators=[]), - Field(item=24, name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, + Field(item="20C", name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, required=True, validators=[]), - Field(item=25, name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, + Field(item="21A", name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, required=True, validators=[]), - Field(item=26, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, + Field(item="21B", name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, required=True, validators=[]), - Field(item=27, name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, + Field(item="22A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, required=True, validators=[]), - Field(item=28, name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, + Field(item="22B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, required=True, validators=[]), - Field(item=29, name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, + Field(item="23A", name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, required=True, validators=[]), - Field(item=30, name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, + Field(item="23B", name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, required=True, validators=[]), - Field(item=31, name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, + Field(item="24AI", name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, required=True, validators=[]), - Field(item=32, name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, + Field(item="24AII", name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, required=True, validators=[]), - Field(item=33, name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, + Field(item="24AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, required=True, validators=[]), - Field(item=34, name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, + Field(item="24AIV", name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, required=True, validators=[]), - Field(item=35, name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, + Field(item="24AV", name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, required=True, validators=[]), - Field(item=36, name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, + Field(item="24AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, required=True, validators=[]), - Field(item=37, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, + Field(item="24AVII", name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, required=True, validators=[]), - Field(item=38, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, + Field(item="24B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, required=True, validators=[]), - Field(item=39, name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, + Field(item="24CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, required=True, validators=[]), - Field(item=40, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, + Field(item="24CII", name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, required=True, validators=[]), - Field(item=41, name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, + Field(item="24CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, required=True, validators=[]), - Field(item=42, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, + Field(item="24CIV", name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, required=True, validators=[]), - Field(item=43, name='BLANK', type='string', startIndex=113, endIndex=150, + Field(item="25", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, + required=True, validators=[]), + Field(item="-1", name='BLANK', type='string', startIndex=113, endIndex=150, required=False, validators=[]), ] ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index fd855181be..cc80e3431c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -13,137 +13,137 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="26", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, + Field(item="27", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[]), - Field(item=6, name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, + Field(item="28", name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, required=True, validators=[]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, + Field(item="29", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, + Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, + Field(item="30B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, + Field(item="30C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_BLACK', type='number', startIndex=41, endIndex=42, + Field(item="30D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, + Field(item="30E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='RACE_WHITE', type='number', startIndex=43, endIndex=44, + Field(item="30F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, + Field(item="31", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, + Field(item="32A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, + Field(item="32B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, + Field(item="32C", name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, required=True, validators=[]), - Field(item=18, name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, + Field(item="32D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=19, name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, + Field(item="32E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[]), - Field(item=20, name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, + Field(item="33", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, validators=[]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, + Field(item="34", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[]), - Field(item=22, name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, + Field(item="35", name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, validators=[]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, + Field(item="36", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, validators=[]), - Field(item=24, name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, + Field(item="37", name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, required=True, validators=[]), - Field(item=25, name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, + Field(item="38", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, validators=[]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, + Field(item="39", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, validators=[]), - Field(item=27, name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, + Field(item="40", name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, required=True, validators=[]), - Field(item=28, name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, + Field(item="41", name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, required=True, validators=[]), - Field(item=29, name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, + Field(item="42", name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, required=True, validators=[]), - Field(item=30, name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, + Field(item="43", name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, required=True, validators=[]), - Field(item=31, name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, + Field(item="44", name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=32, name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, + Field(item="45", name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, required=True, validators=[]), - Field(item=33, name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, + Field(item="46A", name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, required=True, validators=[]), - Field(item=34, name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, + Field(item="46B", name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, required=True, validators=[]), - Field(item=35, name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, + Field(item="46C", name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, required=True, validators=[]), - Field(item=36, name='OJT', type='number', startIndex=76, endIndex=78, + Field(item="47", name='OJT', type='number', startIndex=76, endIndex=78, required=True, validators=[]), - Field(item=37, name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, + Field(item="48A", name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, required=True, validators=[]), - Field(item=38, name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, + Field(item="48B", name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, required=True, validators=[]), - Field(item=39, name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, + Field(item="48C", name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, required=True, validators=[]), - Field(item=40, name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, + Field(item="49A", name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, required=True, validators=[]), - Field(item=41, name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, + Field(item="49B", name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, required=True, validators=[]), - Field(item=42, name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, + Field(item="49C", name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, required=True, validators=[]), - Field(item=43, name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, + Field(item="50A", name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=44, name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, + Field(item="50B", name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, required=True, validators=[]), - Field(item=45, name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, + Field(item="50C", name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, required=True, validators=[]), - Field(item=46, name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, + Field(item="51A", name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, required=True, validators=[]), - Field(item=47, name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, + Field(item="51B", name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, required=True, validators=[]), - Field(item=48, name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, + Field(item="51C", name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, required=True, validators=[]), - Field(item=49, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, + Field(item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, required=True, validators=[]), - Field(item=50, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, + Field(item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, required=True, validators=[]), - Field(item=51, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, + Field(item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, required=True, validators=[]), - Field(item=52, name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, + Field(item="53A", name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, required=True, validators=[]), - Field(item=53, name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, + Field(item="53B", name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, required=True, validators=[]), - Field(item=54, name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, + Field(item="53C", name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, required=True, validators=[]), - Field(item=55, name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, + Field(item="54A", name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, required=True, validators=[]), - Field(item=56, name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, + Field(item="54B", name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, required=True, validators=[]), - Field(item=57, name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, + Field(item="54C", name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, required=True, validators=[]), - Field(item=58, name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, + Field(item="55", name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, required=True, validators=[]), - Field(item=59, name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, + Field(item="56", name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, required=True, validators=[]), - Field(item=60, name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, + Field(item="57", name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, required=True, validators=[]), - Field(item=61, name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, + Field(item="58", name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, required=True, validators=[]), - Field(item=62, name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, + Field(item="59A", name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, required=True, validators=[]), - Field(item=63, name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, + Field(item="59B", name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, required=True, validators=[]), - Field(item=64, name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, + Field(item="59C", name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, required=True, validators=[]), - Field(item=65, name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, + Field(item="59D", name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, required=True, validators=[]), - Field(item=66, name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, + Field(item="59E", name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, required=True, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index c269861ff7..c2f1551cc6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -13,47 +13,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, + Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, required=True, validators=[]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, + Field(item="62", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[]), - Field(item=7, name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, + Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, + Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, + Field(item="63C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_BLACK', type='number', startIndex=40, endIndex=41, + Field(item="63D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, + Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_WHITE', type='number', startIndex=42, endIndex=43, + Field(item="63F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, + Field(item="64", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, + Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, + Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, + Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[]), - Field(item=17, name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, + Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=18, name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, + Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, required=True, validators=[]), - Field(item=19, name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, + Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[]), - Field(item=20, name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, + Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, required=True, validators=[]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, + Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, required=True, validators=[]) ] ) @@ -67,47 +67,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, + Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[]), - Field(item=23, name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, + Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, required=True, validators=[]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, + Field(item="62", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[]), - Field(item=25, name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, + Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[]), - Field(item=26, name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, + Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[]), - Field(item=27, name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, + Field(item="63C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[]), - Field(item=28, name='RACE_BLACK', type='number', startIndex=81, endIndex=82, + Field(item="63D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[]), - Field(item=29, name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, + Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[]), - Field(item=30, name='RACE_WHITE', type='number', startIndex=83, endIndex=84, + Field(item="63F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, + Field(item="64", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[]), - Field(item=32, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, + Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, required=True, validators=[]), - Field(item=33, name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, + Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, required=True, validators=[]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, + Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[]), - Field(item=35, name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, + Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[]), - Field(item=36, name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, + Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=37, name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, + Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[]), - Field(item=38, name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, + Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, required=True, validators=[]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, + Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, required=True, validators=[]) ] ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 483950ce87..25f0b91811 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -13,96 +13,96 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, + Field(item="5", name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[]), - Field(item=7, name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, + Field(item="8", name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[]), - Field(item=8, name='DISPOSITION', type='number', startIndex=30, endIndex=31, + Field(item="9", name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[]), - Field(item=9, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, + Field(item="10", name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[]), - Field(item=10, name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, + Field(item="11", name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[]), - Field(item=11, name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, + Field(item="12", name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[]), - Field(item=12, name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, + Field(item="13", name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[]), - Field(item=13, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, + Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[]), - Field(item=14, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, + Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=15, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, + Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[]), - Field(item=16, name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, + Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=17, name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, + Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[]), - Field(item=18, name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, + Field(item="19", name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[]), - Field(item=19, name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, + Field(item="20", name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[]), - Field(item=20, name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, + Field(item="21A", name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[]), - Field(item=21, name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, + Field(item="21B", name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[]), - Field(item=22, name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, + Field(item="22A", name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[]), - Field(item=23, name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, + Field(item="22B", name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=24, name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, + Field(item="22C", name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[]), - Field(item=25, name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, + Field(item="23A", name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[]), - Field(item=26, name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, + Field(item="23B", name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[]), - Field(item=27, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, + Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[]), - Field(item=28, name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, + Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[]), - Field(item=29, name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, + Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[]), - Field(item=30, name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, + Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[]), - Field(item=31, name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, + Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[]), - Field(item=32, name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, + Field(item="26AII", name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[]), - Field(item=33, name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, + Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[]), - Field(item=34, name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, + Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[]), - Field(item=35, name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, + Field(item="26AV", name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[]), - Field(item=36, name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, + Field(item="26AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[]), - Field(item=37, name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, + Field(item="26AVII", name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[]), - Field(item=38, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, + Field(item="26B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[]), - Field(item=39, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, + Field(item="26CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[]), - Field(item=40, name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, + Field(item="26CII", name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[]), - Field(item=41, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, + Field(item="26CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[]), - Field(item=42, name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, + Field(item="26CIV", name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[]), - Field(item=43, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, + Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[]), - Field(item=44, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, + Field(item="28", name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[]), - Field(item=45, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, + Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[]), - Field(item=46, name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), + Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index dc94263d45..49ec6eff1e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -13,143 +13,143 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="30", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, + Field(item="31", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[]), - Field(item=6, name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, + Field(item="32", name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, required=True, validators=[]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, + Field(item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_HISPANIC', type='string', startIndex=38, endIndex=39, + Field(item="34A", name='RACE_HISPANIC', type='string', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_AMER_INDIAN', type='string', startIndex=39, endIndex=40, + Field(item="34B", name='RACE_AMER_INDIAN', type='string', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_ASIAN', type='string', startIndex=40, endIndex=41, + Field(item="34C", name='RACE_ASIAN', type='string', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_BLACK', type='string', startIndex=41, endIndex=42, + Field(item="34D", name='RACE_BLACK', type='string', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_HAWAIIAN', type='string', startIndex=42, endIndex=43, + Field(item="34E", name='RACE_HAWAIIAN', type='string', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='RACE_WHITE', type='string', startIndex=43, endIndex=44, + Field(item="34F", name='RACE_WHITE', type='string', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, + Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='FED_OASDI_PROGRAM', type='string', startIndex=45, endIndex=46, + Field(item="36A", name='FED_OASDI_PROGRAM', type='string', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='FED_DISABILITY_STATUS', type='string', startIndex=46, endIndex=47, + Field(item="36B", name='FED_DISABILITY_STATUS', type='string', startIndex=46, endIndex=47, required=True, validators=[]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, validators=[]), - Field(item=18, name='AID_AGED_BLIND', type='string', startIndex=48, endIndex=49, + Field(item="36D", name='AID_AGED_BLIND', type='string', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=19, name='RECEIVE_SSI', type='string', startIndex=49, endIndex=50, + Field(item="36E", name='RECEIVE_SSI', type='string', startIndex=49, endIndex=50, required=True, validators=[]), - Field(item=20, name='MARITAL_STATUS', type='string', startIndex=50, endIndex=51, + Field(item="37", name='MARITAL_STATUS', type='string', startIndex=50, endIndex=51, required=True, validators=[]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, + Field(item="38", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[]), - Field(item=22, name='PARENT_WITH_MINOR_CHILD', type='string', startIndex=53, endIndex=54, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='string', startIndex=53, endIndex=54, required=True, validators=[]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='string', startIndex=54, endIndex=55, + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='string', startIndex=54, endIndex=55, required=True, validators=[]), - Field(item=24, name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, validators=[]), - Field(item=25, name='CITIZENSHIP_STATUS', type='string', startIndex=57, endIndex=58, + Field(item="42", name='CITIZENSHIP_STATUS', type='string', startIndex=57, endIndex=58, required=True, validators=[]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='string', startIndex=58, endIndex=59, + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='string', startIndex=58, endIndex=59, required=True, validators=[]), - Field(item=27, name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, validators=[]), - Field(item=28, name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, validators=[]), - Field(item=29, name='CURRENT_MONTH_STATE_EXEMPT', type='string', startIndex=64, endIndex=65, + Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='string', startIndex=64, endIndex=65, required=True, validators=[]), - Field(item=30, name='EMPLOYMENT_STATUS', type='string', startIndex=65, endIndex=66, + Field(item="47", name='EMPLOYMENT_STATUS', type='string', startIndex=65, endIndex=66, required=True, validators=[]), - Field(item=31, name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, + Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=32, name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, validators=[]), - Field(item=33, name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, validators=[]), - Field(item=34, name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, validators=[]), - Field(item=35, name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, validators=[]), - Field(item=36, name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, validators=[]), - Field(item=37, name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, validators=[]), - Field(item=38, name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, validators=[]), - Field(item=39, name='OJT', type='string', startIndex=82, endIndex=84, + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[]), - Field(item=40, name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, validators=[]), - Field(item=41, name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, validators=[]), - Field(item=42, name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, validators=[]), - Field(item=43, name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=44, name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, validators=[]), - Field(item=45, name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, validators=[]), - Field(item=46, name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, + Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, required=False, validators=[]), - Field(item=47, name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, + Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, required=False, validators=[]), - Field(item=48, name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, + Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, required=False, validators=[]), - Field(item=49, name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, + Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, required=False, validators=[]), - Field(item=50, name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, + Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, required=False, validators=[]), - Field(item=51, name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, + Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, required=False, validators=[]), - Field(item=52, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, + Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, required=False, validators=[]), - Field(item=53, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, + Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, required=False, validators=[]), - Field(item=54, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, + Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, required=False, validators=[]), - Field(item=55, name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, + Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, required=False, validators=[]), - Field(item=56, name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, + Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, required=False, validators=[]), - Field(item=57, name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, + Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, required=False, validators=[]), - Field(item=58, name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, + Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, required=False, validators=[]), - Field(item=59, name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, + Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, required=False, validators=[]), - Field(item=60, name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, + Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, required=False, validators=[]), - Field(item=61, name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, + Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, required=False, validators=[]), - Field(item=62, name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, + Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, required=False, validators=[]), - Field(item=63, name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, + Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, required=False, validators=[]), - Field(item=64, name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, + Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=False, validators=[]), - Field(item=65, name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, + Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, required=False, validators=[]), - Field(item=67, name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, + Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=False, validators=[]), - Field(item=68, name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, + Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=False, validators=[]), - Field(item=69, name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, + Field(item="66D", name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, required=False, validators=[]), - Field(item=70, name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, + Field(item="66E", name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index c22dddc7a7..4da311c988 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -13,47 +13,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="67", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, + Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, required=True, validators=[]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, + Field(item="69", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[]), - Field(item=7, name='RACE_HISPANIC', type='string', startIndex=37, endIndex=38, + Field(item="70A", name='RACE_HISPANIC', type='string', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_AMER_INDIAN', type='string', startIndex=38, endIndex=39, + Field(item="70B", name='RACE_AMER_INDIAN', type='string', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_ASIAN', type='string', startIndex=39, endIndex=40, + Field(item="70C", name='RACE_ASIAN', type='string', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_BLACK', type='string', startIndex=40, endIndex=41, + Field(item="70D", name='RACE_BLACK', type='string', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_HAWAIIAN', type='string', startIndex=41, endIndex=42, + Field(item="70E", name='RACE_HAWAIIAN', type='string', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_WHITE', type='string', startIndex=42, endIndex=43, + Field(item="70F", name='RACE_WHITE', type='string', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, + Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=44, endIndex=45, + Field(item="72A", name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='RECEIVE_SSI', type='string', startIndex=45, endIndex=46, + Field(item="72B", name='RECEIVE_SSI', type='string', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, + Field(item="73", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[]), - Field(item=17, name='PARENT_MINOR_CHILD', type='string', startIndex=48, endIndex=49, + Field(item="74", name='PARENT_MINOR_CHILD', type='string', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=18, name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, + Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[]), - Field(item=19, name='CITIZENSHIP_STATUS', type='string', startIndex=51, endIndex=52, + Field(item="76", name='CITIZENSHIP_STATUS', type='string', startIndex=51, endIndex=52, required=True, validators=[]), - Field(item=20, name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, + Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, required=False, validators=[]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, + Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, required=False, validators=[]), ], ) @@ -66,47 +66,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, + Field(item="67", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[]), - Field(item=23, name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, + Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, required=True, validators=[]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, + Field(item="69", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[]), - Field(item=25, name='RACE_HISPANIC', type='string', startIndex=78, endIndex=79, + Field(item="70A", name='RACE_HISPANIC', type='string', startIndex=78, endIndex=79, required=True, validators=[]), - Field(item=26, name='RACE_AMER_INDIAN', type='string', startIndex=79, endIndex=80, + Field(item="70B", name='RACE_AMER_INDIAN', type='string', startIndex=79, endIndex=80, required=True, validators=[]), - Field(item=27, name='RACE_ASIAN', type='string', startIndex=80, endIndex=81, + Field(item="70C", name='RACE_ASIAN', type='string', startIndex=80, endIndex=81, required=True, validators=[]), - Field(item=28, name='RACE_BLACK', type='string', startIndex=81, endIndex=82, + Field(item="70D", name='RACE_BLACK', type='string', startIndex=81, endIndex=82, required=True, validators=[]), - Field(item=29, name='RACE_HAWAIIAN', type='string', startIndex=82, endIndex=83, + Field(item="70E", name='RACE_HAWAIIAN', type='string', startIndex=82, endIndex=83, required=True, validators=[]), - Field(item=30, name='RACE_WHITE', type='string', startIndex=83, endIndex=84, + Field(item="70F", name='RACE_WHITE', type='string', startIndex=83, endIndex=84, required=True, validators=[]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, + Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[]), - Field(item=32, name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=85, endIndex=86, + Field(item="72A", name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=85, endIndex=86, required=True, validators=[]), - Field(item=33, name='RECEIVE_SSI', type='string', startIndex=86, endIndex=87, + Field(item="72B", name='RECEIVE_SSI', type='string', startIndex=86, endIndex=87, required=True, validators=[]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, + Field(item="73", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[]), - Field(item=35, name='PARENT_MINOR_CHILD', type='string', startIndex=89, endIndex=90, + Field(item="74", name='PARENT_MINOR_CHILD', type='string', startIndex=89, endIndex=90, required=True, validators=[]), - Field(item=36, name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, + Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=37, name='CITIZENSHIP_STATUS', type='string', startIndex=92, endIndex=93, + Field(item="76", name='CITIZENSHIP_STATUS', type='string', startIndex=92, endIndex=93, required=True, validators=[]), - Field(item=38, name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, + Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, required=False, validators=[]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, + Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 3657cfc515..086c83e85a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -16,13 +16,13 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ + Field(item="1", name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ validators.matches('TRAILER') ]), - Field(item=2, name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ + Field(item="2", name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ validators.between(0, 9999999) ]), - Field(item=3, name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ + Field(item="-1", name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ validators.matches(' ') ]), ], diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index e45e78be20..4af7a67070 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -68,7 +68,7 @@ class Meta: file = factory.SubFactory(DataFileFactory) row_number = 1 column_number = 1 - item_number = 1 + item_number = "1" field_name = "test field name" case_number = '1' rpt_month_year = 202001 From a623e00e326b7b1c8e8e3a5b5bf006f20496b833 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 27 Jun 2023 11:40:22 -0600 Subject: [PATCH 24/35] pipeline filtering (#2538) * pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md --------- Co-authored-by: George Hudson Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .circleci/README.md | 4 +- .circleci/base_config.yml | 18 ++++++ .circleci/build-and-test/workflows.yml | 62 ++++++++++++++++--- .circleci/config.yml | 39 +++++++++++- .circleci/infrastructure/workflows.yml | 8 ++- .github/workflows/build-all.yml | 58 +++++++++++++++++ .github/workflows/build-backend.yml | 44 +++++++++++++ .github/workflows/build-frontend.yml | 44 +++++++++++++ .github/workflows/build-pr.yml | 54 ++++++++++++++++ .github/workflows/deploy-develop-on-merge.yml | 2 +- .github/workflows/deploy-infrastructure.yml | 46 ++++++++++++++ .../020-pipeline-build-flow.md | 44 +++++++++++++ .../Technical-Documentation/github-actions.md | 5 ++ 13 files changed, 414 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/build-all.yml create mode 100644 .github/workflows/build-backend.yml create mode 100644 .github/workflows/build-frontend.yml create mode 100644 .github/workflows/build-pr.yml create mode 100644 .github/workflows/deploy-infrastructure.yml create mode 100644 docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md create mode 100644 docs/Technical-Documentation/github-actions.md diff --git a/.circleci/README.md b/.circleci/README.md index 39e484871c..1b0c4c0311 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -14,10 +14,10 @@ This script will generate a complete config for building, testing, and deploying ### Directory structure #### build-and-test -Contains workflows, jobs, and commands for building and testing the application. +Contains workflows, jobs, and commands for building and testing the application. For all development side builds, these are now triggered by GitHub Actions that serve as a filter so only the code that's changed is tested. See [build-all](../.github/workflows/build-all.yml), [build-backend](../.github/workflows/build-backend.yml), and [build-frontend](../.github/workflows/build-frontend.yml) #### infrastructure -Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. +Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. For all development side builds, this is now triggered by GitHub Actions that serve as a filter so only runs when infrastructure code is changed. See [deploy-infrastructure](../.github/workflows/deploy-infrastructure.yml) #### deployment Contains workflows, jobs, and commands for deploying the application on Cloud gov. Note: merges to develop now automatically trigger a develop deploy using [deploy-develop-on-merge](../.github/workflows/deploy-develop-on-merge.yml) and deploys to dev environments happen when a label is created on the PR using [deploy-on-label](../.github/workflows/deploy-on-label.yml) diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml index eb4b9af15c..d722abecf1 100644 --- a/.circleci/base_config.yml +++ b/.circleci/base_config.yml @@ -21,6 +21,18 @@ executors: resource_class: large parameters: + build_and_test_all: + type: boolean + default: false + build_and_test_backend: + type: boolean + default: false + build_and_test_frontend: + type: boolean + default: false + deploy_infrastructure: + type: boolean + default: false develop_branch_deploy: type: boolean default: false @@ -36,3 +48,9 @@ parameters: target_env: type: string default: '' + triggered: + type: boolean + default: false + util_make_erd: + type: boolean + default: false diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml index 9122ab1666..7c0e559b0f 100644 --- a/.circleci/build-and-test/workflows.yml +++ b/.circleci/build-and-test/workflows.yml @@ -1,19 +1,67 @@ # workflows: - build-and-test: - unless: - or: - - << pipeline.parameters.run_dev_deployment >> - - << pipeline.parameters.develop_branch_deploy >> - - << pipeline.parameters.run_owasp_scan >> - - << pipeline.parameters.run_nightly_owasp_scan >> + build-and-test-all: + when: << pipeline.parameters.build_and_test_all >> jobs: - secrets-check + - test-backend: + requires: + - secrets-check - test-frontend: requires: - secrets-check + - test-e2e: + requires: + - secrets-check + + ci-build-and-test-all: + jobs: + - secrets-check: + filters: + branches: + only: + - main + - master + - /^release.*/ - test-backend: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - test-frontend: + filters: + branches: + only: + - main + - master + - /^release.*/ requires: - secrets-check - test-e2e: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + + build-and-test-backend: + when: << pipeline.parameters.build_and_test_backend >> + jobs: + - secrets-check + - test-backend: + requires: + - secrets-check + + build-and-test-frontend: + when: << pipeline.parameters.build_and_test_frontend >> + jobs: + - secrets-check + - test-frontend: requires: - secrets-check diff --git a/.circleci/config.yml b/.circleci/config.yml index ea0ddd1c49..5d0be7af65 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,18 @@ orbs: # parameters from github actions parameters: + build_and_test_all: + type: boolean + default: false + build_and_test_backend: + type: boolean + default: false + build_and_test_frontend: + type: boolean + default: false + deploy_infrastructure: + type: boolean + default: false develop_branch_deploy: type: boolean default: false @@ -28,6 +40,12 @@ parameters: target_env: type: string default: '' + triggered: + type: boolean + default: false + util_make_erd: + type: boolean + default: false jobs: setup: @@ -45,6 +63,23 @@ jobs: # our single workflow, that triggers the setup job defined above workflows: - setup: + github-triggered-setup: + when: << pipeline.parameters.triggered >> + jobs: + - setup: + filters: + branches: + ignore: + - main + - master + - /^release.*/ + circle-ci-setup: jobs: - - setup + - setup: + filters: + branches: + only: + - main + - master + - /^release.*/ + diff --git a/.circleci/infrastructure/workflows.yml b/.circleci/infrastructure/workflows.yml index 46f29aec7a..9cb85e8ed8 100644 --- a/.circleci/infrastructure/workflows.yml +++ b/.circleci/infrastructure/workflows.yml @@ -1,6 +1,10 @@ -#workflows: +#workflows: enable-versioning-for-s3-buckets: - unless: << pipeline.parameters.run_nightly_owasp_scan >> + when: + or: + - << pipeline.parameters.deploy_infrastructure >> + - equal: [ 'main', << pipeline.git.branch >> ] + - equal: [ 'master', << pipeline.git.branch >> ] jobs: - enable-versioning: filters: diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml new file mode 100644 index 0000000000..c03d344abb --- /dev/null +++ b/.github/workflows/build-all.yml @@ -0,0 +1,58 @@ +########################################################################### +# GitHub Action Workflow +# On changes to scripts or changes to the pipeline code to any branch +# besides develop, staging and master triggers the full build and test +# pipeline. +# +# NOTE: develop, staging(main) and master are skipped on the push because this +# would be redundant after running the full set of tests from the PR. +# See build-pr.yml for builds that run on code being merged into develop. +# See deploy-develop-on-merge.yml and make_erd for the workflow +# pipelines that run on merge to develop, staging, and master branches. +# HHS (main and master and release/**) branches build all only +# and are managed in CircleCI +# +# Step 0: Make changes on your branch to files in scripts/ .circleci or .github +# and push changes to your remote branch. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend and +# build_and_test_frontend to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build and test All on push when scripts/commands change +on: + push: + branches-ignore: + - develop + - main + - master + - 'release/**' + paths: + - 'scripts/**' + - '.circleci/**' + - '.github/**' +jobs: + build_and_test_all: + runs-on: ubuntu-latest + name: Initiate deploy job in CircleCI + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }} + payload: '{ + "build_and_test_all": true, + "triggered": true + }' diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..26ef5c03eb --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,44 @@ +########################################################################### +# GitHub Action Workflow +# On push to any branch, triggers the back end build and test pipeline +# if the tdrs-backend has changed. +# +# Step 0: make changes on your branch to non-documentation files in +# tdrs-backend and push changes to your remote branch +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend +# to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build Only Backend When tdrs-backend/ Files Change +on: + push: + paths: 'tdrs-backend/**' + branches-ignore: + - develop + - main + - master +jobs: + build_and_test_backend: + runs-on: ubuntu-latest + name: Build and Test Backend + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "build_and_test_backend": true, + "triggered": true + }' diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml new file mode 100644 index 0000000000..b9b60a9141 --- /dev/null +++ b/.github/workflows/build-frontend.yml @@ -0,0 +1,44 @@ +########################################################################### +# GitHub Action Workflow +# On push to any branch, triggers the front end build and test pipeline +# if the tdrs-frontend has changed. +# +# Step 0: make changes on your branch to non-documentation files in +# tdrs-frontend and push changes to your remote branch +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_frontend +# to run the workflow/jobs listed here: +# build-and-test:[ +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build Only Frontend When tdrs-frontend Files Change +on: + push: + paths: 'tdrs-frontend/**' + branches-ignore: + - develop + - main + - master +jobs: + build_and_test_frontend: + runs-on: ubuntu-latest + name: Build and Test Frontend + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "build_and_test_frontend": true, + "triggered": true + }' diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml new file mode 100644 index 0000000000..fa54a2a097 --- /dev/null +++ b/.github/workflows/build-pr.yml @@ -0,0 +1,54 @@ +########################################################################### +# GitHub Action Workflow +# On pull requests requesting review from individuals, besides staging, +# master, and release branches triggers the full build and test pipeline. +# +# NOTE: release branches, staging(main) and master are skipped because +# these branch builds are managed in CircleCI +# +# Step 0: make PR from your branch into develop, then select reviewers. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend and +# build_and_test_frontend to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build and test All for PRs +on: + pull_request: + branches-ignore: #handled in circleci + - main + - master + - 'release/**' + types: [review_requested, ready_for_review, synchronize] + paths-ignore: + - 'docs/**' + - '**.md' + - '**.txt' + - '.gitattributes' + - '.gitignore' + - 'LICENSE' +jobs: + build_and_test_pr: + runs-on: ubuntu-latest + name: Initiate deploy job in CircleCI + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }} + payload: '{ + "build_and_test_all": true, + "triggered": true + }' \ No newline at end of file diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml index 80ad9b3088..a9df826455 100644 --- a/.github/workflows/deploy-develop-on-merge.yml +++ b/.github/workflows/deploy-develop-on-merge.yml @@ -24,7 +24,7 @@ on: push: branches: - develop - paths_ignore: + paths-ignore: - 'docs/**' - '**.md' - '**.txt' diff --git a/.github/workflows/deploy-infrastructure.yml b/.github/workflows/deploy-infrastructure.yml new file mode 100644 index 0000000000..e5eeeb611a --- /dev/null +++ b/.github/workflows/deploy-infrastructure.yml @@ -0,0 +1,46 @@ +########################################################################### +# GitHub Action Workflow +# On push changing terraform files or infrastructure pipelines, triggers the +# terraform deploy pipeline for the appropriate cf space. +# +# Step 0: make changes to non-documentation files in terraform/ or +# .circleci/infrastructure/ and push/merge changes. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon terraform: true flag +# to run the workflow/jobs listed here: +# build-and-test:[ +# enable-versioning-for-s3-buckets +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Run Infrastructure Pipeline When Terraform or Infrastructure Files Change +on: + push: + branches-ignore: #handled in CircleCI + - main + - master + - 'release/**' + paths: + - 'terraform/**' + - '.circleci/infrastructure/**' +jobs: + run_infrastructure_deployment: + runs-on: ubuntu-latest + name: Deploy Infrastructure + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "deploy_infrastructure": true, + "triggered": true + }' + \ No newline at end of file diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md new file mode 100644 index 0000000000..0e5f9b17ce --- /dev/null +++ b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md @@ -0,0 +1,44 @@ +# 20. Pipeline Build Flow + +Date: 2023-06-07 (_Updated 2023-06-13_) + +## Status + +Pending + +## Context + +We use [CircleCI](https://app.circleci.com/pipelines/github/raft-tech/TANF-app)] as our primary pipeline build tool. On the HHS side, the standard pipeline, build all and deploy ran by CircleCI is sufficiently granular to meet all our build needs. However, on the dev Raft-TANF side, where work can often be held up by waiting for build and test pipelines, it's useful to reduce build times by filtering which workflows run based upon which code is changing. In order to do this, GitHub Actions is leveraged to kick off different CircleCI build workflows since Actions has more granular control over what paths are changed. + +These pipelines are closely tied to the [Git workflow](./009-git-workflow.md) and should run build and test as expected based on what code changed, and still deploy to the correct CF spaces based upon the Decision tree in the [Deployment Flow](./008-deployment-flow.md) + +## Build Logic + +For all release branches, the main, and the master branch, CircleCI should run the full build, infrastructure deploy, and app deployment. + +for feature/development branches, only the build and test pertainint to the code changed should be built. +Front end tests should be run if /tdrs-frontent changes +Back end tests should be run if /tdrs-backend changes +the entire build and test all should run if anything pertaining to the pipeline changes +infrastructure deploy should run if /terraform or infrastructure deploy pipeline changes + +Once a pull request is flagged as ready for review and/or has reviewers assigned, a full build and test all should be run (and tests must pass before merge to develop) + +Develop merges trigger a deploy to the develop CF space, and then a special integration end-2-end test that tests against the real development environment instead of a simulated environment on the CircleCI build servers. + +## Consequences + +**Pros** +* reduce time of build and tests by only running the appropriate workflows. +* only run infrastructure deploys when changes are made to infrastructure code. +* only run end-2-end tests when code is ready to be reviewed. +* only run full integration end-2-end testing when the develop environment assets are updated. + +**Risks** +* Increased pipeline logic complexity + +## Notes + +- For the nuanced build triggers, CircleCI documentation recommends using Actions to trigger CircleCI builds and send different flags to tell which workflows should run. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0 + +- This could be an argument for future complete pipeline switchover to GitHub Actions in order to reduce pipeline complexity, while maintaining the desired build granularity. \ No newline at end of file diff --git a/docs/Technical-Documentation/github-actions.md b/docs/Technical-Documentation/github-actions.md new file mode 100644 index 0000000000..05839498a4 --- /dev/null +++ b/docs/Technical-Documentation/github-actions.md @@ -0,0 +1,5 @@ +# How We Use GitHub Actions +For now, the only use case we have for GitHub Actions is to help up trigger CircleCI builds the way we want to. This is actually the preferred method CircleCI advises for branch, path, pull-request, and labelled filtering and job triggering. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0 + +## Path Filtering +We use Actions to filter which workflows are getting run by CircleCI by sending different flags to CircleCI through the promiseofcake CircleCI API trigger. See the individual files in [.github](../../.github/) for detailed instructions for how to use each. From b6d8b777f8b24f1ea4db313daf6fc17050f452cd Mon Sep 17 00:00:00 2001 From: George Hudson Date: Thu, 29 Jun 2023 08:54:40 -0600 Subject: [PATCH 25/35] Hotfix Devops/2457 path filtering for documentation (#2597) * pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md * build and test all on PRs even for documentation --------- Co-authored-by: George Hudson --- .github/workflows/build-pr.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index fa54a2a097..5aef710990 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -28,13 +28,6 @@ on: - master - 'release/**' types: [review_requested, ready_for_review, synchronize] - paths-ignore: - - 'docs/**' - - '**.md' - - '**.txt' - - '.gitattributes' - - '.gitignore' - - 'LICENSE' jobs: build_and_test_pr: runs-on: ubuntu-latest From 0dcd36a026b9f9e51dba12b4c4e5432a5dd87a25 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Thu, 29 Jun 2023 08:05:08 -0700 Subject: [PATCH 26/35] Create sprint-74-summary.md (#2596) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-74-summary.md | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/Sprint-Review/sprint-74-summary.md diff --git a/docs/Sprint-Review/sprint-74-summary.md b/docs/Sprint-Review/sprint-74-summary.md new file mode 100644 index 0000000000..9d64430d39 --- /dev/null +++ b/docs/Sprint-Review/sprint-74-summary.md @@ -0,0 +1,54 @@ +# Sprint 74 Summary +05/24/23 - 06/06/23 + +Velocity: 21 + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX to continue STT onboarding (focusing on onboarding CyberFusion users), errors research sessions, provide copy for 2509 (e-mail notification for data submission and errors/transmission report) +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + +## Tickets + +#### Completed/Merged +* [#2439 - [Design] Video Release — Embed videos in knowledge center +](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2439) +* [#2503 [Design] May Knowledge Center Enhancements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2503) +* [#2531 Bug/Max file size](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2531) +* [#2424 Complete TANF section 1 parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) +* [#2368 errors reporting research](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2368) + + + +#### Submitted (QASP Review, OCIO Review) +* N/A + + +#### Closed (not merged) +* N/A + +### Moved to Next Sprint (Blocked, Raft Review, In Progress) + +#### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +#### Raft Review +* [2486 [Spike - Parser Performance]](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2521 - Update to CFlinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2550 - Deactivation warning emails are missing hyperlinks in staging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2550) + +#### In Progress +* [#2116 [DevOps] Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + + + +### Sprint 74 Demo +* Internal: + * UX walkthrough of Knowledge Center enhancements + * [#2424 Complete TANF section 1 parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) +* External: + * N/A From 9aef162acf32c7a8b33a35ad0b2d36957b04b2c8 Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Wed, 5 Jul 2023 09:52:35 -0400 Subject: [PATCH 27/35] added URL filters (#2580) * added URL filters * allow github to trigger owasp and label deploys (#2601) Co-authored-by: George Hudson --------- Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> Co-authored-by: George Hudson Co-authored-by: George Hudson --- .github/workflows/deploy-develop-on-merge.yml | 6 +- .github/workflows/deploy-on-label.yml | 6 +- .github/workflows/qasp-owasp-scan.yml | 3 +- scripts/zap-scanner.sh | 80 ++++++++++++++++++- 4 files changed, 90 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml index a9df826455..ee4eee0577 100644 --- a/.github/workflows/deploy-develop-on-merge.yml +++ b/.github/workflows/deploy-develop-on-merge.yml @@ -45,4 +45,8 @@ jobs: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} project-slug: ${{ github.repository }} branch: ${{ github.ref_name }} - payload: '{"develop_branch_deploy": true, "target_env": "develop"}' + payload: '{ + "develop_branch_deploy": true, + "target_env": "develop", + "triggered": true + }' diff --git a/.github/workflows/deploy-on-label.yml b/.github/workflows/deploy-on-label.yml index 857dd3a35d..20fbfd8b33 100644 --- a/.github/workflows/deploy-on-label.yml +++ b/.github/workflows/deploy-on-label.yml @@ -66,4 +66,8 @@ jobs: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} project-slug: ${{ github.repository }} branch: ${{ github.head_ref }} - payload: '{"run_dev_deployment": true, "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}"}' + payload: '{ + "run_dev_deployment": true, + "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}", + "triggered": true + }' diff --git a/.github/workflows/qasp-owasp-scan.yml b/.github/workflows/qasp-owasp-scan.yml index 3a4dea99bd..963bf430c7 100644 --- a/.github/workflows/qasp-owasp-scan.yml +++ b/.github/workflows/qasp-owasp-scan.yml @@ -34,5 +34,6 @@ jobs: branch: ${{ github.head_ref }} payload: | { - "run_owasp_scan": ${{ env.HAS_QASP_LABEL }} + "run_owasp_scan": ${{ env.HAS_QASP_LABEL }}, + "triggered": true } diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh index 4e017eb7e2..dbce417a84 100755 --- a/scripts/zap-scanner.sh +++ b/scripts/zap-scanner.sh @@ -59,15 +59,91 @@ ZAP_CLI_OPTIONS="\ -config globalexcludeurl.url_list.url\(0\).regex='.*/robots\.txt.*' \ -config globalexcludeurl.url_list.url\(0\).description='Exclude robots.txt' \ -config globalexcludeurl.url_list.url\(0\).enabled=true \ + -config globalexcludeurl.url_list.url\(1\).regex='^https?://.*\.cdn\.mozilla\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(1\).description='Site - Mozilla CDN (requests such as getpocket)' \ -config globalexcludeurl.url_list.url\(1\).enabled=true \ + -config globalexcludeurl.url_list.url\(2\).regex='^https?://.*\.amazonaws\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(2\).description='TDP S3 buckets' \ -config globalexcludeurl.url_list.url\(2\).enabled=true \ - -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.acf.hhs.gov\/v1\/login\/.*$' \ - -config globalexcludeurl.url_list.url\(3\).description='Site - identity pages' \ + + -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.hhs.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(3\).description='Site - acf.hhs.gov' \ -config globalexcludeurl.url_list.url\(3\).enabled=true \ + + -config globalexcludeurl.url_list.url\(4\).regex='^https:\/\/.*\.google.com\/.*$' \ + -config globalexcludeurl.url_list.url\(4\).description='Site - Google.com' \ + -config globalexcludeurl.url_list.url\(4\).enabled=true \ + + -config globalexcludeurl.url_list.url\(5\).regex='^https:\/\/.*\.youtube.com\/.*$' \ + -config globalexcludeurl.url_list.url\(5\).description='Site - youtube.com' \ + -config globalexcludeurl.url_list.url\(5\).enabled=true \ + + -config globalexcludeurl.url_list.url\(6\).regex='^https:\/\/.*\.monsido.com\/.*$' \ + -config globalexcludeurl.url_list.url\(6\).description='Site - monsido.com' \ + -config globalexcludeurl.url_list.url\(6\).enabled=true \ + + -config globalexcludeurl.url_list.url\(7\).regex='^https:\/\/.*\.crazyegg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(7\).description='Site - crazyegg.com' \ + -config globalexcludeurl.url_list.url\(7\).enabled=true \ + + -config globalexcludeurl.url_list.url\(8\).regex='^https:\/\/.*\.gstatic.com\/.*$' \ + -config globalexcludeurl.url_list.url\(8\).description='Site - gstatic.com' \ + -config globalexcludeurl.url_list.url\(8\).enabled=true \ + + -config globalexcludeurl.url_list.url\(9\).regex='^https:\/\/.*\.googleapis.com\/.*$' \ + -config globalexcludeurl.url_list.url\(9\).description='Site - GoogleAPIs.com' \ + -config globalexcludeurl.url_list.url\(9\).enabled=true \ + + -config globalexcludeurl.url_list.url\(10\).regex='^https:\/\/.*\.crazyegg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(10\).description='Site - CrazyEgg.com' \ + -config globalexcludeurl.url_list.url\(10\).enabled=true \ + + -config globalexcludeurl.url_list.url\(11\).regex='^https:\/\/.*\.doubleclick.net\/.*$' \ + -config globalexcludeurl.url_list.url\(11\).description='Site - DoubleClick.net' \ + -config globalexcludeurl.url_list.url\(11\).enabled=true \ + + -config globalexcludeurl.url_list.url\(12\).regex='^https:\/\/.*unpkg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(12\).description='Site - Unpkg.com' \ + -config globalexcludeurl.url_list.url\(12\).enabled=true \ + + -config globalexcludeurl.url_list.url\(13\).regex='^https:\/\/.*\.readspeaker.com\/.*$' \ + -config globalexcludeurl.url_list.url\(13\).description='Site - ReadSpeaker.com' \ + -config globalexcludeurl.url_list.url\(13\).enabled=true \ + + -config globalexcludeurl.url_list.url\(14\).regex='^https:\/\/.*\.fontawesome.com\/.*$' \ + -config globalexcludeurl.url_list.url\(14\).description='Site - FontAwesome.com' \ + -config globalexcludeurl.url_list.url\(14\).enabled=true \ + + -config globalexcludeurl.url_list.url\(15\).regex='^https:\/\/.*\.cloud.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(15\).description='Site - Cloud.gov' \ + -config globalexcludeurl.url_list.url\(15\).enabled=true \ + + -config globalexcludeurl.url_list.url\(16\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \ + -config globalexcludeurl.url_list.url\(16\).description='Site - googletagmanager.com' \ + -config globalexcludeurl.url_list.url\(16\).enabled=true \ + + -config globalexcludeurl.url_list.url\(17\).regex='^https:\/\/.*\.cloudflare.com\/.*$' \ + -config globalexcludeurl.url_list.url\(17\).description='Site - CloudFlare.com' \ + -config globalexcludeurl.url_list.url\(17\).enabled=true \ + + -config globalexcludeurl.url_list.url\(18\).regex='^https:\/\/.*\.google-analytics.com\/.*$' \ + -config globalexcludeurl.url_list.url\(18\).description='Site - Google-Analytics.com' \ + -config globalexcludeurl.url_list.url\(18\).enabled=true \ + + -config globalexcludeurl.url_list.url\(19\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \ + -config globalexcludeurl.url_list.url\(19\).description='Site - googletagmanager.com' \ + -config globalexcludeurl.url_list.url\(19\).enabled=true \ + + -config globalexcludeurl.url_list.url\(20\).regex='^https:\/\/.*\.digitalgov.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(20\).description='Site - DigitalGov.gov' \ + -config globalexcludeurl.url_list.url\(20\).enabled=true \ + + -config globalexcludeurl.url_list.url\(21\).regex='^https:\/\/.*\.identitysandbox.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(21\).description='Site - IdentitySandbox.gov' \ + -config globalexcludeurl.url_list.url\(21\).enabled=true \ + -config spider.postform=true" # How long ZAP will crawl the app with the spider process From 91cc3313bed30021e316641104adea378ee8ba8c Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Mon, 10 Jul 2023 06:43:09 -0700 Subject: [PATCH 28/35] Create sprint-75-summary.md (#2608) --- docs/Sprint-Review/sprint-75-summary.md | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/Sprint-Review/sprint-75-summary.md diff --git a/docs/Sprint-Review/sprint-75-summary.md b/docs/Sprint-Review/sprint-75-summary.md new file mode 100644 index 0000000000..967b0381b8 --- /dev/null +++ b/docs/Sprint-Review/sprint-75-summary.md @@ -0,0 +1,49 @@ +# Sprint 75 Summary + +06/06/23 - 06/20/23 + +Velocity: Dev (1) + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX to continue STT onboarding (focusing on onboarding CyberFusion users), errors research synthesis, copy for e-mail notification of data submission and errors/transmission reports - 2559 +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [2550 Deactivation warning emails are missing e-mail links in staging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2550) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (In Progress, Blocked, Raft Review) + +### In Progress +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) + +### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +### Raft Review +* [#2563 Assess OWASP scan accuracy and URL filter](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2563) +* [#2551 [Bug] - All users are not returned in API response](https://](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2551)) +* [#2564 Adjust TANF and SSP Section 1 item numbers](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2564) +* [#2457 Path filtering for CI Builds](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2457) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2516 Errors Research Synthesis ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2516) + + From baffaba6c83ba204d56763bef3e394ab69477672 Mon Sep 17 00:00:00 2001 From: Smithh-Co <121890311+Smithh-Co@users.noreply.github.com> Date: Mon, 10 Jul 2023 06:54:17 -0700 Subject: [PATCH 29/35] Create sprint-76-summary.md (#2609) Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- docs/Sprint-Review/sprint-76-summary.md | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/Sprint-Review/sprint-76-summary.md diff --git a/docs/Sprint-Review/sprint-76-summary.md b/docs/Sprint-Review/sprint-76-summary.md new file mode 100644 index 0000000000..0d35a73143 --- /dev/null +++ b/docs/Sprint-Review/sprint-76-summary.md @@ -0,0 +1,54 @@ +# Sprint 76 Summary + +06/21/23 - 07/04/23 + +Velocity: Dev (17) + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX errors template, follow-on research, onboarding +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [#2563 Assess OWASP scan accuracy and URL filter](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2563) +* [#2564 Adjust TANF and SSP Section 1 item numbers](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2564) +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2551 [Bug] - All users are not returned in API response](https://](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2551)) +* [#2457 Path filtering for CI Builds](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2457) +* [#2516 Errors Research Synthesis ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2516) +* [#2527 Error Research informed excel prototype](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2527) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1784 - Email Relay](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1784) +* [#2347 decouple backend apps spike](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2347) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2598 HHS Staging Deployment Failure](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2598) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) + + +### Blocked +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) + + +### Raft Review +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + + + From b9056678ad39ed60f79796d0fd22520fa41fab1a Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 10:44:17 -0600 Subject: [PATCH 30/35] - Resolved failing tests --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index dbab071656..6329fef498 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -64,7 +64,6 @@ def test_parse_small_correct_file(test_datafile, dfs): def test_parse_section_mismatch(test_datafile, dfs): """Test parsing of small_correct_file where the DataFile section doesn't match the rawfile section.""" test_datafile.section = 'Closed Case Data' - test_datafile.save() dfs.datafile = test_datafile dfs.save() @@ -518,6 +517,8 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 + for e in parser_errors: + print(e.error_type, e.error_message) row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE From 29fa8b8953d1e61bf53808d04fdf5e20b1486616 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 11:54:41 -0600 Subject: [PATCH 31/35] - Corrected merge thrash --- tdrs-backend/tdpservice/parsers/parse.py | 33 +++++++++++++------ .../tdpservice/parsers/test/test_parse.py | 1 + 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index 66a18a2dca..fba6d4d962 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -152,25 +152,38 @@ def parse_datafile_lines(datafile, program_type, section): def parse_multi_record_line(line, schema, generate_error): """Parse and validate a datafile line using MultiRecordRowSchema.""" - records = schema.parse_and_validate(line, generate_error) + if schema: + records = schema.parse_and_validate(line, generate_error) - for r in records: - record, record_is_valid, record_errors = r + for r in records: + record, record_is_valid, record_errors = r - if record: - record.save() + if record: + record.save() - return records + return records + + return [(None, False, [ + generate_error( + schema=None, + error_category=ParserErrorCategoryChoices.PRE_CHECK, + error_message="Record Type is missing from record.", + record=None, + field=None + ) + ])] def parse_datafile_line(line, schema, generate_error): """Parse and validate a datafile line and save any errors to the model.""" - record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) + if schema: + record, record_is_valid, record_errors = schema.parse_and_validate(line, generate_error) + + if record: + record.save() - if record: - record.save() + return record_is_valid, record_errors - return record_is_valid, record_errors return (False, [ generate_error( schema=None, diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 6329fef498..4fbe0ae1e5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -64,6 +64,7 @@ def test_parse_small_correct_file(test_datafile, dfs): def test_parse_section_mismatch(test_datafile, dfs): """Test parsing of small_correct_file where the DataFile section doesn't match the rawfile section.""" test_datafile.section = 'Closed Case Data' + test_datafile.save() dfs.datafile = test_datafile dfs.save() From ac9b0659a62f64c4114c41faf0baa659a92be07c Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 13:28:45 -0600 Subject: [PATCH 32/35] - Using randbits to generate pk to get around confilcting sequence pks --- tdrs-backend/tdpservice/stts/test/factories.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/stts/test/factories.py b/tdrs-backend/tdpservice/stts/test/factories.py index 2e20b62cf5..19f5a279de 100644 --- a/tdrs-backend/tdpservice/stts/test/factories.py +++ b/tdrs-backend/tdpservice/stts/test/factories.py @@ -2,6 +2,7 @@ import factory from ..models import STT, Region +import random class RegionFactory(factory.django.DjangoModelFactory): @@ -12,7 +13,7 @@ class Meta: model = "stts.Region" - id = factory.Sequence(int) + id = random.getrandbits(16) @classmethod def _create(cls, model_class, *args, **kwargs): From 8a01767865db9cad134879fb8c81e68080bc72e3 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Mon, 10 Jul 2023 13:39:14 -0600 Subject: [PATCH 33/35] =?UTF-8?q?setup=20can=20trigger=20by=20circleci=20f?= =?UTF-8?q?or=20develop=20branch=20for=20nightly=20scans=20to=E2=80=A6=20(?= =?UTF-8?q?#2610)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * setup can trigger by circleci for develop branch for nightly scans to run now * quick test * removing hotfix branch from tracked branches after successful test --------- Co-authored-by: George Hudson --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5d0be7af65..8b8a62ee79 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,6 +79,7 @@ workflows: filters: branches: only: + - develop - main - master - /^release.*/ From 6c8b3c4bbf1eed9f3b11a5da2c5d2d4c0b923483 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 13:42:02 -0600 Subject: [PATCH 34/35] Revert "- Using randbits to generate pk to get around confilcting sequence pks" This reverts commit ac9b0659a62f64c4114c41faf0baa659a92be07c. --- tdrs-backend/tdpservice/stts/test/factories.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/stts/test/factories.py b/tdrs-backend/tdpservice/stts/test/factories.py index 19f5a279de..2e20b62cf5 100644 --- a/tdrs-backend/tdpservice/stts/test/factories.py +++ b/tdrs-backend/tdpservice/stts/test/factories.py @@ -2,7 +2,6 @@ import factory from ..models import STT, Region -import random class RegionFactory(factory.django.DjangoModelFactory): @@ -13,7 +12,7 @@ class Meta: model = "stts.Region" - id = random.getrandbits(16) + id = factory.Sequence(int) @classmethod def _create(cls, model_class, *args, **kwargs): From 88d4b62ca57092acd2c4fe717e8fc3bd749f9ea7 Mon Sep 17 00:00:00 2001 From: Eric Lipe Date: Mon, 10 Jul 2023 13:52:46 -0600 Subject: [PATCH 35/35] - Updating region in fixture instead of factory - letting django handle transaction for test --- tdrs-backend/tdpservice/parsers/test/test_parse.py | 2 ++ tdrs-backend/tdpservice/parsers/test/test_summary.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 4fbe0ae1e5..d51a1a3009 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -29,6 +29,8 @@ def create_test_datafile(filename, stt_user, stt, section='Active Case Data'): @pytest.fixture def test_datafile(stt_user, stt): """Fixture for small_correct_file.""" + stt.region = None + stt.save() return create_test_datafile('small_correct_file', stt_user, stt) @pytest.fixture diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index 4f92ab8651..14ca122ba2 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -24,7 +24,7 @@ def test_dfs_model(dfs): """Test that the model is created and populated correctly.""" assert dfs.case_aggregates['Jan']['accepted'] == 100 -@pytest.mark.django_db(transaction=True) +@pytest.mark.django_db def test_dfs_rejected(test_datafile, dfs): """Ensure that an invalid file generates a rejected status.""" test_datafile.section = 'Closed Case Data'