diff --git a/.circleci/README.md b/.circleci/README.md index 39e484871c..1b0c4c0311 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -14,10 +14,10 @@ This script will generate a complete config for building, testing, and deploying ### Directory structure #### build-and-test -Contains workflows, jobs, and commands for building and testing the application. +Contains workflows, jobs, and commands for building and testing the application. For all development side builds, these are now triggered by GitHub Actions that serve as a filter so only the code that's changed is tested. See [build-all](../.github/workflows/build-all.yml), [build-backend](../.github/workflows/build-backend.yml), and [build-frontend](../.github/workflows/build-frontend.yml) #### infrastructure -Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. +Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. For all development side builds, this is now triggered by GitHub Actions that serve as a filter so only runs when infrastructure code is changed. See [deploy-infrastructure](../.github/workflows/deploy-infrastructure.yml) #### deployment Contains workflows, jobs, and commands for deploying the application on Cloud gov. Note: merges to develop now automatically trigger a develop deploy using [deploy-develop-on-merge](../.github/workflows/deploy-develop-on-merge.yml) and deploys to dev environments happen when a label is created on the PR using [deploy-on-label](../.github/workflows/deploy-on-label.yml) diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml index eb4b9af15c..d722abecf1 100644 --- a/.circleci/base_config.yml +++ b/.circleci/base_config.yml @@ -21,6 +21,18 @@ executors: resource_class: large parameters: + build_and_test_all: + type: boolean + default: false + build_and_test_backend: + type: boolean + default: false + build_and_test_frontend: + type: boolean + default: false + deploy_infrastructure: + type: boolean + default: false develop_branch_deploy: type: boolean default: false @@ -36,3 +48,9 @@ parameters: target_env: type: string default: '' + triggered: + type: boolean + default: false + util_make_erd: + type: boolean + default: false diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml index 9122ab1666..7c0e559b0f 100644 --- a/.circleci/build-and-test/workflows.yml +++ b/.circleci/build-and-test/workflows.yml @@ -1,19 +1,67 @@ # workflows: - build-and-test: - unless: - or: - - << pipeline.parameters.run_dev_deployment >> - - << pipeline.parameters.develop_branch_deploy >> - - << pipeline.parameters.run_owasp_scan >> - - << pipeline.parameters.run_nightly_owasp_scan >> + build-and-test-all: + when: << pipeline.parameters.build_and_test_all >> jobs: - secrets-check + - test-backend: + requires: + - secrets-check - test-frontend: requires: - secrets-check + - test-e2e: + requires: + - secrets-check + + ci-build-and-test-all: + jobs: + - secrets-check: + filters: + branches: + only: + - main + - master + - /^release.*/ - test-backend: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + - test-frontend: + filters: + branches: + only: + - main + - master + - /^release.*/ requires: - secrets-check - test-e2e: + filters: + branches: + only: + - main + - master + - /^release.*/ + requires: + - secrets-check + + build-and-test-backend: + when: << pipeline.parameters.build_and_test_backend >> + jobs: + - secrets-check + - test-backend: + requires: + - secrets-check + + build-and-test-frontend: + when: << pipeline.parameters.build_and_test_frontend >> + jobs: + - secrets-check + - test-frontend: requires: - secrets-check diff --git a/.circleci/config.yml b/.circleci/config.yml index ea0ddd1c49..65715debc2 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,18 @@ orbs: # parameters from github actions parameters: + build_and_test_all: + type: boolean + default: false + build_and_test_backend: + type: boolean + default: false + build_and_test_frontend: + type: boolean + default: false + deploy_infrastructure: + type: boolean + default: false develop_branch_deploy: type: boolean default: false @@ -28,6 +40,12 @@ parameters: target_env: type: string default: '' + triggered: + type: boolean + default: false + util_make_erd: + type: boolean + default: false jobs: setup: @@ -45,6 +63,24 @@ jobs: # our single workflow, that triggers the setup job defined above workflows: - setup: + github-triggered-setup: + when: << pipeline.parameters.triggered >> + jobs: + - setup: + filters: + branches: + ignore: + - main + - master + - /^release.*/ + circle-ci-setup: jobs: - - setup + - setup: + filters: + branches: + only: + - develop + - main + - master + - /^release.*/ + diff --git a/.circleci/infrastructure/workflows.yml b/.circleci/infrastructure/workflows.yml index 46f29aec7a..9cb85e8ed8 100644 --- a/.circleci/infrastructure/workflows.yml +++ b/.circleci/infrastructure/workflows.yml @@ -1,6 +1,10 @@ -#workflows: +#workflows: enable-versioning-for-s3-buckets: - unless: << pipeline.parameters.run_nightly_owasp_scan >> + when: + or: + - << pipeline.parameters.deploy_infrastructure >> + - equal: [ 'main', << pipeline.git.branch >> ] + - equal: [ 'master', << pipeline.git.branch >> ] jobs: - enable-versioning: filters: diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml new file mode 100644 index 0000000000..c03d344abb --- /dev/null +++ b/.github/workflows/build-all.yml @@ -0,0 +1,58 @@ +########################################################################### +# GitHub Action Workflow +# On changes to scripts or changes to the pipeline code to any branch +# besides develop, staging and master triggers the full build and test +# pipeline. +# +# NOTE: develop, staging(main) and master are skipped on the push because this +# would be redundant after running the full set of tests from the PR. +# See build-pr.yml for builds that run on code being merged into develop. +# See deploy-develop-on-merge.yml and make_erd for the workflow +# pipelines that run on merge to develop, staging, and master branches. +# HHS (main and master and release/**) branches build all only +# and are managed in CircleCI +# +# Step 0: Make changes on your branch to files in scripts/ .circleci or .github +# and push changes to your remote branch. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend and +# build_and_test_frontend to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build and test All on push when scripts/commands change +on: + push: + branches-ignore: + - develop + - main + - master + - 'release/**' + paths: + - 'scripts/**' + - '.circleci/**' + - '.github/**' +jobs: + build_and_test_all: + runs-on: ubuntu-latest + name: Initiate deploy job in CircleCI + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }} + payload: '{ + "build_and_test_all": true, + "triggered": true + }' diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..26ef5c03eb --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,44 @@ +########################################################################### +# GitHub Action Workflow +# On push to any branch, triggers the back end build and test pipeline +# if the tdrs-backend has changed. +# +# Step 0: make changes on your branch to non-documentation files in +# tdrs-backend and push changes to your remote branch +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend +# to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build Only Backend When tdrs-backend/ Files Change +on: + push: + paths: 'tdrs-backend/**' + branches-ignore: + - develop + - main + - master +jobs: + build_and_test_backend: + runs-on: ubuntu-latest + name: Build and Test Backend + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "build_and_test_backend": true, + "triggered": true + }' diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml new file mode 100644 index 0000000000..b9b60a9141 --- /dev/null +++ b/.github/workflows/build-frontend.yml @@ -0,0 +1,44 @@ +########################################################################### +# GitHub Action Workflow +# On push to any branch, triggers the front end build and test pipeline +# if the tdrs-frontend has changed. +# +# Step 0: make changes on your branch to non-documentation files in +# tdrs-frontend and push changes to your remote branch +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_frontend +# to run the workflow/jobs listed here: +# build-and-test:[ +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build Only Frontend When tdrs-frontend Files Change +on: + push: + paths: 'tdrs-frontend/**' + branches-ignore: + - develop + - main + - master +jobs: + build_and_test_frontend: + runs-on: ubuntu-latest + name: Build and Test Frontend + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "build_and_test_frontend": true, + "triggered": true + }' diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml new file mode 100644 index 0000000000..5aef710990 --- /dev/null +++ b/.github/workflows/build-pr.yml @@ -0,0 +1,47 @@ +########################################################################### +# GitHub Action Workflow +# On pull requests requesting review from individuals, besides staging, +# master, and release branches triggers the full build and test pipeline. +# +# NOTE: release branches, staging(main) and master are skipped because +# these branch builds are managed in CircleCI +# +# Step 0: make PR from your branch into develop, then select reviewers. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon build_and_test_backend and +# build_and_test_frontend to run the workflow/jobs listed here: +# build-and-test:[ +# test-backend, +# test-frontend, +# test-e2e +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Build and test All for PRs +on: + pull_request: + branches-ignore: #handled in circleci + - main + - master + - 'release/**' + types: [review_requested, ready_for_review, synchronize] +jobs: + build_and_test_pr: + runs-on: ubuntu-latest + name: Initiate deploy job in CircleCI + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }} + payload: '{ + "build_and_test_all": true, + "triggered": true + }' \ No newline at end of file diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml index 80ad9b3088..ee4eee0577 100644 --- a/.github/workflows/deploy-develop-on-merge.yml +++ b/.github/workflows/deploy-develop-on-merge.yml @@ -24,7 +24,7 @@ on: push: branches: - develop - paths_ignore: + paths-ignore: - 'docs/**' - '**.md' - '**.txt' @@ -45,4 +45,8 @@ jobs: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} project-slug: ${{ github.repository }} branch: ${{ github.ref_name }} - payload: '{"develop_branch_deploy": true, "target_env": "develop"}' + payload: '{ + "develop_branch_deploy": true, + "target_env": "develop", + "triggered": true + }' diff --git a/.github/workflows/deploy-infrastructure.yml b/.github/workflows/deploy-infrastructure.yml new file mode 100644 index 0000000000..e5eeeb611a --- /dev/null +++ b/.github/workflows/deploy-infrastructure.yml @@ -0,0 +1,46 @@ +########################################################################### +# GitHub Action Workflow +# On push changing terraform files or infrastructure pipelines, triggers the +# terraform deploy pipeline for the appropriate cf space. +# +# Step 0: make changes to non-documentation files in terraform/ or +# .circleci/infrastructure/ and push/merge changes. +# +# Step 1: Makes a request to the V2 CircleCI API to initiate the project, +# which will filter based upon terraform: true flag +# to run the workflow/jobs listed here: +# build-and-test:[ +# enable-versioning-for-s3-buckets +# ] +# +# Leverages the open source GitHub Action: +# https://github.com/promiseofcake/circleci-trigger-action +########################################################################### +name: Run Infrastructure Pipeline When Terraform or Infrastructure Files Change +on: + push: + branches-ignore: #handled in CircleCI + - main + - master + - 'release/**' + paths: + - 'terraform/**' + - '.circleci/infrastructure/**' +jobs: + run_infrastructure_deployment: + runs-on: ubuntu-latest + name: Deploy Infrastructure + steps: + - uses: actions/checkout@v2 + - name: Circle CI Deployment Trigger + id: curl-circle-ci + uses: promiseofcake/circleci-trigger-action@v1 + with: + user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} + project-slug: ${{ github.repository }} + branch: ${{ github.ref_name }} + payload: '{ + "deploy_infrastructure": true, + "triggered": true + }' + \ No newline at end of file diff --git a/.github/workflows/deploy-on-label.yml b/.github/workflows/deploy-on-label.yml index 857dd3a35d..20fbfd8b33 100644 --- a/.github/workflows/deploy-on-label.yml +++ b/.github/workflows/deploy-on-label.yml @@ -66,4 +66,8 @@ jobs: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} project-slug: ${{ github.repository }} branch: ${{ github.head_ref }} - payload: '{"run_dev_deployment": true, "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}"}' + payload: '{ + "run_dev_deployment": true, + "target_env": "${{steps.extract-deploy-env.outputs.DEPLOY_ENV}}", + "triggered": true + }' diff --git a/.github/workflows/qasp-owasp-scan.yml b/.github/workflows/qasp-owasp-scan.yml index 3a4dea99bd..963bf430c7 100644 --- a/.github/workflows/qasp-owasp-scan.yml +++ b/.github/workflows/qasp-owasp-scan.yml @@ -34,5 +34,6 @@ jobs: branch: ${{ github.head_ref }} payload: | { - "run_owasp_scan": ${{ env.HAS_QASP_LABEL }} + "run_owasp_scan": ${{ env.HAS_QASP_LABEL }}, + "triggered": true } diff --git a/docs/Sprint-Review/sprint-74-summary.md b/docs/Sprint-Review/sprint-74-summary.md new file mode 100644 index 0000000000..9d64430d39 --- /dev/null +++ b/docs/Sprint-Review/sprint-74-summary.md @@ -0,0 +1,54 @@ +# Sprint 74 Summary +05/24/23 - 06/06/23 + +Velocity: 21 + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX to continue STT onboarding (focusing on onboarding CyberFusion users), errors research sessions, provide copy for 2509 (e-mail notification for data submission and errors/transmission report) +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + +## Tickets + +#### Completed/Merged +* [#2439 - [Design] Video Release — Embed videos in knowledge center +](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2439) +* [#2503 [Design] May Knowledge Center Enhancements](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2503) +* [#2531 Bug/Max file size](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2531) +* [#2424 Complete TANF section 1 parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) +* [#2368 errors reporting research](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2368) + + + +#### Submitted (QASP Review, OCIO Review) +* N/A + + +#### Closed (not merged) +* N/A + +### Moved to Next Sprint (Blocked, Raft Review, In Progress) + +#### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +#### Raft Review +* [2486 [Spike - Parser Performance]](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2521 - Update to CFlinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2550 - Deactivation warning emails are missing hyperlinks in staging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2550) + +#### In Progress +* [#2116 [DevOps] Container Registry creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + + + +### Sprint 74 Demo +* Internal: + * UX walkthrough of Knowledge Center enhancements + * [#2424 Complete TANF section 1 parsing](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2424) +* External: + * N/A diff --git a/docs/Sprint-Review/sprint-75-summary.md b/docs/Sprint-Review/sprint-75-summary.md new file mode 100644 index 0000000000..967b0381b8 --- /dev/null +++ b/docs/Sprint-Review/sprint-75-summary.md @@ -0,0 +1,49 @@ +# Sprint 75 Summary + +06/06/23 - 06/20/23 + +Velocity: Dev (1) + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX to continue STT onboarding (focusing on onboarding CyberFusion users), errors research synthesis, copy for e-mail notification of data submission and errors/transmission reports - 2559 +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [2550 Deactivation warning emails are missing e-mail links in staging](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2550) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (In Progress, Blocked, Raft Review) + +### In Progress +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) + +### Blocked +* [#2115 [DevOps] Create utility image(s) for CircleCI pipeline](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2115) + +### Raft Review +* [#2563 Assess OWASP scan accuracy and URL filter](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2563) +* [#2551 [Bug] - All users are not returned in API response](https://](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2551)) +* [#2564 Adjust TANF and SSP Section 1 item numbers](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2564) +* [#2457 Path filtering for CI Builds](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2457) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2516 Errors Research Synthesis ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2516) + + diff --git a/docs/Sprint-Review/sprint-76-summary.md b/docs/Sprint-Review/sprint-76-summary.md new file mode 100644 index 0000000000..0d35a73143 --- /dev/null +++ b/docs/Sprint-Review/sprint-76-summary.md @@ -0,0 +1,54 @@ +# Sprint 76 Summary + +06/21/23 - 07/04/23 + +Velocity: Dev (17) + +## Sprint Goal +* Continue parsing engine development for Section 1 and close out integration test epic (310). +* UX errors template, follow-on research, onboarding +* DevOps to resolve utility images for CircleCI and container registry and close out path filtering for CI builds + + + +## Tickets +### Completed/Merged +* [#2563 Assess OWASP scan accuracy and URL filter](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2563) +* [#2564 Adjust TANF and SSP Section 1 item numbers](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2564) +* [#2521 - Update cflinuxfs4](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2521) +* [#2551 [Bug] - All users are not returned in API response](https://](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2551)) +* [#2457 Path filtering for CI Builds](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2457) +* [#2516 Errors Research Synthesis ](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2516) +* [#2527 Error Research informed excel prototype](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2527) + + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* N/A + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#1784 - Email Relay](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1784) +* [#2347 decouple backend apps spike](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2347) +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) +* [#2598 HHS Staging Deployment Failure](https://app.zenhub.com/workspaces/product-board-5f2c6cdc7c0bb1001bdc43a5/issues/gh/raft-tech/tanf-app/2598) +* [#2116 Container Registry Creation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2116) +* [#2486 Parser Performance](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2486) +* [#2282 As tech lead, I want a file upload integration test](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2282) + + +### Blocked +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) + + +### Raft Review +* [#2369 As tech lead, we need the parsing engine to run quailty checks across TANF section 1](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2369) + + + diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md new file mode 100644 index 0000000000..0e5f9b17ce --- /dev/null +++ b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md @@ -0,0 +1,44 @@ +# 20. Pipeline Build Flow + +Date: 2023-06-07 (_Updated 2023-06-13_) + +## Status + +Pending + +## Context + +We use [CircleCI](https://app.circleci.com/pipelines/github/raft-tech/TANF-app)] as our primary pipeline build tool. On the HHS side, the standard pipeline, build all and deploy ran by CircleCI is sufficiently granular to meet all our build needs. However, on the dev Raft-TANF side, where work can often be held up by waiting for build and test pipelines, it's useful to reduce build times by filtering which workflows run based upon which code is changing. In order to do this, GitHub Actions is leveraged to kick off different CircleCI build workflows since Actions has more granular control over what paths are changed. + +These pipelines are closely tied to the [Git workflow](./009-git-workflow.md) and should run build and test as expected based on what code changed, and still deploy to the correct CF spaces based upon the Decision tree in the [Deployment Flow](./008-deployment-flow.md) + +## Build Logic + +For all release branches, the main, and the master branch, CircleCI should run the full build, infrastructure deploy, and app deployment. + +for feature/development branches, only the build and test pertainint to the code changed should be built. +Front end tests should be run if /tdrs-frontent changes +Back end tests should be run if /tdrs-backend changes +the entire build and test all should run if anything pertaining to the pipeline changes +infrastructure deploy should run if /terraform or infrastructure deploy pipeline changes + +Once a pull request is flagged as ready for review and/or has reviewers assigned, a full build and test all should be run (and tests must pass before merge to develop) + +Develop merges trigger a deploy to the develop CF space, and then a special integration end-2-end test that tests against the real development environment instead of a simulated environment on the CircleCI build servers. + +## Consequences + +**Pros** +* reduce time of build and tests by only running the appropriate workflows. +* only run infrastructure deploys when changes are made to infrastructure code. +* only run end-2-end tests when code is ready to be reviewed. +* only run full integration end-2-end testing when the develop environment assets are updated. + +**Risks** +* Increased pipeline logic complexity + +## Notes + +- For the nuanced build triggers, CircleCI documentation recommends using Actions to trigger CircleCI builds and send different flags to tell which workflows should run. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0 + +- This could be an argument for future complete pipeline switchover to GitHub Actions in order to reduce pipeline complexity, while maintaining the desired build granularity. \ No newline at end of file diff --git a/docs/Technical-Documentation/github-actions.md b/docs/Technical-Documentation/github-actions.md new file mode 100644 index 0000000000..05839498a4 --- /dev/null +++ b/docs/Technical-Documentation/github-actions.md @@ -0,0 +1,5 @@ +# How We Use GitHub Actions +For now, the only use case we have for GitHub Actions is to help up trigger CircleCI builds the way we want to. This is actually the preferred method CircleCI advises for branch, path, pull-request, and labelled filtering and job triggering. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0 + +## Path Filtering +We use Actions to filter which workflows are getting run by CircleCI by sending different flags to CircleCI through the promiseofcake CircleCI API trigger. See the individual files in [.github](../../.github/) for detailed instructions for how to use each. diff --git a/scripts/zap-scanner.sh b/scripts/zap-scanner.sh index 4e017eb7e2..8e1147e565 100755 --- a/scripts/zap-scanner.sh +++ b/scripts/zap-scanner.sh @@ -59,15 +59,90 @@ ZAP_CLI_OPTIONS="\ -config globalexcludeurl.url_list.url\(0\).regex='.*/robots\.txt.*' \ -config globalexcludeurl.url_list.url\(0\).description='Exclude robots.txt' \ -config globalexcludeurl.url_list.url\(0\).enabled=true \ + -config globalexcludeurl.url_list.url\(1\).regex='^https?://.*\.cdn\.mozilla\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(1\).description='Site - Mozilla CDN (requests such as getpocket)' \ -config globalexcludeurl.url_list.url\(1\).enabled=true \ + -config globalexcludeurl.url_list.url\(2\).regex='^https?://.*\.amazonaws\.(?:com|org|net)/.*$' \ -config globalexcludeurl.url_list.url\(2\).description='TDP S3 buckets' \ -config globalexcludeurl.url_list.url\(2\).enabled=true \ - -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.acf.hhs.gov\/v1\/login\/.*$' \ - -config globalexcludeurl.url_list.url\(3\).description='Site - identity pages' \ + + -config globalexcludeurl.url_list.url\(3\).regex='^https:\/\/.*\.hhs.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(3\).description='Site - acf.hhs.gov' \ -config globalexcludeurl.url_list.url\(3\).enabled=true \ + + -config globalexcludeurl.url_list.url\(4\).regex='^https:\/\/.*\.google.com\/.*$' \ + -config globalexcludeurl.url_list.url\(4\).description='Site - Google.com' \ + -config globalexcludeurl.url_list.url\(4\).enabled=true \ + + -config globalexcludeurl.url_list.url\(5\).regex='^https:\/\/.*\.youtube.com\/.*$' \ + -config globalexcludeurl.url_list.url\(5\).description='Site - youtube.com' \ + -config globalexcludeurl.url_list.url\(5\).enabled=true \ + + -config globalexcludeurl.url_list.url\(6\).regex='^https:\/\/.*\.monsido.com\/.*$' \ + -config globalexcludeurl.url_list.url\(6\).description='Site - monsido.com' \ + -config globalexcludeurl.url_list.url\(6\).enabled=true \ + + -config globalexcludeurl.url_list.url\(7\).regex='^https:\/\/.*\.crazyegg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(7\).description='Site - crazyegg.com' \ + -config globalexcludeurl.url_list.url\(7\).enabled=true \ + + -config globalexcludeurl.url_list.url\(8\).regex='^https:\/\/.*\.gstatic.com\/.*$' \ + -config globalexcludeurl.url_list.url\(8\).description='Site - gstatic.com' \ + -config globalexcludeurl.url_list.url\(8\).enabled=true \ + + -config globalexcludeurl.url_list.url\(9\).regex='^https:\/\/.*\.googleapis.com\/.*$' \ + -config globalexcludeurl.url_list.url\(9\).description='Site - GoogleAPIs.com' \ + -config globalexcludeurl.url_list.url\(9\).enabled=true \ + + -config globalexcludeurl.url_list.url\(10\).regex='^https:\/\/.*\.crazyegg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(10\).description='Site - CrazyEgg.com' \ + -config globalexcludeurl.url_list.url\(10\).enabled=true \ + + -config globalexcludeurl.url_list.url\(11\).regex='^https:\/\/.*\.doubleclick.net\/.*$' \ + -config globalexcludeurl.url_list.url\(11\).description='Site - DoubleClick.net' \ + -config globalexcludeurl.url_list.url\(11\).enabled=true \ + + -config globalexcludeurl.url_list.url\(12\).regex='^https:\/\/.*unpkg.com\/.*$' \ + -config globalexcludeurl.url_list.url\(12\).description='Site - Unpkg.com' \ + -config globalexcludeurl.url_list.url\(12\).enabled=true \ + + -config globalexcludeurl.url_list.url\(13\).regex='^https:\/\/.*\.readspeaker.com\/.*$' \ + -config globalexcludeurl.url_list.url\(13\).description='Site - ReadSpeaker.com' \ + -config globalexcludeurl.url_list.url\(13\).enabled=true \ + + -config globalexcludeurl.url_list.url\(14\).regex='^https:\/\/.*\.fontawesome.com\/.*$' \ + -config globalexcludeurl.url_list.url\(14\).description='Site - FontAwesome.com' \ + -config globalexcludeurl.url_list.url\(14\).enabled=true \ + + -config globalexcludeurl.url_list.url\(15\).regex='^https:\/\/.*\.cloud.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(15\).description='Site - Cloud.gov' \ + -config globalexcludeurl.url_list.url\(15\).enabled=true \ + + -config globalexcludeurl.url_list.url\(16\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \ + -config globalexcludeurl.url_list.url\(16\).description='Site - googletagmanager.com' \ + -config globalexcludeurl.url_list.url\(16\).enabled=true \ + + -config globalexcludeurl.url_list.url\(17\).regex='^https:\/\/.*\.cloudflare.com\/.*$' \ + -config globalexcludeurl.url_list.url\(17\).description='Site - CloudFlare.com' \ + -config globalexcludeurl.url_list.url\(17\).enabled=true \ + + -config globalexcludeurl.url_list.url\(18\).regex='^https:\/\/.*\.google-analytics.com\/.*$' \ + -config globalexcludeurl.url_list.url\(18\).description='Site - Google-Analytics.com' \ + -config globalexcludeurl.url_list.url\(18\).enabled=true \ + + -config globalexcludeurl.url_list.url\(19\).regex='^https:\/\/.*\.googletagmanager.com\/.*$' \ + -config globalexcludeurl.url_list.url\(19\).description='Site - googletagmanager.com' \ + -config globalexcludeurl.url_list.url\(19\).enabled=true \ + + -config globalexcludeurl.url_list.url\(20\).regex='^https:\/\/.*\.digitalgov.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(20\).description='Site - DigitalGov.gov' \ + -config globalexcludeurl.url_list.url\(20\).enabled=true \ + + -config globalexcludeurl.url_list.url\(21\).regex='^https:\/\/.*\.identitysandbox.gov\/.*$' \ + -config globalexcludeurl.url_list.url\(21\).description='Site - IdentitySandbox.gov' \ + -config globalexcludeurl.url_list.url\(21\).enabled=true \ -config spider.postform=true" # How long ZAP will crawl the app with the spider process diff --git a/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py b/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py new file mode 100644 index 0000000000..79607377ed --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/migrations/0006_alter_parsererror_item_number.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-06-15 18:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0005_auto_20230601_1510'), + ] + + operations = [ + migrations.AlterField( + model_name='parsererror', + name='item_number', + field=models.CharField(max_length=8, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/parsers/models.py b/tdrs-backend/tdpservice/parsers/models.py index 064993b2df..0adeb96d36 100644 --- a/tdrs-backend/tdpservice/parsers/models.py +++ b/tdrs-backend/tdpservice/parsers/models.py @@ -35,7 +35,7 @@ class Meta: ) row_number = models.IntegerField(null=False) column_number = models.IntegerField(null=True) - item_number = models.IntegerField(null=True) + item_number = models.CharField(null=True, max_length=8) field_name = models.TextField(null=True, max_length=128) rpt_month_year = models.IntegerField(null=True, blank=False) case_number = models.TextField(null=True, max_length=128) diff --git a/tdrs-backend/tdpservice/parsers/parse.py b/tdrs-backend/tdpservice/parsers/parse.py index ecf860dd5a..f224da2b66 100644 --- a/tdrs-backend/tdpservice/parsers/parse.py +++ b/tdrs-backend/tdpservice/parsers/parse.py @@ -188,6 +188,7 @@ def parse_datafile_line(line, schema, generate_error, datafile): return record_is_valid, record_errors + return (False, [ generate_error( schema=None, @@ -197,4 +198,3 @@ def parse_datafile_line(line, schema, generate_error, datafile): field=None ) ]) - diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/header.py b/tdrs-backend/tdpservice/parsers/schema_defs/header.py index b51c433ede..3183142521 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/header.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/header.py @@ -16,34 +16,34 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ + Field(item="2", name='title', type='string', startIndex=0, endIndex=6, required=True, validators=[ validators.matches('HEADER'), ]), - Field(item=2, name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ + Field(item="4", name='year', type='number', startIndex=6, endIndex=10, required=True, validators=[ validators.between(2000, 2099) ]), - Field(item=3, name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ + Field(item="5", name='quarter', type='string', startIndex=10, endIndex=11, required=True, validators=[ validators.oneOf(['1', '2', '3', '4']) ]), - Field(item=4, name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ + Field(item="6", name='type', type='string', startIndex=11, endIndex=12, required=True, validators=[ validators.oneOf(['A', 'C', 'G', 'S']) ]), - Field(item=5, name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ + Field(item="1", name='state_fips', type='string', startIndex=12, endIndex=14, required=True, validators=[ validators.between(0, 99) ]), - Field(item=6, name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ + Field(item="3", name='tribe_code', type='string', startIndex=14, endIndex=17, required=False, validators=[ validators.between(0, 999) ]), - Field(item=7, name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ + Field(item="7", name='program_type', type='string', startIndex=17, endIndex=20, required=True, validators=[ validators.oneOf(['TAN', 'SSP']) ]), - Field(item=8, name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ + Field(item="8", name='edit', type='string', startIndex=20, endIndex=21, required=True, validators=[ validators.oneOf(['1', '2']) ]), - Field(item=9, name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ + Field(item="9", name='encryption', type='string', startIndex=21, endIndex=22, required=False, validators=[ validators.matches('E') ]), - Field(item=10, name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ + Field(item="10", name='update', type='string', startIndex=22, endIndex=23, required=True, validators=[ validators.oneOf(['N', 'D', 'U']) ]), ], diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 79b9c8bc83..590ca4ea1e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -13,89 +13,91 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, + Field(item="4", name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + Field(item="6", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[]), - Field(item=7, name='DISPOSITION', type='number', startIndex=29, endIndex=30, + Field(item="7", name='DISPOSITION', type='number', startIndex=29, endIndex=30, required=True, validators=[]), - Field(item=8, name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, + Field(item="8", name='NBR_FAMILY_MEMBERS', type='number', startIndex=30, endIndex=32, required=True, validators=[]), - Field(item=10, name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, + Field(item="9", name='FAMILY_TYPE', type='number', startIndex=32, endIndex=33, required=True, validators=[]), - Field(item=11, name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, + Field(item="10", name='TANF_ASST_IN_6MONTHS', type='number', startIndex=33, endIndex=34, required=True, validators=[]), - Field(item=12, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, + Field(item="11", name='RECEIVES_SUB_HOUSING', type='number', startIndex=34, endIndex=35, required=True, validators=[]), - Field(item=13, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, + Field(item="12", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=35, endIndex=36, required=True, validators=[]), - Field(item=14, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, + Field(item="13", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=36, endIndex=37, required=True, validators=[]), - Field(item=15, name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, + Field(item="14", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=37, endIndex=41, required=True, validators=[]), - Field(item=16, name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, + Field(item="15", name='RECEIVES_SUB_CC', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=17, name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, + Field(item="16", name='AMT_SUB_CC', type='number', startIndex=42, endIndex=46, required=True, validators=[]), - Field(item=18, name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, + Field(item="17", name='CHILD_SUPPORT_AMT', type='number', startIndex=46, endIndex=50, required=True, validators=[]), - Field(item=19, name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, + Field(item="18", name='FAMILY_CASH_RESOURCES', type='number', startIndex=50, endIndex=54, required=True, validators=[]), - Field(item=20, name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, + Field(item="19A", name='CASH_AMOUNT', type='number', startIndex=54, endIndex=58, required=True, validators=[]), - Field(item=21, name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, + Field(item="19B", name='NBR_MONTHS', type='number', startIndex=58, endIndex=61, required=True, validators=[]), - Field(item=22, name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, + Field(item="20A", name='CC_AMOUNT', type='number', startIndex=61, endIndex=65, required=True, validators=[]), - Field(item=23, name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, + Field(item="20B", name='CHILDREN_COVERED', type='number', startIndex=65, endIndex=67, required=True, validators=[]), - Field(item=24, name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, + Field(item="20C", name='CC_NBR_MONTHS', type='number', startIndex=67, endIndex=70, required=True, validators=[]), - Field(item=25, name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, + Field(item="21A", name='TRANSP_AMOUNT', type='number', startIndex=70, endIndex=74, required=True, validators=[]), - Field(item=26, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, + Field(item="21B", name='TRANSP_NBR_MONTHS', type='number', startIndex=74, endIndex=77, required=True, validators=[]), - Field(item=27, name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, + Field(item="22A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=77, endIndex=81, required=True, validators=[]), - Field(item=28, name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, + Field(item="22B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=81, endIndex=84, required=True, validators=[]), - Field(item=29, name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, + Field(item="23A", name='OTHER_AMOUNT', type='number', startIndex=84, endIndex=88, required=True, validators=[]), - Field(item=30, name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, + Field(item="23B", name='OTHER_NBR_MONTHS', type='number', startIndex=88, endIndex=91, required=True, validators=[]), - Field(item=31, name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, + Field(item="24AI", name='SANC_REDUCTION_AMT', type='number', startIndex=91, endIndex=95, required=True, validators=[]), - Field(item=32, name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, + Field(item="24AII", name='WORK_REQ_SANCTION', type='number', startIndex=95, endIndex=96, required=True, validators=[]), - Field(item=33, name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, + Field(item="24AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=96, endIndex=97, required=True, validators=[]), - Field(item=34, name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, + Field(item="24AIV", name='SANC_TEEN_PARENT', type='number', startIndex=97, endIndex=98, required=True, validators=[]), - Field(item=35, name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, + Field(item="24AV", name='NON_COOPERATION_CSE', type='number', startIndex=98, endIndex=99, required=True, validators=[]), - Field(item=36, name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, + Field(item="24AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=99, endIndex=100, required=True, validators=[]), - Field(item=37, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, + Field(item="24AVII", name='OTHER_SANCTION', type='number', startIndex=100, endIndex=101, required=True, validators=[]), - Field(item=38, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, + Field(item="24B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=101, endIndex=105, required=True, validators=[]), - Field(item=39, name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, + Field(item="24CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=105, endIndex=109, required=True, validators=[]), - Field(item=40, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, + Field(item="24CII", name='FAMILY_CAP', type='number', startIndex=109, endIndex=110, required=True, validators=[]), - Field(item=41, name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, + Field(item="24CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=110, endIndex=111, required=True, validators=[]), - Field(item=42, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, + Field(item="24CIV", name='OTHER_NON_SANCTION', type='number', startIndex=111, endIndex=112, required=True, validators=[]), - Field(item=43, name='BLANK', type='string', startIndex=113, endIndex=150, + Field(item="25", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=112, endIndex=113, + required=True, validators=[]), + Field(item="-1", name='BLANK', type='string', startIndex=113, endIndex=150, required=False, validators=[]), ] ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index fd855181be..cc80e3431c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -13,137 +13,137 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="26", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, + Field(item="27", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[]), - Field(item=6, name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, + Field(item="28", name='DATE_OF_BIRTH', type='string', startIndex=21, endIndex=29, required=True, validators=[]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, + Field(item="29", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, + Field(item="30A", name='RACE_HISPANIC', type='number', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, + Field(item="30B", name='RACE_AMER_INDIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, + Field(item="30C", name='RACE_ASIAN', type='number', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_BLACK', type='number', startIndex=41, endIndex=42, + Field(item="30D", name='RACE_BLACK', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, + Field(item="30E", name='RACE_HAWAIIAN', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='RACE_WHITE', type='number', startIndex=43, endIndex=44, + Field(item="30F", name='RACE_WHITE', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, + Field(item="31", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, + Field(item="32A", name='FED_OASDI_PROGRAM', type='number', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, + Field(item="32B", name='FED_DISABILITY_STATUS', type='number', startIndex=46, endIndex=47, required=True, validators=[]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, + Field(item="32C", name='DISABLED_TITLE_XIVAPDT', type='number', startIndex=47, endIndex=48, required=True, validators=[]), - Field(item=18, name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, + Field(item="32D", name='AID_AGED_BLIND', type='number', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=19, name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, + Field(item="32E", name='RECEIVE_SSI', type='number', startIndex=49, endIndex=50, required=True, validators=[]), - Field(item=20, name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, + Field(item="33", name='MARITAL_STATUS', type='number', startIndex=50, endIndex=51, required=True, validators=[]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, + Field(item="34", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[]), - Field(item=22, name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, + Field(item="35", name='PARENT_MINOR_CHILD', type='number', startIndex=53, endIndex=54, required=True, validators=[]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, + Field(item="36", name='NEEDS_PREGNANT_WOMAN', type='number', startIndex=54, endIndex=55, required=True, validators=[]), - Field(item=24, name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, + Field(item="37", name='EDUCATION_LEVEL', type='number', startIndex=55, endIndex=57, required=True, validators=[]), - Field(item=25, name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, + Field(item="38", name='CITIZENSHIP_STATUS', type='number', startIndex=57, endIndex=58, required=True, validators=[]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, + Field(item="39", name='COOPERATION_CHILD_SUPPORT', type='number', startIndex=58, endIndex=59, required=True, validators=[]), - Field(item=27, name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, + Field(item="40", name='EMPLOYMENT_STATUS', type='number', startIndex=59, endIndex=60, required=True, validators=[]), - Field(item=28, name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, + Field(item="41", name='WORK_ELIGIBLE_INDICATOR', type='number', startIndex=60, endIndex=62, required=True, validators=[]), - Field(item=29, name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, + Field(item="42", name='WORK_PART_STATUS', type='number', startIndex=62, endIndex=64, required=True, validators=[]), - Field(item=30, name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, + Field(item="43", name='UNSUB_EMPLOYMENT', type='number', startIndex=64, endIndex=66, required=True, validators=[]), - Field(item=31, name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, + Field(item="44", name='SUB_PRIVATE_EMPLOYMENT', type='number', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=32, name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, + Field(item="45", name='SUB_PUBLIC_EMPLOYMENT', type='number', startIndex=68, endIndex=70, required=True, validators=[]), - Field(item=33, name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, + Field(item="46A", name='WORK_EXPERIENCE_HOP', type='number', startIndex=70, endIndex=72, required=True, validators=[]), - Field(item=34, name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, + Field(item="46B", name='WORK_EXPERIENCE_EA', type='number', startIndex=72, endIndex=74, required=True, validators=[]), - Field(item=35, name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, + Field(item="46C", name='WORK_EXPERIENCE_HOL', type='number', startIndex=74, endIndex=76, required=True, validators=[]), - Field(item=36, name='OJT', type='number', startIndex=76, endIndex=78, + Field(item="47", name='OJT', type='number', startIndex=76, endIndex=78, required=True, validators=[]), - Field(item=37, name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, + Field(item="48A", name='JOB_SEARCH_HOP', type='number', startIndex=78, endIndex=80, required=True, validators=[]), - Field(item=38, name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, + Field(item="48B", name='JOB_SEARCH_EA', type='number', startIndex=80, endIndex=82, required=True, validators=[]), - Field(item=39, name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, + Field(item="48C", name='JOB_SEARCH_HOL', type='number', startIndex=82, endIndex=84, required=True, validators=[]), - Field(item=40, name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, + Field(item="49A", name='COMM_SERVICES_HOP', type='number', startIndex=84, endIndex=86, required=True, validators=[]), - Field(item=41, name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, + Field(item="49B", name='COMM_SERVICES_EA', type='number', startIndex=86, endIndex=88, required=True, validators=[]), - Field(item=42, name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, + Field(item="49C", name='COMM_SERVICES_HOL', type='number', startIndex=88, endIndex=90, required=True, validators=[]), - Field(item=43, name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, + Field(item="50A", name='VOCATIONAL_ED_TRAINING_HOP', type='number', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=44, name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, + Field(item="50B", name='VOCATIONAL_ED_TRAINING_EA', type='number', startIndex=92, endIndex=94, required=True, validators=[]), - Field(item=45, name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, + Field(item="50C", name='VOCATIONAL_ED_TRAINING_HOL', type='number', startIndex=94, endIndex=96, required=True, validators=[]), - Field(item=46, name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, + Field(item="51A", name='JOB_SKILLS_TRAINING_HOP', type='number', startIndex=96, endIndex=98, required=True, validators=[]), - Field(item=47, name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, + Field(item="51B", name='JOB_SKILLS_TRAINING_EA', type='number', startIndex=98, endIndex=100, required=True, validators=[]), - Field(item=48, name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, + Field(item="51C", name='JOB_SKILLS_TRAINING_HOL', type='number', startIndex=100, endIndex=102, required=True, validators=[]), - Field(item=49, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, + Field(item="52A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='number', startIndex=102, endIndex=104, required=True, validators=[]), - Field(item=50, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, + Field(item="52B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='number', startIndex=104, endIndex=106, required=True, validators=[]), - Field(item=51, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, + Field(item="52C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='number', startIndex=106, endIndex=108, required=True, validators=[]), - Field(item=52, name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, + Field(item="53A", name='SCHOOL_ATTENDENCE_HOP', type='number', startIndex=108, endIndex=110, required=True, validators=[]), - Field(item=53, name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, + Field(item="53B", name='SCHOOL_ATTENDENCE_EA', type='number', startIndex=110, endIndex=112, required=True, validators=[]), - Field(item=54, name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, + Field(item="53C", name='SCHOOL_ATTENDENCE_HOL', type='number', startIndex=112, endIndex=114, required=True, validators=[]), - Field(item=55, name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, + Field(item="54A", name='PROVIDE_CC_HOP', type='number', startIndex=114, endIndex=116, required=True, validators=[]), - Field(item=56, name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, + Field(item="54B", name='PROVIDE_CC_EA', type='number', startIndex=116, endIndex=118, required=True, validators=[]), - Field(item=57, name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, + Field(item="54C", name='PROVIDE_CC_HOL', type='number', startIndex=118, endIndex=120, required=True, validators=[]), - Field(item=58, name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, + Field(item="55", name='OTHER_WORK_ACTIVITIES', type='number', startIndex=120, endIndex=122, required=True, validators=[]), - Field(item=59, name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, + Field(item="56", name='DEEMED_HOURS_FOR_OVERALL', type='number', startIndex=122, endIndex=124, required=True, validators=[]), - Field(item=60, name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, + Field(item="57", name='DEEMED_HOURS_FOR_TWO_PARENT', type='number', startIndex=124, endIndex=126, required=True, validators=[]), - Field(item=61, name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, + Field(item="58", name='EARNED_INCOME', type='number', startIndex=126, endIndex=130, required=True, validators=[]), - Field(item=62, name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, + Field(item="59A", name='UNEARNED_INCOME_TAX_CREDIT', type='number', startIndex=130, endIndex=134, required=True, validators=[]), - Field(item=63, name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, + Field(item="59B", name='UNEARNED_SOCIAL_SECURITY', type='number', startIndex=134, endIndex=138, required=True, validators=[]), - Field(item=64, name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, + Field(item="59C", name='UNEARNED_SSI', type='number', startIndex=138, endIndex=142, required=True, validators=[]), - Field(item=65, name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, + Field(item="59D", name='UNEARNED_WORKERS_COMP', type='number', startIndex=142, endIndex=146, required=True, validators=[]), - Field(item=66, name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, + Field(item="59E", name='OTHER_UNEARNED_INCOME', type='number', startIndex=146, endIndex=150, required=True, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index c269861ff7..c2f1551cc6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -13,47 +13,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, + Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=20, endIndex=28, required=True, validators=[]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, + Field(item="62", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[]), - Field(item=7, name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, + Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, + Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, + Field(item="63C", name='RACE_ASIAN', type='number', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_BLACK', type='number', startIndex=40, endIndex=41, + Field(item="63D", name='RACE_BLACK', type='number', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, + Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_WHITE', type='number', startIndex=42, endIndex=43, + Field(item="63F", name='RACE_WHITE', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, + Field(item="64", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, + Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, + Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, + Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[]), - Field(item=17, name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, + Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=18, name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, + Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=49, endIndex=51, required=True, validators=[]), - Field(item=19, name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, + Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=51, endIndex=52, required=True, validators=[]), - Field(item=20, name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, + Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=52, endIndex=56, required=True, validators=[]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, + Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=56, endIndex=60, required=True, validators=[]) ] ) @@ -67,47 +67,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="3", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="5", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, + Field(item="60", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[]), - Field(item=23, name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, + Field(item="61", name='DATE_OF_BIRTH', type='string', startIndex=61, endIndex=69, required=True, validators=[]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, + Field(item="62", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[]), - Field(item=25, name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, + Field(item="63A", name='RACE_HISPANIC', type='number', startIndex=78, endIndex=79, required=True, validators=[]), - Field(item=26, name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, + Field(item="63B", name='RACE_AMER_INDIAN', type='number', startIndex=79, endIndex=80, required=True, validators=[]), - Field(item=27, name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, + Field(item="63C", name='RACE_ASIAN', type='number', startIndex=80, endIndex=81, required=True, validators=[]), - Field(item=28, name='RACE_BLACK', type='number', startIndex=81, endIndex=82, + Field(item="63D", name='RACE_BLACK', type='number', startIndex=81, endIndex=82, required=True, validators=[]), - Field(item=29, name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, + Field(item="63E", name='RACE_HAWAIIAN', type='number', startIndex=82, endIndex=83, required=True, validators=[]), - Field(item=30, name='RACE_WHITE', type='number', startIndex=83, endIndex=84, + Field(item="63F", name='RACE_WHITE', type='number', startIndex=83, endIndex=84, required=True, validators=[]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, + Field(item="64", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[]), - Field(item=32, name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, + Field(item="65A", name='RECEIVE_NONSSI_BENEFITS', type='number', startIndex=85, endIndex=86, required=True, validators=[]), - Field(item=33, name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, + Field(item="65B", name='RECEIVE_SSI', type='number', startIndex=86, endIndex=87, required=True, validators=[]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, + Field(item="66", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[]), - Field(item=35, name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, + Field(item="67", name='PARENT_MINOR_CHILD', type='number', startIndex=89, endIndex=90, required=True, validators=[]), - Field(item=36, name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, + Field(item="68", name='EDUCATION_LEVEL', type='number', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=37, name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, + Field(item="69", name='CITIZENSHIP_STATUS', type='number', startIndex=92, endIndex=93, required=True, validators=[]), - Field(item=38, name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, + Field(item="70A", name='UNEARNED_SSI', type='number', startIndex=93, endIndex=97, required=True, validators=[]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, + Field(item="70B", name='OTHER_UNEARNED_INCOME', type='number', startIndex=97, endIndex=101, required=True, validators=[]) ] ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index 483950ce87..25f0b91811 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -13,96 +13,96 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, + Field(item="2", name='COUNTY_FIPS_CODE', type='string', startIndex=19, endIndex=22, required=True, validators=[]), - Field(item=5, name='STRATUM', type='number', startIndex=22, endIndex=24, + Field(item="5", name='STRATUM', type='number', startIndex=22, endIndex=24, required=True, validators=[]), - Field(item=6, name='ZIP_CODE', type='string', startIndex=24, endIndex=29, + Field(item="7", name='ZIP_CODE', type='string', startIndex=24, endIndex=29, required=True, validators=[]), - Field(item=7, name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, + Field(item="8", name='FUNDING_STREAM', type='number', startIndex=29, endIndex=30, required=True, validators=[]), - Field(item=8, name='DISPOSITION', type='number', startIndex=30, endIndex=31, + Field(item="9", name='DISPOSITION', type='number', startIndex=30, endIndex=31, required=True, validators=[]), - Field(item=9, name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, + Field(item="10", name='NEW_APPLICANT', type='number', startIndex=31, endIndex=32, required=True, validators=[]), - Field(item=10, name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, + Field(item="11", name='NBR_FAMILY_MEMBERS', type='number', startIndex=32, endIndex=34, required=True, validators=[]), - Field(item=11, name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, + Field(item="12", name='FAMILY_TYPE', type='number', startIndex=34, endIndex=35, required=True, validators=[]), - Field(item=12, name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, + Field(item="13", name='RECEIVES_SUB_HOUSING', type='number', startIndex=35, endIndex=36, required=True, validators=[]), - Field(item=13, name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, + Field(item="14", name='RECEIVES_MED_ASSISTANCE', type='number', startIndex=36, endIndex=37, required=True, validators=[]), - Field(item=14, name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, + Field(item="15", name='RECEIVES_FOOD_STAMPS', type='number', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=15, name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, + Field(item="16", name='AMT_FOOD_STAMP_ASSISTANCE', type='number', startIndex=38, endIndex=42, required=True, validators=[]), - Field(item=16, name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, + Field(item="17", name='RECEIVES_SUB_CC', type='number', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=17, name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, + Field(item="18", name='AMT_SUB_CC', type='number', startIndex=43, endIndex=47, required=True, validators=[]), - Field(item=18, name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, + Field(item="19", name='CHILD_SUPPORT_AMT', type='number', startIndex=47, endIndex=51, required=True, validators=[]), - Field(item=19, name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, + Field(item="20", name='FAMILY_CASH_RESOURCES', type='number', startIndex=51, endIndex=55, required=True, validators=[]), - Field(item=20, name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, + Field(item="21A", name='CASH_AMOUNT', type='number', startIndex=55, endIndex=59, required=True, validators=[]), - Field(item=21, name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, + Field(item="21B", name='NBR_MONTHS', type='number', startIndex=59, endIndex=62, required=True, validators=[]), - Field(item=22, name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, + Field(item="22A", name='CC_AMOUNT', type='number', startIndex=62, endIndex=66, required=True, validators=[]), - Field(item=23, name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, + Field(item="22B", name='CHILDREN_COVERED', type='number', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=24, name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, + Field(item="22C", name='CC_NBR_MONTHS', type='number', startIndex=68, endIndex=71, required=True, validators=[]), - Field(item=25, name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, + Field(item="23A", name='TRANSP_AMOUNT', type='number', startIndex=71, endIndex=75, required=True, validators=[]), - Field(item=26, name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, + Field(item="23B", name='TRANSP_NBR_MONTHS', type='number', startIndex=75, endIndex=78, required=True, validators=[]), - Field(item=27, name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, + Field(item="24A", name='TRANSITION_SERVICES_AMOUNT', type='number', startIndex=78, endIndex=82, required=True, validators=[]), - Field(item=28, name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, + Field(item="24B", name='TRANSITION_NBR_MONTHS', type='number', startIndex=82, endIndex=85, required=True, validators=[]), - Field(item=29, name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, + Field(item="25A", name='OTHER_AMOUNT', type='number', startIndex=85, endIndex=89, required=True, validators=[]), - Field(item=30, name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, + Field(item="25B", name='OTHER_NBR_MONTHS', type='number', startIndex=89, endIndex=92, required=True, validators=[]), - Field(item=31, name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, + Field(item="26AI", name='SANC_REDUCTION_AMT', type='number', startIndex=92, endIndex=96, required=True, validators=[]), - Field(item=32, name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, + Field(item="26AII", name='WORK_REQ_SANCTION', type='number', startIndex=96, endIndex=97, required=True, validators=[]), - Field(item=33, name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, + Field(item="26AIII", name='FAMILY_SANC_ADULT', type='number', startIndex=97, endIndex=98, required=True, validators=[]), - Field(item=34, name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, + Field(item="26AIV", name='SANC_TEEN_PARENT', type='number', startIndex=98, endIndex=99, required=True, validators=[]), - Field(item=35, name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, + Field(item="26AV", name='NON_COOPERATION_CSE', type='number', startIndex=99, endIndex=100, required=True, validators=[]), - Field(item=36, name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, + Field(item="26AVI", name='FAILURE_TO_COMPLY', type='number', startIndex=100, endIndex=101, required=True, validators=[]), - Field(item=37, name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, + Field(item="26AVII", name='OTHER_SANCTION', type='number', startIndex=101, endIndex=102, required=True, validators=[]), - Field(item=38, name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, + Field(item="26B", name='RECOUPMENT_PRIOR_OVRPMT', type='number', startIndex=102, endIndex=106, required=True, validators=[]), - Field(item=39, name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, + Field(item="26CI", name='OTHER_TOTAL_REDUCTIONS', type='number', startIndex=106, endIndex=110, required=True, validators=[]), - Field(item=40, name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, + Field(item="26CII", name='FAMILY_CAP', type='number', startIndex=110, endIndex=111, required=True, validators=[]), - Field(item=41, name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, + Field(item="26CIII", name='REDUCTIONS_ON_RECEIPTS', type='number', startIndex=111, endIndex=112, required=True, validators=[]), - Field(item=42, name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, + Field(item="26CIV", name='OTHER_NON_SANCTION', type='number', startIndex=112, endIndex=113, required=True, validators=[]), - Field(item=43, name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, + Field(item="27", name='WAIVER_EVAL_CONTROL_GRPS', type='number', startIndex=113, endIndex=114, required=True, validators=[]), - Field(item=44, name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, + Field(item="28", name='FAMILY_EXEMPT_TIME_LIMITS', type='number', startIndex=114, endIndex=116, required=True, validators=[]), - Field(item=45, name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, + Field(item="29", name='FAMILY_NEW_CHILD', type='number', startIndex=116, endIndex=117, required=True, validators=[]), - Field(item=46, name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), + Field(item="-1", name='BLANK', type='string', startIndex=117, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index dc94263d45..49ec6eff1e 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -13,143 +13,143 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="30", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, + Field(item="31", name='NONCUSTODIAL_PARENT', type='number', startIndex=20, endIndex=21, required=True, validators=[]), - Field(item=6, name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, + Field(item="32", name='DATE_OF_BIRTH', type='number', startIndex=21, endIndex=29, required=True, validators=[]), - Field(item=7, name='SSN', type='string', startIndex=29, endIndex=38, + Field(item="33", name='SSN', type='string', startIndex=29, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_HISPANIC', type='string', startIndex=38, endIndex=39, + Field(item="34A", name='RACE_HISPANIC', type='string', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_AMER_INDIAN', type='string', startIndex=39, endIndex=40, + Field(item="34B", name='RACE_AMER_INDIAN', type='string', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_ASIAN', type='string', startIndex=40, endIndex=41, + Field(item="34C", name='RACE_ASIAN', type='string', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_BLACK', type='string', startIndex=41, endIndex=42, + Field(item="34D", name='RACE_BLACK', type='string', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_HAWAIIAN', type='string', startIndex=42, endIndex=43, + Field(item="34E", name='RACE_HAWAIIAN', type='string', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='RACE_WHITE', type='string', startIndex=43, endIndex=44, + Field(item="34F", name='RACE_WHITE', type='string', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='GENDER', type='number', startIndex=44, endIndex=45, + Field(item="35", name='GENDER', type='number', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='FED_OASDI_PROGRAM', type='string', startIndex=45, endIndex=46, + Field(item="36A", name='FED_OASDI_PROGRAM', type='string', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='FED_DISABILITY_STATUS', type='string', startIndex=46, endIndex=47, + Field(item="36B", name='FED_DISABILITY_STATUS', type='string', startIndex=46, endIndex=47, required=True, validators=[]), - Field(item=17, name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, + Field(item="36C", name='DISABLED_TITLE_XIVAPDT', type='string', startIndex=47, endIndex=48, required=True, validators=[]), - Field(item=18, name='AID_AGED_BLIND', type='string', startIndex=48, endIndex=49, + Field(item="36D", name='AID_AGED_BLIND', type='string', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=19, name='RECEIVE_SSI', type='string', startIndex=49, endIndex=50, + Field(item="36E", name='RECEIVE_SSI', type='string', startIndex=49, endIndex=50, required=True, validators=[]), - Field(item=20, name='MARITAL_STATUS', type='string', startIndex=50, endIndex=51, + Field(item="37", name='MARITAL_STATUS', type='string', startIndex=50, endIndex=51, required=True, validators=[]), - Field(item=21, name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, + Field(item="38", name='RELATIONSHIP_HOH', type='number', startIndex=51, endIndex=53, required=True, validators=[]), - Field(item=22, name='PARENT_WITH_MINOR_CHILD', type='string', startIndex=53, endIndex=54, + Field(item="39", name='PARENT_WITH_MINOR_CHILD', type='string', startIndex=53, endIndex=54, required=True, validators=[]), - Field(item=23, name='NEEDS_PREGNANT_WOMAN', type='string', startIndex=54, endIndex=55, + Field(item="40", name='NEEDS_PREGNANT_WOMAN', type='string', startIndex=54, endIndex=55, required=True, validators=[]), - Field(item=24, name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, + Field(item="41", name='EDUCATION_LEVEL', type='string', startIndex=55, endIndex=57, required=True, validators=[]), - Field(item=25, name='CITIZENSHIP_STATUS', type='string', startIndex=57, endIndex=58, + Field(item="42", name='CITIZENSHIP_STATUS', type='string', startIndex=57, endIndex=58, required=True, validators=[]), - Field(item=26, name='COOPERATION_CHILD_SUPPORT', type='string', startIndex=58, endIndex=59, + Field(item="43", name='COOPERATION_CHILD_SUPPORT', type='string', startIndex=58, endIndex=59, required=True, validators=[]), - Field(item=27, name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, + Field(item="44", name='MONTHS_FED_TIME_LIMIT', type='string', startIndex=59, endIndex=62, required=True, validators=[]), - Field(item=28, name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, + Field(item="45", name='MONTHS_STATE_TIME_LIMIT', type='string', startIndex=62, endIndex=64, required=True, validators=[]), - Field(item=29, name='CURRENT_MONTH_STATE_EXEMPT', type='string', startIndex=64, endIndex=65, + Field(item="46", name='CURRENT_MONTH_STATE_EXEMPT', type='string', startIndex=64, endIndex=65, required=True, validators=[]), - Field(item=30, name='EMPLOYMENT_STATUS', type='string', startIndex=65, endIndex=66, + Field(item="47", name='EMPLOYMENT_STATUS', type='string', startIndex=65, endIndex=66, required=True, validators=[]), - Field(item=31, name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, + Field(item="48", name='WORK_ELIGIBLE_INDICATOR', type='string', startIndex=66, endIndex=68, required=True, validators=[]), - Field(item=32, name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, + Field(item="49", name='WORK_PART_STATUS', type='string', startIndex=68, endIndex=70, required=True, validators=[]), - Field(item=33, name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, + Field(item="50", name='UNSUB_EMPLOYMENT', type='string', startIndex=70, endIndex=72, required=True, validators=[]), - Field(item=34, name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, + Field(item="51", name='SUB_PRIVATE_EMPLOYMENT', type='string', startIndex=72, endIndex=74, required=True, validators=[]), - Field(item=35, name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, + Field(item="52", name='SUB_PUBLIC_EMPLOYMENT', type='string', startIndex=74, endIndex=76, required=True, validators=[]), - Field(item=36, name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, + Field(item="53A", name='WORK_EXPERIENCE_HOP', type='string', startIndex=76, endIndex=78, required=True, validators=[]), - Field(item=37, name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, + Field(item="53B", name='WORK_EXPERIENCE_EA', type='string', startIndex=78, endIndex=80, required=True, validators=[]), - Field(item=38, name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, + Field(item="53C", name='WORK_EXPERIENCE_HOL', type='string', startIndex=80, endIndex=82, required=True, validators=[]), - Field(item=39, name='OJT', type='string', startIndex=82, endIndex=84, + Field(item="54", name='OJT', type='string', startIndex=82, endIndex=84, required=True, validators=[]), - Field(item=40, name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, + Field(item="55A", name='JOB_SEARCH_HOP', type='string', startIndex=84, endIndex=86, required=True, validators=[]), - Field(item=41, name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, + Field(item="55B", name='JOB_SEARCH_EA', type='string', startIndex=86, endIndex=88, required=True, validators=[]), - Field(item=42, name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, + Field(item="55C", name='JOB_SEARCH_HOL', type='string', startIndex=88, endIndex=90, required=True, validators=[]), - Field(item=43, name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, + Field(item="56A", name='COMM_SERVICES_HOP', type='string', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=44, name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, + Field(item="56B", name='COMM_SERVICES_EA', type='string', startIndex=92, endIndex=94, required=True, validators=[]), - Field(item=45, name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, + Field(item="56C", name='COMM_SERVICES_HOL', type='string', startIndex=94, endIndex=96, required=True, validators=[]), - Field(item=46, name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, + Field(item="57A", name='VOCATIONAL_ED_TRAINING_HOP', type='string', startIndex=96, endIndex=98, required=False, validators=[]), - Field(item=47, name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, + Field(item="57B", name='VOCATIONAL_ED_TRAINING_EA', type='string', startIndex=98, endIndex=100, required=False, validators=[]), - Field(item=48, name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, + Field(item="57C", name='VOCATIONAL_ED_TRAINING_HOL', type='string', startIndex=100, endIndex=102, required=False, validators=[]), - Field(item=49, name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, + Field(item="58A", name='JOB_SKILLS_TRAINING_HOP', type='string', startIndex=102, endIndex=104, required=False, validators=[]), - Field(item=50, name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, + Field(item="58B", name='JOB_SKILLS_TRAINING_EA', type='string', startIndex=104, endIndex=106, required=False, validators=[]), - Field(item=51, name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, + Field(item="58C", name='JOB_SKILLS_TRAINING_HOL', type='string', startIndex=106, endIndex=108, required=False, validators=[]), - Field(item=52, name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, + Field(item="59A", name='ED_NO_HIGH_SCHOOL_DIPL_HOP', type='string', startIndex=108, endIndex=110, required=False, validators=[]), - Field(item=53, name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, + Field(item="59B", name='ED_NO_HIGH_SCHOOL_DIPL_EA', type='string', startIndex=110, endIndex=112, required=False, validators=[]), - Field(item=54, name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, + Field(item="59C", name='ED_NO_HIGH_SCHOOL_DIPL_HOL', type='string', startIndex=112, endIndex=114, required=False, validators=[]), - Field(item=55, name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, + Field(item="60A", name='SCHOOL_ATTENDENCE_HOP', type='string', startIndex=114, endIndex=116, required=False, validators=[]), - Field(item=56, name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, + Field(item="60B", name='SCHOOL_ATTENDENCE_EA', type='string', startIndex=116, endIndex=118, required=False, validators=[]), - Field(item=57, name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, + Field(item="60C", name='SCHOOL_ATTENDENCE_HOL', type='string', startIndex=118, endIndex=120, required=False, validators=[]), - Field(item=58, name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, + Field(item="61A", name='PROVIDE_CC_HOP', type='string', startIndex=120, endIndex=122, required=False, validators=[]), - Field(item=59, name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, + Field(item="61B", name='PROVIDE_CC_EA', type='string', startIndex=122, endIndex=124, required=False, validators=[]), - Field(item=60, name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, + Field(item="61C", name='PROVIDE_CC_HOL', type='string', startIndex=124, endIndex=126, required=False, validators=[]), - Field(item=61, name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, + Field(item="62", name='OTHER_WORK_ACTIVITIES', type='string', startIndex=126, endIndex=128, required=False, validators=[]), - Field(item=62, name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, + Field(item="63", name='DEEMED_HOURS_FOR_OVERALL', type='string', startIndex=128, endIndex=130, required=False, validators=[]), - Field(item=63, name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, + Field(item="64", name='DEEMED_HOURS_FOR_TWO_PARENT', type='string', startIndex=130, endIndex=132, required=False, validators=[]), - Field(item=64, name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, + Field(item="65", name='EARNED_INCOME', type='string', startIndex=132, endIndex=136, required=False, validators=[]), - Field(item=65, name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, + Field(item="66A", name='UNEARNED_INCOME_TAX_CREDIT', type='string', startIndex=136, endIndex=140, required=False, validators=[]), - Field(item=67, name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, + Field(item="66B", name='UNEARNED_SOCIAL_SECURITY', type='string', startIndex=140, endIndex=144, required=False, validators=[]), - Field(item=68, name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, + Field(item="66C", name='UNEARNED_SSI', type='string', startIndex=144, endIndex=148, required=False, validators=[]), - Field(item=69, name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, + Field(item="66D", name='UNEARNED_WORKERS_COMP', type='string', startIndex=148, endIndex=152, required=False, validators=[]), - Field(item=70, name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, + Field(item="66E", name='OTHER_UNEARNED_INCOME', type='string', startIndex=152, endIndex=156, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index c22dddc7a7..4da311c988 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -13,47 +13,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=4, name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, + Field(item="67", name='FAMILY_AFFILIATION', type='number', startIndex=19, endIndex=20, required=True, validators=[]), - Field(item=5, name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, + Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=20, endIndex=28, required=True, validators=[]), - Field(item=6, name='SSN', type='string', startIndex=28, endIndex=37, + Field(item="69", name='SSN', type='string', startIndex=28, endIndex=37, required=True, validators=[]), - Field(item=7, name='RACE_HISPANIC', type='string', startIndex=37, endIndex=38, + Field(item="70A", name='RACE_HISPANIC', type='string', startIndex=37, endIndex=38, required=True, validators=[]), - Field(item=8, name='RACE_AMER_INDIAN', type='string', startIndex=38, endIndex=39, + Field(item="70B", name='RACE_AMER_INDIAN', type='string', startIndex=38, endIndex=39, required=True, validators=[]), - Field(item=9, name='RACE_ASIAN', type='string', startIndex=39, endIndex=40, + Field(item="70C", name='RACE_ASIAN', type='string', startIndex=39, endIndex=40, required=True, validators=[]), - Field(item=10, name='RACE_BLACK', type='string', startIndex=40, endIndex=41, + Field(item="70D", name='RACE_BLACK', type='string', startIndex=40, endIndex=41, required=True, validators=[]), - Field(item=11, name='RACE_HAWAIIAN', type='string', startIndex=41, endIndex=42, + Field(item="70E", name='RACE_HAWAIIAN', type='string', startIndex=41, endIndex=42, required=True, validators=[]), - Field(item=12, name='RACE_WHITE', type='string', startIndex=42, endIndex=43, + Field(item="70F", name='RACE_WHITE', type='string', startIndex=42, endIndex=43, required=True, validators=[]), - Field(item=13, name='GENDER', type='number', startIndex=43, endIndex=44, + Field(item="71", name='GENDER', type='number', startIndex=43, endIndex=44, required=True, validators=[]), - Field(item=14, name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=44, endIndex=45, + Field(item="72A", name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=44, endIndex=45, required=True, validators=[]), - Field(item=15, name='RECEIVE_SSI', type='string', startIndex=45, endIndex=46, + Field(item="72B", name='RECEIVE_SSI', type='string', startIndex=45, endIndex=46, required=True, validators=[]), - Field(item=16, name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, + Field(item="73", name='RELATIONSHIP_HOH', type='number', startIndex=46, endIndex=48, required=True, validators=[]), - Field(item=17, name='PARENT_MINOR_CHILD', type='string', startIndex=48, endIndex=49, + Field(item="74", name='PARENT_MINOR_CHILD', type='string', startIndex=48, endIndex=49, required=True, validators=[]), - Field(item=18, name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, + Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=49, endIndex=51, required=True, validators=[]), - Field(item=19, name='CITIZENSHIP_STATUS', type='string', startIndex=51, endIndex=52, + Field(item="76", name='CITIZENSHIP_STATUS', type='string', startIndex=51, endIndex=52, required=True, validators=[]), - Field(item=20, name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, + Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=52, endIndex=56, required=False, validators=[]), - Field(item=21, name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, + Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=56, endIndex=60, required=False, validators=[]), ], ) @@ -66,47 +66,47 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='RecordType', type='string', startIndex=0, endIndex=2, + Field(item="0", name='RecordType', type='string', startIndex=0, endIndex=2, required=True, validators=[]), - Field(item=2, name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, + Field(item="4", name='RPT_MONTH_YEAR', type='number', startIndex=2, endIndex=8, required=True, validators=[]), - Field(item=3, name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, + Field(item="6", name='CASE_NUMBER', type='string', startIndex=8, endIndex=19, required=True, validators=[]), - Field(item=22, name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, + Field(item="67", name='FAMILY_AFFILIATION', type='number', startIndex=60, endIndex=61, required=True, validators=[]), - Field(item=23, name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, + Field(item="68", name='DATE_OF_BIRTH', type='number', startIndex=61, endIndex=69, required=True, validators=[]), - Field(item=24, name='SSN', type='string', startIndex=69, endIndex=78, + Field(item="69", name='SSN', type='string', startIndex=69, endIndex=78, required=True, validators=[]), - Field(item=25, name='RACE_HISPANIC', type='string', startIndex=78, endIndex=79, + Field(item="70A", name='RACE_HISPANIC', type='string', startIndex=78, endIndex=79, required=True, validators=[]), - Field(item=26, name='RACE_AMER_INDIAN', type='string', startIndex=79, endIndex=80, + Field(item="70B", name='RACE_AMER_INDIAN', type='string', startIndex=79, endIndex=80, required=True, validators=[]), - Field(item=27, name='RACE_ASIAN', type='string', startIndex=80, endIndex=81, + Field(item="70C", name='RACE_ASIAN', type='string', startIndex=80, endIndex=81, required=True, validators=[]), - Field(item=28, name='RACE_BLACK', type='string', startIndex=81, endIndex=82, + Field(item="70D", name='RACE_BLACK', type='string', startIndex=81, endIndex=82, required=True, validators=[]), - Field(item=29, name='RACE_HAWAIIAN', type='string', startIndex=82, endIndex=83, + Field(item="70E", name='RACE_HAWAIIAN', type='string', startIndex=82, endIndex=83, required=True, validators=[]), - Field(item=30, name='RACE_WHITE', type='string', startIndex=83, endIndex=84, + Field(item="70F", name='RACE_WHITE', type='string', startIndex=83, endIndex=84, required=True, validators=[]), - Field(item=31, name='GENDER', type='number', startIndex=84, endIndex=85, + Field(item="71", name='GENDER', type='number', startIndex=84, endIndex=85, required=True, validators=[]), - Field(item=32, name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=85, endIndex=86, + Field(item="72A", name='RECEIVE_NONSSA_BENEFITS', type='string', startIndex=85, endIndex=86, required=True, validators=[]), - Field(item=33, name='RECEIVE_SSI', type='string', startIndex=86, endIndex=87, + Field(item="72B", name='RECEIVE_SSI', type='string', startIndex=86, endIndex=87, required=True, validators=[]), - Field(item=34, name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, + Field(item="73", name='RELATIONSHIP_HOH', type='number', startIndex=87, endIndex=89, required=True, validators=[]), - Field(item=35, name='PARENT_MINOR_CHILD', type='string', startIndex=89, endIndex=90, + Field(item="74", name='PARENT_MINOR_CHILD', type='string', startIndex=89, endIndex=90, required=True, validators=[]), - Field(item=36, name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, + Field(item="75", name='EDUCATION_LEVEL', type='string', startIndex=90, endIndex=92, required=True, validators=[]), - Field(item=37, name='CITIZENSHIP_STATUS', type='string', startIndex=92, endIndex=93, + Field(item="76", name='CITIZENSHIP_STATUS', type='string', startIndex=92, endIndex=93, required=True, validators=[]), - Field(item=38, name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, + Field(item="77A", name='UNEARNED_SSI', type='string', startIndex=93, endIndex=97, required=False, validators=[]), - Field(item=39, name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, + Field(item="77B", name='OTHER_UNEARNED_INCOME', type='string', startIndex=97, endIndex=101, required=False, validators=[]), ], ) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py index 3657cfc515..086c83e85a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/trailer.py @@ -16,13 +16,13 @@ ], postparsing_validators=[], fields=[ - Field(item=1, name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ + Field(item="1", name='title', type='string', startIndex=0, endIndex=7, required=True, validators=[ validators.matches('TRAILER') ]), - Field(item=2, name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ + Field(item="2", name='record_count', type='number', startIndex=7, endIndex=14, required=True, validators=[ validators.between(0, 9999999) ]), - Field(item=3, name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ + Field(item="-1", name='blank', type='string', startIndex=14, endIndex=23, required=False, validators=[ validators.matches(' ') ]), ], diff --git a/tdrs-backend/tdpservice/parsers/test/factories.py b/tdrs-backend/tdpservice/parsers/test/factories.py index e45e78be20..4af7a67070 100644 --- a/tdrs-backend/tdpservice/parsers/test/factories.py +++ b/tdrs-backend/tdpservice/parsers/test/factories.py @@ -68,7 +68,7 @@ class Meta: file = factory.SubFactory(DataFileFactory) row_number = 1 column_number = 1 - item_number = 1 + item_number = "1" field_name = "test field name" case_number = '1' rpt_month_year = 202001 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index e3a373bd5d..d583a2d4c5 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -555,6 +555,8 @@ def test_parse_bad_tfs1_missing_required(bad_tanf_s1__row_missing_required_field parser_errors = ParserError.objects.filter(file=bad_tanf_s1__row_missing_required_field) assert parser_errors.count() == 4 + for e in parser_errors: + print(e.error_type, e.error_message) row_2_error = parser_errors.get(row_number=2) assert row_2_error.error_type == ParserErrorCategoryChoices.FIELD_VALUE diff --git a/tdrs-backend/tdpservice/parsers/test/test_summary.py b/tdrs-backend/tdpservice/parsers/test/test_summary.py index db267d95d4..b4d96c746f 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_summary.py +++ b/tdrs-backend/tdpservice/parsers/test/test_summary.py @@ -24,7 +24,7 @@ def test_dfs_model(dfs): """Test that the model is created and populated correctly.""" assert dfs.case_aggregates['Jan']['accepted'] == 100 -@pytest.mark.django_db(transaction=True) +@pytest.mark.django_db def test_dfs_rejected(test_datafile, dfs): """Ensure that an invalid file generates a rejected status.""" test_datafile.section = 'Closed Case Data' @@ -44,7 +44,7 @@ def test_dfs_set_status(dfs): parser_errors = [] for i in range(2, 4): - parser_errors.append(ParserErrorFactory(row_number=i, category=str(i))) + parser_errors.append(ParserErrorFactory(row_number=i, error_type=str(i))) dfs.status = dfs.get_status(errors={'document': parser_errors}) diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 2625f3d447..c3b58b8d10 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -117,9 +117,24 @@ def validate_single_header_trailer(datafile): return is_valid, error -def validate_header_section_matches_submission(datafile, section): +def validate_header_section_matches_submission(datafile, program_type, section): """Validate header section matches submission section.""" - is_valid = datafile.section == section + section_names = { + 'TAN': { + 'A': DataFile.Section.ACTIVE_CASE_DATA, + 'C': DataFile.Section.CLOSED_CASE_DATA, + 'G': DataFile.Section.AGGREGATE_DATA, + 'S': DataFile.Section.STRATUM_DATA, + }, + 'SSP': { + 'A': DataFile.Section.SSP_ACTIVE_CASE_DATA, + 'C': DataFile.Section.SSP_CLOSED_CASE_DATA, + 'G': DataFile.Section.SSP_AGGREGATE_DATA, + 'S': DataFile.Section.SSP_STRATUM_DATA, + }, + } + + is_valid = datafile.section == section_names.get(program_type, {}).get(section) error = None if not is_valid: