Merge branch 'develop' of https://github.com/raft-tech/TANF-app into …

…friendlynamefixes2

docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md

@@ -9,9 +9,11 @@ Accepted
 
 Applications need to be configured differently depending on where they are running. For example, the backend running locally will have different configuration then the backend running in production.
 
+Further, environment variables can be designated "secret" or not; the term "secret key" is often used in place of secret environment variables. Secret keys are sometimes (but not always) shared between different deployment environments, which makes it useful to have a central "single source of truth" where a secret key can be kept and copied out to different environments. CircleCI solves this use case for us, allowing secret keys to be managed by the project's Environment Variables, and accessed in the deployment process to write to cloud.gov applications.
+
 ## Decision
 
-We will use environment variables to configure applications.
+We will use environment variables to configure applications. We will use Environment Variables in CircleCI to store and manage secret keys.
 
 ## Consequences
 

scripts/deploy-backend.sh

@@ -62,6 +62,7 @@ set_cf_envs()
   "REDIS_URI"
   "JWT_KEY"
   "STAGING_JWT_KEY"
+  "SENDGRID_API_KEY"
   )
 
   echo "Setting environment variables for $CGAPPNAME_BACKEND"

tdrs-backend/docs/api/authentication.md

@@ -4,3 +4,14 @@ For clients to authenticate, they have to authenticate with Login.gov via the ba
  This will allow the backend to identify the browser which requested access and authorize them based on the cookie they provide in their API calls. 
 
  The secured portion of this authorization is due to the httpOnly cookie being inaccessible to the client's local browser. 
+
+# Generating API token
+
+In order to use APIs, an activated `OFA Sys Admin` user has to generate a new token and use it in the API request following these steps:
+1. User has to first login using frontend and going through the normal login process
+2. After user is logged in, user can grab a token at `/v1/security/get-token`
+3. The token then can be used in authorization header. As an example:
+
+```curl --location 'http://{host}/v1/users/' --header 'Authorization: Token {token}'```
+
+Note: the authentication token is available for 24 hours by default but this can be overridden using the `TOKEN_EXPIRATION_HOURS` environment variable.  

tdrs-backend/tdpservice/data_files/test/test_api.py

@@ -96,23 +96,23 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response):
         """Assert the error report file contents match expected with friendly names."""
         ws = DataFileAPITestBase.get_spreadsheet(response)
 
-        COL_ERROR_MESSAGE = 5
+        COL_ERROR_MESSAGE = 4
 
-        assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \
-            + " be in touch when it's ready to use!For now please refer to the reports you receive via email"
-        assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "if Cash Amount :873 validator1 passed then " +  \
+        assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \
+            + "instructions (linked below) when looking up items and allowable values during the data revision process"
+        assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "if Cash Amount :873 validator1 passed then " +  \
             "Cash and Cash Equivalents: Number of Months T1: 0 is not larger than 0."
 
     @staticmethod
     def assert_error_report_ssp_file_content_matches_with_friendly_names(response):
         """Assert the error report file contents match expected with friendly names."""
         ws = DataFileAPITestBase.get_spreadsheet(response)
 
-        COL_ERROR_MESSAGE = 5
+        COL_ERROR_MESSAGE = 4
 
-        assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \
-            + " be in touch when it's ready to use!For now please refer to the reports you receive via email"
-        assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \
+        assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \
+            + "instructions (linked below) when looking up items and allowable values during the data revision process"
+        assert ws.cell(row=7, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \
             "but must be 23."
 
     @staticmethod
@@ -128,11 +128,11 @@ def assert_error_report_file_content_matches_without_friendly_names(response):
         wb = openpyxl.load_workbook('mycls.xlsx')
         ws = wb.get_sheet_by_name('Sheet1')
 
-        COL_ERROR_MESSAGE = 5
+        COL_ERROR_MESSAGE = 4
 
-        assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \
-            + " be in touch when it's ready to use!For now please refer to the reports you receive via email"
-        assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then "
+        assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \
+            + "instructions (linked below) when looking up items and allowable values during the data revision process"
+        assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then "
                                                                   "NBR_MONTHS T1: 0 is not larger than 0.")
 
     @staticmethod

tdrs-backend/tdpservice/data_files/util.py

@@ -36,27 +36,51 @@ def format_error_msg(x):
     output = BytesIO()
     workbook = xlsxwriter.Workbook(output)
     worksheet = workbook.add_worksheet()
+
     report_columns = [
         ('case_number', lambda x: x['case_number']),
         ('year', lambda x: str(x['rpt_month_year'])[0:4] if x['rpt_month_year'] else None),
         ('month', lambda x: calendar.month_name[
             int(str(x['rpt_month_year'])[4:])
             ] if x['rpt_month_year'] else None),
-        ('error_type', lambda x: x['error_type']),
         ('error_message', lambda x: format_error_msg(chk(x))),
         ('item_number', lambda x: x['item_number']),
         ('item_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].values()])),
         ('internal_variable_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].keys()])),
         ('row_number', lambda x: x['row_number']),
-        ('column_number', lambda x: x['column_number'])
     ]
 
     # write beta banner
-    worksheet.write(row, col,
-                    "Error reporting in TDP is still in development." +
-                    "We'll be in touch when it's ready to use!" +
-                    "For now please refer to the reports you receive via email")
+    worksheet.write(
+        row, col,
+        "Please refer to the most recent versions of the coding " +
+        "instructions (linked below) when looking up items " +
+        "and allowable values during the data revision process"
+    )
+
+    row, col = 1, 0
+    worksheet.write_url(
+        row, col,
+        'https://www.acf.hhs.gov/ofa/policy-guidance/tribal-tanf-data-coding-instructions',
+        string='For Tribal TANF data reports: Tribal TANF Instructions',
+    )
+
     row, col = 2, 0
+    worksheet.write_url(
+        row, col,
+        'https://www.acf.hhs.gov/ofa/policy-guidance/acf-ofa-pi-23-04',
+        string='For TANF and SSP-MOE data reports: TANF / SSP-MOE (ACF-199 / ACF-209) Instructions'
+    )
+
+    row, col = 3, 0
+    worksheet.write_url(
+        row, col,
+        'https://tdp-project-updates.app.cloud.gov/knowledge-center/viewing-error-reports.html',
+        string='Visit the Knowledge Center for further guidance on reviewing error reports'
+    )
+
+    row, col = 5, 0
+
     # write csv header
     bold = workbook.add_format({'bold': True})
 
@@ -68,7 +92,7 @@ def format_header(header_list: list):
     [worksheet.write(row, col, format_header(key[0]), bold) for col, key in enumerate(report_columns)]
 
     [
-        worksheet.write(row + 3, col, key[1](data_i)) for col, key in enumerate(report_columns)
+        worksheet.write(row + 6, col, key[1](data_i)) for col, key in enumerate(report_columns)
         for row, data_i in enumerate(data)
     ]
 

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py

@@ -1,7 +1,7 @@
 """Schema for SSP M1 record type."""
 
-
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M1DataSubmissionDocument
@@ -133,15 +133,16 @@
                     required=True,
                     validators=[validators.notEmpty()]
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name='COUNTY_FIPS_CODE',
                     friendly_name="County FIPS code",
                     type='string',
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[validators.isNumber(),]
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="4",

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py

@@ -369,14 +369,14 @@
                     item="37",
                     name='EDUCATION_LEVEL',
                     friendly_name="Educational Level",
-                    type='number',
+                    type='string',
                     startIndex=55,
                     endIndex=57,
                     required=False,
                     validators=[
                         validators.or_validators(
-                            validators.isInLimits(0, 16), validators.isInLimits(98, 99)
-                        )
+                            validators.isInStringRange(1, 16), validators.isInStringRange(98, 99)
+                        ),
                     ]
                 ),
                 Field(
@@ -387,7 +387,7 @@
                     startIndex=57,
                     endIndex=58,
                     required=False,
-                    validators=[validators.oneOf([0, 1, 2, 3, 9])]
+                    validators=[validators.oneOf([1, 2, 3, 9])]
                 ),
                 Field(
                     item="39",
@@ -397,7 +397,7 @@
                     startIndex=58,
                     endIndex=59,
                     required=False,
-                    validators=[validators.oneOf([0, 1, 2, 9])]
+                    validators=[validators.oneOf([1, 2, 9])]
                 ),
                 Field(
                     item="40",

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py

@@ -281,15 +281,15 @@
             item="68",
             name='EDUCATION_LEVEL',
             friendly_name="Educational Level",
-            type='number',
+            type='string',
             startIndex=49,
             endIndex=51,
             required=True,
             validators=[
                 validators.or_validators(
-                    validators.isInStringRange(0, 16),
+                    validators.isInStringRange(1, 16),
                     validators.isInStringRange(98, 99)
-                )
+                ),
             ]
         ),
         Field(
@@ -300,7 +300,7 @@
             startIndex=51,
             endIndex=52,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 3, 9])]
+            validators=[validators.oneOf([1, 2, 3, 9])]
         ),
         Field(
             item="70A",
@@ -595,13 +595,13 @@
             item="68",
             name='EDUCATION_LEVEL',
             friendly_name="Educational Level",
-            type='number',
+            type='string',
             startIndex=90,
             endIndex=92,
             required=True,
             validators=[
                 validators.or_validators(
-                    validators.isInStringRange(0, 16),
+                    validators.isInStringRange(1, 16),
                     validators.isInStringRange(98, 99)
                 )
             ]
@@ -614,7 +614,7 @@
             startIndex=92,
             endIndex=93,
             required=False,
-            validators=[validators.oneOf([0, 1, 2, 3, 9])]
+            validators=[validators.oneOf([1, 2, 3, 9])]
         ),
         Field(
             item="70A",

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py

@@ -1,7 +1,7 @@
 """Schema for SSP M1 record type."""
 
-
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M4DataSubmissionDocument
@@ -54,15 +54,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="County FIPS code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[validators.isInStringRange(0, 999)],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="4",

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py

@@ -347,7 +347,8 @@
                         validators.or_validators(
                             validators.isInStringRange(0, 16),
                             validators.isInStringRange(98, 99),
-                        )
+                        ),
+                        validators.notMatches("00")
                     ],
                 ),
                 Field(
@@ -359,10 +360,7 @@
                     endIndex=57,
                     required=False,
                     validators=[
-                        validators.or_validators(
-                            validators.isInLimits(0, 3),
-                            validators.matches(9)
-                        )
+                        validators.oneOf([1, 2, 3, 9]),
                     ],
                 ),
                 Field(

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py

@@ -1,10 +1,10 @@
 """Schema for HEADER row of all submission types."""
 
 
-from ...transforms import calendar_quarter_to_rpt_month_year
-from ...fields import Field, TransformField
-from ...row_schema import RowSchema, SchemaManager
-from ... import validators
+from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument
 
 s1 = RowSchema(

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py

@@ -1,9 +1,9 @@
 """Schema for TANF T7 Row."""
 
-from ...fields import Field, TransformField
-from ...row_schema import RowSchema, SchemaManager
-from ...transforms import calendar_quarter_to_rpt_month_year
-from ... import validators
+from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year
+from tdpservice.parsers.fields import Field, TransformField
+from tdpservice.parsers.row_schema import RowSchema, SchemaManager
+from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument
 
 schemas = []

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py

@@ -1,6 +1,7 @@
 """Schema for t1 record types."""
 
-from tdpservice.parsers.fields import Field
+from tdpservice.parsers.transforms import zero_pad
+from tdpservice.parsers.fields import Field, TransformField
 from tdpservice.parsers.row_schema import RowSchema, SchemaManager
 from tdpservice.parsers import validators
 from tdpservice.search_indexes.documents.tanf import TANF_T1DataSubmissionDocument
@@ -157,17 +158,16 @@
                     required=True,
                     validators=[validators.notEmpty()],
                 ),
-                Field(
+                TransformField(
+                    zero_pad(3),
                     item="2",
                     name="COUNTY_FIPS_CODE",
                     friendly_name="County FIPS Code",
                     type="string",
                     startIndex=19,
                     endIndex=22,
                     required=True,
-                    validators=[
-                        validators.isNumber(),
-                    ],
+                    validators=[validators.isNumber()],
                 ),
                 Field(
                     item="5",

tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py

@@ -98,7 +98,6 @@
                     result_field_name="COOPERATION_CHILD_SUPPORT",
                     result_function=validators.oneOf((1, 2, 9)),
                 ),
-                validators.validate__FAM_AFF__HOH__Fed_Time(),
                 validators.if_then_validator(
                     condition_field_name="FAMILY_AFFILIATION",
                     condition_function=validators.isInLimits(1, 3),