Regenerate Publications

.gitignore

@@ -17,7 +17,7 @@
 .env.production.local
 cypress.env.json
 cypress/videos
-cypress/snapshots
+cypress/screenshots
 
 npm-debug.log*
 yarn-debug.log*

cypress/integration/publication.spec.js

@@ -0,0 +1,59 @@
+const {
+  HH_HOST,
+  HH_USERNAME,
+  HH_PASSWORD,
+  HH_INSTITUTION,
+  HH_AUTH_URL,
+  HH_AUTH_REALM,
+  HH_AUTH_CLIENT_ID
+} = Cypress.env()
+
+describe('HMDA Help', () => {
+  beforeEach(() => {
+    cy.logout({ root: HH_AUTH_URL, realm: HH_AUTH_REALM })
+    cy.login({
+      root: HH_AUTH_URL,
+      realm: HH_AUTH_REALM,
+      client_id: HH_AUTH_CLIENT_ID,
+      redirect_uri: HH_HOST,
+      username: HH_USERNAME,
+      password: HH_PASSWORD
+    })
+    cy.viewport(1600, 900)
+    cy.visit(HH_HOST)
+  })
+
+  it('Can trigger Publication regeneration', () => {
+    cy.on('window:confirm', () => true)
+    let row = 0
+
+    // Search for existing Instititution
+    cy.findByLabelText("LEI").type(HH_INSTITUTION)
+    cy.findByText('Search publications').click()
+
+    // Can't generate Publication for future year
+    cy.get('#publications table tbody tr').eq(row).as('mlarRow')
+    cy.get('@mlarRow').contains('td', '2020')
+    cy.get('@mlarRow').contains('td', 'Modified LAR')
+    cy.get('@mlarRow').contains('td', 'No file')
+    cy.get('@mlarRow').contains('td', 'Regenerate')
+    cy.findAllByText('Regenerate').eq(row).should('have.class', 'disabled')
+
+    // Can generate Publication for past year
+    row = 3
+    cy.get('#publications table tbody tr').eq(row).as('irsRow')
+    cy.get('@irsRow').contains('td', '2019')
+    cy.get('@irsRow').contains('td', 'IRS')
+    cy.get('@irsRow').contains('td', 'Download')
+    cy.get('@irsRow').contains('td', 'Regenerate')
+    cy.findAllByText('Regenerate').eq(row).click()
+    cy.get('@irsRow').contains('Regeneration of 2019 IRS triggered!')
+
+    // Has valid Download links
+    cy.findAllByText('Download').each(link => {
+      cy.get(link).hasValidHref().then(({ status }) => {
+        assert.isTrue(status, `${link.text()} is a valid link`)
+      })
+    })
+  })
+})

cypress/support/index.js

@@ -5,3 +5,20 @@
 
 import '@testing-library/cypress/add-commands'
 import 'cypress-keycloak';
+
+function urlExists(url) {
+  return new Promise((resolve) => {
+    const xhr = new XMLHttpRequest()
+
+    xhr.onreadystatechange = function () {
+      if (xhr.readyState === 4) resolve({ url, status: xhr.status === 200 })
+    }
+
+    xhr.open('HEAD', url)
+    xhr.send()
+  })
+}
+
+Cypress.Commands.add("hasValidHref", { prevSubject: true }, anchor => {
+  return urlExists(anchor.attr("href"))
+})

src/InputSubmit.css

@@ -41,3 +41,30 @@
 .inputSubmit:visited:disabled:active {
   background-color: #d6d7d9;
 }
+
+
+.inputSubmit.secondary {
+  background-color: #5d3a85;
+  margin-left: 1rem;
+}
+.inputSubmit.secondary:hover,
+.inputSubmit.secondary:visited:hover {
+  background-color: #4a039b;
+}
+.inputSubmit.secondary:active,
+.inputSubmit.secondary:visited:active {
+  background-color: #7f54b0;
+}
+.inputSubmit.secondary:disabled,
+.inputSubmit.secondary:visited:disabled {
+  background-color: #aa9dba;
+  pointer-events: none;
+}
+.inputSubmit.secondary:disabled:hover,
+.inputSubmit.secondary:disabled:active,
+.inputSubmit.secondary:disabled:focus,
+.inputSubmit.secondary:visited:disabled:hover,
+.inputSubmit.secondary:visited:disabled:focus,
+.inputSubmit.secondary:visited:disabled:active {
+  background-color: #d6d7d9;
+}

src/InputSubmit.js

@@ -6,29 +6,44 @@ import './InputSubmit.css'
 const values = {
   search: 'Search institutions',
   add: 'Add the institution',
-  update: 'Update the institution'
+  update: 'Update the institution',
+  publications: 'Search publications'
 }
 
 class InputSubmit extends Component {
   render() {
+    let cname = "inputSubmit"
+    const options = {}
+    const { addClass, actionType, disabled, onClick } = this.props
+
+    if(addClass) cname = `${cname} ${addClass}`
+    if(onClick) options.onClick = onClick
+
     return (
       <input
-        className="inputSubmit"
+        id={actionType}
+        className={cname}
         type="submit"
-        value={values[this.props.actionType]}
-        disabled={this.props.disabled}
+        value={values[actionType]}
+        disabled={disabled}
+        {...options}
       />
     )
   }
 }
 
 InputSubmit.defaultProps = {
-  disabled: false
+  disabled: false,
+  className: 'inputSubmit',
+  onClick: () => null
 }
 
 InputSubmit.propTypes = {
   actionType: PropTypes.string.isRequired,
-  disabled: PropTypes.bool
+  disabled: PropTypes.bool,
+  className: PropTypes.string,
+  onClick: PropTypes.func,
+  addClass: PropTypes.string
 }
 
 export default InputSubmit

src/publications/DownloadButton.jsx

@@ -0,0 +1,7 @@
+import React from 'react'
+
+export const DownloadButton = ({ url }) => (
+  <a href={url} target="_blank" rel="noopener noreferrer">
+    Download
+  </a>
+)

src/publications/Message.jsx

@@ -0,0 +1,10 @@
+import React from 'react'
+
+const Message = ({ isError, message }) => {
+  if (!message) return null
+  const className = isError ? 'error' : 'success'
+
+  return <span className={`message ${className}`}>{message}</span>
+}
+
+export default Message

src/publications/PublicationRow.jsx

@@ -0,0 +1,117 @@
+import React, { useState, useEffect } from 'react'
+import LoadingIcon from '../Loading'
+import { LABELS, TOPICS } from './constants'
+import { fetchData } from '../utils/api'
+import { DownloadButton } from './DownloadButton'
+import { RegenerateButton } from './RegenerateButton'
+
+const defaultState = {
+  waiting: false,
+  error: false,
+  message: null
+}
+
+const regenMsg = (label) => 'Begin the regeneration process for ' + label + '?'
+
+export const PublicationRow = ({
+  notFound,
+  fetched,
+  institution,
+  token,
+  type,
+  url,
+}) => {
+  const label = LABELS[type]
+  const topic = TOPICS[type]
+
+  const { lei, respondentName, activityYear: year } = institution
+  const latestURL = `/v2/filing/institutions/${lei}/filings/${year}/submissions/latest`
+  const headers = { Authorization: `Bearer ${token}` }
+
+  const [state, setState] = useState(defaultState)
+  const [seqNum, setSeqNum] = useState(null)
+
+  const updateState = newState => setState((oldState) => ({...oldState, ...newState }))
+  const saveError = message => updateState({ waiting: false, error: true, message})
+
+  // Determine if we are able to trigger a Regeneration
+  useEffect(() => {
+    fetchSequenceNumber(latestURL, { headers }, setSeqNum)
+  }, [headers, setSeqNum, latestURL])
+
+  const handleRegeneration = () => {
+    if (window.confirm(regenMsg(label))) {
+      updateState({ ...defaultState, waiting: true })
+
+      triggerRegeneration(saveError, updateState, {
+        seqNum,
+        topic,
+        lei,
+        year,
+        label,
+        headers,
+      })
+    }
+  }
+
+  return (
+    <tr>
+      <td>{year}</td>
+      <td>{respondentName}</td>
+      <td>{label}</td>
+      <td>
+        {!fetched ? (
+          <LoadingIcon />
+        ) : fetched && notFound ? (
+          "No file"
+        ) : (
+          <DownloadButton url={url} />
+        )}
+      </td>
+      <td>
+        <RegenerateButton
+          onClick={handleRegeneration}
+          error={state.error}
+          message={state.message}
+          waiting={state.waiting}
+          disabled={[null, undefined].indexOf(seqNum) > -1}
+        />
+      </td>
+    </tr>
+  )
+}
+
+// Sequence Number is required for Regeneration
+function fetchSequenceNumber(url, options, setResult) {
+  return fetchData(url, options)
+    .then(({ error, response }) => {
+      if (error) return {}
+      return response.json()
+    })
+    .then((json) => {
+      const sequenceNumber = json && json.id && json.id.sequenceNumber
+      setResult(sequenceNumber)
+    })
+}
+
+// Send a Kafka topic
+function triggerRegeneration(onError, onSuccess, data) {
+  const { seqNum, topic, lei, year, headers, label } = data
+  const regenerationUrl = `/v2/admin/publish/${topic}/institutions/${lei}/filings/${year}/submissions/${seqNum}`
+
+  // Trigger Publication regeneration
+  return fetchData(regenerationUrl, { method: 'POST', headers })
+    .then(({ error, message }) => {
+      if (error) {
+        onError(message)
+        return
+      }
+
+      onSuccess({
+        waiting: false,
+        error: false,
+        message: `Regeneration of ${year} ${label} triggered!`
+      })
+    })
+    .catch((err) => onError(`Some other error: ${err}`))
+}

src/publications/PublicationRows.jsx

@@ -0,0 +1,65 @@
+import React, { useEffect, useState } from 'react'
+import { PublicationRow } from './PublicationRow'
+import { fileExists } from '../utils/file'
+
+const defaultPubState = { fetched: false, url: null, notFound: null }
+
+const PublicationRows = ({ institution, token }) => {
+  const [mlar, setMlar] = useState({ ...defaultPubState })
+  const [irs, setIrs] = useState({ ...defaultPubState })
+  const [loading, setLoading] = useState(true)
+  const { lei, activityYear } = institution
+
+  useEffect(() => {
+    if (!loading) return
+    const env = !!window.location.host.match(/^ffiec/) ? 'prod' : 'dev'
+    const baseUrl = "https://s3.amazonaws.com/cfpb-hmda-public/"
+
+    const irsUrl = `${baseUrl}${env}/reports/disclosure/${activityYear}/${lei}/nationwide/IRS.csv`
+    const mlarUrl = `${baseUrl}${env}/modified-lar/${activityYear}/${lei}.txt`
+
+    const targets = [
+      { url: irsUrl, setter: setIrs },
+      { url: mlarUrl, setter: setMlar },
+    ]
+
+    // Check if Publications exist
+    targets.forEach(({ url, setter}) => {
+      fileExists(url)
+        .then(() =>
+          setter((state) => ({
+            ...state,
+            fetched: true,
+            notFound: null,
+            url,
+          }))
+        )
+        .catch(() =>
+          setter((state) => ({ ...state, fetched: true, notFound: true }))
+        )
+    })
+  }, [lei, activityYear, loading])
+
+  useEffect(() => {
+    if (irs.fetched && mlar.fetched) setLoading(false)
+  }, [irs, mlar, setLoading])
+
+  return (
+    <>
+      <PublicationRow
+        type="mlar"
+        institution={institution}
+        token={token}
+        {...mlar}
+      />
+      <PublicationRow
+        type="irs"
+        institution={institution}
+        token={token}
+        {...irs}
+      />
+    </>
+  )
+}
+
+export default PublicationRows