2429/single clamav

.circleci/deployment/commands.yml

@@ -37,16 +37,40 @@
           backend-appname: <<parameters.backend-appname>>
           frontend-appname: <<parameters.frontend-appname>>
           cf-space: <<parameters.cf-space>>
-      - deploy-clamav:
-          backend-appname: <<parameters.backend-appname>>
-          cf-org: <<parameters.cf-org>>
-          cf-space: <<parameters.cf-space>>
       - deploy-frontend:
           environment: <<parameters.environment>>
           backend-appname: <<parameters.backend-appname>>
           frontend-appname: <<parameters.frontend-appname>>
           cf-space: <<parameters.cf-space>>
 
+  clamav-cloud-dot-gov:
+    parameters:
+      backend-appname:
+        default: tdp-backend
+        type: string
+      cf-password:
+        default: CF_PASSWORD_DEV
+        type: env_var_name
+      cf-org:
+        default: CF_ORG
+        type: env_var_name
+      cf-space:
+        default: tanf-dev
+        type: string
+      cf-username:
+        default: CF_USERNAME_DEV
+        type: env_var_name
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-org: <<parameters.cf-org>>
+          cf-space: <<parameters.cf-space>>
+          cf-username: <<parameters.cf-username>>
+      - deploy-clamav
+
   deploy-backend:
     parameters:
       backend-appname:
@@ -71,30 +95,12 @@
               <<parameters.cf-space>>
 
   deploy-clamav:
-    parameters:
-      backend-appname:
-        default: tdp-backend
-        type: string
-      cf-org:
-        default: CF_ORG
-        type: env_var_name
-      cf-space:
-        default: tanf-dev
-        type: string
     steps:
       - run:
           name: Deploy ClamAV REST application
           command: |
             cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \
-              --var cf-space=<<parameters.cf-space>> \
-      - run:
-          name: Enable internal route between backend and clamav-rest app
-          command: |
-            cf add-network-policy <<parameters.backend-appname>> clamav-rest \
-              -s <<parameters.cf-space>> \
-              -o ${<<parameters.cf-org>>} \
-              --protocol tcp \
-              --port 9000
+              --var cf-space=tanf-prod
 
   deploy-frontend:
     parameters:

.circleci/deployment/jobs.yml

@@ -151,3 +151,12 @@
           cf-password: CF_PASSWORD_PROD
           cf-space: tanf-prod
           cf-username: CF_USERNAME_PROD
+  prod-deploy-clamav:
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - clamav-cloud-dot-gov:
+          backend-appname: tdp-backend-prod
+          cf-password: CF_PASSWORD_PROD
+          cf-space: tanf-prod
+          cf-username: CF_USERNAME_PROD

.circleci/deployment/workflows.yml

@@ -118,6 +118,13 @@
             branches:
               only:
                 - master
+      - prod-deploy-clamav:
+          requires:
+            - deploy-infrastructure-production
+          filters:
+            branches:
+              only:
+                - master
       - make_erd: # from ../util folder
           filters:
             branches:

docs/Technical-Documentation/README.md

@@ -12,6 +12,7 @@ This directory contains system and architecture documentation including diagrams
 - [buildpack-changelog.md](./buildpack-changelog.md) : A running log of updates to our Cloud.gov buildpacks in use.
 - [circle-ci-audit-template.md](./circle-ci-audit-template.md) : This is a checklist document used during audits of our continuous intergration pipeline tool.
 - [circle-ci.md](./circle-ci.md) : Overview of our CI/CD platform jobs.
+- [clamav.md](./clamav.md) : How to access ClamAV from different apps/spaces.
 - [cypress-integration-tests.md](./cypress-integration-tests.md) : Shows how we use Cypress to manage our end to end integration testing.
 - [data-file-downloads.md](./data-file-downloads.md) : Provides an architecture-level view of data file storage and downloading.
 - [django-admin-logging.md](./django-admin-logging.md) : Outlines sections of the Django Administrator Console and details what should be logged.

docs/Technical-Documentation/clamav.md

@@ -0,0 +1,48 @@
+# CLAMAV 
+
+In order to have one CLAMAV instance, existing in prod, the Nginx router is created
+for CLAMAV to forward the traffic from 'dev' and 'staging' spaces into
+prod space, where the CLAMAV service exists.
+
+## Deploy Nginx instance
+To route the clamav traffic to clamav in prod, each space needs to have one instance of _Nginx Router_ which routes traffic to clamav.
+
+In order to deploy the nginx router instance, change your directory to `tdrs-backend/clamav-router/` and run the following command while logged into the target space:
+
+
+>`cf push {nginx_instance_name} -f manifest.yml --no-route`
+
+, where _nginx_instance_name_ can be : _tdp-clamav-nginx_. 
+
+The instance name then will be set as an environment variable to redirect each instance traffic. This will deploy the nginx instance to the target environment.
+
+## Further communication configurations
+
+### Setup Individual Instances
+
+First, set the environment variable __AV_SCAN_URL__ as follows:
+```
+Environment variable name: AV_SCAN_URL
+Environment variable value: http://{nginx_instance}.apps.internal:9000/scan
+```
+
+### Add network policy from _{tdp-clamav-nginx}_ to clamav in prod
+To enable traffic between the "__nginx instance__" and "__clamav instance in production__", we need to add the network policiy and route between the two:
+
+>`cf add-network-policy {nginx_instance} "clamav-rest" -s "tanf-prod" --protocol tcp --port 9000`
+e.g: `{nginx_instance_name} = tdp-clamav-nginx-dev`
+
+### Add network policy from _{backend_instance}_ to _tdp-clamav-nginx_
+
+>`cf add-network-policy {backend_instance} {nginx_instance} --protocol tcp --port 9000`
+
+where e.g: `backend_instance = tdp-backend-develop`
+
+### Add route for _tdp-clamav-nginx_
+
+ Note: Make sure to delete (if existing) routes that are not being used. In some rare cases, a mal-assigned network policy can interfere with outgoing traffic. As an example, a policy like 
+ >`cf delete-route app.cloud.gov --hostname tdp-frontend-staging`
+
+Add route:
+
+>`cf map-route {nginx_instance} apps.internal --hostname {nginx_instance}`

scripts/deploy-backend.sh

@@ -40,7 +40,6 @@ set_cf_envs()
   "AMS_CLIENT_ID"
   "AMS_CLIENT_SECRET"
   "AMS_CONFIGURATION_ENDPOINT"
-  "AV_SCAN_URL"
   "BASE_URL"
   "CLAMAV_NEEDED"
   "CYPRESS_TOKEN"
@@ -107,6 +106,17 @@ update_backend()
     # Add network policy to allow frontend to access backend
     cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080
 
+    if ["$CF_SPACE" = "tanf-prod" ]; then
+      # Add network policy to allow backend to access tanf-prod services
+      cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000
+      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan"
+    else
+      cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$env --protocol tcp --port 9000
+
+      # Add environment varilables for clamav
+      cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "https://tdp-clamav-nginx-$env.apps.internal:9000/scan"
+    fi
+
     cd ..
 }
 

tdrs-backend/clamav-router/manifest.yml

@@ -0,0 +1,9 @@
+version: 1
+applications:
+- name: tdp-clamav-nginx
+  buildpacks:
+    - https://github.com/cloudfoundry/nginx-buildpack.git#v1.2.6
+  memory: 32M
+  instances: 1
+  disk_quota: 64M
+  timeout: 180

tdrs-backend/clamav-router/nginx.conf

@@ -0,0 +1,20 @@
+events { worker_connections 1024;
+}
+
+# This opens a route to clamav prod
+http{ 
+    server {
+        listen {{port}};
+        location /scan {
+            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+            proxy_pass_request_headers      on;
+        }
+    }
+    server {
+        listen 9000;
+        location /scan {
+            proxy_pass                      http://tanf-prod-clamav-rest.apps.internal:9000/scan;
+            proxy_pass_request_headers      on;
+        }
+    }
+}

tdrs-backend/manifest.clamav.yml

@@ -9,4 +9,4 @@ applications:
   env:
     MAX_FILE_SIZE: 200M
   routes:
-    - route: ((cf-space))-clamav-rest.apps.internal
+    - route: tanf-prod-clamav-rest.apps.internal

tdrs-backend/manifest.yml

@@ -6,5 +6,3 @@ applications:
   disk_quota: 2G
   docker:
     image: ((docker-backend))
-  env:
-    AV_SCAN_URL: http://((cf-space))-clamav-rest.apps.internal:9000/scan

tdrs-backend/tdpservice/settings/cloudgov.py

@@ -43,7 +43,6 @@ class CloudGov(Common):
 
     cloudgov_space = cloudgov_app.get('space_name', 'tanf-dev')
     cloudgov_space_suffix = cloudgov_space.strip('tanf-')
-    AV_SCAN_URL = f'http://tanf-{cloudgov_space_suffix}-clamav-rest.apps.internal:9000/scan'
     cloudgov_name = cloudgov_app.get('name').split("-")[-1]  # converting "tdp-backend-name" to just "name"
     services_basename = cloudgov_name if (
         cloudgov_name == "develop" and cloudgov_space_suffix == "staging"

tdrs-backend/tdpservice/settings/common.py

@@ -326,7 +326,7 @@ class Common(Configuration):
     logger.debug("RAW_CLAMAV: " + str(RAW_CLAMAV))
     CLAMAV_NEEDED = bool(strtobool(RAW_CLAMAV))
 
-    # The URL endpoint to send AV scan requests to (clamav-rest)
+    # The URL endpoint to send AV scan requests to (clamav-rest/clamav-nginx-proxy)
     AV_SCAN_URL = os.getenv('AV_SCAN_URL')
 
     # The factor used to determine how long to wait before retrying failed scans