From 6fe57188879683e0aca124e1939c01fa0300784a Mon Sep 17 00:00:00 2001
From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com>
Date: Mon, 14 Aug 2023 13:55:00 -0400
Subject: [PATCH 1/2] Add cloud-init to change the CloudWatch Agent
 configuration file

Add a cloud-init configuration file template that will write a new
configuration file for the CloudWatch Agent. The template will allow
for the configuration of the base of the CloudWatch log group name and
since we have configured the Agent to start after cloud-init has run it
should be updated before any logs have been pushed. Appropriate locals
have been created to provide standardized base log group names for both
the cyhy and bod portions of the configuration. These include the
Terraform workspace being used resulting in the format:
/instance-logs/<Terraform workspace>/(bod|cyhy)
---
 terraform/README.md                           |   1 +
 terraform/bod_bastion_cloud_init.tf           |   9 +
 terraform/bod_docker_cloud_init.tf            |   9 +
 .../configure_cloudwatch_agent.tpl.yml        | 513 ++++++++++++++++++
 terraform/cyhy_bastion_cloud_init.tf          |   9 +
 terraform/cyhy_dashboard_cloud_init.tf        |   9 +
 terraform/cyhy_mongo_cloud_init.tf            |   9 +
 terraform/cyhy_nessus_cloud_init.tf           |   9 +
 terraform/cyhy_nmap_cloud_init.tf             |   9 +
 terraform/cyhy_reporter_cloud_init.tf         |   9 +
 terraform/locals.tf                           |   8 +
 11 files changed, 594 insertions(+)
 create mode 100644 terraform/cloud-init/configure_cloudwatch_agent.tpl.yml

diff --git a/terraform/README.md b/terraform/README.md
index 3816d016..2537b7ce 100644
--- a/terraform/README.md
+++ b/terraform/README.md
@@ -163,6 +163,7 @@ terraform apply -var-file=<your_workspace>.tfvars
 
 | Name | Source | Version |
 |------|--------|---------|
+| bod\_bastion\_ansible\_provisioner | github.com/cloudposse/terraform-null-ansible | n/a |
 | bod\_docker\_ansible\_provisioner | github.com/cloudposse/terraform-null-ansible | n/a |
 | cyhy\_bastion\_ansible\_provisioner | github.com/cloudposse/terraform-null-ansible | n/a |
 | cyhy\_dashboard\_ansible\_provisioner | github.com/cloudposse/terraform-null-ansible | n/a |
diff --git a/terraform/bod_bastion_cloud_init.tf b/terraform/bod_bastion_cloud_init.tf
index ade680b8..537697fc 100644
--- a/terraform/bod_bastion_cloud_init.tf
+++ b/terraform/bod_bastion_cloud_init.tf
@@ -15,4 +15,13 @@ data "cloudinit_config" "bod_bastion_cloud_init_tasks" {
     filename     = "set_hostname.yml"
     merge_type   = "list(append)+dict(recurse_array)+str()"
   }
+
+  part {
+    content = templatefile("${path.module}/cloud-init/configure_cloudwatch_agent.tpl.yml", {
+      cloudwatch_agent_log_group_base_name = local.bod_cloudwatch_agent_log_group_base
+    })
+    content_type = "text/cloud-config"
+    filename     = "configure_cloudwatch_agent.yml"
+    merge_type   = "list(append)+dict(recurse_array)+str()"
+  }
 }
diff --git a/terraform/bod_docker_cloud_init.tf b/terraform/bod_docker_cloud_init.tf
index af1adf39..433fce2e 100644
--- a/terraform/bod_docker_cloud_init.tf
+++ b/terraform/bod_docker_cloud_init.tf
@@ -17,6 +17,15 @@ data "cloudinit_config" "bod_docker_cloud_init_tasks" {
     merge_type   = "list(append)+dict(recurse_array)+str()"
   }
 
+  part {
+    content = templatefile("${path.module}/cloud-init/configure_cloudwatch_agent.tpl.yml", {
+      cloudwatch_agent_log_group_base_name = local.bod_cloudwatch_agent_log_group_base
+    })
+    content_type = "text/cloud-config"
+    filename     = "configure_cloudwatch_agent.yml"
+    merge_type   = "list(append)+dict(recurse_array)+str()"
+  }
+
   part {
     content = templatefile("${path.module}/cloud-init/chown_directory.tpl.sh", {
       group          = "cyhy"
diff --git a/terraform/cloud-init/configure_cloudwatch_agent.tpl.yml b/terraform/cloud-init/configure_cloudwatch_agent.tpl.yml
new file mode 100644
index 00000000..be438a5e
--- /dev/null
+++ b/terraform/cloud-init/configure_cloudwatch_agent.tpl.yml
@@ -0,0 +1,513 @@
+---
+# overwrite the CloudWatch agent configuration file on the instance
+write_files:
+  - content: |
+      {
+        "agent": {
+          "metrics_collection_interval": 60
+        },
+        "logs": {
+          "logs_collected": {
+            "files": {
+              "collect_list": [
+                {
+                  "file_path": "/var/log/aide/aide.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "aide"
+                },
+                {
+                  "file_path": "/var/alternatives.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "alternatives",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "update-alternatives %Y-%m-%d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/amazon/amazon-cloudwatch-agent/amazon-cloudwatch-agent.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "amazon-cloudwatch-agent",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/amazon/ssm/*.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "amazon-ssm-agent",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/apache2/access_log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "apache2/access"
+                },
+                {
+                  "file_path": "/var/log/apache2/error_log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "apache2/error",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "[%a %b %d %H:%M:%S.%f %Y]"
+                },
+                {
+                  "file_path": "/var/log/apache2/ssl_request_log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "apache2/ssl_request",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "[%d/%b/%Y:%H:%M:%S.%f %z]"
+                },
+                {
+                  "file_path": "/var/log/apt/history.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "apt/history",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "Start-Date: %Y-%m-%d  %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/audit/audit.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "audit",
+                  "multi_line_start_pattern": "^type="
+                },
+                {
+                  "file_path": "/var/log/audit/audit.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/audit",
+                  "log_stream_name": "{local_hostname}",
+                  "multi_line_start_pattern": "^type="
+                },
+                {
+                  "file_path": "/var/log/auth.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "auth",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/clamav/freshclam.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "clamav/freshclam",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%a %b %-d %H:%M:%S %Y"
+                },
+                {
+                  "file_path": "/var/log/freshclam.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "clamav/freshclam",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%a %b %-d %H:%M:%S %Y"
+                },
+                {
+                  "file_path": "/var/log/clamav/lastscan.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "clamav/lastscan"
+                },
+                {
+                  "file_path": "/var/log/cloud-init.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "cloud-init",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
+                },
+                {
+                  "file_path": "/var/log/cloud-init-output.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "cloud-init-output"
+                },
+                {
+                  "file_path": "/var/log/daemon.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "daemon",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/debug",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "debug",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/dirsrv/slapd-*/access",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "dirsrv/slapd/access",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%d/%b/%Y:%H:%M:%S.%f %z"
+                },
+                {
+                  "file_path": "/var/log/dirsrv/slapd-*/audit",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "dirsrv/slapd/audit",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%d/%b/%Y:%H:%M:%S.%f %z"
+                },
+                {
+                  "file_path": "/var/log/dirsrv/slapd-*/errors",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "dirsrv/slapd/errors",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%d/%b/%Y:%H:%M:%S.%f %z"
+                },
+                {
+                  "file_path": "/var/log/dnf.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "dnf",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%S%z"
+                },
+                {
+                  "file_path": "/var/log/dnf.librepo.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "dnf/librepo",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%S%z"
+                },
+                {
+                  "file_path": "/var/log/dnf.rpm.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "dnf/rpm",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%S%z"
+                },
+                {
+                  "file_path": "/var/log/dpkg.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "dpkg",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/hawkey.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "hawkey",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%S%z"
+                },
+                {
+                  "file_path": "/var/log/httpd/access_log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "httpd/access"
+                },
+                {
+                  "file_path": "/var/log/httpd/error_log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "httpd/error",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "[%a %b %d %H:%M:%S.%f %Y]"
+                },
+                {
+                  "file_path": "/var/log/httpd/ssl_request_log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "httpd/ssl_request",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "[%d/%b/%Y:%H:%M:%S.%f %z]"
+                },
+                {
+                  "file_path": "/var/log/ipa/cli.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ipa/cli",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/ipa/ipactl.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ipa/ipactl",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/ipa-custodia.audit.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ipa-custodia/audit",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/ipaclient-install.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ipaclient-install",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/ipaclient-uninstall.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ipaclient-uninstall",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/ipareplica-conncheck.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ipareplica-conncheck",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/ipareplica-install.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ipareplica-install",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/ipaserver-install.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ipaserver-install",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%dT%H:%M:%SZ",
+                  "timezone": "UTC"
+                },
+                {
+                  "file_path": "/var/log/kadmind.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "kadmind",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/kern.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "kern",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/krb5kdc.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "krb5kdc",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/messages",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "messages",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/messages",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/messages",
+                  "log_stream_name": "{local_hostname}",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/secure",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "secure",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/secure",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/secure",
+                  "log_stream_name": "{local_hostname}",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/sssd/ldap_child.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "sssd/ldap_child",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "(%Y-%m-%d %H:%M:%S):"
+                },
+                {
+                  "file_path": "/var/log/sssd/sssd*.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "sssd",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "(%Y-%m-%d %H:%M:%S):"
+                },
+                {
+                  "file_path": "/var/log/sudo.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "sudo",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S : "
+                },
+                {
+                  "file_path": "/var/log/syslog",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "syslog",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/ufw.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "ufw",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/var/log/unattended-upgrades/*.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "unattended-upgrades",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%Y-%m-%d %H:%M:%S,%f"
+                },
+                {
+                  "file_path": "/var/log/user.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "user",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%b %-d %H:%M:%S"
+                },
+                {
+                  "file_path": "/home/vnc/.vnc/*.log",
+                  "log_group_name": "${cloudwatch_agent_log_group_base_name}/{local_hostname}",
+                  "log_stream_name": "vnc",
+                  "multi_line_start_pattern": "{timestamp_format}",
+                  "timestamp_format": "%a %b %-d %H:%M:%S %Y"
+                }
+              ]
+            }
+          }
+        },
+        "metrics": {
+          "aggregation_dimensions": [
+            [
+              "AutoScalingGroupName"
+            ],
+            [
+              "ImageId"
+            ],
+            [
+              "InstanceId"
+            ],
+            [
+              "InstanceType"
+            ]
+          ],
+          "append_dimensions": {
+            "AutoScalingGroupName": "$${aws:AutoScalingGroupName}",
+            "ImageId": "$${aws:ImageId}",
+            "InstanceId": "$${aws:InstanceId}",
+            "InstanceType": "$${aws:InstanceType}"
+          },
+          "metrics_collected": {
+            "cpu": {
+              "measurement": [
+                "cpu_usage_active",
+                "cpu_usage_guest",
+                "cpu_usage_guest_nice",
+                "cpu_usage_idle",
+                "cpu_usage_iowait",
+                "cpu_usage_irq",
+                "cpu_usage_nice",
+                "cpu_usage_softirq",
+                "cpu_usage_steal",
+                "cpu_usage_system",
+                "cpu_usage_user"
+              ],
+              "resources": [
+                "*"
+              ]
+            },
+            "disk": {
+              "ignore_file_system_types": [
+                "devtmpfs",
+                "sysfs",
+                "tmpfs"
+              ],
+              "measurement": [
+                "disk_total",
+                "disk_used",
+                "disk_used_percent"
+              ],
+              "resources": [
+                "*"
+              ]
+            },
+            "diskio": {
+              "measurement": [
+                "diskio_io_time",
+                "diskio_iops_in_progress",
+                "diskio_read_bytes",
+                "diskio_read_time",
+                "diskio_reads",
+                "diskio_write_bytes",
+                "diskio_write_time",
+                "diskio_writes"
+              ],
+              "resources": [
+                "*"
+              ]
+            },
+            "ethtool": {
+              "interface_exclude": [
+                "tun0"
+              ],
+              "metrics_include": [
+                "bw_in_allowance_exceeded",
+                "bw_out_allowance_exceeded",
+                "conntrack_allowance_exceeded",
+                "linklocal_allowance_exceeded",
+                "pps_allowance_exceeded"
+              ]
+            },
+            "mem": {
+              "measurement": [
+                "mem_active",
+                "mem_available",
+                "mem_available_percent",
+                "mem_buffered",
+                "mem_cached",
+                "mem_free",
+                "mem_inactive",
+                "mem_total",
+                "mem_used",
+                "mem_used_percent"
+              ]
+            },
+            "net": {
+              "measurement": [
+                "net_bytes_recv",
+                "net_bytes_sent",
+                "net_drop_in",
+                "net_drop_out",
+                "net_err_in",
+                "net_err_out",
+                "net_packets_sent",
+                "net_packets_recv"
+              ]
+            },
+            "netstat": {
+              "measurement": [
+                "netstat_tcp_close",
+                "netstat_tcp_close_wait",
+                "netstat_tcp_closing",
+                "netstat_tcp_established",
+                "netstat_tcp_fin_wait1",
+                "netstat_tcp_fin_wait2",
+                "netstat_tcp_last_ack",
+                "netstat_tcp_listen",
+                "netstat_tcp_none",
+                "netstat_tcp_syn_recv",
+                "netstat_tcp_syn_sent",
+                "netstat_tcp_time_wait",
+                "udp_socket"
+              ]
+            },
+            "swap": {
+              "measurement": [
+                "swap_used",
+                "swap_used_percent"
+              ]
+            }
+          }
+        }
+      }
+    path: /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
+    permissions: "0600"
diff --git a/terraform/cyhy_bastion_cloud_init.tf b/terraform/cyhy_bastion_cloud_init.tf
index a3697b71..f7bf1194 100644
--- a/terraform/cyhy_bastion_cloud_init.tf
+++ b/terraform/cyhy_bastion_cloud_init.tf
@@ -15,4 +15,13 @@ data "cloudinit_config" "cyhy_bastion_cloud_init_tasks" {
     filename     = "set_hostname.yml"
     merge_type   = "list(append)+dict(recurse_array)+str()"
   }
+
+  part {
+    content = templatefile("${path.module}/cloud-init/configure_cloudwatch_agent.tpl.yml", {
+      cloudwatch_agent_log_group_base_name = local.cyhy_cloudwatch_agent_log_group_base
+    })
+    content_type = "text/cloud-config"
+    filename     = "configure_cloudwatch_agent.yml"
+    merge_type   = "list(append)+dict(recurse_array)+str()"
+  }
 }
diff --git a/terraform/cyhy_dashboard_cloud_init.tf b/terraform/cyhy_dashboard_cloud_init.tf
index 453bbb13..e49097e5 100644
--- a/terraform/cyhy_dashboard_cloud_init.tf
+++ b/terraform/cyhy_dashboard_cloud_init.tf
@@ -16,6 +16,15 @@ data "cloudinit_config" "cyhy_dashboard_cloud_init_tasks" {
     merge_type   = "list(append)+dict(recurse_array)+str()"
   }
 
+  part {
+    content = templatefile("${path.module}/cloud-init/configure_cloudwatch_agent.tpl.yml", {
+      cloudwatch_agent_log_group_base_name = local.cyhy_cloudwatch_agent_log_group_base
+    })
+    content_type = "text/cloud-config"
+    filename     = "configure_cloudwatch_agent.yml"
+    merge_type   = "list(append)+dict(recurse_array)+str()"
+  }
+
   part {
     content = templatefile("${path.module}/cloud-init/chown_directory.tpl.sh", {
       group          = "cyhy"
diff --git a/terraform/cyhy_mongo_cloud_init.tf b/terraform/cyhy_mongo_cloud_init.tf
index 49807012..ccfcfd27 100644
--- a/terraform/cyhy_mongo_cloud_init.tf
+++ b/terraform/cyhy_mongo_cloud_init.tf
@@ -19,6 +19,15 @@ data "cloudinit_config" "cyhy_mongo_cloud_init_tasks" {
     merge_type   = "list(append)+dict(recurse_array)+str()"
   }
 
+  part {
+    content = templatefile("${path.module}/cloud-init/configure_cloudwatch_agent.tpl.yml", {
+      cloudwatch_agent_log_group_base_name = local.cyhy_cloudwatch_agent_log_group_base
+    })
+    content_type = "text/cloud-config"
+    filename     = "configure_cloudwatch_agent.yml"
+    merge_type   = "list(append)+dict(recurse_array)+str()"
+  }
+
   part {
     content = templatefile("${path.module}/cloud-init/chown_directory.tpl.sh", {
       group          = "cyhy"
diff --git a/terraform/cyhy_nessus_cloud_init.tf b/terraform/cyhy_nessus_cloud_init.tf
index f64882ae..3eb89945 100644
--- a/terraform/cyhy_nessus_cloud_init.tf
+++ b/terraform/cyhy_nessus_cloud_init.tf
@@ -19,6 +19,15 @@ data "cloudinit_config" "cyhy_nessus_cloud_init_tasks" {
     merge_type   = "list(append)+dict(recurse_array)+str()"
   }
 
+  part {
+    content = templatefile("${path.module}/cloud-init/configure_cloudwatch_agent.tpl.yml", {
+      cloudwatch_agent_log_group_base_name = local.cyhy_cloudwatch_agent_log_group_base
+    })
+    content_type = "text/cloud-config"
+    filename     = "configure_cloudwatch_agent.yml"
+    merge_type   = "list(append)+dict(recurse_array)+str()"
+  }
+
   part {
     content = templatefile("${path.module}/cloud-init/chown_directory.tpl.sh", {
       group          = "cyhy"
diff --git a/terraform/cyhy_nmap_cloud_init.tf b/terraform/cyhy_nmap_cloud_init.tf
index c228b147..c941b5cb 100644
--- a/terraform/cyhy_nmap_cloud_init.tf
+++ b/terraform/cyhy_nmap_cloud_init.tf
@@ -19,6 +19,15 @@ data "cloudinit_config" "cyhy_nmap_cloud_init_tasks" {
     merge_type   = "list(append)+dict(recurse_array)+str()"
   }
 
+  part {
+    content = templatefile("${path.module}/cloud-init/configure_cloudwatch_agent.tpl.yml", {
+      cloudwatch_agent_log_group_base_name = local.cyhy_cloudwatch_agent_log_group_base
+    })
+    content_type = "text/cloud-config"
+    filename     = "configure_cloudwatch_agent.yml"
+    merge_type   = "list(append)+dict(recurse_array)+str()"
+  }
+
   part {
     content = templatefile("${path.module}/cloud-init/chown_directory.tpl.sh", {
       group          = "cyhy"
diff --git a/terraform/cyhy_reporter_cloud_init.tf b/terraform/cyhy_reporter_cloud_init.tf
index 24a8bddd..660128f6 100644
--- a/terraform/cyhy_reporter_cloud_init.tf
+++ b/terraform/cyhy_reporter_cloud_init.tf
@@ -17,6 +17,15 @@ data "cloudinit_config" "cyhy_reporter_cloud_init_tasks" {
     merge_type   = "list(append)+dict(recurse_array)+str()"
   }
 
+  part {
+    content = templatefile("${path.module}/cloud-init/configure_cloudwatch_agent.tpl.yml", {
+      cloudwatch_agent_log_group_base_name = local.cyhy_cloudwatch_agent_log_group_base
+    })
+    content_type = "text/cloud-config"
+    filename     = "configure_cloudwatch_agent.yml"
+    merge_type   = "list(append)+dict(recurse_array)+str()"
+  }
+
   part {
     content = templatefile("${path.module}/cloud-init/chown_directory.tpl.sh", {
       group          = "cyhy"
diff --git a/terraform/locals.tf b/terraform/locals.tf
index 5b6a61bd..6f325d02 100644
--- a/terraform/locals.tf
+++ b/terraform/locals.tf
@@ -116,6 +116,14 @@ locals {
   bod_public_subdomain  = "bod.ncats."
   mgmt_public_subdomain = "mgmt.ncats."
 
+  # This base will be used by all instances for their CloudWatch Agent
+  # configuration
+  cloudwatch_agent_log_group_base = "/instance-logs/${terraform.workspace}"
+  # CloudWatch Agent log group name base for cyhy instances
+  cyhy_cloudwatch_agent_log_group_base = "${local.cloudwatch_agent_log_group_base}/${local.cyhy_private_domain}"
+  # CloudWatch Agent log group name base for bod instances
+  bod_cloudwatch_agent_log_group_base = "${local.cloudwatch_agent_log_group_base}/${local.bod_private_domain}"
+
   # DNS zone calculations based on requested instances.  The numbers
   # represent the count of IP addresses in a subnet.
   #

From ccd6651638aec498606f86485695b3b2cb81d1a3 Mon Sep 17 00:00:00 2001
From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com>
Date: Fri, 18 Aug 2023 18:33:12 -0400
Subject: [PATCH 2/2] Update the database instance CloudWatch log group

This log group that is created should match the format that we are now
pushing to the instances' CloudWatch Agent configuration.
---
 terraform/README.md                 | 1 -
 terraform/nvdsync_failure_alarms.tf | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/terraform/README.md b/terraform/README.md
index 2537b7ce..3816d016 100644
--- a/terraform/README.md
+++ b/terraform/README.md
@@ -163,7 +163,6 @@ terraform apply -var-file=<your_workspace>.tfvars
 
 | Name | Source | Version |
 |------|--------|---------|
-| bod\_bastion\_ansible\_provisioner | github.com/cloudposse/terraform-null-ansible | n/a |
 | bod\_docker\_ansible\_provisioner | github.com/cloudposse/terraform-null-ansible | n/a |
 | cyhy\_bastion\_ansible\_provisioner | github.com/cloudposse/terraform-null-ansible | n/a |
 | cyhy\_dashboard\_ansible\_provisioner | github.com/cloudposse/terraform-null-ansible | n/a |
diff --git a/terraform/nvdsync_failure_alarms.tf b/terraform/nvdsync_failure_alarms.tf
index 85987113..f0a596a2 100644
--- a/terraform/nvdsync_failure_alarms.tf
+++ b/terraform/nvdsync_failure_alarms.tf
@@ -7,7 +7,7 @@ resource "aws_cloudwatch_log_group" "instance_logs" {
   #
   # We have to account for the fact that the local hostname on the
   # instance drops the local domain name.
-  name = "/instance-logs/${split(".", each.value)[0]}"
+  name = "${local.cloudwatch_agent_log_group_base}/${split(".", each.value)[1]}/${split(".", each.value)[0]}"
 }
 
 # Create a log metric filter that bumps a metric when a syslog