for idaholab#389, multiarch build, fix capa

.github/workflows/api-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/api.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'

.github/workflows/arkime-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/arkime.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/dashboards-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/dashboards.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/dashboards-helper-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/dashboards-helper.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/file-upload-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/file-upload.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/filebeat-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/filebeat.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/freq-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/freq.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/htadmin-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/htadmin.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/logstash-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/logstash.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml

@@ -8,6 +8,7 @@ on:
     paths:
       - 'malcolm-iso/**'
       - 'shared/bin/*'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/configure-capture.py'
       - '!shared/bin/extracted_files_http_server.py'
       - '!shared/bin/web-ui-asset-download.sh'

.github/workflows/netbox-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/netbox.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/nginx-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/nginx.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/extracted_files_http_server.py'

.github/workflows/opensearch-build-and-push-ghcr.yml

@@ -9,6 +9,7 @@ on:
       - 'Dockerfiles/opensearch.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'

.github/workflows/pcap-capture-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/pcap-capture.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'

.github/workflows/pcap-monitor-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/pcap-monitor.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'

.github/workflows/postgresql-build-and-push-ghcr.yml

@@ -9,6 +9,7 @@ on:
       - 'Dockerfiles/postgresql.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'

.github/workflows/redis-build-and-push-ghcr.yml

@@ -9,6 +9,7 @@ on:
       - 'Dockerfiles/redis.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'

.github/workflows/suricata-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/suricata.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'

.github/workflows/zeek-build-and-push-ghcr.yml

@@ -10,6 +10,7 @@ on:
       - 'Dockerfiles/zeek.Dockerfile'
       - 'shared/bin/*'
       - '!shared/bin/agg-init.sh'
+      - '!shared/bin/capa-build.sh'
       - '!shared/bin/common-init.sh'
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/os-disk-config.py'

Dockerfiles/file-monitor.Dockerfile

@@ -92,10 +92,6 @@ ENV YARA_VERSION "4.5.0"
 ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
 ENV YARA_RULES_SRC_DIR "/yara-rules-src"
 ENV YARA_RULES_DIR "/yara-rules"
-ENV CAPA_VERSION "7.0.1"
-ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip"
-ENV CAPA_DIR "/opt/capa"
-ENV CAPA_BIN "${CAPA_DIR}/capa"
 ENV EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR "/opt/assets"
 ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG
 ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE
@@ -109,6 +105,7 @@ ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$
 ENV SUPERCRONIC_CRONTAB "/etc/crontab"
 
 COPY --chmod=755 shared/bin/yara_rules_setup.sh /usr/local/bin/
+COPY --chmod=777 shared/bin/capa-build.sh /usr/local/bin/
 ADD nginx/landingpage/css "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css"
 ADD nginx/landingpage/js "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/js"
 ADD --chmod=644 docs/images/logo/Malcolm_background.png "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/assets/img/bg-masthead.png"
@@ -146,6 +143,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
       libzmq5 \
       psmisc \
       python3 \
+      python3-venv \
       python3-bs4 \
       python3-dev \
       python3-pip \
@@ -183,12 +181,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
     cd /tmp && \
       /usr/local/bin/web-ui-asset-download.sh -o "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css" && \
     cd /tmp && \
-      curl -fsSL -o ./capa.zip "${CAPA_URL}" && \
-      unzip ./capa.zip && \
-      chmod 755 ./capa && \
-      mkdir -p "${CAPA_DIR}" && \
-      mv ./capa "${CAPA_BIN}" && \
-      rm -f ./capa.zip && \
+      /usr/local/bin/capa-build.sh && \
     apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove \
         automake \
         build-essential \
@@ -201,13 +194,14 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
         libssl-dev \
         libtool \
         make \
-        python3-dev && \
+        python3-dev \
+        python3-venv && \
     mkdir -p /var/log/clamav "${CLAMAV_RULES_DIR}" && \
     groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
       useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
       usermod -a -G tty ${PUSER} && \
-    chown -R ${PUSER}:${PGROUP} /var/log/clamav "${CLAMAV_RULES_DIR}" "${CAPA_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" && \
-    find /var/log/clamav "${CLAMAV_RULES_DIR}" "${CAPA_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" -type d -exec chmod 750 "{}" \; && \
+    chown -R ${PUSER}:${PGROUP} /var/log/clamav "${CLAMAV_RULES_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" && \
+    find /var/log/clamav "${CLAMAV_RULES_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" -type d -exec chmod 750 "{}" \; && \
     sed -i 's/^Foreground .*$/Foreground true/g' /etc/clamav/clamd.conf && \
       sed -i "s/^User .*$/User ${PUSER}/g" /etc/clamav/clamd.conf && \
       sed -i "s|^LocalSocket .*$|LocalSocket $CLAMD_SOCKET_FILE|g" /etc/clamav/clamd.conf && \
@@ -223,7 +217,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
       ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/clam_scan.py && \
       ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/yara_scan.py && \
       ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/capa_scan.py && \
-      echo "0 */6 * * * /bin/bash /usr/local/bin/capa-update.sh\n0 */6 * * * /usr/local/bin/yara_rules_setup.sh -r \"${YARA_RULES_SRC_DIR}\" -y \"${YARA_RULES_DIR}\"" > ${SUPERCRONIC_CRONTAB} && \
+      echo "0 */6 * * * /usr/local/bin/yara_rules_setup.sh -r \"${YARA_RULES_SRC_DIR}\" -y \"${YARA_RULES_DIR}\"" > ${SUPERCRONIC_CRONTAB} && \
   apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/* /tmp/*
@@ -248,9 +242,6 @@ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic
 
 WORKDIR /zeek/extract_files
 
-ENV PATH "${CAPA_DIR}:${PATH}"
-
-VOLUME ["$CAPA_DIR"]
 VOLUME ["$CLAMAV_RULES_DIR"]
 VOLUME ["$YARA_RULES_DIR"]
 VOLUME ["$YARA_RULES_SRC_DIR"]

hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot

@@ -23,7 +23,6 @@ GITHUB_API_CURL_ARGS+=( "Accept: application/vnd.github.v3+json" )
 
 SURICATA_RULES_DIR="/etc/suricata/rules"
 
-CAPA_RELEASE_URL="https://api.github.com/repos/fireeye/capa/releases/latest"
 YQ_RELEASE_URL="https://api.github.com/repos/mikefarah/yq/releases/latest"
 SUPERCRONIC_RELEASE_URL="https://api.github.com/repos/aptible/supercronic/releases/latest"
 CROC_RELEASE_URL="https://api.github.com/repos/schollz/croc/releases/latest"
@@ -183,47 +182,7 @@ mv ./yara-rules-src-hedgehog.tar.gz /opt/hedgehog_install_artifacts/
 ###
 
 # capa
-cd /tmp
-
-capa_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$CAPA_RELEASE_URL" | jq '.assets_url' | tr -d '"')"
-
-if [[ "${ARCH,,}" =~ ^arm ]]; then
-	#Build from source for ARM...
-	#Not sure if there is an easier way to get the latest release tag
-	capa_latest_ver=$(curl "${GITHUB_API_CURL_ARGS[@]}" "$capa_assets_url" | jq ".[] | select(.name | contains(\"-linux.zip\")) | .name")
-	# Retrieves strings like "capa-v6.1.0-linux.zip"; below trims out the x.x.x
-	capa_latest_ver=$(echo ${capa_latest_ver#*v})
-	capa_latest_ver=$(echo ${capa_latest_ver%%-*})
-	capa_latest_src_url="https://github.com/mandiant/capa/archive/refs/tags/v${capa_latest_ver}.zip"
-
-	python3 -m venv capa
-	source capa/bin/activate
-	cd capa
-
-	curl "${GITHUB_API_CURL_ARGS[@]}" "${capa_latest_src_url}" -o capa.zip
-	unzip -q capa.zip
-
-	cd capa-${capa_latest_ver}
-	python3 -m pip install -e .[build]
-	python scripts/cache-ruleset.py rules/ cache/
-	pyinstaller .github/pyinstaller/pyinstaller.spec
-	mv dist/capa /usr/local/bin/capa
-
-	deactivate
-
-else
-	# Assume 64-bit Linux otherwise
-	capa_zip_url=$(curl "${GITHUB_API_CURL_ARGS[@]}" "$capa_assets_url" | jq ".[] | select(.browser_download_url | contains(\"-linux.zip\")) | .browser_download_url" | tr -d '"')
-	curl -o capa.zip "${GITHUB_API_CURL_ARGS[@]}" "${capa_zip_url}"
-	unzip ./capa.zip
-	mv ./capa /usr/local/bin/capa
-
-fi
-
-chmod 755 /usr/local/bin/capa
-rm -rf /tmp/capa*
-
-cp /usr/local/bin/capa /opt/hedgehog_install_artifacts/
+/usr/local/bin/build-capa.sh
 ###
 
 # yq

hedgehog-iso/config/package-lists/python.list.chroot

@@ -16,4 +16,5 @@ python3-tz
 python3-wheel
 python3-yaml
 python3-yara
+python3-venv
 python3-zmq

hedgehog-raspi/sensor_install.sh

@@ -50,7 +50,7 @@ BUILD_DEPS+='meson ninja-build python3-dev re2c ruby ruby-dev ruby-rubygems '
 # Build dependencies we're leaving in place after installation (for building new Zeek plugins in the wild, mostly)
 BUILD_DEPS_KEEP='build-essential ccache cmake flex gcc g++ git libfl-dev libgoogle-perftools-dev '
 BUILD_DEPS_KEEP+='libgoogle-perftools4 libkrb5-3 libkrb5-dev libmaxminddb-dev libpcap-dev libssl-dev libtcmalloc-minimal4 '
-BUILD_DEPS_KEEP+='make patch pkg-config python3-git python3-pip python3-semantic-version python3-setuptools python3-venv swig wget zlib1g-dev '
+BUILD_DEPS_KEEP+='make patch pkg-config python3-git python3-pip python3-semantic-version python3-setuptools swig wget zlib1g-dev '
 
 BUILD_ERROR_CODE=1
 

scripts/build.sh

@@ -111,6 +111,7 @@ FILES_IN_IMAGES=(
   "/usr/share/filebeat/filebeat.yml;filebeat-oss"
   "/var/www/upload/filepond/dist/filepond.js;file-upload"
   "/opt/freq_server/freq_server.py;freq"
+  "/usr/local/bin/capa;file-monitor"
   "/var/www/htadmin/htadmin.php;htadmin"
   "/etc/ip_protocol_name_to_number.yaml;logstash"
   "/etc/ja3.yaml;logstash"

shared/bin/capa-build.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+export CAPA_VERSION="7.0.1"
+export CAPA_SRC_URL="https://github.com/mandiant/capa/archive/refs/tags/v${CAPA_VERSION}.zip"
+export CAPA_RULES_URL="https://github.com/mandiant/capa-rules/archive/refs/tags/v${CAPA_VERSION}.zip"
+
+cd /tmp
+mkdir ./capa
+python3 -m venv capa
+. ./capa/bin/activate
+cd ./capa
+curl -fsSL -o ./capa.zip "${CAPA_SRC_URL}"
+unzip -q ./capa.zip
+cd capa-${CAPA_VERSION}
+python3 -m pip install -e .[build]
+curl -fsSL -o ./rules.zip "${CAPA_RULES_URL}"
+unzip -q ./rules.zip
+mv ./capa-rules-${CAPA_VERSION}/* ./rules/
+python3 ./scripts/cache-ruleset.py rules/ cache
+pyinstaller ./.github/pyinstaller/pyinstaller.spec
+mv ./dist/capa /usr/local/bin/capa
+chmod 755 /usr/local/bin/capa
+deactivate
+rm -rf /tmp/capa*