From b792a4127ab6e157803dda411636f1b2d0092148 Mon Sep 17 00:00:00 2001 From: Grzegorz Siewruk Date: Fri, 20 Sep 2024 12:10:06 +0200 Subject: [PATCH] sbom generation --- backend/Dockerfile | 12 +++ .../mixewayflowapi/db/entity/CodeRepo.java | 9 +- .../coderepo/UpdateCodeRepoService.java | 5 + .../DependencyTrackApiClientService.java | 15 ++- .../scanner/sca/service/CdxGenService.java | 99 +++++++++++++++++++ .../scanmanager/scheduler/ScanScheduler.java | 4 +- .../service/ScanManagerService.java | 1 + 7 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/service/CdxGenService.java diff --git a/backend/Dockerfile b/backend/Dockerfile index 377a5f6..6172bce 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -14,7 +14,19 @@ WORKDIR /app # Copy the built JAR file from the previous stage to the container COPY --from=maven_build /app/target/MixewayFlowAPI-0.0.1-SNAPSHOT.jar flowapi.jar +# Install dependencies and Node.js +RUN apt-get update && \ + apt-get install -y curl && \ + curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ + apt-get install -y \ + wget \ + unzip \ + git \ + nodejs \ + && apt-get clean +# Install cdxgen globally +RUN npm install -g @cyclonedx/cdxgen # Install dependencies RUN apt-get update && apt-get install -y \ diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/db/entity/CodeRepo.java b/backend/src/main/java/io/mixeway/mixewayflowapi/db/entity/CodeRepo.java index 74a922f..ee1cdaa 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/db/entity/CodeRepo.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/db/entity/CodeRepo.java @@ -17,7 +17,7 @@ public final class CodeRepo { public enum ScanStatus { - SUCCESS, DANGER, WARNING, NOT_PERFORMED + SUCCESS, DANGER, WARNING, NOT_PERFORMED, RUNNING } public enum RepoType { GITLAB, GITHUB @@ -173,6 +173,13 @@ public void updateSecretsScanStatus(ScanStatus status) { this.secretsScan = status; } + public void startScan(){ + this.secretsScan = ScanStatus.RUNNING; + this.iacScan = ScanStatus.RUNNING; + this.scaScan = ScanStatus.RUNNING; + this.sastScan = ScanStatus.RUNNING; + } + public String getGitHostUrl() throws MalformedURLException { URL url = new URL(this.repourl); String port = (url.getPort() == -1) ? "" : ":" + url.getPort(); diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/UpdateCodeRepoService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/UpdateCodeRepoService.java index 9ef12d5..e221ecf 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/UpdateCodeRepoService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/UpdateCodeRepoService.java @@ -195,4 +195,9 @@ private int countCriticalFindings(Finding.Source source, CodeRepo codeRepo, Code .filter(finding -> finding.getSeverity() == Finding.Severity.CRITICAL) .count(); } + + public void setScanRunning(CodeRepo codeRepo) { + codeRepo.startScan(); + codeRepoRepository.save(codeRepo); + } } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/apiclient/DependencyTrackApiClientService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/apiclient/DependencyTrackApiClientService.java index c997d86..496bb67 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/apiclient/DependencyTrackApiClientService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/apiclient/DependencyTrackApiClientService.java @@ -3,11 +3,14 @@ import io.mixeway.mixewayflowapi.db.entity.CodeRepo; import io.mixeway.mixewayflowapi.db.entity.CodeRepoBranch; import io.mixeway.mixewayflowapi.db.entity.Settings; +import io.mixeway.mixewayflowapi.db.repository.CodeRepoRepository; import io.mixeway.mixewayflowapi.domain.coderepo.UpdateCodeRepoService; import io.mixeway.mixewayflowapi.domain.dtrack.ProcessDTrackVulnDataService; import io.mixeway.mixewayflowapi.exceptions.ScanException; import io.mixeway.mixewayflowapi.integrations.scanner.sca.dto.*; +import io.mixeway.mixewayflowapi.integrations.scanner.sca.service.CdxGenService; import io.mixeway.mixewayflowapi.utils.Constants; +import jakarta.persistence.EntityNotFoundException; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Value; @@ -38,6 +41,8 @@ public class DependencyTrackApiClientService { private final UpdateCodeRepoService updateCodeRepoService; private final ProcessDTrackVulnDataService processDTrackVulnDataService; + private final CdxGenService cdxGenService; + private final CodeRepoRepository codeRepoRepository; @Value("${dependency-track.url}") private String dependencyTrackUrl; @@ -224,13 +229,18 @@ private File findSbom(String dir) { */ public boolean runScan(String dir, CodeRepo codeRepo, Settings settings, CodeRepoBranch codeRepoBranch) throws IOException, InterruptedException { File sbomFile = findSbom(dir); + if (sbomFile == null) { + cdxGenService.generateBom(dir,codeRepo,codeRepoBranch); + } + sbomFile = findSbom(dir); if (sbomFile != null) { log.info("[Dependency Track] SBOM detected in {}, proceeding with SCA scan...", codeRepo.getRepourl()); sendBomToDTrack(codeRepo, sbomFile.getPath(), settings); - TimeUnit.SECONDS.sleep(5); + TimeUnit.SECONDS.sleep(15); loadVulnerabilities(codeRepo, settings, codeRepoBranch); return true; } else { + codeRepo.updateScaScanStatus(CodeRepo.ScanStatus.NOT_PERFORMED); log.info("[Dependency Track] No SBOM in {}, skipping SCA scan", codeRepo.getRepourl()); return false; } @@ -324,6 +334,9 @@ private List loadVulnerabilities(CodeRepo codeRepo, Se * @param settings The settings containing the API key. */ private void getComponents(CodeRepo codeRepo, Settings settings) { + codeRepo = codeRepoRepository.findById(codeRepo.getId()) + .orElseThrow(() -> new EntityNotFoundException("CodeRepo not found with ID")) ; + WebClient webClient = WebClient.builder() .baseUrl(dependencyTrackUrl + Constants.DEPENDENCYTRACK_URL_GET_COMPONENTS + codeRepo.getScaUUID() + "?limit=2000&offset=0") .defaultHeader(Constants.DEPENDENCYTRACK_APIKEY_HEADER, settings.getScaApiKey()) diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/service/CdxGenService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/service/CdxGenService.java new file mode 100644 index 0000000..279dee4 --- /dev/null +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/service/CdxGenService.java @@ -0,0 +1,99 @@ +package io.mixeway.mixewayflowapi.integrations.scanner.sca.service; + +import ch.qos.logback.core.spi.ScanException; +import io.mixeway.mixewayflowapi.db.entity.CodeRepo; +import io.mixeway.mixewayflowapi.db.entity.CodeRepoBranch; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Service for generating Software Bill of Materials (SBOM) using the cdxgen tool. + * + *

This service provides methods to generate SBOM files for code repositories + * by executing the cdxgen command-line tool. It handles process execution, + * output redirection, timeout management, environment variable configuration, + * and validation of the generated SBOM file.

+ */ +@Service +@Log4j2 +@RequiredArgsConstructor +public class CdxGenService { + + /** + * Generates the SBOM (Software Bill of Materials) file using the cdxgen tool. + * + *

This method executes the cdxgen command in the specified repository directory, + * redirecting both standard output and error streams to prevent blocking. + * It conditionally sets environment variables for proxy configuration if the + * system properties proxy.host and proxy.port are provided. + * The method waits for the process to complete, with a timeout of 2 minutes. + * If the process exceeds the timeout, it is forcibly terminated. + * After the process completes, the method validates the generated bom.json + * file by checking for its existence and ensuring it has content.

+ * + * @param repoDir the directory of the repository where cdxgen will run + * @param codeRepo the code repository entity + * @param codeRepoBranch the branch of the code repository + * @throws IOException if an I/O error occurs when starting or communicating with the process + * @throws InterruptedException if the current thread is interrupted while waiting for the process to finish + * @throws ScanException if a scanning error occurs (note: not currently thrown in this method) + */ + public void generateBom(String repoDir, CodeRepo codeRepo, CodeRepoBranch codeRepoBranch) + throws IOException, InterruptedException { + log.info("[CdxGen] Starting SBOM generation for: {} branch: {}", codeRepo.getName(), codeRepoBranch.getName()); + + String proxyHost = System.getProperty("proxy.host"); + String proxyPort = System.getProperty("proxy.port"); + String command; + + if (proxyHost != null && proxyPort != null) { + // Construct the command with environment variables inline + command = "HTTP_PROXY=http://" + proxyHost + ":" + proxyPort + " " + + "HTTPS_PROXY=http://" + proxyHost + ":" + proxyPort + " " + + "cdxgen -o sbom.json"; + log.info("[CdxGen] Proxy settings applied: {}:{}", proxyHost, proxyPort); + } else { + command = "cdxgen -o sbom.json"; + } + + // Use 'sh -c' to execute the command in a shell + ProcessBuilder pb = new ProcessBuilder("sh", "-c", command); + pb.directory(new File(repoDir)); + // pb.redirectOutput(ProcessBuilder.Redirect.DISCARD); + pb.redirectError(ProcessBuilder.Redirect.DISCARD); + + Process process = pb.start(); + + // Wait for the process to finish with a timeout of 2 minutes + boolean finished = process.waitFor(5, TimeUnit.MINUTES); + if (!finished) { + process.destroyForcibly(); // Terminate the process + log.warn("[CdxGen] Process timed out and was terminated"); + } else { + // Check the exit code if the process finished normally + int exitCode = process.exitValue(); + if (exitCode != 0) { + log.warn("[CdxGen] Process exited with code {}", exitCode); + } + } + + // Validate the bom.json file + File bomFile = new File(repoDir, "bom.json"); + if (bomFile.exists()) { + if (bomFile.length() > 0) { + log.info("[CdxGen] SBOM generated successfully: {}", bomFile.getAbsolutePath()); + } else { + log.warn("[CdxGen] SBOM file is empty: {}", bomFile.getAbsolutePath()); + } + } else { + log.warn("[CdxGen] SBOM file not found: {}", bomFile.getAbsolutePath()); + } + } + +} diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/scheduler/ScanScheduler.java b/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/scheduler/ScanScheduler.java index e402b2d..c30c659 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/scheduler/ScanScheduler.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/scheduler/ScanScheduler.java @@ -3,12 +3,14 @@ import ch.qos.logback.core.spi.ScanException; import io.mixeway.mixewayflowapi.db.entity.CodeRepo; import io.mixeway.mixewayflowapi.db.repository.CodeRepoRepository; +import io.mixeway.mixewayflowapi.domain.coderepo.FindCodeRepoService; import io.mixeway.mixewayflowapi.integrations.repo.service.GetCodeRepoInfoService; import io.mixeway.mixewayflowapi.integrations.scanner.sca.service.SCAService; import io.mixeway.mixewayflowapi.scanmanager.service.ScanManagerService; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.aspectj.apache.bcel.classfile.Code; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -44,7 +46,7 @@ public class ScanScheduler { * @throws URISyntaxException If an error occurs related to URI syntax during initialization. */ @PostConstruct - public void runAfterStartup() throws URISyntaxException { + public void runAfterStartup() { scaService.initialize(); } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java index 44e19a4..f8f395e 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java @@ -65,6 +65,7 @@ public class ScanManagerService { public void scanRepository(CodeRepo codeRepo, CodeRepoBranch codeRepoBranch, String commitId, Long iid) throws IOException, InterruptedException, ScanException { + updateCodeRepoService.setScanRunning(codeRepo); // Acquire a lock specific to the codeRepo Lock lock = repoLocks.computeIfAbsent(codeRepo.getId(), k -> new ReentrantLock());