From d04aeb2d35af30f474091495cb17b113d52e5983 Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Tue, 10 Oct 2023 18:08:06 +0200 Subject: [PATCH] refactor(scanner)!: Use the configurable plugin API for scanner wrappers Use the `TypedConfigurablePluginFactory` interface as base for the `ScannerWrapperFactory` to further align the plugin APIs. While at it, extract the logic to parse the scanner matcher properties to a central place to not have to repeat the properties in the configuration classes for all scanner wrappers. This also makes it possible to use `Unit` as configuration class for scanners that do not offer any configuration options. As this change makes the configuration classes of the scanner wrappers part of the public API, they must not be `internal` anymore. Scanners that provide secret configuration options will be migrated to read them from the `secrets` map instead of the `options` map in later commits. Signed-off-by: Martin Nonnenmacher --- .../src/main/kotlin/RequirementsCommand.kt | 5 +- .../scanner/src/main/kotlin/ScannerCommand.kt | 4 +- .../src/funTest/kotlin/AskalonoFunTest.kt | 3 +- .../askalono/src/main/kotlin/Askalono.kt | 12 ++- .../src/funTest/kotlin/BoyterLcFunTest.kt | 3 +- .../boyterlc/src/main/kotlin/BoyterLc.kt | 12 ++- .../scanners/fossid/src/main/kotlin/FossId.kt | 9 +- .../fossid/src/main/kotlin/FossIdConfig.kt | 2 +- .../src/main/kotlin/FossIdNamingProvider.kt | 2 +- .../src/main/kotlin/FossIdUrlProvider.kt | 2 +- .../src/funTest/kotlin/LicenseeFunTest.kt | 3 +- .../licensee/src/main/kotlin/Licensee.kt | 12 ++- .../funTest/kotlin/ScanCodeScannerFunTest.kt | 3 +- .../scancode/src/main/kotlin/ScanCode.kt | 23 +++-- .../src/main/kotlin/ScanCodeConfig.kt | 37 +++++++ .../scancode/src/test/kotlin/ScanCodeTest.kt | 30 +++--- .../scanoss/src/main/kotlin/ScanOss.kt | 21 ++-- .../scanoss/src/main/kotlin/ScanOssConfig.kt | 2 +- .../kotlin/ScanOssScannerDirectoryTest.kt | 5 +- .../src/test/kotlin/ScanOssScannerFileTest.kt | 5 +- scanner/src/main/kotlin/ScannerMatcher.kt | 96 +++++++++++++------ .../src/main/kotlin/ScannerWrapperFactory.kt | 18 +++- .../kotlin/storages/ClearlyDefinedStorage.kt | 2 +- scanner/src/test/kotlin/ScannerMatcherTest.kt | 49 ++++++++-- 24 files changed, 257 insertions(+), 103 deletions(-) create mode 100644 plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt diff --git a/plugins/commands/requirements/src/main/kotlin/RequirementsCommand.kt b/plugins/commands/requirements/src/main/kotlin/RequirementsCommand.kt index 2c3c31671731..2bda71bbf612 100644 --- a/plugins/commands/requirements/src/main/kotlin/RequirementsCommand.kt +++ b/plugins/commands/requirements/src/main/kotlin/RequirementsCommand.kt @@ -33,6 +33,7 @@ import org.ossreviewtoolkit.model.config.AnalyzerConfiguration import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.plugins.commands.api.OrtCommand import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.spdx.scanCodeLicenseTextDir @@ -88,8 +89,8 @@ class RequirementsCommand : OrtCommand( logger.debug { "$it is a $category." } it.getDeclaredConstructor( String::class.java, - Map::class.java - ).newInstance("", emptyMap()) + ScannerMatcherConfig::class.java + ).newInstance("", ScannerMatcherConfig.EMPTY) } VersionControlSystem::class.java.isAssignableFrom(it) -> { diff --git a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt index b164d73b979c..7cf99277f5f9 100644 --- a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt +++ b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt @@ -177,10 +177,10 @@ class ScannerCommand : OrtCommand( ): OrtResult { val packageScannerWrappers = scannerWrapperFactories .takeIf { PackageType.PACKAGE in packageTypes }.orEmpty() - .map { it.create(ortConfig.scanner.options?.get(it.type).orEmpty()) } + .map { it.create(ortConfig.scanner.options?.get(it.type).orEmpty(), emptyMap()) } val projectScannerWrappers = projectScannerWrapperFactories .takeIf { PackageType.PROJECT in packageTypes }.orEmpty() - .map { it.create(ortConfig.scanner.options?.get(it.type).orEmpty()) } + .map { it.create(ortConfig.scanner.options?.get(it.type).orEmpty(), emptyMap()) } if (projectScannerWrappers.isNotEmpty()) { echo("Scanning projects with:") diff --git a/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt b/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt index 9dd4aae58182..24c209d3ea67 100644 --- a/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt +++ b/plugins/scanners/askalono/src/funTest/kotlin/AskalonoFunTest.kt @@ -21,10 +21,11 @@ package org.ossreviewtoolkit.plugins.scanners.askalono import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.scanners.AbstractPathScannerWrapperFunTest class AskalonoFunTest : AbstractPathScannerWrapperFunTest() { - override val scanner = Askalono("Askalono", emptyMap()) + override val scanner = Askalono("Askalono", ScannerMatcherConfig.EMPTY) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 1.0f) diff --git a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt index 43efb56414d4..62906029a77d 100644 --- a/plugins/scanners/askalono/src/main/kotlin/Askalono.kt +++ b/plugins/scanners/askalono/src/main/kotlin/Askalono.kt @@ -36,6 +36,7 @@ import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerMatcher +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os @@ -44,14 +45,17 @@ private const val CONFIDENCE_NOTICE = "Confidence threshold not high enough for private val JSON = Json { ignoreUnknownKeys = true } -class Askalono internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { - class Factory : ScannerWrapperFactory("Askalono") { - override fun create(options: Options) = Askalono(type, options) +class Askalono internal constructor(name: String, private val matcherConfig: ScannerMatcherConfig) : + CommandLinePathScannerWrapper(name) { + class Factory : ScannerWrapperFactory("Askalono") { + override fun create(config: Unit, matcherConfig: ScannerMatcherConfig) = Askalono(type, matcherConfig) + + override fun parseConfig(options: Options, secrets: Options) = Unit } override val configuration = "" - override val matcher by lazy { ScannerMatcher.create(details, options) } + override val matcher by lazy { ScannerMatcher.create(details, matcherConfig) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "askalono.exe" else "askalono").joinToString(File.separator) diff --git a/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt b/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt index ba26fe9cee60..e57832a8bab9 100644 --- a/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt +++ b/plugins/scanners/boyterlc/src/funTest/kotlin/BoyterLcFunTest.kt @@ -21,10 +21,11 @@ package org.ossreviewtoolkit.plugins.scanners.boyterlc import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.scanners.AbstractPathScannerWrapperFunTest class BoyterLcFunTest : AbstractPathScannerWrapperFunTest() { - override val scanner = BoyterLc("BoyterLc", emptyMap()) + override val scanner = BoyterLc("BoyterLc", ScannerMatcherConfig.EMPTY) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 0.98388565f), diff --git a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt index fe2cccb2ea01..a6ebcaf84828 100644 --- a/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt +++ b/plugins/scanners/boyterlc/src/main/kotlin/BoyterLc.kt @@ -35,6 +35,7 @@ import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerMatcher +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os @@ -43,7 +44,8 @@ import org.ossreviewtoolkit.utils.ort.createOrtTempDir private val JSON = Json { ignoreUnknownKeys = true } -class BoyterLc internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { +class BoyterLc internal constructor(name: String, private val matcherConfig: ScannerMatcherConfig) : + CommandLinePathScannerWrapper(name) { companion object { val CONFIGURATION_OPTIONS = listOf( "--confidence", "0.95", // Cut-off value to only get most relevant matches. @@ -51,13 +53,15 @@ class BoyterLc internal constructor(name: String, private val options: Options) ) } - class Factory : ScannerWrapperFactory("BoyterLc") { - override fun create(options: Options) = BoyterLc(type, options) + class Factory : ScannerWrapperFactory("BoyterLc") { + override fun create(config: Unit, matcherConfig: ScannerMatcherConfig) = BoyterLc(type, matcherConfig) + + override fun parseConfig(options: Options, secrets: Options) = Unit } override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val matcher by lazy { ScannerMatcher.create(details, options) } + override val matcher by lazy { ScannerMatcher.create(details, matcherConfig) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "lc.exe" else "lc").joinToString(File.separator) diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index 284709923df6..3e6e19e2214f 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -78,6 +78,7 @@ import org.ossreviewtoolkit.scanner.PackageScannerWrapper import org.ossreviewtoolkit.scanner.ProvenanceScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerMatcher +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.enumSetOf @@ -168,14 +169,16 @@ class FossId internal constructor( ) } - class Factory : ScannerWrapperFactory("FossId") { - override fun create(options: Options) = FossId(type, FossIdConfig.create(options)) + class Factory : ScannerWrapperFactory("FossId") { + override fun create(config: FossIdConfig, matcherConfig: ScannerMatcherConfig) = FossId(type, config) + + override fun parseConfig(options: Options, secrets: Options) = FossIdConfig.create(options) } /** * The qualifier of a scan when delta scans are enabled. */ - internal enum class DeltaTag { + enum class DeltaTag { /** * Qualifier used when there is no scan and the first one is created. */ diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt index e37fa93f56f6..04147dafc9ec 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt @@ -73,7 +73,7 @@ import org.ossreviewtoolkit.utils.common.Options * * every repository URL would be added credentials. Mappings are applied in the order they are defined. */ -internal data class FossIdConfig( +data class FossIdConfig( /** The URL where the FossID service is running. */ val serverUrl: String, diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdNamingProvider.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdNamingProvider.kt index 729223fb9452..596380369f41 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdNamingProvider.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdNamingProvider.kt @@ -38,7 +38,7 @@ import org.apache.logging.log4j.kotlin.logger * * **deltaTag** (scan code only): If delta scans is enabled, this qualifies the scan as an *origin* scan or a *delta* * scan. */ -internal class FossIdNamingProvider( +class FossIdNamingProvider( private val namingProjectPattern: String?, private val namingScanPattern: String?, private val namingConventionVariables: Map diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdUrlProvider.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdUrlProvider.kt index 0ed23ac08c24..ca653577b4b8 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdUrlProvider.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdUrlProvider.kt @@ -35,7 +35,7 @@ import org.ossreviewtoolkit.utils.ort.requestPasswordAuthentication * The URLs used by FossId can sometimes be different from the normal package URLs. For instance, credentials may need * to be added, or a different protocol may be used. This class takes care of such mappings. */ -internal class FossIdUrlProvider private constructor( +class FossIdUrlProvider private constructor( /** * The URL mapping. URLs matched by a key [Regex] are replaced by the URL in the value. The replacement string can * refer to matched groups of the [Regex]. It can also contain the variables [VAR_USERNAME] and [VAR_PASSWORD] to diff --git a/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt b/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt index d852b8d54b49..6b19fa0daad0 100644 --- a/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt +++ b/plugins/scanners/licensee/src/funTest/kotlin/LicenseeFunTest.kt @@ -21,11 +21,12 @@ package org.ossreviewtoolkit.plugins.scanners.licensee import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.scanners.AbstractPathScannerWrapperFunTest import org.ossreviewtoolkit.utils.test.ExpensiveTag class LicenseeFunTest : AbstractPathScannerWrapperFunTest(setOf(ExpensiveTag)) { - override val scanner = Licensee("Licensee", emptyMap()) + override val scanner = Licensee("Licensee", ScannerMatcherConfig.EMPTY) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", TextLocation.UNKNOWN_LINE), 100.0f) diff --git a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt index bbc07f18eb2d..2de3cc157671 100644 --- a/plugins/scanners/licensee/src/main/kotlin/Licensee.kt +++ b/plugins/scanners/licensee/src/main/kotlin/Licensee.kt @@ -36,6 +36,7 @@ import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanException import org.ossreviewtoolkit.scanner.ScannerMatcher +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os @@ -45,18 +46,21 @@ private val JSON = Json { namingStrategy = JsonNamingStrategy.SnakeCase } -class Licensee internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { +class Licensee internal constructor(name: String, private val matcherConfig: ScannerMatcherConfig) : + CommandLinePathScannerWrapper(name) { companion object { val CONFIGURATION_OPTIONS = listOf("--json") } - class Factory : ScannerWrapperFactory("Licensee") { - override fun create(options: Options) = Licensee(type, options) + class Factory : ScannerWrapperFactory("Licensee") { + override fun create(config: Unit, matcherConfig: ScannerMatcherConfig) = Licensee(type, matcherConfig) + + override fun parseConfig(options: Options, secrets: Options) = Unit } override val configuration = CONFIGURATION_OPTIONS.joinToString(" ") - override val matcher by lazy { ScannerMatcher.create(details, options) } + override val matcher by lazy { ScannerMatcher.create(details, matcherConfig) } override fun command(workingDir: File?) = listOfNotNull(workingDir, if (Os.isWindows) "licensee.bat" else "licensee").joinToString(File.separator) diff --git a/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt b/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt index 8db3e3430ddb..1f64ee94e279 100644 --- a/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt +++ b/plugins/scanners/scancode/src/funTest/kotlin/ScanCodeScannerFunTest.kt @@ -26,13 +26,14 @@ import io.kotest.matchers.string.startWith import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.scanners.AbstractPathScannerWrapperFunTest import org.ossreviewtoolkit.utils.ort.createOrtTempDir import org.ossreviewtoolkit.utils.spdx.getLicenseText import org.ossreviewtoolkit.utils.test.ExpensiveTag class ScanCodeScannerFunTest : AbstractPathScannerWrapperFunTest(setOf(ExpensiveTag)) { - override val scanner = ScanCode("ScanCode", emptyMap()) + override val scanner = ScanCode("ScanCode", ScanCodeConfig.EMPTY, ScannerMatcherConfig.EMPTY) override val expectedFileLicenses = listOf( LicenseFinding("Apache-2.0", TextLocation("LICENSE", 1, 187), 100.0f), diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt index aee1e4f48e1a..875ef83b762a 100644 --- a/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCode.kt @@ -33,6 +33,7 @@ import org.ossreviewtoolkit.scanner.CommandLinePathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScanResultsStorage import org.ossreviewtoolkit.scanner.ScannerMatcher +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os @@ -57,7 +58,14 @@ import org.semver4j.Semver * * **"commandLineNonConfig":** Command line options that do not modify the result and should therefore not be * considered in [configuration], like "--processes". Defaults to [DEFAULT_NON_CONFIGURATION_OPTIONS]. */ -class ScanCode internal constructor(name: String, private val options: Options) : CommandLinePathScannerWrapper(name) { +class ScanCode internal constructor( + name: String, + config: ScanCodeConfig, + private val matcherConfig: ScannerMatcherConfig +) : CommandLinePathScannerWrapper(name) { + // This constructor is required by the `RequirementsCommand`. + constructor(name: String, matcherConfig: ScannerMatcherConfig) : this(name, ScanCodeConfig.EMPTY, matcherConfig) + companion object { const val SCANNER_NAME = "ScanCode" @@ -91,11 +99,14 @@ class ScanCode internal constructor(name: String, private val options: Options) } } - class Factory : ScannerWrapperFactory(SCANNER_NAME) { - override fun create(options: Options) = ScanCode(type, options) + class Factory : ScannerWrapperFactory(SCANNER_NAME) { + override fun create(config: ScanCodeConfig, matcherConfig: ScannerMatcherConfig) = + ScanCode(type, config, matcherConfig) + + override fun parseConfig(options: Options, secrets: Options) = ScanCodeConfig.create(options) } - override val matcher by lazy { ScannerMatcher.create(details, options) } + override val matcher by lazy { ScannerMatcher.create(details, matcherConfig) } override val configuration by lazy { buildList { @@ -104,8 +115,8 @@ class ScanCode internal constructor(name: String, private val options: Options) }.joinToString(" ") } - private val configurationOptions = options["commandLine"]?.splitOnWhitespace() ?: DEFAULT_CONFIGURATION_OPTIONS - private val nonConfigurationOptions = options["commandLineNonConfig"]?.splitOnWhitespace() + private val configurationOptions = config.commandLine?.splitOnWhitespace() ?: DEFAULT_CONFIGURATION_OPTIONS + private val nonConfigurationOptions = config.commandLineNonConfig?.splitOnWhitespace() ?: DEFAULT_NON_CONFIGURATION_OPTIONS internal fun getCommandLineOptions(version: String) = diff --git a/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt b/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt new file mode 100644 index 000000000000..715dc394396b --- /dev/null +++ b/plugins/scanners/scancode/src/main/kotlin/ScanCodeConfig.kt @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.scanners.scancode + +import org.ossreviewtoolkit.utils.common.Options + +data class ScanCodeConfig( + val commandLine: String?, + val commandLineNonConfig: String? +) { + companion object { + val EMPTY = ScanCodeConfig(null, null) + + private const val COMMAND_LINE_PROPERTY = "commandLine" + private const val COMMAND_LINE_NON_CONFIG_PROPERTY = "commandLineNonConfig" + + fun create(options: Options) = + ScanCodeConfig(options[COMMAND_LINE_PROPERTY], options[COMMAND_LINE_NON_CONFIG_PROPERTY]) + } +} diff --git a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt index 3c1bbf3c92ab..d636b731f6fd 100644 --- a/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt +++ b/plugins/scanners/scancode/src/test/kotlin/ScanCodeTest.kt @@ -35,10 +35,11 @@ import java.io.File import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.scanner.ScanContext +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.utils.common.ProcessCapture class ScanCodeTest : WordSpec({ - val scanner = ScanCode("ScanCode", emptyMap()) + val scanner = ScanCode("ScanCode", ScanCodeConfig.EMPTY, ScannerMatcherConfig.EMPTY) "configuration" should { "return the default values if the scanner configuration is empty" { @@ -48,10 +49,11 @@ class ScanCodeTest : WordSpec({ "return the non-config values from the scanner configuration" { val scannerWithConfig = ScanCode( "ScanCode", - mapOf( - "commandLine" to "--command --line", - "commandLineNonConfig" to "--commandLineNonConfig" - ) + ScanCodeConfig( + commandLine = "--command --line", + commandLineNonConfig = "--commandLineNonConfig" + ), + ScannerMatcherConfig.EMPTY ) scannerWithConfig.configuration shouldBe "--command --line --json-pp" @@ -69,10 +71,11 @@ class ScanCodeTest : WordSpec({ "contain the values from the scanner configuration" { val scannerWithConfig = ScanCode( "ScanCode", - mapOf( - "commandLine" to "--command --line", - "commandLineNonConfig" to "--commandLineNonConfig" - ) + ScanCodeConfig( + commandLine = "--command --line", + commandLineNonConfig = "--commandLineNonConfig" + ), + ScannerMatcherConfig.EMPTY ) scannerWithConfig.getCommandLineOptions("31.2.4").joinToString(" ") shouldBe @@ -82,10 +85,11 @@ class ScanCodeTest : WordSpec({ "be handled correctly when containing multiple spaces" { val scannerWithConfig = ScanCode( "ScanCode", - mapOf( - "commandLine" to " --command --line ", - "commandLineNonConfig" to " -n -c " - ) + ScanCodeConfig( + commandLine = " --command --line ", + commandLineNonConfig = " -n -c " + ), + ScannerMatcherConfig.EMPTY ) scannerWithConfig.getCommandLineOptions("31.2.4") shouldBe listOf("--command", "--line", "-n", "-c") diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index 2d6d0dfe908e..d0e02b83af8a 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -41,6 +41,7 @@ import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.scanner.PathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.scanner.ScannerMatcher +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES @@ -49,13 +50,17 @@ import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES private const val FAKE_WFP_FILE_NAME = "fake.wfp" private const val ARG_FIELD_NAME = "file" -class ScanOss internal constructor(override val name: String, private val options: Options) : PathScannerWrapper { - class Factory : ScannerWrapperFactory("SCANOSS") { - override fun create(options: Options) = ScanOss(type, options) - } - - private val config = ScanOssConfig.create(options).also { - logger.info { "The $name API URL is ${it.apiUrl}." } +class ScanOss internal constructor( + override val name: String, + config: ScanOssConfig, + private val matcherConfig: ScannerMatcherConfig +) : PathScannerWrapper { + class Factory : ScannerWrapperFactory("SCANOSS") { + override fun create(config: ScanOssConfig, matcherConfig: ScannerMatcherConfig) = + ScanOss(type, config, matcherConfig) + + override fun parseConfig(options: Options, secrets: Options) = + ScanOssConfig.create(options).also { logger.info { "The $type API URL is ${it.apiUrl}." } } } private val service = ScanOssService.create(config.apiUrl) @@ -69,7 +74,7 @@ class ScanOss internal constructor(override val name: String, private val option override val configuration = "" - override val matcher by lazy { ScannerMatcher.create(details, options) } + override val matcher by lazy { ScannerMatcher.create(details, matcherConfig) } /** * The name of the file corresponding to the fingerprints can be sent to SCANOSS for more precise matches. diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt index 749d9c6ca057..8ddb7e1a8270 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt @@ -28,7 +28,7 @@ import org.ossreviewtoolkit.utils.common.Options * created from the options contained in a [ScannerConfiguration] object under the key _ScanOss_. It offers the * following configuration options: */ -internal data class ScanOssConfig( +data class ScanOssConfig( /** URL of the ScanOSS server. */ val apiUrl: String, diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt index f08cde25ad37..cd9ccffe0c83 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt @@ -43,6 +43,7 @@ import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.scanner.ScanContext +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig import org.ossreviewtoolkit.utils.spdx.SpdxExpression private val TEST_DIRECTORY_TO_SCAN = File("src/test/assets/filesToScan") @@ -61,8 +62,8 @@ class ScanOssScannerDirectoryTest : StringSpec({ beforeSpec { server.start() - val options = mapOf(ScanOssConfig.API_URL_PROPERTY to "http://localhost:${server.port()}") - scanner = spyk(ScanOss.Factory().create(options)) + val config = ScanOssConfig(apiUrl = "http://localhost:${server.port()}", apiKey = "") + scanner = spyk(ScanOss.Factory().create(config, ScannerMatcherConfig.EMPTY)) } afterSpec { diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt index aecfc7a46fd4..beb83513a5cb 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerFileTest.kt @@ -37,6 +37,7 @@ import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.scanner.ScanContext +import org.ossreviewtoolkit.scanner.ScannerMatcherConfig private val TEST_FILE_TO_SCAN = File("src/test/assets/filesToScan/ScannerFactory.kt") @@ -54,8 +55,8 @@ class ScanOssScannerFileTest : StringSpec({ beforeSpec { server.start() - val options = mapOf(ScanOssConfig.API_URL_PROPERTY to "http://localhost:${server.port()}") - scanner = spyk(ScanOss.Factory().create(options)) + val config = ScanOssConfig(apiUrl = "http://localhost:${server.port()}", apiKey = "") + scanner = spyk(ScanOss.Factory().create(config, ScannerMatcherConfig.EMPTY)) } afterSpec { diff --git a/scanner/src/main/kotlin/ScannerMatcher.kt b/scanner/src/main/kotlin/ScannerMatcher.kt index f97867e94212..440e40c85757 100644 --- a/scanner/src/main/kotlin/ScannerMatcher.kt +++ b/scanner/src/main/kotlin/ScannerMatcher.kt @@ -62,43 +62,18 @@ data class ScannerMatcher( val configuration: String? ) { companion object { - /** - * The name of the property defining the regular expression for the scanner name as part of [ScannerMatcher]. - * The [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. - */ - const val PROP_CRITERIA_NAME = "regScannerName" - - /** - * The name of the property defining the minimum version of the scanner as part of [ScannerMatcher]. The - * [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. - */ - const val PROP_CRITERIA_MIN_VERSION = "minVersion" - - /** - * The name of the property defining the maximum version of the scanner as part of [ScannerMatcher]. The - * [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. - */ - const val PROP_CRITERIA_MAX_VERSION = "maxVersion" - - /** - * The name of the property defining the configuration of the scanner as part of [ScannerMatcher]. The - * [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. - */ - const val PROP_CRITERIA_CONFIGURATION = "configuration" /** * Return a [ScannerMatcher] instance that is to be used when looking up existing scan results from a * [ScanResultsStorage]. By default, the properties of this instance are initialized to match the scanner - * [details]. These defaults can be overridden by the provided [options]. The keys of the option map must match - * names of the [ScannerMatcher] class. For example, to specify that a specific minimum version of the scanner - * is allowed, set this option: `minVersion=3.0.2`. + * [details]. These defaults can be overridden by the provided [config]. */ - fun create(details: ScannerDetails, options: Options = emptyMap()): ScannerMatcher { + fun create(details: ScannerDetails, config: ScannerMatcherConfig = ScannerMatcherConfig.EMPTY): ScannerMatcher { val scannerVersion = Semver(normalizeVersion(details.version)) - val minVersion = parseVersion(options[PROP_CRITERIA_MIN_VERSION]) ?: scannerVersion - val maxVersion = parseVersion(options[PROP_CRITERIA_MAX_VERSION]) ?: minVersion.nextMinor() - val name = options[PROP_CRITERIA_NAME] ?: details.name - val configuration = options[PROP_CRITERIA_CONFIGURATION] ?: details.configuration + val minVersion = parseVersion(config.minVersion) ?: scannerVersion + val maxVersion = parseVersion(config.maxVersion) ?: minVersion.nextMinor() + val name = config.regScannerName ?: details.name + val configuration = config.configuration ?: details.configuration return ScannerMatcher(name, minVersion, maxVersion, configuration) } @@ -126,6 +101,65 @@ data class ScannerMatcher( } } +/** + * A holder class for the [ScannerMatcher] configuration. + */ +data class ScannerMatcherConfig( + val regScannerName: String? = null, + val minVersion: String? = null, + val maxVersion: String? = null, + val configuration: String? = null +) { + companion object { + val EMPTY = ScannerMatcherConfig(null, null, null, null) + + /** + * The name of the property defining the regular expression for the scanner name as part of [ScannerMatcher]. + * The [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. + */ + internal const val PROP_CRITERIA_NAME = "regScannerName" + + /** + * The name of the property defining the minimum version of the scanner as part of [ScannerMatcher]. The + * [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. + */ + internal const val PROP_CRITERIA_MIN_VERSION = "minVersion" + + /** + * The name of the property defining the maximum version of the scanner as part of [ScannerMatcher]. The + * [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. + */ + internal const val PROP_CRITERIA_MAX_VERSION = "maxVersion" + + /** + * The name of the property defining the configuration of the scanner as part of [ScannerMatcher]. The + * [scanner details][ScannerDetails] of the corresponding scanner must match the criteria. + */ + internal const val PROP_CRITERIA_CONFIGURATION = "configuration" + + private val properties = listOf( + PROP_CRITERIA_NAME, PROP_CRITERIA_MIN_VERSION, PROP_CRITERIA_MAX_VERSION, PROP_CRITERIA_CONFIGURATION + ) + + /** + * Create a [ScannerMatcherConfig] from the provided options. Return the created config and the options without + * the properties that are used to configure the matcher. + */ + fun create(options: Options): Pair { + val filteredOptions = options.filterKeys { it !in properties } + + val matcherConfig = ScannerMatcherConfig( + regScannerName = options[PROP_CRITERIA_NAME], + minVersion = options[PROP_CRITERIA_MIN_VERSION], + maxVersion = options[PROP_CRITERIA_MAX_VERSION], + configuration = options[PROP_CRITERIA_CONFIGURATION] + ) + + return matcherConfig to filteredOptions + } + } +} + /** * Parse the given [versionStr] to a [Semver] while trying to be failure tolerant. */ diff --git a/scanner/src/main/kotlin/ScannerWrapperFactory.kt b/scanner/src/main/kotlin/ScannerWrapperFactory.kt index 2d1834163040..b6e8e87d851a 100644 --- a/scanner/src/main/kotlin/ScannerWrapperFactory.kt +++ b/scanner/src/main/kotlin/ScannerWrapperFactory.kt @@ -22,16 +22,26 @@ package org.ossreviewtoolkit.scanner import java.util.ServiceLoader import org.ossreviewtoolkit.utils.common.Options -import org.ossreviewtoolkit.utils.common.Plugin +import org.ossreviewtoolkit.utils.common.TypedConfigurablePluginFactory /** * A common abstract class for use with [ServiceLoader] that all [ScannerWrapperFactory] classes need to implement. */ -abstract class ScannerWrapperFactory(override val type: String) : Plugin { +abstract class ScannerWrapperFactory(override val type: String) : + TypedConfigurablePluginFactory { + override fun create(options: Options, secrets: Options): ScannerWrapper { + val (matcherConfig, filteredOptions) = ScannerMatcherConfig.create(options) + return create(parseConfig(filteredOptions, secrets), matcherConfig) + } + + final override fun create(config: CONFIG): ScannerWrapper { + throw UnsupportedOperationException("Use 'create(CONFIG, ScannerMatcherConfig)' instead.") + } + /** - * Create a [ScannerWrapper] using the provided [options]. + * Create a [ScannerWrapper] from the provided [config] and [matcherConfig]. */ - abstract fun create(options: Options): T + abstract fun create(config: CONFIG, matcherConfig: ScannerMatcherConfig): ScannerWrapper /** * Return the scanner wrapper's name here to allow Clikt to display something meaningful when listing the scanners diff --git a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt index 79f558c3e56f..454c6c6c5a76 100644 --- a/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt +++ b/scanner/src/main/kotlin/storages/ClearlyDefinedStorage.kt @@ -117,7 +117,7 @@ class ClearlyDefinedStorage( val supportedScanners = toolVersionsByName.mapNotNull { (name, versions) -> // For the ClearlyDefined tool names see https://github.com/clearlydefined/service#tool-name-registry. ScannerWrapper.ALL[name]?.let { factory -> - val scanner = factory.create(emptyMap()) + val scanner = factory.create(emptyMap(), emptyMap()) (scanner as? CommandLinePathScannerWrapper)?.let { cliScanner -> cliScanner to versions.last() } }.also { if (it == null) logger.debug { "Unsupported tool '$name' for coordinates '$coordinates'." } diff --git a/scanner/src/test/kotlin/ScannerMatcherTest.kt b/scanner/src/test/kotlin/ScannerMatcherTest.kt index e949e417ba37..75f24fb12397 100644 --- a/scanner/src/test/kotlin/ScannerMatcherTest.kt +++ b/scanner/src/test/kotlin/ScannerMatcherTest.kt @@ -20,6 +20,7 @@ package org.ossreviewtoolkit.scanner import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.maps.shouldContainExactly import io.kotest.matchers.shouldBe import org.ossreviewtoolkit.model.ScannerDetails @@ -38,14 +39,14 @@ class ScannerMatcherTest : WordSpec({ } "obtain values from the configuration" { - val options = mapOf( - ScannerMatcher.PROP_CRITERIA_NAME to "foo", - ScannerMatcher.PROP_CRITERIA_MIN_VERSION to "1.2.3", - ScannerMatcher.PROP_CRITERIA_MAX_VERSION to "4.5.6", - ScannerMatcher.PROP_CRITERIA_CONFIGURATION to "config" + val config = ScannerMatcherConfig( + regScannerName = "foo", + minVersion = "1.2.3", + maxVersion = "4.5.6", + configuration = "config" ) - val matcher = ScannerMatcher.create(testDetails, options) + val matcher = ScannerMatcher.create(testDetails, config) matcher.regScannerName shouldBe "foo" matcher.minVersion.version shouldBe "1.2.3" @@ -54,9 +55,9 @@ class ScannerMatcherTest : WordSpec({ } "parse versions in a lenient way" { - val options = mapOf( - ScannerMatcher.PROP_CRITERIA_MIN_VERSION to "1", - ScannerMatcher.PROP_CRITERIA_MAX_VERSION to "3.7" + val options = ScannerMatcherConfig( + minVersion = "1", + maxVersion = "3.7" ) val matcher = ScannerMatcher.create(testDetails, options) @@ -113,6 +114,36 @@ class ScannerMatcherTest : WordSpec({ matcher.matches(testDetails) shouldBe true } } + + "ScannerMatcherConfig.create()" should { + "obtain values from the options" { + val options = mapOf( + ScannerMatcherConfig.PROP_CRITERIA_NAME to "foo", + ScannerMatcherConfig.PROP_CRITERIA_MIN_VERSION to "1.2.3", + ScannerMatcherConfig.PROP_CRITERIA_MAX_VERSION to "4.5.6", + ScannerMatcherConfig.PROP_CRITERIA_CONFIGURATION to "config" + ) + + with(ScannerMatcherConfig.create(options).first) { + regScannerName shouldBe "foo" + minVersion shouldBe "1.2.3" + maxVersion shouldBe "4.5.6" + configuration shouldBe "config" + } + } + + "filter matcher properties from the options" { + val options = mapOf( + ScannerMatcherConfig.PROP_CRITERIA_NAME to "foo", + ScannerMatcherConfig.PROP_CRITERIA_MIN_VERSION to "1.2.3", + ScannerMatcherConfig.PROP_CRITERIA_MAX_VERSION to "4.5.6", + ScannerMatcherConfig.PROP_CRITERIA_CONFIGURATION to "config", + "other" to "value" + ) + + ScannerMatcherConfig.create(options).second shouldContainExactly mapOf("other" to "value") + } + } }) private const val SCANNER_NAME = "ScannerMatcherTest"