From 95ed7333154372fb42ce9123447e1408d7141de3 Mon Sep 17 00:00:00 2001 From: Marcel Bochtler Date: Tue, 2 Mar 2021 09:05:52 +0100 Subject: [PATCH] LicenseInfoResolver: Apply license choices by package Apply license choices specified for a package defined in the repository configuration file. The choice reflects in the ResolvedLicense where only the chosen licenses get added to the final list of resolved licenses. The information of the non-chosen license is not lost as it is still present in the `originalLicenseExpressions`. Relates to #3396. Signed-off-by: Stephanie Neubauer Signed-off-by: Marcel Bochtler --- .../main/kotlin/commands/EvaluatorCommand.kt | 3 +- .../kotlin/licenses/LicenseInfoResolver.kt | 50 ++++++-- .../licenses/LicenseInfoResolverTest.kt | 111 ++++++++++++++++-- 3 files changed, 141 insertions(+), 23 deletions(-) diff --git a/cli/src/main/kotlin/commands/EvaluatorCommand.kt b/cli/src/main/kotlin/commands/EvaluatorCommand.kt index d72c477090822..8c50d4553de40 100644 --- a/cli/src/main/kotlin/commands/EvaluatorCommand.kt +++ b/cli/src/main/kotlin/commands/EvaluatorCommand.kt @@ -259,7 +259,8 @@ class EvaluatorCommand : CliktCommand(name = "evaluate", help = "Evaluate ORT re provider = DefaultLicenseInfoProvider(finalOrtResult, packageConfigurationProvider), copyrightGarbage = copyrightGarbage, archiver = globalOptionsForSubcommands.config.scanner?.archive.createFileArchiver(), - licenseFilenamePatterns = LicenseFilenamePatterns.getInstance() + licenseFilenamePatterns = LicenseFilenamePatterns.getInstance(), + licenseChoices = finalOrtResult.repository.config.licenseChoices ) val licenseClassifications = diff --git a/model/src/main/kotlin/licenses/LicenseInfoResolver.kt b/model/src/main/kotlin/licenses/LicenseInfoResolver.kt index 4848176cb74c7..577825c23e478 100644 --- a/model/src/main/kotlin/licenses/LicenseInfoResolver.kt +++ b/model/src/main/kotlin/licenses/LicenseInfoResolver.kt @@ -28,9 +28,7 @@ import org.ossreviewtoolkit.model.CopyrightFinding import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.LicenseSource import org.ossreviewtoolkit.model.Provenance -import org.ossreviewtoolkit.model.config.CopyrightGarbage -import org.ossreviewtoolkit.model.config.LicenseFilenamePatterns -import org.ossreviewtoolkit.model.config.PathExclude +import org.ossreviewtoolkit.model.config.* import org.ossreviewtoolkit.model.utils.FileArchiver import org.ossreviewtoolkit.model.utils.FindingCurationMatcher import org.ossreviewtoolkit.model.utils.FindingsMatcher @@ -44,7 +42,8 @@ class LicenseInfoResolver( val provider: LicenseInfoProvider, val copyrightGarbage: CopyrightGarbage, val archiver: FileArchiver?, - val licenseFilenamePatterns: LicenseFilenamePatterns = LicenseFilenamePatterns.DEFAULT + val licenseFilenamePatterns: LicenseFilenamePatterns = LicenseFilenamePatterns.DEFAULT, + val licenseChoices: LicenseChoices = LicenseChoices() ) { private val resolvedLicenseInfo: ConcurrentMap = ConcurrentHashMap() private val resolvedLicenseFiles: ConcurrentMap = ConcurrentHashMap() @@ -67,8 +66,17 @@ class LicenseInfoResolver( private fun createLicenseInfo(id: Identifier): ResolvedLicenseInfo { val licenseInfo = provider.get(id) - val concludedLicenses = licenseInfo.concludedLicenseInfo.concludedLicense?.decompose().orEmpty() - val declaredLicenses = licenseInfo.declaredLicenseInfo.processed.spdxExpression?.decompose().orEmpty() + val packageLicenseChoice = licenseChoices.packageLicenseChoices.singleOrNull { it.packageId == id } + + val concludedLicenses = applyChoiceIfApplicable( + packageLicenseChoice, + licenseInfo.concludedLicenseInfo.concludedLicense + )?.decompose().orEmpty() + + val declaredLicenses = applyChoiceIfApplicable( + packageLicenseChoice, + licenseInfo.declaredLicenseInfo.processed.spdxExpression + )?.decompose().orEmpty() val resolvedLicenses = mutableMapOf() @@ -103,7 +111,7 @@ class LicenseInfoResolver( licenseInfo.detectedLicenseInfo.filterCopyrightGarbage(copyrightGarbageFindings) val unmatchedCopyrights = mutableMapOf>() - val resolvedLocations = resolveLocations(filteredDetectedLicenseInfo, unmatchedCopyrights) + val resolvedLocations = resolveLocations(filteredDetectedLicenseInfo, unmatchedCopyrights, packageLicenseChoice) resolvedLocations.keys.forEach { license -> license.builder().apply { @@ -124,6 +132,26 @@ class LicenseInfoResolver( ) } + private fun applyChoiceIfApplicable( + packageLicenseChoice: PackageLicenseChoice?, + license: SpdxExpression? + ): SpdxExpression? { + if (packageLicenseChoice != null) { + val licenseChoice = packageLicenseChoice.licenseChoices.singleOrNull { it.given == license } + + if (licenseChoice != null) { + return if (licenseChoice.subExpression != null) { + license?.applyChoice(licenseChoice.choice, licenseChoice.subExpression) + } else { + license?.applyChoice(licenseChoice.choice) + } + } + } + + return license + } + + private fun DetectedLicenseInfo.filterCopyrightGarbage( copyrightGarbageFindings: MutableMap> ): DetectedLicenseInfo { @@ -139,7 +167,8 @@ class LicenseInfoResolver( private fun resolveLocations( detectedLicenseInfo: DetectedLicenseInfo, - unmatchedCopyrights: MutableMap> + unmatchedCopyrights: MutableMap>, + packageLicenseChoice: PackageLicenseChoice? ): Map> { val resolvedLocations = mutableMapOf>() val curationMatcher = FindingCurationMatcher() @@ -174,7 +203,10 @@ class LicenseInfoResolver( it.matches(licenseFinding.location.prependPath(findings.relativeFindingsPath)) } - licenseFinding.license.decompose().forEach { singleLicense -> + val chosenLicenses = + applyChoiceIfApplicable(packageLicenseChoice, licenseFinding.license)?.decompose().orEmpty() + + chosenLicenses.forEach { singleLicense -> resolvedLocations.getOrPut(singleLicense) { mutableSetOf() } += ResolvedLicenseLocation( findings.provenance, licenseFinding.location, diff --git a/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt b/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt index 8506b12c371d4..b834b867f7d44 100644 --- a/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt +++ b/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt @@ -23,9 +23,7 @@ import io.kotest.assertions.show.show import io.kotest.core.spec.style.WordSpec import io.kotest.matchers.Matcher import io.kotest.matchers.MatcherResult -import io.kotest.matchers.collections.beEmpty -import io.kotest.matchers.collections.containExactly -import io.kotest.matchers.collections.haveSize +import io.kotest.matchers.collections.* import io.kotest.matchers.neverNullMatcher import io.kotest.matchers.should import io.kotest.matchers.shouldBe @@ -44,12 +42,7 @@ import org.ossreviewtoolkit.model.RemoteArtifact import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.model.config.CopyrightGarbage -import org.ossreviewtoolkit.model.config.LicenseFilenamePatterns -import org.ossreviewtoolkit.model.config.LicenseFindingCuration -import org.ossreviewtoolkit.model.config.LicenseFindingCurationReason -import org.ossreviewtoolkit.model.config.PathExclude -import org.ossreviewtoolkit.model.config.PathExcludeReason +import org.ossreviewtoolkit.model.config.* import org.ossreviewtoolkit.model.utils.FileArchiver import org.ossreviewtoolkit.spdx.SpdxExpression import org.ossreviewtoolkit.spdx.SpdxSingleLicenseExpression @@ -506,6 +499,76 @@ class LicenseInfoResolverTest : WordSpec() { result should containNumberOfLocationsForLicense(gplLicense, 2) result should containNumberOfLocationsForLicense(bsdLicense, 4) } + + "contain only chosen licenses" { + val mitLicense = "MIT" + val apacheLicense = "Apache-2.0 WITH LLVM-exception" + val gplLicense = "GPL-2.0-only" + val bsdLicense = "0BSD" + + val licenseInfos = listOf( + createLicenseInfo( + id = pkgId, + declaredLicenses = setOf("$apacheLicense or $gplLicense", mitLicense), + detectedLicenses = listOf( + Findings( + provenance = provenance, + licenses = mapOf( + "$gplLicense OR $bsdLicense" to listOf( + TextLocation("LICENSE", 1), + TextLocation("LICENSE", 21) + ), + bsdLicense to listOf( + TextLocation("LICENSE", 31), + TextLocation("LICENSE", 41) + ) + ).toFindingsSet(), + copyrights = setOf( + CopyrightFinding( + "Copyright GPL 2.0 OR BSD Zero Clause", + TextLocation("LICENSE", 1) + ), + CopyrightFinding("Copyright BSD Zero Clause", TextLocation("LICENSE", 31)) + ), + licenseFindingCurations = emptyList(), + pathExcludes = emptyList(), + relativeFindingsPath = "" + ) + ), + concludedLicense = "$apacheLicense OR $gplLicense".toSpdx() + ) + ) + + val licenseChoice = LicenseChoices( + listOf( + PackageLicenseChoice( + pkgId, + listOf( + LicenseChoice( + "($apacheLicense OR $gplLicense) AND $mitLicense".toSpdx(), + apacheLicense.toSpdx(), + "$apacheLicense OR $gplLicense".toSpdx() + ), + LicenseChoice("$apacheLicense OR $gplLicense".toSpdx(), apacheLicense.toSpdx()), + LicenseChoice("$gplLicense OR $bsdLicense".toSpdx(), bsdLicense.toSpdx()) + ) + ) + ) + ) + + val resolver = createResolver(data = licenseInfos, licenseChoices = licenseChoice) + + val result = resolver.resolveLicenseInfo(pkgId) + + result should containLicenseExactlyBySource( + LicenseSource.DECLARED, apacheLicense.toSpdx(), + mitLicense.toSpdx() + ) + + result should containLicenseExactlyBySource(LicenseSource.DETECTED, bsdLicense.toSpdx()) + + result should containLicenseExactlyBySource(LicenseSource.CONCLUDED, apacheLicense.toSpdx()) + } } "resolveLicenseFiles()" should { @@ -571,11 +634,13 @@ class LicenseInfoResolverTest : WordSpec() { private fun createResolver( data: List, copyrightGarbage: Set = emptySet(), - archiver: FileArchiver = FileArchiver.createDefault() + archiver: FileArchiver = FileArchiver.createDefault(), + licenseChoices: LicenseChoices = LicenseChoices() ) = LicenseInfoResolver( - SimpleLicenseInfoProvider(data), - CopyrightGarbage(copyrightGarbage.toSortedSet()), - archiver + provider = SimpleLicenseInfoProvider(data), + copyrightGarbage = CopyrightGarbage(copyrightGarbage.toSortedSet()), + archiver = archiver, + licenseChoices = licenseChoices ) private fun createLicenseInfo( @@ -739,6 +804,26 @@ fun containLicenseExpressionsExactlyBySource( ) } +fun containLicenseExactlyBySource( + source: LicenseSource, + vararg licences: SpdxExpression? +): Matcher = + neverNullMatcher { resolvedLicenseInfo -> + val actualLicenses = resolvedLicenseInfo.licenses + .filter { it.sources.contains(source) } + .map { it.license } + .toSet() + val expectedLicenses = licences.toSet() + + MatcherResult( + expectedLicenses == actualLicenses, + "ResolvedLicenseInfo for ${source.show().value} licenses should contain exactly " + + "${expectedLicenses.show().value}, but has ${actualLicenses.show().value}.", + "ResolvedLicenseInfo for ${source.show().value} licenses should not contain exactly " + + "${expectedLicenses.show().value}, but has ${actualLicenses.show().value}." + ) + } + fun containLicensesExactly(vararg licenses: String): Matcher?> = neverNullMatcher { value -> val expected = licenses.map { SpdxExpression.parse(it) as SpdxSingleLicenseExpression }.toSet()