Skip to content

Commit

Permalink
SCANMAVEN-216 scanAll includes Java and Kotlin files if `sonar.java.b…
Browse files Browse the repository at this point in the history 
…inaries` and `sonar.java.libraries` are set (#222)
  • Loading branch information
@alban-auzeill
alban-auzeill authored May 21, 2024
1 parent b772d4c commit e3cc5ec
Show file tree
Hide file tree
Showing 10 changed files with 200 additions and 74 deletions.
4 changes: 2 additions & 2 deletions src/main/java/org/sonarsource/scanner/maven/bootstrap/MavenProjectConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ public class MavenProjectConverter {

public static final String FINDBUGS_EXCLUDE_FILTERS = "sonar.findbugs.excludeFilters";

private static final String JAVA_PROJECT_MAIN_BINARY_DIRS = "sonar.java.binaries";
public static final String JAVA_PROJECT_MAIN_BINARY_DIRS = "sonar.java.binaries";

private static final String JAVA_PROJECT_MAIN_LIBRARIES = "sonar.java.libraries";
public static final String JAVA_PROJECT_MAIN_LIBRARIES = "sonar.java.libraries";

private static final String SONAR_JAVA_JDK_HOME_PROPERTY = "sonar.java.jdkHome";

Expand Down
15 changes: 9 additions & 6 deletions src/main/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,15 +124,15 @@ Map<String, String> collectProperties()
Properties userProperties = session.getUserProperties();
Map<String, String> props = mavenProjectConverter.configure(sortedProjects, topLevelProject, userProperties);
props.putAll(propertyDecryptor.decryptProperties(props));

if (shouldCollectAllSources(userProperties)) {
log.info("Parameter " + MavenScannerProperties.PROJECT_SCAN_ALL_SOURCES + " is enabled. The scanner will attempt to collect additional sources.");
if (mavenProjectConverter.isSourceDirsOverridden()) {
log.warn(notCollectingAdditionalSourcesBecauseOf(ScanProperties.PROJECT_SOURCE_DIRS));
} else if (mavenProjectConverter.isTestDirsOverridden()) {
log.warn(notCollectingAdditionalSourcesBecauseOf(ScanProperties.PROJECT_TEST_DIRS));
} else {
collectAllSources(props);
boolean shouldCollectJavaAndKotlinSources = isUserDefinedJavaBinaries(userProperties);
collectAllSources(props, shouldCollectJavaAndKotlinSources);
}
}

Expand Down Expand Up @@ -162,7 +162,7 @@ private static Set<Path> excludedReportFiles(Map<String, String> props) {
}

@VisibleForTesting
void collectAllSources(Map<String, String> props) {
void collectAllSources(Map<String, String> props, boolean shouldCollectJavaAndKotlinSources) {
String projectBasedir = props.get(ScanProperties.PROJECT_BASEDIR);
// Exclude the files and folders covered by sonar.sources and sonar.tests (and sonar.exclusions) as computed by the MavenConverter
// Combine all the sonar.sources at the top-level and by module
Expand All @@ -178,9 +178,7 @@ void collectAllSources(Map<String, String> props) {
Set<Path> existingSources = coveredSources.stream()
.map(Paths::get)
.collect(Collectors.toSet());
Set<Path> excludedFiles = excludedReportFiles(props);

SourceCollector visitor = new SourceCollector(existingSources, mavenProjectConverter.getSkippedBasedDirs(), excludedFiles);
SourceCollector visitor = new SourceCollector(existingSources, mavenProjectConverter.getSkippedBasedDirs(), excludedReportFiles(props), shouldCollectJavaAndKotlinSources);
Files.walkFileTree(Paths.get(projectBasedir), visitor);
collectedSources = visitor.getCollectedSources().stream()
.map(file -> file.toAbsolutePath().toString())
Expand All @@ -194,6 +192,11 @@ void collectAllSources(Map<String, String> props) {
}
}

private static boolean isUserDefinedJavaBinaries(Properties userProperties) {
return userProperties.containsKey(MavenProjectConverter.JAVA_PROJECT_MAIN_LIBRARIES) &&
userProperties.containsKey(MavenProjectConverter.JAVA_PROJECT_MAIN_BINARY_DIRS);
}

private void checkSQVersion() {
if (isVersionPriorTo("5.6")) {
throw new UnsupportedOperationException(UNSUPPORTED_BELOW_SONARQUBE_56_MESSAGE);
Expand Down
86 changes: 46 additions & 40 deletions src/main/java/org/sonarsource/scanner/maven/bootstrap/SourceCollector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

public class SourceCollector implements FileVisitor<Path> {
Expand All @@ -44,54 +46,59 @@ public class SourceCollector implements FileVisitor<Path> {
)
);

private static final Set<String> EXCLUDED_EXTENSIONS = new HashSet<>(
Arrays.asList(
"jar",
"war",
"class",
"ear",
"nar",
// Archives
"DS_Store",
"zip",
"7z",
"rar",
"gz",
"tar",
"xz",
// log
"log",
// temp files
"bak",
"tmp",
"swp",
// ide files
"iml",
"ipr",
"iws",
"nib",
"log",
"java",
"jav",
"kt",
"scala"
)
);
private static final Set<String> EXCLUDED_EXTENSIONS_WITH_JAVA_AND_KOTLIN = Stream.of(
".jar",
".war",
".class",
".ear",
".nar",
// Archives
".DS_Store",
".zip",
".7z",
".rar",
".gz",
".tar",
".xz",
// log
".log",
// temp files
".bak",
".tmp",
".swp",
// ide files
".iml",
".ipr",
".iws",
".nib",
".log")
.map(ext -> ext.toLowerCase(Locale.ROOT))
.collect(Collectors.toSet());

private static final Set<String> EXCLUDED_EXTENSIONS_WITHOUT_JAVA_AND_KOTLIN = Stream.concat(EXCLUDED_EXTENSIONS_WITH_JAVA_AND_KOTLIN.stream(), Stream.of(
".java",
".jav",
".kt")).map(ext -> ext.toLowerCase(Locale.ROOT))
.collect(Collectors.toSet());

private final Set<Path> existingSources;
private final Set<Path> directoriesToIgnore;
private final Set<Path> excludedFiles;
private final Set<String> excludedExtensions;

public Set<Path> getCollectedSources() {
return collectedSources;
}

private final Set<Path> collectedSources = new HashSet<>();

public SourceCollector(Set<Path> existingSources, Set<Path> directoriesToIgnore, Set<Path> excludedFiles) {
public SourceCollector(Set<Path> existingSources, Set<Path> directoriesToIgnore, Set<Path> excludedFiles, boolean shouldCollectJavaAndKotlinSources) {
this.existingSources = existingSources;
this.directoriesToIgnore = directoriesToIgnore;
this.excludedFiles = excludedFiles;
this.excludedExtensions = shouldCollectJavaAndKotlinSources ? EXCLUDED_EXTENSIONS_WITH_JAVA_AND_KOTLIN : EXCLUDED_EXTENSIONS_WITHOUT_JAVA_AND_KOTLIN;
}

@Override
public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
if (
Expand Down Expand Up @@ -120,12 +127,11 @@ private boolean isCoveredByExistingSources(Path path) {

@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) {
if (
!excludedFiles.contains(path) &&
EXCLUDED_EXTENSIONS.stream().noneMatch(ext -> path.toString().endsWith(ext)) &&
existingSources.stream().noneMatch(path::equals)
) {
collectedSources.add(path);
if (!excludedFiles.contains(path) && existingSources.stream().noneMatch(path::equals)) {
String lowerCaseFileName = path.getFileName().toString().toLowerCase(Locale.ROOT);
if (excludedExtensions.stream().noneMatch(lowerCaseFileName::endsWith)) {
collectedSources.add(path);
}
}
return FileVisitResult.CONTINUE;
}
Expand Down
76 changes: 61 additions & 15 deletions src/test/java/org/sonarsource/scanner/maven/SonarQubeMojoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Map;
import java.util.Properties;
Expand Down Expand Up @@ -104,6 +105,50 @@ public void shouldExportOverridenWarWebSource() throws Exception {
+ new File(baseDir, "src/main/java").getAbsolutePath()));
}

@Test
public void project_with_java_files_not_in_src_should_not_be_collected() throws Exception {
File baseDir = executeProject(
"project-with-java-files-not-in-src",
"sonar.maven.scanAll", "true");
Set<String> actualListOfSources = extractSonarSources("target/dump.properties", baseDir.toPath());
assertThat(actualListOfSources).containsExactlyInAnyOrder(
"/pom.xml", "/src/main/java");
}

@Test
public void project_with_java_files_not_in_src_should_be_collected_when_user_define_binaries_and_libraries() throws Exception {
File baseDir = executeProject(
"project-with-java-files-not-in-src",
"sonar.maven.scanAll", "true",
"sonar.java.binaries", "target/classes",
"sonar.java.libraries", "target/lib/logger.jar");
Set<String> actualListOfSources = extractSonarSources("target/dump.properties", baseDir.toPath());
assertThat(actualListOfSources).containsExactlyInAnyOrder(
"/pom.xml", "/src/main/java", "/Hello.java", "/Hello.kt");
}

@Test
public void project_with_java_files_not_in_src_should_not_be_collected_when_user_define_only_binaries() throws Exception {
File baseDir = executeProject(
"project-with-java-files-not-in-src",
"sonar.maven.scanAll", "true",
"sonar.java.binaries", "target/classes");
Set<String> actualListOfSources = extractSonarSources("target/dump.properties", baseDir.toPath());
assertThat(actualListOfSources).containsExactlyInAnyOrder(
"/pom.xml", "/src/main/java");
}

@Test
public void project_with_java_files_not_in_src_should_not_be_collected_when_user_define_only_libraries() throws Exception {
File baseDir = executeProject(
"project-with-java-files-not-in-src",
"sonar.maven.scanAll", "true",
"sonar.java.libraries", "target/lib/logger.jar");
Set<String> actualListOfSources = extractSonarSources("target/dump.properties", baseDir.toPath());
assertThat(actualListOfSources).containsExactlyInAnyOrder(
"/pom.xml", "/src/main/java");
}

// MSONAR-113
@Test
public void shouldExportSurefireReportsPath() throws Exception {
Expand All @@ -130,19 +175,7 @@ public void reuse_findbugs_exclusions_from_reporting() throws IOException, Excep
@Test
public void exclude_report_paths_from_scanAll() throws Exception {
File projectBarDir = executeProject("project-with-external-reports", "sonar.maven.scanAll", "true");

String sources = readProps("target/dump.properties")
.entrySet()
.stream()
.filter(e -> e.getKey().toString().equals("sonar.sources"))
.map(Map.Entry::getValue)
.findFirst()
.orElse(null);

Set<String> actualListOfSources = Arrays.stream(sources.split(","))
.map(file -> file.replace(projectBarDir.toString(), "").replace("\\", "/"))
.collect(Collectors.toSet());

Set<String> actualListOfSources = extractSonarSources("target/dump.properties", projectBarDir.toPath());
assertThat(actualListOfSources).containsExactlyInAnyOrder("/other.xml", "/pom.xml");
}

Expand Down Expand Up @@ -193,11 +226,11 @@ private File executeProject(String projectName, String... properties) throws Exc
}

@SafeVarargs
private final void assertPropsContains(MapEntry<String, String>... entries) throws FileNotFoundException, IOException {
private final void assertPropsContains(MapEntry<String, String>... entries) throws IOException {
assertThat(readProps("target/dump.properties")).contains(entries);
}

private Map<String, String> readProps(String filePath) throws FileNotFoundException, IOException {
private static Map<String, String> readProps(String filePath) throws IOException {
FileInputStream fis = null;
try {
File dump = new File(filePath);
Expand All @@ -210,4 +243,17 @@ private Map<String, String> readProps(String filePath) throws FileNotFoundExcept
}
}

private static Set<String> extractSonarSources(String propertiesPath, Path projectBarDir) throws IOException {
String sources = readProps(propertiesPath)
.entrySet()
.stream()
.filter(e -> e.getKey().endsWith("sonar.sources"))
.map(Map.Entry::getValue)
.findFirst()
.orElse(null);

return Arrays.stream(sources.split(","))
.map(file -> file.replace(projectBarDir.toString(), "").replace("\\", "/"))
.collect(Collectors.toSet());
}
}
33 changes: 28 additions & 5 deletions src/test/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapperTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ void scanAll_property_is_applied_by_default() throws MojoExecutionException {
});

verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources.");
verify(scannerBootstrapper, times(1)).collectAllSources(any());
verify(scannerBootstrapper, times(1)).collectAllSources(any(), eq(false));
}

@Test
Expand All @@ -179,7 +179,7 @@ void scanAll_property_is_not_applied_when_set_explicitly() throws MojoExecutionE
});

verify(log, never()).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources.");
verify(scannerBootstrapper, never()).collectAllSources(any());
verify(scannerBootstrapper, never()).collectAllSources(any(), eq(false));
}

@Test
Expand All @@ -194,7 +194,22 @@ void scanAll_property_is_applied_when_set_explicitly() throws MojoExecutionExcep
});

verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources.");
verify(scannerBootstrapper, times(1)).collectAllSources(any());
verify(scannerBootstrapper, times(1)).collectAllSources(any(), eq(false));
}

@Test
void scanAll_should_also_collect_java_and_kotlin_sources_when_binaries_and_libraries_are_explicitly_set() throws MojoExecutionException {
setSonarScannerScanAllAndBinariesAndLibraries();

verifyCollectedSources(sourceDirs -> {
assertThat(sourceDirs).hasSize(3);
assertThat(sourceDirs[0]).endsWith(Paths.get("src", "main", "java").toString());
assertThat(sourceDirs[1]).endsWith(Paths.get("pom.xml").toString());
assertThat(sourceDirs[2]).endsWith(Paths.get("src", "main", "resources", "index.js").toString());
});

verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources.");
verify(scannerBootstrapper, times(1)).collectAllSources(any(), eq(true));
}

@Test
Expand All @@ -212,7 +227,7 @@ void should_not_collect_all_sources_when_sonar_sources_is_overridden() throws Mo

verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources.");
verify(log, times(1)).warn("Parameter sonar.maven.scanAll is enabled but the scanner will not collect additional sources because sonar.sources has been overridden.");
verify(scannerBootstrapper, never()).collectAllSources(any());
verify(scannerBootstrapper, never()).collectAllSources(any(), eq(false));
}

@Test
Expand All @@ -231,7 +246,7 @@ void should_not_collect_all_sources_when_sonar_tests_is_overridden() throws Mojo

verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources.");
verify(log, times(1)).warn("Parameter sonar.maven.scanAll is enabled but the scanner will not collect additional sources because sonar.tests has been overridden.");
verify(scannerBootstrapper, never()).collectAllSources(any());
verify(scannerBootstrapper, never()).collectAllSources(any(), eq(false));
}

@Test
Expand Down Expand Up @@ -334,6 +349,14 @@ private void setSonarScannerScanAllTo(String value) {
when(session.getUserProperties()).thenReturn(withScanAllSet);
}

private void setSonarScannerScanAllAndBinariesAndLibraries() {
Properties withScanAllSet = new Properties();
withScanAllSet.put(MavenScannerProperties.PROJECT_SCAN_ALL_SOURCES, "true");
withScanAllSet.put(MavenProjectConverter.JAVA_PROJECT_MAIN_BINARY_DIRS, "target/classes");
withScanAllSet.put(MavenProjectConverter.JAVA_PROJECT_MAIN_LIBRARIES, "target/lib/log.jar");
when(session.getUserProperties()).thenReturn(withScanAllSet);
}

private void verifyCollectedSources(Consumer<String[]> sourceDirsAssertions) throws MojoExecutionException {
Map<String, String> collectedProperties = scannerBootstrapper.collectProperties();
assertThat(collectedProperties).containsKey(ScanProperties.PROJECT_SOURCE_DIRS);
Expand Down
Loading

0 comments on commit e3cc5ec

Please sign in to comment.