Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: correct detection of obsolete files #776

Merged
merged 5 commits into from
May 9, 2024
Merged

fix: correct detection of obsolete files #776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9dbbe6b
Fix detecting of obsolete files
anbraten Apr 26, 2024
301bcf0
fix list
anbraten Apr 29, 2024
f2e0812
adjust code and test
anbraten May 7, 2024
5068328
normalize paths
anbraten May 7, 2024
4578ccb
cleanup var namings
anbraten May 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions src/main/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,23 @@ public class ObsoleteSourcesUtils {

public static Map<String, File> findObsoleteProjectFiles(
@NonNull Map<String, File> projectFiles, boolean preserveHierarchy,
@NonNull List<String> filesToUpload, @NonNull String pattern, @NonNull String exportPattern, List<String> ignorePattern
@NonNull List<String> filesToUpload, @NonNull String sourcePattern, @NonNull String exportPattern, List<String> ignorePatterns
) {
Predicate<String> patternCheck =
ProjectFilesUtils.isProjectFilePathSatisfiesPatterns(pattern, ignorePattern, preserveHierarchy);
ProjectFilesUtils.isProjectFilePathSatisfiesPatterns(sourcePattern, ignorePatterns, preserveHierarchy);
return projectFiles.entrySet().stream()
.filter(entry -> patternCheck.test(entry.getKey()) && checkExportPattern(exportPattern, entry.getValue()))
.filter(entry -> isFileDontHaveUpdate(filesToUpload, entry.getKey(), preserveHierarchy))
.filter(entry -> patternCheck.test(entry.getKey()))
.filter(entry -> checkExportPattern(exportPattern, entry.getValue()))
.filter(entry -> isFileNotInList(filesToUpload, entry.getKey(), preserveHierarchy))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}

private static boolean checkExportPattern(String exportPattern, File file) {
String fileExportPattern = ProjectFilesUtils.getExportPattern(file.getExportOptions());
return exportPattern.equals(fileExportPattern != null ? Utils.normalizePath(fileExportPattern) : null);
if (fileExportPattern == null) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@andrii-bodnar What do you think of this approach?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this mean that files with the null export pattern will be considered obsolete and deleted, or the opposite?

Copy link
Contributor Author

@anbraten anbraten May 8, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It basically uses all projectFiles (files received by the api) and checks for each file:

  • if that file matches the config patterns (source-pattern, ignore and preserveHierarchy)
  • if it matches the exportPattern (if the api returns null for the export pattern we ignore it now by this change)
  • and if the file has isFileDontHaveUpdate=true (which seems to check if the file is not part of the local files anymore)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I adjusted some var and method names so they match the rest of the code and are hopefully more descriptive.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@anbraten great, I just tested it locally and the fix works for me, thank you! 🚀

return true;
}
return exportPattern.equals(Utils.normalizePath(fileExportPattern));
}

public static SortedMap<String, Long> findObsoleteProjectDirectories(
Expand Down Expand Up @@ -71,7 +75,7 @@ public static SortedMap<String, Long> findObsoleteProjectDirectories(
return obsoleteDirs;
}

private static boolean isFileDontHaveUpdate(List<String> filesToUpload, String filePath, boolean preserveHierarchy) {
private static boolean isFileNotInList(List<String> filesToUpload, String filePath, boolean preserveHierarchy) {
String filePathRegex = "^" + (preserveHierarchy ? "" : Utils.PRESERVE_HIERARCHY_REGEX_PART) + Utils.regexPath(filePath) + "$";
return filesToUpload.stream()
.noneMatch(Pattern.compile(filePathRegex).asPredicate());
Expand Down
41 changes: 41 additions & 0 deletions src/test/java/com/crowdin/cli/commands/functionality/ObsoleteSourcesUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package com.crowdin.cli.commands.functionality;

import com.crowdin.cli.utils.Utils;
import com.crowdin.client.sourcefiles.model.File;

import org.junit.jupiter.api.Test;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class ObsoleteSourcesUtilsTest {

@Test
public void testFindObsoleteProjectFiles() {
Map<String, File> projectFilesFromApi = new HashMap<String, File>() {
{
put(Utils.normalizePath("test/en/test.md"), new File());
put(Utils.normalizePath("test/en/support.md"), new File());
put(Utils.normalizePath("test/en/help.md"), new File());
}
};
boolean preserveHierarchy = true;
List<String> filesToUpload = Arrays.asList(Utils.normalizePath("test/en/test.md"),
Utils.normalizePath("test/en/help.md"));
String sourcePattern = Utils.normalizePath("/test/en/*.md");
String exportPattern = Utils.normalizePath("/test/%two_letters_code%/%original_path%/%original_file_name%");
List<String> ignorePatterns = Arrays.asList(Utils.normalizePath("**/.*"));

Map<String, File> obsoleteFiles = ObsoleteSourcesUtils.findObsoleteProjectFiles(projectFilesFromApi,
preserveHierarchy,
filesToUpload, sourcePattern, exportPattern, ignorePatterns);

assertEquals(1, obsoleteFiles.size());
anbraten marked this conversation as resolved.
Show resolved Hide resolved
assertEquals(true, obsoleteFiles.containsKey(Utils.normalizePath("test/en/support.md")));
}

}
Loading