Merge pull request #39805 from tienifr/fix/38549

Add support for Latin characters in emoji suggestion
Expensify · May 30, 2024 · a99ce82 · a99ce82
2 parents c115d69 + 30fd7bb
commit a99ce82
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 9 deletions.
diff --git a/src/CONST.ts b/src/CONST.ts
@@ -1930,8 +1930,8 @@ const CONST = {
         // Extract attachment's source from the data's html string
         ATTACHMENT_DATA: /(data-expensify-source|data-name)="([^"]+)"/g,
 
-        EMOJI_NAME: /:[\w+-]+:/g,
-        EMOJI_SUGGESTIONS: /:[a-zA-Z0-9_+-]{1,40}$/,
+        EMOJI_NAME: /:[\p{L}0-9_+-]+:/gu,
+        EMOJI_SUGGESTIONS: /:[\p{L}0-9_+-]{1,40}$/u,
         AFTER_FIRST_LINE_BREAK: /\n.*/g,
         LINE_BREAK: /\r\n|\r|\n/g,
         CODE_2FA: /^\d{6}$/,

diff --git a/src/libs/EmojiTrie.ts b/src/libs/EmojiTrie.ts
@@ -3,6 +3,7 @@ import type {Emoji, HeaderEmoji, PickerEmoji} from '@assets/emojis/types';
 import CONST from '@src/CONST';
 import type {Locale} from '@src/types/onyx';
 import Timing from './actions/Timing';
+import StringUtils from './StringUtils';
 import Trie from './Trie';
 
 type EmojiMetaData = {
@@ -33,15 +34,25 @@ type EmojiTrie = {
 function addKeywordsToTrie(trie: Trie<EmojiMetaData>, keywords: string[], item: Emoji, name: string, shouldPrependKeyword = false) {
     keywords.forEach((keyword) => {
         const keywordNode = trie.search(keyword);
+        const normalizedKeyword = StringUtils.normalizeAccents(keyword);
+
         if (!keywordNode) {
-            trie.add(keyword, {suggestions: [{code: item.code, types: item.types, name}]});
+            const metadata = {suggestions: [{code: item.code, types: item.types, name}]};
+            if (normalizedKeyword !== keyword) {
+                trie.add(normalizedKeyword, metadata);
+            }
+            trie.add(keyword, metadata);
         } else {
             const suggestion = {code: item.code, types: item.types, name};
             const suggestions = shouldPrependKeyword ? [suggestion, ...(keywordNode.metaData.suggestions ?? [])] : [...(keywordNode.metaData.suggestions ?? []), suggestion];
-            trie.update(keyword, {
+            const newMetadata = {
                 ...keywordNode.metaData,
                 suggestions,
-            });
+            };
+            if (normalizedKeyword !== keyword) {
+                trie.update(normalizedKeyword, newMetadata);
+            }
+            trie.update(keyword, newMetadata);
         }
     });
 }
@@ -68,12 +79,21 @@ function createTrie(lang: SupportedLanguage = CONST.LOCALES.DEFAULT): Trie<Emoji
         .forEach((item: Emoji) => {
             const englishName = item.name;
             const localeName = langEmojis?.[item.code]?.name ?? englishName;
+            const normalizedName = StringUtils.normalizeAccents(localeName);
 
             const node = trie.search(localeName);
             if (!node) {
-                trie.add(localeName, {code: item.code, types: item.types, name: localeName, suggestions: []});
+                const metadata = {code: item.code, types: item.types, name: localeName, suggestions: []};
+                if (normalizedName !== localeName) {
+                    trie.add(normalizedName, metadata);
+                }
+                trie.add(localeName, metadata);
             } else {
-                trie.update(localeName, {code: item.code, types: item.types, name: localeName, suggestions: node.metaData.suggestions});
+                const newMetadata = {code: item.code, types: item.types, name: localeName, suggestions: node.metaData.suggestions};
+                if (normalizedName !== localeName) {
+                    trie.update(normalizedName, newMetadata);
+                }
+                trie.update(localeName, newMetadata);
             }
 
             const nameParts = getNameParts(localeName).slice(1); // We remove the first part because we already index the full name.

diff --git a/src/libs/GetStyledTextArray.ts b/src/libs/GetStyledTextArray.ts
@@ -1,4 +1,5 @@
 import Str from 'expensify-common/lib/str';
+import StringUtils from './StringUtils';
 
 type StyledText = {
     text: string;
@@ -8,7 +9,9 @@ type StyledText = {
 const getStyledTextArray = (name: string, prefix: string): StyledText[] => {
     const texts = [];
     const prefixLowercase = prefix.toLowerCase();
-    const prefixLocation = name.toLowerCase().search(Str.escapeForRegExp(prefixLowercase));
+    const prefixLocation = StringUtils.normalizeAccents(name)
+        .toLowerCase()
+        .search(Str.escapeForRegExp(StringUtils.normalizeAccents(prefixLowercase)));
 
     if (prefixLocation === 0 && prefix.length === name.length) {
         texts.push({text: name, isColored: true});

diff --git a/src/libs/StringUtils.ts b/src/libs/StringUtils.ts
@@ -63,6 +63,15 @@ function removeInvisibleCharacters(value: string): string {
     return result.trim();
 }
 
+/**
+ * Remove accents/diacritics
+ * @param text - The input string
+ * @returns The string with all accents/diacritics removed
+ */
+function normalizeAccents(text: string) {
+    return text.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
+}
+
 /**
  *  Replace all CRLF with LF
  *  @param value - The input string
@@ -96,4 +105,4 @@ function lineBreaksToSpaces(text = '') {
     return text.replace(CONST.REGEX.LINE_BREAK, ' ');
 }
 
-export default {sanitizeString, isEmptyString, removeInvisibleCharacters, normalizeCRLF, getAcronym, lineBreaksToSpaces};
+export default {sanitizeString, isEmptyString, removeInvisibleCharacters, normalizeAccents, normalizeCRLF, getAcronym, lineBreaksToSpaces};