TH_Tokenized: more fixes for getPosInfoAtOffset (mb-aware string len...)

src/TypeHandlers/TH_Tokenized.cpp

@@ -20,6 +20,14 @@
 #include <regex>
 #include <fstream>
 
+static size_t strlen_mb(const std::string& s)
+{
+    size_t len = 0;
+    for (const char* p = s.data(); *p != 0; ++p)
+        len += (*p & 0xc0) != 0x80;
+    return len;
+}
+
 namespace tivars
 {
     namespace TH_Tokenized
@@ -438,11 +446,12 @@ namespace tivars
                 tokStr = std::regex_replace(tokStr, toPrettifyRX, "$1");
             }
 
-            posinfo.column += (uint16_t)tokStr.size();
+            const size_t tokStrLen = strlen_mb(tokStr);
+            posinfo.column += (uint16_t)tokStrLen;
 
             if (posinfo.len == 0 && ((currIdx == byteOffset && !is2ByteTok) || (currIdx >= byteOffset-1 && is2ByteTok)))
             {
-                posinfo.len = (uint8_t)tokStr.size();
+                posinfo.len = (uint8_t)tokStrLen;
                 posinfo.column -= posinfo.len; // column will be the beginning of the token
             }
         }

src/main_emscripten.cpp

@@ -11,12 +11,15 @@
 #include "TIVarTypes.h"
 #include "TypeHandlers/TypeHandlers.h"
 
+#include <locale.h>
 #include <emscripten.h>
 
 using namespace tivars;
 
 extern "C" int EMSCRIPTEN_KEEPALIVE main(int, char**)
 {
+    setlocale(LC_ALL, ".UTF-8");
+
     TIModels::initTIModelsArray();
     TIVarTypes::initTIVarTypesArray();
     TH_Tokenized::initTokens();

tests.cpp

@@ -111,6 +111,14 @@ int main(int argc, char** argv)
         assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
     }
 
+    {
+        TH_Tokenized::token_posinfo actual{}, expected{};
+        const std::string hexStr = "010004";
+        actual = TH_Tokenized::getPosInfoAtOffsetFromHexStr(hexStr, 2);
+        expected = { 0, 0, 1 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+    }
+
     {
         // Test string interpolation behaviour
         TIVarFile testPrgm = TIVarFile::createNew("Program", "INTERP");