add various utility functions and tests.

- TIVarFile::getRawContentHexStr
- TH_Tokenized::oneTokenBytesToString
- TH_Tokenized::getPosInfoAtOffsetFromHexStr

src/TIVarFile.cpp

@@ -6,12 +6,16 @@
  */
 
 #include "TIVarFile.h"
+
+#include <iomanip>
+
 #include "tivarslib_utils.h"
 #include "TIModels.h"
 
 #include <stdexcept>
 #include <numeric>
 #include <regex>
+#include <sstream>
 
 namespace tivars
 {
@@ -338,6 +342,17 @@ namespace tivars
         return this->getRawContent(0);
     }
 
+    std::string TIVarFile::getRawContentHexStr()
+    {
+        const data_t rawContent = getRawContent();
+        std::ostringstream result;
+        for (const auto& v : rawContent)
+        {
+            result << std::setfill('0') << std::setw(sizeof(v) * 2) << std::hex << +v;
+        }
+        return result.str();
+    }
+
     std::string TIVarFile::getReadableContent(const options_t& options, uint16_t entryIdx)
     {
         const auto& entry = this->entries[entryIdx];
@@ -474,6 +489,7 @@ namespace tivars
                     .function("setArchived"              , select_overload<void(bool)>(&tivars::TIVarFile::setArchived))
                     .function("isCorrupt"                , &tivars::TIVarFile::isCorrupt)
                     .function("getRawContent"            , select_overload<data_t(void)>(&tivars::TIVarFile::getRawContent))
+                    .function("getRawContentHexStr"      , &tivars::TIVarFile::getRawContentHexStr)
                     .function("getReadableContent"       , select_overload<std::string(const options_t&)>(&tivars::TIVarFile::getReadableContent))
                     .function("getReadableContent"       , select_overload<std::string(void)>(&tivars::TIVarFile::getReadableContent))
 

src/TIVarFile.h

@@ -91,6 +91,8 @@ namespace tivars
         data_t getRawContent(uint16_t entryIdx);
         data_t getRawContent();
 
+        std::string getRawContentHexStr();
+
         std::string getReadableContent(const options_t& options, uint16_t entryIdx);
         std::string getReadableContent(const options_t& options);
         std::string getReadableContent();

src/TypeHandlers/TH_Tokenized.cpp

@@ -351,6 +351,27 @@ namespace tivars
         return tokStr;
     }
 
+    std::string TH_Tokenized::oneTokenBytesToString(uint16_t tokenBytes)
+    {
+        if (tokenBytes < 0xFF && is_in_vector(firstByteOfTwoByteTokens, (uint8_t)(tokenBytes & 0xFF)))
+        {
+            std::cerr << "[Warning] Encountered an unfinished two-byte token!";
+            return "";
+        }
+
+        std::string tokStr;
+        if (tokens_BytesToName.find(tokenBytes) != tokens_BytesToName.end())
+        {
+            tokStr = tokens_BytesToName[tokenBytes][LANG_EN];
+        } else {
+            tokStr = " [???] ";
+        }
+
+        tokStr = std::regex_replace(tokStr, toPrettifyRX, "$1");
+
+        return tokStr;
+    }
+
     TH_Tokenized::token_posinfo TH_Tokenized::getPosInfoAtOffset(const data_t& data, uint16_t byteOffset, const options_t& options)
     {
         const size_t dataSize = data.size();
@@ -429,6 +450,21 @@ namespace tivars
         return posinfo;
     }
 
+    TH_Tokenized::token_posinfo TH_Tokenized::getPosInfoAtOffsetFromHexStr(const std::string& hexBytesStr, uint16_t byteOffset)
+    {
+        const size_t strLen = hexBytesStr.length();
+        if (strLen % 2 != 0 || strLen > 65500 * 2) // todo: actual len?
+        {
+            throw std::invalid_argument("invalid hexBytesStr length!");
+        }
+
+        data_t data;
+        for (size_t i = 0; i < strLen; i += 2) {
+            data.push_back((char) strtol(hexBytesStr.substr(i, 2).c_str(), nullptr, 16));
+        }
+
+        return getPosInfoAtOffset(data, byteOffset, { { "prettify", 1 } });
+    }
 
     void TH_Tokenized::initTokens()
     {
@@ -488,3 +524,18 @@ namespace tivars
     }
 
 }
+
+#ifdef __EMSCRIPTEN__
+    #include <emscripten/bind.h>
+    using namespace emscripten;
+    EMSCRIPTEN_BINDINGS(_thtokenized) {
+
+        value_object<tivars::TH_Tokenized::token_posinfo>("token_posinfo")
+            .field("line",   &tivars::TH_Tokenized::token_posinfo::line)
+            .field("column", &tivars::TH_Tokenized::token_posinfo::column)
+            .field("len",    &tivars::TH_Tokenized::token_posinfo::len);
+
+        function("TH_Tokenized_getPosInfoAtOffsetFromHexStr", &tivars::TH_Tokenized::getPosInfoAtOffsetFromHexStr);
+        function("TH_Tokenized_oneTokenBytesToString"       , &tivars::TH_Tokenized::oneTokenBytesToString);
+    }
+#endif

src/TypeHandlers/TypeHandlers.h

@@ -94,7 +94,9 @@ namespace tivars
         struct token_posinfo { uint16_t line; uint16_t column; uint8_t len; };
         std::string reindentCodeString(const std::string& str_orig, const options_t& options = options_t());
         token_posinfo getPosInfoAtOffset(const data_t& data, uint16_t byteOffset, const options_t& options = options_t());
+        token_posinfo getPosInfoAtOffsetFromHexStr(const std::string& hexBytesStr, uint16_t byteOffset);
         std::string tokenToString(const data_t& data, int *incr, const options_t& options);
+        std::string oneTokenBytesToString(uint16_t tokenBytes);
         void initTokens();
         void initTokensFromCSVFilePath(const std::string& csvFilePath);
         void initTokensFromCSVContent(const std::string& csvFileStr);

tests.cpp

@@ -64,6 +64,42 @@ int main(int argc, char** argv)
         assert(trim(testPrgmStr1.getReadableContent({{"prettify", true}, {"reindent", true}})) == "\"42→Str1\nStr2\n123");
     }
 
+    {
+        assert(TH_Tokenized::oneTokenBytesToString(0x00) == "");
+        assert(TH_Tokenized::oneTokenBytesToString(0xBB) == "");
+        assert(TH_Tokenized::oneTokenBytesToString(0x3F) == "\n");
+        assert(TH_Tokenized::oneTokenBytesToString(0xAD) == "getKey");
+        assert(TH_Tokenized::oneTokenBytesToString(0xEF97) == "toString(");
+    }
+
+    {
+        TH_Tokenized::token_posinfo actual{}, expected{};
+        const data_t data = { 0x12,0x00,0x41,0x40,0x42,0x3f,0xde,0x2a,0x41,0x29,0xbb,0xb0,0xbb,0xbe,0xbb,0xb3,0x29,0x42,0x2a,0x3f };
+        actual = TH_Tokenized::getPosInfoAtOffset(data, 2);
+        expected = { 0, 0, 1 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+        actual = TH_Tokenized::getPosInfoAtOffset(data, 3);
+        expected = { 0, 1, 5 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+        actual = TH_Tokenized::getPosInfoAtOffset(data, 6);
+        expected = { 1, 0, 5 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+    }
+
+    {
+        TH_Tokenized::token_posinfo actual{}, expected{};
+        const std::string hexStr = "12004140423fde2a4129bbb0bbbebbb329422a3f";
+        actual = TH_Tokenized::getPosInfoAtOffsetFromHexStr(hexStr, 2);
+        expected = { 0, 0, 1 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+        actual = TH_Tokenized::getPosInfoAtOffsetFromHexStr(hexStr, 3);
+        expected = { 0, 1, 5 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+        actual = TH_Tokenized::getPosInfoAtOffsetFromHexStr(hexStr, 6);
+        expected = { 1, 0, 5 };
+        assert(memcmp(&actual, &expected, sizeof(actual)) == 0);
+    }
+
     {
         // Test string interpolation behaviour
         TIVarFile testPrgm = TIVarFile::createNew("Program", "INTERP");
@@ -133,6 +169,7 @@ int main(int argc, char** argv)
     {
         TIVarFile testReal = TIVarFile::loadFromFile("testData/Real.8xn");
         assert(testReal.getRawContent() == data_t({0x80,0x81,0x42,0x13,0x37,0x00,0x00,0x00,0x00}));
+        assert(testReal.getRawContentHexStr() == "808142133700000000");
         assert(testReal.getReadableContent() == "-42.1337");
         testReal.setContentFromString("5");
         cout << "testReal.getReadableContent() : " << testReal.getReadableContent() << endl;