fix: set bufferSize.

b4rtaz · Jul 27, 2024 · 9a729c9 · 9a729c9
1 parent dc0e94f
commit 9a729c9
Showing 1 changed file with 5 additions and 4 deletions.
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
@@ -522,16 +522,17 @@ EosDetector::~EosDetector() {
 
 EosDetectorType EosDetector::append(int tokenId, const char* piece) {
     int pieceLength = strlen(piece);
-    int length = bufferPos + pieceLength + 1;
-    if (length > bufferSize) {
-        char* newBuffer = new char[length];
+    int newSize = bufferPos + pieceLength + 1;
+    if (newSize > bufferSize) {
+        char* newBuffer = new char[newSize];
         if (bufferPos > 0)
             memcpy(newBuffer, buffer, bufferPos);
         if (bufferSize > 0)
             delete[] buffer;
         buffer = newBuffer;
+        bufferSize = newSize;
     }
-    memcpy(buffer + bufferPos, piece, pieceLength + 1);
+    memcpy(&buffer[bufferPos], piece, pieceLength + 1);
     bufferPos += pieceLength;
 
     // detection