NKF.guess may return ASCII for binary strings

shugo · Jan 16, 2024 · 4832967 · 4832967
1 parent a69ba04
commit 4832967
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 1 deletion.
diff --git a/lib/textbringer/buffer.rb b/lib/textbringer/buffer.rb
@@ -44,7 +44,11 @@ class Buffer
       e = NKF.guess(s)
       case e
       when Encoding::US_ASCII
-        Encoding::UTF_8
+        if s.ascii_only?
+          Encoding::UTF_8
+        else
+          Encoding::ASCII_8BIT
+        end
       when Encoding::ASCII_8BIT
         s.force_encoding(Encoding::UTF_8)
         if s.valid_encoding?

diff --git a/test/textbringer/test_buffer.rb b/test/textbringer/test_buffer.rb
@@ -8,6 +8,21 @@ class TestBuffer < Textbringer::TestCase
     KILL_RING.clear
   end
 
+  def test_nkf_detect_encoding
+    assert_equal(Encoding::UTF_8,
+                 Buffer::NKF_DETECT_ENCODING.call(""))
+    assert_equal(Encoding::UTF_8,
+                 Buffer::NKF_DETECT_ENCODING.call("foo"))
+    assert_equal(Encoding::UTF_8,
+                 Buffer::NKF_DETECT_ENCODING.call("あいう"))
+    assert_equal(Encoding::EUC_JP,
+                 Buffer::NKF_DETECT_ENCODING.call("あいう".encode("EUC-JP")))
+    assert_equal(Encoding::Shift_JIS,
+                 Buffer::NKF_DETECT_ENCODING.call("あいう".encode("Shift_JIS")))
+    assert_equal(Encoding::ASCII_8BIT,
+                 Buffer::NKF_DETECT_ENCODING.call("\xab"))
+  end
+
   def test_s_list
     Buffer.new_buffer("foo")
     Buffer.new_buffer("bar")