Allow Rust to store unicode as well as bytes (#6108)

This is currently unused, but will be used to land
#6103 with
DirectoryDigest.fingerprint being a `unicode` not a `bytes`.

src/python/pants/engine/native.py

@@ -10,11 +10,13 @@
 import sys
 import sysconfig
 import traceback
+from builtins import str
 from contextlib import closing
 
 import cffi
 import pkg_resources
 import six
+from future.utils import native
 
 from pants.engine.selectors import Get, constraint_for
 from pants.util.contextutil import temporary_dir
@@ -110,6 +112,7 @@
 typedef _Bool               (*extern_ptr_satisfied_by_type)(ExternContext*, Handle*, TypeId*);
 typedef Handle              (*extern_ptr_store_tuple)(ExternContext*, Handle**, uint64_t);
 typedef Handle              (*extern_ptr_store_bytes)(ExternContext*, uint8_t*, uint64_t);
+typedef Handle              (*extern_ptr_store_utf8)(ExternContext*, uint8_t*, uint64_t);
 typedef Handle              (*extern_ptr_store_i64)(ExternContext*, int64_t);
 typedef HandleBuffer        (*extern_ptr_project_multi)(ExternContext*, Handle*, uint8_t*, uint64_t);
 typedef Handle              (*extern_ptr_project_ignoring_type)(ExternContext*, Handle*, uint8_t*, uint64_t);
@@ -155,6 +158,7 @@
                  extern_ptr_satisfied_by_type,
                  extern_ptr_store_tuple,
                  extern_ptr_store_bytes,
+                 extern_ptr_store_utf8,
                  extern_ptr_store_i64,
                  extern_ptr_project_ignoring_type,
                  extern_ptr_project_multi,
@@ -264,6 +268,7 @@
   _Bool               extern_satisfied_by_type(ExternContext*, Handle*, TypeId*);
   Handle              extern_store_tuple(ExternContext*, Handle**, uint64_t);
   Handle              extern_store_bytes(ExternContext*, uint8_t*, uint64_t);
+  Handle              extern_store_utf8(ExternContext*, uint8_t*, uint64_t);
   Handle              extern_store_i64(ExternContext*, int64_t);
   Handle              extern_project_ignoring_type(ExternContext*, Handle*, uint8_t*, uint64_t);
   HandleBuffer        extern_project_multi(ExternContext*, Handle*, uint8_t*, uint64_t);
@@ -424,6 +429,12 @@ def extern_store_bytes(context_handle, bytes_ptr, bytes_len):
     c = ffi.from_handle(context_handle)
     return c.to_value(bytes(ffi.buffer(bytes_ptr, bytes_len)))
 
+  @ffi.def_extern()
+  def extern_store_utf8(context_handle, utf8_ptr, utf8_len):
+    """Given a context and UTF8 bytes, return a new Handle to represent the content."""
+    c = ffi.from_handle(context_handle)
+    return c.to_value(native(str(ffi.buffer(utf8_ptr, utf8_len))))
+
   @ffi.def_extern()
   def extern_store_i64(context_handle, i64):
     """Given a context and int32_t, return a new Handle to represent the int32_t."""
@@ -687,6 +698,7 @@ def init_externs():
                            self.ffi_lib.extern_satisfied_by_type,
                            self.ffi_lib.extern_store_tuple,
                            self.ffi_lib.extern_store_bytes,
+                           self.ffi_lib.extern_store_utf8,
                            self.ffi_lib.extern_store_i64,
                            self.ffi_lib.extern_project_ignoring_type,
                            self.ffi_lib.extern_project_multi,

src/rust/engine/src/externs.rs

@@ -68,10 +68,20 @@ pub fn store_tuple(values: &[Value]) -> Value {
   with_externs(|e| (e.store_tuple)(e.context, handles.as_ptr(), handles.len() as u64).into())
 }
 
+///
+/// Store an opqaue buffer of bytes to pass to Python. This will end up as a Python `bytes`.
+///
 pub fn store_bytes(bytes: &[u8]) -> Value {
   with_externs(|e| (e.store_bytes)(e.context, bytes.as_ptr(), bytes.len() as u64).into())
 }
 
+///
+/// Store an buffer of utf8 bytes to pass to Python. This will end up as a Python `unicode`.
+///
+pub fn store_utf8(utf8: &str) -> Value {
+  with_externs(|e| (e.store_utf8)(e.context, utf8.as_ptr(), utf8.len() as u64).into())
+}
+
 pub fn store_i64(val: i64) -> Value {
   with_externs(|e| (e.store_i64)(e.context, val).into())
 }
@@ -262,6 +272,7 @@ pub struct Externs {
   pub satisfied_by_type: SatisfiedByTypeExtern,
   pub store_tuple: StoreTupleExtern,
   pub store_bytes: StoreBytesExtern,
+  pub store_utf8: StoreUtf8Extern,
   pub store_i64: StoreI64Extern,
   pub project_ignoring_type: ProjectIgnoringTypeExtern,
   pub project_multi: ProjectMultiExtern,
@@ -297,6 +308,9 @@ pub type StoreTupleExtern =
 
 pub type StoreBytesExtern = extern "C" fn(*const ExternContext, *const u8, u64) -> Handle;
 
+pub type StoreUtf8Extern = extern "C" fn(*const ExternContext, *const u8, u64) -> Handle;
+
+
 pub type StoreI64Extern = extern "C" fn(*const ExternContext, i64) -> Handle;
 
 ///

src/rust/engine/src/lib.rs

@@ -46,8 +46,8 @@ use externs::{
   Buffer, BufferBuffer, CallExtern, CloneValExtern, CreateExceptionExtern, DropHandlesExtern,
   EqualsExtern, EvalExtern, ExternContext, Externs, GeneratorSendExtern, IdentifyExtern, LogExtern,
   ProjectIgnoringTypeExtern, ProjectMultiExtern, PyResult, SatisfiedByExtern,
-  SatisfiedByTypeExtern, StoreBytesExtern, StoreI64Extern, StoreTupleExtern, TypeIdBuffer,
-  TypeToStrExtern, ValToStrExtern,
+  SatisfiedByTypeExtern, StoreBytesExtern, StoreI64Extern, StoreTupleExtern, StoreUtf8Extern,
+  TypeIdBuffer, TypeToStrExtern, ValToStrExtern,
 };
 use futures::Future;
 use handles::Handle;
@@ -141,6 +141,7 @@ pub extern "C" fn externs_set(
   satisfied_by_type: SatisfiedByTypeExtern,
   store_tuple: StoreTupleExtern,
   store_bytes: StoreBytesExtern,
+  store_utf8: StoreUtf8Extern,
   store_i64: StoreI64Extern,
   project_ignoring_type: ProjectIgnoringTypeExtern,
   project_multi: ProjectMultiExtern,
@@ -164,6 +165,7 @@ pub extern "C" fn externs_set(
     satisfied_by_type,
     store_tuple,
     store_bytes,
+    store_utf8,
     store_i64,
     project_ignoring_type,
     project_multi,