Add RcBuf variant to DataPayload

experimental/provider_static/src/lib.rs

@@ -70,7 +70,7 @@ impl<'de> SerdeDeDataProvider<'de> for StaticDataProvider {
         receiver: &mut dyn SerdeDeDataReceiver<'de>,
     ) -> Result<DataResponseMetadata, DataError> {
         let file = self.get_file(req)?;
-        receiver.receive_deserializer(&mut erased_serde::Deserializer::erase(
+        receiver.receive_static(&mut erased_serde::Deserializer::erase(
             &mut serde_json::Deserializer::from_reader(file.as_bytes()),
         ))?;
 

ffi/capi/src/provider.rs

@@ -42,7 +42,7 @@ impl ICU4XDataProvider {
     }
 
     /// Construct a [`ICU4XDataProvider`] this from a boxed [`SerdeDeDataProvider`]
-    pub fn from_boxed(x: Box<dyn SerdeDeDataProvider<'static>>) -> Self {
+    pub fn from_boxed(x: Box<dyn SerdeDeDataProvider>) -> Self {
         unsafe {
             // If the layout changes this will error
             // Once Rust gets pointer metadata APIs we should switch to using those
@@ -51,15 +51,15 @@ impl ICU4XDataProvider {
     }
 
     /// Obtain the original boxed Rust [`SerdeDeDataProvider`] for this
-    pub fn into_boxed(self) -> Box<dyn SerdeDeDataProvider<'static>> {
+    pub fn into_boxed(self) -> Box<dyn SerdeDeDataProvider> {
         debug_assert!(self._field1 != 0);
         // If the layout changes this will error
         // Once Rust gets pointer metadata APIs we should switch to using those
         unsafe { mem::transmute(self) }
     }
 
     /// Convert a borrowed reference to a borrowed [`SerdeDeDataProvider`]
-    pub fn as_dyn_ref(&self) -> &dyn SerdeDeDataProvider<'static> {
+    pub fn as_dyn_ref(&self) -> &dyn SerdeDeDataProvider {
         debug_assert!(self._field1 != 0);
         unsafe {
             // &dyn Trait and Box<dyn Trait> have the same layout

provider/core/src/data_provider.rs

@@ -98,6 +98,7 @@ where
     Borrowed(Yoke<M::Yokeable, &'d M::Cart>),
     RcStruct(Yoke<M::Yokeable, Rc<M::Cart>>),
     Owned(Yoke<M::Yokeable, ()>),
+    RcBuf(Yoke<M::Yokeable, Rc<[u8]>>),
 }
 
 /// A wrapper around the payload returned in a [`DataResponse`].
@@ -149,6 +150,7 @@ where
             Borrowed(yoke) => Borrowed(yoke.clone()),
             RcStruct(yoke) => RcStruct(yoke.clone()),
             Owned(yoke) => Owned(yoke.clone()),
+            RcBuf(yoke) => RcBuf(yoke.clone()),
         };
         Self { inner: new_inner }
     }
@@ -219,6 +221,49 @@ where
     }
 }
 
+impl<'d, 's, M> DataPayload<'d, 's, M>
+where
+    M: DataMarker<'s>,
+{
+    /// Convert a byte buffer into a [`DataPayload`]. A function must be provided to perform the
+    /// conversion. This can often be a Serde deserialization operation.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[cfg(feature = "provider_serde")] {
+    /// use icu_provider::prelude::*;
+    /// use icu_provider::hello_world::*;
+    /// use std::rc::Rc;
+    /// use icu_provider::yoke::Yokeable;
+    ///
+    /// let json_text = "{\"message\":\"Hello World\"}";
+    /// let json_rc_buffer: Rc<[u8]> = json_text.as_bytes().into();
+    ///
+    /// let helper: for<'de> fn(_: &'de [u8]) -> Result<<HelloWorldV1<'static> as Yokeable<'de>>::Output, serde_json::Error> = |bytes| {
+    ///     serde_json::from_slice(bytes)
+    /// };
+    /// let payload = DataPayload::<HelloWorldV1Marker>::try_from_rc_buffer(
+    ///     json_rc_buffer.clone(),
+    ///     helper
+    /// )
+    /// .expect("JSON is valid");
+    ///
+    /// assert_eq!("Hello World", payload.get().message);
+    /// # } // feature = "provider_serde"
+    /// ```
+    #[inline]
+    pub fn try_from_rc_buffer<E>(
+        buffer: Rc<[u8]>,
+        f: impl for<'de> FnOnce(&'de [u8]) -> Result<<M::Yokeable as Yokeable<'de>>::Output, E>,
+    ) -> Result<Self, E> {
+        let yoke = Yoke::try_attach_to_cart(buffer, f)?;
+        Ok(Self {
+            inner: DataPayloadInner::RcBuf(yoke),
+        })
+    }
+}
+
 impl<'d, 's, M> DataPayload<'d, 's, M>
 where
     M: DataMarker<'s>,
@@ -289,6 +334,7 @@ where
             Borrowed(yoke) => yoke.with_mut(f),
             RcStruct(yoke) => yoke.with_mut(f),
             Owned(yoke) => yoke.with_mut(f),
+            RcBuf(yoke) => yoke.with_mut(f),
         }
     }
 
@@ -314,6 +360,7 @@ where
             Borrowed(yoke) => yoke.get(),
             RcStruct(yoke) => yoke.get(),
             Owned(yoke) => yoke.get(),
+            RcBuf(yoke) => yoke.get(),
         }
     }
 }

provider/core/src/erased.rs

@@ -109,6 +109,11 @@ where
                 let cart: Rc<dyn ErasedDataStruct> = Rc::from(yoke);
                 DataPayload::from_partial_owned(cart)
             }
+            RcBuf(yoke) => {
+                // Case 4: Cast the whole RcBuf Yoke to the trait object.
+                let cart: Rc<dyn ErasedDataStruct> = Rc::from(yoke);
+                DataPayload::from_partial_owned(cart)
+            }
         }
     }
 }
@@ -203,15 +208,27 @@ impl<'d> DataPayload<'d, 'static, ErasedDataStructMarker> {
                     },
                     Err(any_rc) => any_rc,
                 };
+                // Check for Case 4: an RcBuf Yoke.
+                let y2 = any_rc.downcast::<Yoke<M::Yokeable, Rc<[u8]>>>();
+                let any_rc = match y2 {
+                    Ok(rc_yoke) => match Rc::try_unwrap(rc_yoke) {
+                        Ok(yoke) => return Ok(DataPayload { inner: RcBuf(yoke) }),
+                        // Note: We could consider cloning the Yoke instead of erroring out.
+                        Err(_) => return Err(Error::MultipleReferences),
+                    },
+                    Err(any_rc) => any_rc,
+                };
                 // None of the downcasts succeeded; return an error.
                 Err(Error::MismatchedType {
                     actual: Some(any_rc.type_id()),
                     generic: Some(TypeId::of::<M::Cart>()),
                 })
             }
-            // This is unreachable because ErasedDataStructMarker cannot be fully owned, since it
+            // This is unreachable because ErasedDataStruct cannot be fully owned, since it
             // contains a reference.
             Owned(_) => unreachable!(),
+            // This is unreachable because ErasedDataStruct needs to reference an object.
+            RcBuf(_) => unreachable!(),
         }
     }
 }

provider/core/src/serde.rs

@@ -30,76 +30,89 @@ use yoke::*;
 /// Lifetimes:
 ///
 /// - `'de` = deserializer lifetime; can usually be `'_`
-pub trait SerdeDeDataReceiver<'de> {
-    /// Consumes a Serde Deserializer into this SerdeDeDataReceiver as owned data.
-    ///
-    /// This method results in an owned payload, but the payload could have non-static references
-    /// according to the deserializer lifetime.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use icu_provider::prelude::*;
-    /// use icu_provider::serde::SerdeDeDataReceiver;
-    ///
-    /// const JSON: &'static str = "\"hello world\"";
-    ///
-    /// let mut receiver: Option<&str> = None;
-    /// let mut d = serde_json::Deserializer::from_str(JSON);
-    /// receiver.receive_deserializer(&mut erased_serde::Deserializer::erase(&mut d))
-    ///     .expect("Deserialization should be successful");
-    ///
-    /// assert!(matches!(receiver, Some(_)));
-    /// assert_eq!(receiver, Some("hello world"));
-    /// ```
-    fn receive_deserializer(
+pub trait SerdeDeDataReceiver {
+    /// Receives a reference-counted byte buffer.
+    /// 
+    /// Upon calling this function, the receiver sends byte buffer back to the caller as the first
+    /// argument of `f1`. The caller should then map the byte buffer to an
+    /// [`erased_serde::Deserializer`] and pass it back to the receiver via `f2`.
+    fn receive_rc_bytes(
         &mut self,
-        deserializer: &mut dyn erased_serde::Deserializer<'de>,
+        rc_bytes: Rc<[u8]>,
+        f1: for<'de> fn(bytes: &'de [u8], f2: &mut dyn FnMut(&mut dyn erased_serde::Deserializer<'de>)),
+    ) -> Result<(), Error>;
+
+    /// Receives a `&'static` byte buffer via an [`erased_serde::Deserializer`].
+    /// 
+    /// Note: Since the purpose of this function is to handle zero-copy deserialization of static
+    /// byte buffers, we want `Deserializer<'static>` as opposed to `DeserializeOwned`.
+    fn receive_static(
+        &mut self,
+        deserializer: &mut dyn erased_serde::Deserializer<'static>,
     ) -> Result<(), Error>;
 }
 
-impl<'de, T> SerdeDeDataReceiver<'de> for Option<T>
+impl<'d, 's, M> SerdeDeDataReceiver for Option<DataPayload<'d, 's, M>>
 where
-    T: serde::Deserialize<'de>,
+    M: DataMarker<'s>,
+    M::Yokeable: serde::de::Deserialize<'static>,
+    for<'de> <M::Yokeable as Yokeable<'de>>::Output: serde::de::Deserialize<'de>,
 {
-    fn receive_deserializer(
+    fn receive_rc_bytes(
+        &mut self,
+        rc_bytes: Rc<[u8]>,
+        f1: for<'de> fn(bytes: &'de [u8], f2: &mut dyn FnMut(&mut dyn erased_serde::Deserializer<'de>)),
+    ) -> Result<(), Error> {
+        self.replace(DataPayload::try_from_rc_buffer(rc_bytes, move |bytes| {
+            let mut holder = None;
+            f1(bytes, &mut |deserializer| {
+                holder.replace(erased_serde::deserialize(deserializer));
+            });
+            // The holder is guaranteed to be populated so long as the lambda function was invoked,
+            // which is in the contract of `receive_rc_bytes`.
+            holder.unwrap()
+        })?);
+        Ok(())
+    }
+
+    fn receive_static(
         &mut self,
-        deserializer: &mut dyn erased_serde::Deserializer<'de>,
+        deserializer: &mut dyn erased_serde::Deserializer<'static>,
     ) -> Result<(), Error> {
-        let obj: T = erased_serde::deserialize(deserializer)?;
-        self.replace(obj);
+        let obj: M::Yokeable = erased_serde::deserialize(deserializer)?;
+        self.replace(DataPayload::from_owned(obj));
         Ok(())
     }
 }
 
 /// A type-erased data provider that loads payloads from a Serde Deserializer.
 ///
 /// Uses [`erased_serde`] to allow the trait to be object-safe.
-pub trait SerdeDeDataProvider<'de> {
+pub trait SerdeDeDataProvider {
     /// Query the provider for data, loading it into a [`SerdeDeDataReceiver`].
     ///
     /// Returns Ok if the request successfully loaded data. If data failed to load, returns an
     /// Error with more information.
     fn load_to_receiver(
         &self,
         req: &DataRequest,
-        receiver: &mut dyn SerdeDeDataReceiver<'de>,
+        receiver: &mut dyn SerdeDeDataReceiver,
     ) -> Result<DataResponseMetadata, Error>;
 }
 
-impl<'d, 's, M> DataProvider<'d, 's, M> for dyn SerdeDeDataProvider<'s> + 'd
+impl<'d, 's, M> DataProvider<'d, 's, M> for dyn SerdeDeDataProvider + 'd
 where
     M: DataMarker<'s>,
-    M::Cart: serde::Deserialize<'s>,
-    M::Yokeable: ZeroCopyFrom<M::Cart>,
+    M::Yokeable: serde::de::Deserialize<'static>,
+    for<'de> <M::Yokeable as Yokeable<'de>>::Output: serde::de::Deserialize<'de>,
 {
     /// Serve objects implementing [`serde::Deserialize<'s>`] from a [`SerdeDeDataProvider`].
     fn load_payload(&self, req: &DataRequest) -> Result<DataResponse<'d, 's, M>, Error> {
         let mut payload = None;
         let metadata = self.load_to_receiver(req, &mut payload)?;
         Ok(DataResponse {
             metadata,
-            payload: payload.map(|obj| DataPayload::from_partial_owned(Rc::new(obj))),
+            payload: payload,
         })
     }
 }
@@ -193,6 +206,7 @@ where
             Borrowed(_) => todo!("#752"),
             RcStruct(yoke) => Rc::from(yoke),
             Owned(yoke) => Rc::from(yoke),
+            RcBuf(yoke) => Rc::from(yoke),
         };
         DataPayload::from_partial_owned(cart)
     }

provider/core/tests/data_receiver.rs

@@ -5,66 +5,50 @@
 use icu_provider::serde::SerdeDeDataReceiver;
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
-
-#[derive(Deserialize, Serialize, Clone, Debug, Default, PartialEq)]
-struct DataStruct<'a> {
-    #[serde(borrow)]
-    pub value: Cow<'a, str>,
-}
+use std::rc::Rc;
+use icu_provider::hello_world::*;
+use icu_provider::prelude::*;
 
 #[allow(clippy::redundant_static_lifetimes)]
 const DATA_JSON: &'static str = r#"{
-    "value": "abc"
+    "message": "abc"
 }"#;
 
 #[test]
 fn test_deserializer_static() {
     // Deserialize from a string to create static references.
     let deserializer = &mut serde_json::Deserializer::from_str(DATA_JSON);
-    let mut receiver = None;
-    receiver
-        .receive_deserializer(&mut erased_serde::Deserializer::erase(deserializer))
-        .expect("Well-formed data");
-
-    assert!(matches!(
-        receiver,
-        Some(DataStruct {
-            value: Cow::Borrowed(_)
-        })
-    ));
-}
-
-#[test]
-fn test_deserializer_borrowed() {
-    // Deserialize from a local string to create non-static references.
-    let local_data = DATA_JSON.to_string();
-    let deserializer = &mut serde_json::Deserializer::from_str(&local_data);
-    let mut receiver = None;
+    let mut receiver: Option<DataPayload<HelloWorldV1Marker>> = None;
     receiver
-        .receive_deserializer(&mut erased_serde::Deserializer::erase(deserializer))
+        .receive_static(&mut erased_serde::Deserializer::erase(deserializer))
         .expect("Well-formed data");
+    let payload = receiver.expect("Data is present");
 
     assert!(matches!(
-        receiver,
-        Some(DataStruct {
-            value: Cow::Borrowed(_)
-        })
+        payload.get(),
+        &HelloWorldV1 {
+            message: Cow::Borrowed(_)
+        }
     ));
 }
 
 #[test]
 fn test_deserializer_owned() {
-    // Deserialize from a reader to create owned data.
-    let deserializer = &mut serde_json::Deserializer::from_reader(DATA_JSON.as_bytes());
-    let mut receiver = None;
+    // Deserialize from a reference-counted buffer.
+    let rc_bytes: Rc<[u8]> = DATA_JSON.as_bytes().into();
+    let mut receiver: Option<DataPayload<HelloWorldV1Marker>> = None;
     receiver
-        .receive_deserializer(&mut erased_serde::Deserializer::erase(deserializer))
+        .receive_rc_bytes(rc_bytes, |bytes, f2| {
+            let mut d = serde_json::Deserializer::from_slice(bytes);
+            f2(&mut erased_serde::Deserializer::erase(&mut d))
+        })
         .expect("Well-formed data");
+    let payload = receiver.expect("Data is present");
 
     assert!(matches!(
-        receiver,
-        Some(DataStruct {
-            value: Cow::Owned(_)
-        })
+        payload.get(),
+        &HelloWorldV1 {
+            message: Cow::Owned(_)
+        }
     ));
 }