Use scale-typegen as a backend for the codegen (#1260)

* integrate scale-typegen, remove types mod * reintroduce default substitutes and derives * support runtime_types only again * generating polkadot.rs ok * update scale-typegen to discrete error types * scale-typegen-api-changes * add note about UncheckedExtrinsic in default substitutes * add resursive attributes and derives * adjust example where Clone bound recursive * move scale-typegen dependency to workspace * expose default typegen settings * lightclient: Fix wasm socket closure called after being dropped (#1289) * lightclient: Close wasm socket while dropping from connecting state Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Construct one time only closures Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * testing: Enable console logs for lightclient WASM testing Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Separate wakes and check connectivity on poll_read Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Close the socket depending on internal state Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Revert "lightclient: Separate wakes and check connectivity on poll_read" This reverts commit 8660940. * lightclient: Return pending if socket is opening from poll_read Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Close the socket on `poll_close` Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Reset closures on Drop to avoid recursive invokation Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Close the socket if not already closing Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> --------- Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * workflows: Install rustup component for building substrate (#1295) Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Command to fetch chainSpec and optimise its size (#1278) * cli: Add chainSpec command Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli/chainSpec: Move to dedicated module Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Compute the state root hash Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Remove code substitutes Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * artifacts: Update polkadot.json Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * scripts: Generate the chain spec Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Remove testing artifacts Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Fix clippy Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Apply rustfmt Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Introduce feature flag for smoldot dependency Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * cli: Rename chain-spec to chain-spec-pruning Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * scripts: Update chain-spec command Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> --------- Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * remove comments and unused args * Update substrate- and signer-related dependencies (#1297) * update crypto dependencies, adjust keypair * add scale_info::TypeInfo derive in some places * add multi signature derive * fix lock file * fix lock file again :| * adjust to new interface in scale-typegen * use released scale typegen * reintroduce type aliases * introduce type aliases again using scale-typegen * cargo fmt and clippy * reconcile changes with master branch * update polkadot.rs * bump scale-typgen to fix substitution * implemented Alex suggestions, regenerated polkadot.rs (did not change) * resolve conflicts in Cargo.lock * make expect messages more clear * correct typos --------- Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
paritytech · Jan 11, 2024 · fc5a18a · fc5a18a
1 parent d004789
commit fc5a18a
Show file tree

Hide file tree

Showing 30 changed files with 1,834 additions and 4,886 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -17,13 +17,7 @@ members = [
 # We exclude any crates that would depend on non mutually
 # exclusive feature flags and thus can't compile with the
 # workspace:
-exclude = [
-    "testing/wasm-rpc-tests",
-    "testing/wasm-lightclient-tests",
-    "signer/wasm-tests",
-    "examples/wasm-example",
-    "examples/parachain-example"
-]
+exclude = ["testing/wasm-rpc-tests", "testing/wasm-lightclient-tests", "signer/wasm-tests", "examples/wasm-example", "examples/parachain-example"]
 resolver = "2"
 
 [workspace.package]
@@ -100,6 +94,8 @@ url = "2.5.0"
 wabt = "0.10.0"
 wasm-bindgen-test = "0.3.24"
 which = "5.0.0"
+scale-typegen-description = "0.1.0"
+scale-typegen = "0.1.1"
 
 # Light client support:
 smoldot = { version = "0.16.0", default-features = false }

diff --git a/cli/src/commands/codegen.rs b/cli/src/commands/codegen.rs
@@ -173,7 +173,8 @@ fn codegen(
             .map_err(|e| eyre!("Cannot parse derive for type {ty_str}: {e}"))?;
         let derive = syn::parse_str(&derive)
             .map_err(|e| eyre!("Cannot parse derive for type {ty_str}: {e}"))?;
-        codegen.add_derives_for_type(ty, std::iter::once(derive));
+        // Note: recursive derives and attributes not supported in the CLI => recursive: false
+        codegen.add_derives_for_type(ty, std::iter::once(derive), false);
     }
 
     // Configure attribtues:
@@ -190,7 +191,8 @@ fn codegen(
             .map_err(|e| eyre!("Cannot parse attribute for type {ty_str}: {e}"))?;
         let attribute: OuterAttribute = syn::parse_str(&attr)
             .map_err(|e| eyre!("Cannot parse attribute for type {ty_str}: {e}"))?;
-        codegen.add_attributes_for_type(ty, std::iter::once(attribute.0));
+        // Note: recursive derives and attributes not supported in the CLI => recursive: false
+        codegen.add_attributes_for_type(ty, std::iter::once(attribute.0), false);
     }
 
     // Insert type substitutions:

diff --git a/cli/src/commands/diff.rs b/cli/src/commands/diff.rs
@@ -228,11 +228,11 @@ impl StorageEntryDiff {
         let value_1_ty_id = storage_entry_1.entry_type().value_ty();
         let value_1_hash = metadata_1
             .type_hash(value_1_ty_id)
-            .expect("type should be present");
+            .expect("type is in metadata; qed");
         let value_2_ty_id = storage_entry_2.entry_type().value_ty();
         let value_2_hash = metadata_1
             .type_hash(value_2_ty_id)
-            .expect("type should be present");
+            .expect("type is in metadata; qed");
         let value_different = value_1_hash != value_2_hash;
 
         let key_1_hash = storage_entry_1
@@ -241,7 +241,7 @@ impl StorageEntryDiff {
             .map(|key_ty| {
                 metadata_1
                     .type_hash(key_ty)
-                    .expect("type should be present")
+                    .expect("type is in metadata; qed")
             })
             .unwrap_or_default();
         let key_2_hash = storage_entry_2
@@ -250,7 +250,7 @@ impl StorageEntryDiff {
             .map(|key_ty| {
                 metadata_2
                     .type_hash(key_ty)
-                    .expect("type should be present")
+                    .expect("type is in metadata; qed")
             })
             .unwrap_or_default();
         let key_different = key_1_hash != key_2_hash;
@@ -309,12 +309,12 @@ fn storage_differences<'a>(
         |e| {
             pallet_metadata_1
                 .storage_hash(e.name())
-                .expect("storage entry should be present")
+                .expect("storage entry is in metadata; qed")
         },
         |e| {
             pallet_metadata_2
                 .storage_hash(e.name())
-                .expect("storage entry should be present")
+                .expect("storage entry is in metadata; qed")
         },
         |e| e.name(),
     )
@@ -330,12 +330,12 @@ fn calls_differences<'a>(
         |e| {
             pallet_metadata_1
                 .call_hash(&e.name)
-                .expect("call should be present")
+                .expect("call is in metadata; qed")
         },
         |e| {
             pallet_metadata_2
                 .call_hash(&e.name)
-                .expect("call should be present")
+                .expect("call is in metadata; qed")
         },
         |e| &e.name,
     );
@@ -351,12 +351,12 @@ fn constants_differences<'a>(
         |e| {
             pallet_metadata_1
                 .constant_hash(e.name())
-                .expect("constant should be present")
+                .expect("constant is in metadata; qed")
         },
         |e| {
             pallet_metadata_2
                 .constant_hash(e.name())
-                .expect("constant should be present")
+                .expect("constant is in metadata; qed")
         },
         |e| e.name(),
     )

diff --git a/cli/src/commands/explore/mod.rs b/cli/src/commands/explore/mod.rs
@@ -142,7 +142,7 @@ pub async fn run(opts: Opts, output: &mut impl std::io::Write) -> color_eyre::Re
             explore_constants(command, &metadata, pallet_metadata, output)
         }
         PalletSubcommand::Storage(command) => {
-            // if the metadata came from some url, we use that same url to make storage calls against.
+            // if the metadata is in some url, we use that same url to make storage calls against.
             let node_url = opts.file_or_url.url.map(|url| url.to_string());
             explore_storage(command, &metadata, pallet_metadata, node_url, output).await
         }

diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
@@ -31,12 +31,10 @@ jsonrpsee = { workspace = true, features = ["async-client", "client-ws-transport
 hex = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread"], optional = true }
 thiserror = { workspace = true }
+scale-typegen = { workspace = true }
 
 # Included if "web" feature is enabled, to enable its js feature.
 getrandom = { workspace = true, optional = true }
 
 [dev-dependencies]
-bitvec = { workspace = true }
 scale-info = { workspace = true, features = ["bit-vec"] }
-pretty_assertions = { workspace = true }
-frame-metadata = { workspace = true }
diff --git a/codegen/src/api/calls.rs b/codegen/src/api/calls.rs
@@ -3,92 +3,80 @@
 // see LICENSE for license details.
 
 use super::CodegenError;
-use crate::types::{CompositeDefFields, TypeGenerator};
 use heck::{ToSnakeCase as _, ToUpperCamelCase as _};
 use proc_macro2::TokenStream as TokenStream2;
 use quote::{format_ident, quote};
+use scale_typegen::{typegen::ir::type_ir::CompositeIRKind, TypeGenerator};
 use subxt_metadata::PalletMetadata;
 
 /// Generate calls from the provided pallet's metadata. Each call returns a `StaticTxPayload`
 /// that can be passed to the subxt client to submit/sign/encode.
 ///
 /// # Arguments
 ///
-/// - `metadata` - Runtime metadata from which the calls are generated.
-/// - `type_gen` - The type generator containing all types defined by metadata.
+/// - `type_gen` - [`scale_typegen::TypeGenerator`] that contains settings and all types from the runtime metadata.
 /// - `pallet` - Pallet metadata from which the calls are generated.
-/// - `types_mod_ident` - The ident of the base module that we can use to access the generated types from.
+/// - `crate_path` - The crate path under which subxt is located, e.g. `::subxt` when using subxt as a dependency.
 pub fn generate_calls(
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata,
-    types_mod_ident: &syn::Ident,
     crate_path: &syn::Path,
-    should_gen_docs: bool,
 ) -> Result<TokenStream2, CodegenError> {
     // Early return if the pallet has no calls.
     let Some(call_ty) = pallet.call_ty_id() else {
         return Ok(quote!());
     };
 
-    let mut struct_defs = super::generate_structs_from_variants(
+    let variant_names_and_struct_defs = super::generate_structs_from_variants(
         type_gen,
-        types_mod_ident,
         call_ty,
         |name| name.to_upper_camel_case().into(),
         "Call",
-        crate_path,
-        should_gen_docs,
     )?;
-
-    let result = struct_defs
-        .iter_mut()
-        .map(|(variant_name, struct_def, aliases)| {
-            let fn_name = format_ident!("{}", variant_name.to_snake_case());
-
-            let result: Vec<_> = match struct_def.fields {
-                CompositeDefFields::Named(ref named_fields) => named_fields
+    let (call_structs, call_fns): (Vec<_>, Vec<_>) = variant_names_and_struct_defs
+        .into_iter()
+        .map(|var| {
+            let (call_fn_args, call_args): (Vec<_>, Vec<_>) = match &var.composite.kind {
+                CompositeIRKind::Named(named_fields) => named_fields
                     .iter()
                     .map(|(name, field)| {
-                        let call_arg = if field.is_boxed() {
+                        // Note: fn_arg_type this is relative the type path of the type alias when prefixed with `types::`, e.g. `set_max_code_size::New`
+                        let fn_arg_type = &field.type_path;
+                        let call_arg = if field.is_boxed {
                             quote! { #name: ::std::boxed::Box::new(#name) }
                         } else {
                             quote! { #name }
                         };
-
-                        let alias_name =
-                            format_ident!("{}", name.to_string().to_upper_camel_case());
-
-                        (quote!( #name: types::#fn_name::#alias_name ), call_arg)
+                        (quote!( #name: types::#fn_arg_type ), call_arg)
                     })
-                    .collect(),
-                CompositeDefFields::NoFields => Default::default(),
-                CompositeDefFields::Unnamed(_) => {
+                    .unzip(),
+                CompositeIRKind::NoFields => Default::default(),
+                CompositeIRKind::Unnamed(_) => {
                     return Err(CodegenError::InvalidCallVariant(call_ty))
                 }
             };
 
-            let call_fn_args = result.iter().map(|(call_fn_arg, _)| call_fn_arg);
-            let call_args = result.iter().map(|(_, call_arg)| call_arg);
-
             let pallet_name = pallet.name();
-            let call_name = &variant_name;
-            let struct_name = &struct_def.name;
+            let call_name = &var.variant_name;
+            let struct_name = &var.composite.name;
             let Some(call_hash) = pallet.call_hash(call_name) else {
                 return Err(CodegenError::MissingCallMetadata(
                     pallet_name.into(),
                     call_name.to_string(),
                 ));
             };
-
+            let fn_name = format_ident!("{}", var.variant_name.to_snake_case());
             // Propagate the documentation just to `TransactionApi` methods, while
             // draining the documentation of inner call structures.
-            let docs = should_gen_docs.then_some(struct_def.docs.take()).flatten();
+            let docs = &var.composite.docs;
 
+            // this converts the composite into a full struct type. No Type Parameters needed here.
+            let struct_def = type_gen.upcast_composite(&var.composite);
+            let alias_mod = var.type_alias_mod;
             // The call structure's documentation was stripped above.
             let call_struct = quote! {
                 #struct_def
-
-                #aliases
+                #alias_mod
 
                 impl #crate_path::blocks::StaticExtrinsic for #struct_name {
                     const PALLET: &'static str = #pallet_name;
@@ -113,17 +101,15 @@ pub fn generate_calls(
 
             Ok((call_struct, client_fn))
         })
-        .collect::<Result<Vec<_>, _>>()?;
+        .collect::<Result<Vec<_>, _>>()?
+        .into_iter()
+        .unzip();
 
-    let call_structs = result.iter().map(|(call_struct, _)| call_struct);
-    let call_fns = result.iter().map(|(_, client_fn)| client_fn);
+    let call_type = type_gen.resolve_type_path(call_ty)?;
+    let call_ty = type_gen.resolve_type(call_ty)?;
+    let docs = type_gen.docs_from_scale_info(&call_ty.docs);
 
-    let call_type = type_gen.resolve_type_path(call_ty);
-    let call_ty = type_gen.resolve_type(call_ty);
-    let docs = &call_ty.docs;
-    let docs = should_gen_docs
-        .then_some(quote! { #( #[doc = #docs ] )* })
-        .unwrap_or_default();
+    let types_mod_ident = type_gen.types_mod_ident();
 
     Ok(quote! {
         #docs

diff --git a/codegen/src/api/constants.rs b/codegen/src/api/constants.rs
@@ -2,10 +2,10 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
-use crate::types::TypeGenerator;
 use heck::ToSnakeCase as _;
 use proc_macro2::TokenStream as TokenStream2;
 use quote::{format_ident, quote};
+use scale_typegen::TypeGenerator;
 use subxt_metadata::PalletMetadata;
 
 use super::CodegenError;
@@ -29,16 +29,13 @@ use super::CodegenError;
 ///
 /// # Arguments
 ///
-/// - `metadata` - Runtime metadata from which the calls are generated.
-/// - `type_gen` - The type generator containing all types defined by metadata
-/// - `pallet` - Pallet metadata from which the calls are generated.
-/// - `types_mod_ident` - The ident of the base module that we can use to access the generated types from.
+/// - `type_gen` - [`scale_typegen::TypeGenerator`] that contains settings and all types from the runtime metadata.
+/// - `pallet` - Pallet metadata from which the constants are generated.
+/// - `crate_path` - The crate path under which subxt is located, e.g. `::subxt` when using subxt as a dependency.
 pub fn generate_constants(
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata,
-    types_mod_ident: &syn::Ident,
     crate_path: &syn::Path,
-    should_gen_docs: bool,
 ) -> Result<TokenStream2, CodegenError> {
     // Early return if the pallet has no constants.
     if pallet.constants().len() == 0 {
@@ -58,9 +55,11 @@ pub fn generate_constants(
                 ));
             };
 
-            let return_ty = type_gen.resolve_type_path(constant.ty());
+            let return_ty = type_gen.resolve_type_path(constant.ty())?;
             let docs = constant.docs();
-            let docs = should_gen_docs
+            let docs = type_gen
+                .settings()
+                .should_gen_docs
                 .then_some(quote! { #( #[doc = #docs ] )* })
                 .unwrap_or_default();
 
@@ -77,6 +76,8 @@ pub fn generate_constants(
         })
         .collect::<Result<Vec<_>, _>>()?;
 
+    let types_mod_ident = type_gen.types_mod_ident();
+
     Ok(quote! {
         pub mod constants {
             use super::#types_mod_ident;

diff --git a/codegen/src/api/custom_values.rs b/codegen/src/api/custom_values.rs
@@ -2,10 +2,9 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
-use std::collections::HashSet;
-
-use crate::types::TypeGenerator;
 use heck::ToSnakeCase as _;
+use scale_typegen::TypeGenerator;
+use std::collections::HashSet;
 use subxt_metadata::{CustomValueMetadata, Metadata};
 
 use proc_macro2::TokenStream as TokenStream2;
@@ -60,6 +59,7 @@ fn generate_custom_value_fn(
     let (return_ty, decodable) = if type_is_valid {
         let return_ty = type_gen
             .resolve_type_path(custom_value.type_id())
+            .expect("type is in metadata; qed")
             .to_token_stream();
         let decodable = quote!(#crate_path::custom_values::Yes);
         (return_ty, decodable)

diff --git a/codegen/src/api/errors.rs b/codegen/src/api/errors.rs
@@ -4,26 +4,26 @@
 
 use proc_macro2::TokenStream as TokenStream2;
 use quote::quote;
+use scale_typegen::TypeGenerator;
 use subxt_metadata::PalletMetadata;
 
-use crate::types::TypeGenerator;
-
 use super::CodegenError;
 
 /// Generate error type alias from the provided pallet metadata.
 pub fn generate_error_type_alias(
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata,
-    should_gen_docs: bool,
 ) -> Result<TokenStream2, CodegenError> {
     let Some(error_ty) = pallet.error_ty_id() else {
         return Ok(quote!());
     };
 
-    let error_type = type_gen.resolve_type_path(error_ty);
-    let error_ty = type_gen.resolve_type(error_ty);
+    let error_type = type_gen.resolve_type_path(error_ty)?;
+    let error_ty = type_gen.resolve_type(error_ty)?;
     let docs = &error_ty.docs;
-    let docs = should_gen_docs
+    let docs = type_gen
+        .settings()
+        .should_gen_docs
         .then_some(quote! { #( #[doc = #docs ] )* })
         .unwrap_or_default();
     Ok(quote! {