Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add runtime_metadata_url to pull metadata directly from a node #689

Merged
merged 4 commits into from
Oct 19, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 11 additions & 13 deletions cli/src/commands/codegen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,7 @@

use clap::Parser as ClapParser;
use color_eyre::eyre;
use frame_metadata::RuntimeMetadataPrefixed;
use jsonrpsee::client_transport::ws::Uri;
use scale::{
Decode,
Input,
};
use std::{
fs,
io::Read,
Expand Down Expand Up @@ -48,7 +43,7 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
let mut file = fs::File::open(file)?;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes)?;
codegen(&mut &bytes[..], opts.derives, opts.crate_path)?;
codegen(&bytes, opts.derives, opts.crate_path)?;
return Ok(())
}

Expand All @@ -57,18 +52,16 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
.parse::<Uri>()
.expect("default url is valid")
});
let (_, bytes) = super::metadata::fetch_metadata(&url).await?;
codegen(&mut &bytes[..], opts.derives, opts.crate_path)?;
let bytes = subxt_codegen::utils::fetch_metadata_bytes(&url).await?;
codegen(&bytes, opts.derives, opts.crate_path)?;
Ok(())
}

fn codegen<I: Input>(
encoded: &mut I,
fn codegen(
metadata_bytes: &[u8],
raw_derives: Vec<String>,
crate_path: Option<String>,
) -> color_eyre::Result<()> {
let metadata = <RuntimeMetadataPrefixed as Decode>::decode(encoded)?;
let generator = subxt_codegen::RuntimeGenerator::new(metadata);
let item_mod = syn::parse_quote!(
pub mod api {}
);
Expand All @@ -82,7 +75,12 @@ fn codegen<I: Input>(
let mut derives = DerivesRegistry::new(&crate_path);
derives.extend_for_all(p.into_iter());

let runtime_api = generator.generate_runtime(item_mod, derives, crate_path);
let runtime_api = subxt_codegen::generate_runtime_api_from_bytes(
item_mod,
metadata_bytes,
derives,
crate_path,
);
println!("{}", runtime_api);
Ok(())
}
2 changes: 1 addition & 1 deletion cli/src/commands/compatibility.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ async fn handle_full_metadata(nodes: &[Uri]) -> color_eyre::Result<()> {
}

async fn fetch_runtime_metadata(url: &Uri) -> color_eyre::Result<RuntimeMetadataV14> {
let (_, bytes) = super::metadata::fetch_metadata(url).await?;
let bytes = subxt_codegen::utils::fetch_metadata_bytes(url).await?;

let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
if metadata.0 != META_RESERVED {
Expand Down
60 changes: 8 additions & 52 deletions cli/src/commands/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,13 @@
use clap::Parser as ClapParser;
use color_eyre::eyre;
use frame_metadata::RuntimeMetadataPrefixed;
use jsonrpsee::{
async_client::ClientBuilder,
client_transport::ws::{
Uri,
WsTransportClientBuilder,
},
core::{
client::ClientT,
Error,
},
http_client::HttpClientBuilder,
rpc_params,
};
use jsonrpsee::client_transport::ws::Uri;
use scale::Decode;
use std::io::{
self,
Write,
};
use subxt_codegen::utils::fetch_metadata_hex;

/// Download metadata from a substrate node, for use with `subxt` codegen.
#[derive(Debug, ClapParser)]
Expand All @@ -41,10 +30,11 @@ pub struct Opts {
}

pub async fn run(opts: Opts) -> color_eyre::Result<()> {
let (hex_data, bytes) = fetch_metadata(&opts.url).await?;
let hex_data = fetch_metadata_hex(&opts.url).await?;

match opts.format.as_str() {
"json" => {
let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
let json = serde_json::to_string_pretty(&metadata)?;
println!("{}", json);
Expand All @@ -54,7 +44,10 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
println!("{}", hex_data);
Ok(())
}
"bytes" => Ok(io::stdout().write_all(&bytes)?),
"bytes" => {
let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
Ok(io::stdout().write_all(&bytes)?)
}
_ => {
Err(eyre::eyre!(
"Unsupported format `{}`, expected `json`, `hex` or `bytes`",
Expand All @@ -63,40 +56,3 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
}
}
}

pub async fn fetch_metadata(url: &Uri) -> color_eyre::Result<(String, Vec<u8>)> {
let hex_data = match url.scheme_str() {
Some("http") => fetch_metadata_http(url).await,
Some("ws") | Some("wss") => fetch_metadata_ws(url).await,
invalid_scheme => {
let scheme = invalid_scheme.unwrap_or("no scheme");
Err(eyre::eyre!(format!(
"`{}` not supported, expects 'http', 'ws', or 'wss'",
scheme
)))
}
}?;

let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;

Ok((hex_data, bytes))
}

async fn fetch_metadata_ws(url: &Uri) -> color_eyre::Result<String> {
let (sender, receiver) = WsTransportClientBuilder::default()
.build(url.to_string().parse::<Uri>().unwrap())
.await
.map_err(|e| Error::Transport(e.into()))?;

let client = ClientBuilder::default()
.max_notifs_per_subscription(4096)
.build_with_tokio(sender, receiver);

Ok(client.request("state_getMetadata", rpc_params![]).await?)
}

async fn fetch_metadata_http(url: &Uri) -> color_eyre::Result<String> {
let client = HttpClientBuilder::default().build(url.to_string())?;

Ok(client.request::<String>("state_getMetadata", None).await?)
}
3 changes: 3 additions & 0 deletions codegen/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ quote = "1.0.8"
syn = "1.0.58"
scale-info = { version = "2.0.0", features = ["bit-vec"] }
subxt-metadata = { version = "0.24.0", path = "../metadata" }
jsonrpsee = { version = "0.15.1", features = ["async-client", "client-ws-transport", "http-client"] }
hex = "0.4.3"
tokio = { version = "1.8", features = ["macros", "rt-multi-thread"] }

[dev-dependencies]
bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
Expand Down
49 changes: 48 additions & 1 deletion codegen/src/api/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ use crate::{
CompositeDefFields,
TypeGenerator,
},
utils::{
fetch_metadata_bytes_blocking,
Uri,
},
CratePath,
};
use codec::Decode;
Expand Down Expand Up @@ -50,9 +54,10 @@ use syn::parse_quote;
/// * `item_mod` - The module declaration for which the API is implemented.
/// * `path` - The path to the scale encoded metadata of the runtime node.
/// * `derives` - Provide custom derives for the generated types.
/// * `crate_path` - Path to the `subxt` crate.
///
/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
pub fn generate_runtime_api<P>(
pub fn generate_runtime_api_from_path<P>(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: we could also update the RuntimeGenerator doc to reflect this change

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ooh good point; our docs CI thing caught that also :)

item_mod: syn::ItemMod,
path: P,
derives: DerivesRegistry,
Expand All @@ -69,6 +74,48 @@ where
file.read_to_end(&mut bytes)
.unwrap_or_else(|e| abort_call_site!("Failed to read metadata file: {}", e));

generate_runtime_api_from_bytes(item_mod, &bytes, derives, crate_path)
}

/// Generates the API for interacting with a substrate runtime, using metadata
/// that can be downloaded from a node at the provided URL.
///
/// # Arguments
///
/// * `item_mod` - The module declaration for which the API is implemented.
/// * `url` - HTTP/WS URL to the substrate node you'd like to pull metadata from.
/// * `derives` - Provide custom derives for the generated types.
/// * `crate_path` - Path to the `subxt` crate.
///
/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
pub fn generate_runtime_api_from_url(
item_mod: syn::ItemMod,
url: &Uri,
derives: DerivesRegistry,
crate_path: CratePath,
) -> TokenStream2 {
let bytes = fetch_metadata_bytes_blocking(url)
.unwrap_or_else(|e| abort_call_site!("Failed to obtain metadata: {}", e));

generate_runtime_api_from_bytes(item_mod, &bytes, derives, crate_path)
}

/// Generates the API for interacting with a substrate runtime, using metadata bytes
///
/// # Arguments
///
/// * `item_mod` - The module declaration for which the API is implemented.
/// * `url` - HTTP/WS URL to the substrate node you'd like to pull metadata from.
/// * `derives` - Provide custom derives for the generated types.
/// * `crate_path` - Path to the `subxt` crate.
///
/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases.
pub fn generate_runtime_api_from_bytes(
item_mod: syn::ItemMod,
bytes: &[u8],
derives: DerivesRegistry,
crate_path: CratePath,
) -> TokenStream2 {
let metadata = frame_metadata::RuntimeMetadataPrefixed::decode(&mut &bytes[..])
.unwrap_or_else(|e| abort_call_site!("Failed to decode metadata: {}", e));

Expand Down
6 changes: 5 additions & 1 deletion codegen/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,13 @@ mod api;
mod ir;
mod types;

pub mod utils;

pub use self::{
api::{
generate_runtime_api,
generate_runtime_api_from_bytes,
generate_runtime_api_from_path,
generate_runtime_api_from_url,
RuntimeGenerator,
},
types::{
Expand Down
107 changes: 107 additions & 0 deletions codegen/src/utils/fetch_metadata.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
use jsonrpsee::{
async_client::ClientBuilder,
client_transport::ws::{
Uri,
WsTransportClientBuilder,
},
core::{
client::ClientT,
Error,
},
http_client::HttpClientBuilder,
rpc_params,
};

/// Returns the metadata bytes from the provided URL, blocking the current thread.
pub fn fetch_metadata_bytes_blocking(url: &Uri) -> Result<Vec<u8>, FetchMetadataError> {
tokio::runtime::Builder::new_multi_thread()
Copy link
Member

@niklasad1 niklasad1 Oct 18, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

right, tokio is not initialized here because it's a compile time however you could have another helper function to avoid repeating the tokio runtime init:

fn tokio_block_on<T, Fut: Future<Output = T>>(fut: Fut) -> Result<T, FetchMetadataError> {
     tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .build()
        .unwrap()
        .block_on(fut)
}

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good idea!

.enable_all()
.build()
.unwrap()
.block_on(fetch_metadata_bytes(url))
}

/// Returns the raw, 0x prefixed metadata hex from the provided URL, blocking the current thread.
pub fn fetch_metadata_hex_blocking(url: &Uri) -> Result<String, FetchMetadataError> {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap()
.block_on(fetch_metadata_hex(url))
}

/// Returns the metadata bytes from the provided URL.
pub async fn fetch_metadata_bytes(url: &Uri) -> Result<Vec<u8>, FetchMetadataError> {
let hex = fetch_metadata_hex(url).await?;
let bytes = hex::decode(hex.trim_start_matches("0x"))?;
Ok(bytes)
}

/// Returns the raw, 0x prefixed metadata hex from the provided URL.
pub async fn fetch_metadata_hex(url: &Uri) -> Result<String, FetchMetadataError> {
let hex_data = match url.scheme_str() {
Some("http") => fetch_metadata_http(url).await,
niklasad1 marked this conversation as resolved.
Show resolved Hide resolved
Some("ws") | Some("wss") => fetch_metadata_ws(url).await,
invalid_scheme => {
let scheme = invalid_scheme.unwrap_or("no scheme");
Err(FetchMetadataError::InvalidScheme(scheme.to_owned()))
}
}?;
Ok(hex_data)
}

async fn fetch_metadata_ws(url: &Uri) -> Result<String, FetchMetadataError> {
let (sender, receiver) = WsTransportClientBuilder::default()
.build(url.to_string().parse::<Uri>().unwrap())
.await
.map_err(|e| Error::Transport(e.into()))?;

let client = ClientBuilder::default()
.max_notifs_per_subscription(4096)
.build_with_tokio(sender, receiver);

Ok(client.request("state_getMetadata", rpc_params![]).await?)
}

async fn fetch_metadata_http(url: &Uri) -> Result<String, FetchMetadataError> {
let client = HttpClientBuilder::default().build(url.to_string())?;

Ok(client.request::<String>("state_getMetadata", None).await?)
}

#[derive(Debug)]
pub enum FetchMetadataError {
DecodeError(hex::FromHexError),
RequestError(jsonrpsee::core::Error),
InvalidScheme(String),
}

impl std::fmt::Display for FetchMetadataError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FetchMetadataError::DecodeError(e) => {
write!(f, "Cannot decode hex value: {e}")
}
FetchMetadataError::RequestError(e) => write!(f, "Request error: {e}"),
FetchMetadataError::InvalidScheme(s) => {
write!(
f,
"'{s}' not supported, supported URl schemes are http, ws or wss."
)
}
}
}
}

impl std::error::Error for FetchMetadataError {}

impl From<hex::FromHexError> for FetchMetadataError {
fn from(e: hex::FromHexError) -> Self {
FetchMetadataError::DecodeError(e)
}
}
impl From<jsonrpsee::core::Error> for FetchMetadataError {
fn from(e: jsonrpsee::core::Error) -> Self {
FetchMetadataError::RequestError(e)
}
}
12 changes: 12 additions & 0 deletions codegen/src/utils/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
mod fetch_metadata;

// easy access to this type needed for fetching metadata:
pub use jsonrpsee::client_transport::ws::Uri;

pub use fetch_metadata::{
fetch_metadata_bytes,
fetch_metadata_bytes_blocking,
fetch_metadata_hex,
fetch_metadata_hex_blocking,
FetchMetadataError,
};
11 changes: 11 additions & 0 deletions examples/examples/custom_metadata_url.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
// Copyright 2019-2022 Parity Technologies (UK) Ltd.
// This file is dual-licensed as Apache-2.0 or GPL-3.0.
// see LICENSE for license details.

// If you'd like to use metadata directly from a running node, you
// can provide a URL to that node here. Note that if the metadata cannot
// be retrieved from this node URL at compile time, compilation will fail.
#[subxt::subxt(runtime_metadata_url = "wss://rpc.polkadot.io:443")]
pub mod polkadot {}

fn main() {}
Loading