Stabilize V16 metadata (#8443)

This PR bumps frame-metadata 23.0.0, which stabilizes V16 metadata. This
was previously available as the "unstable" metadata.

The main additions provided by V16 metadata are:
- Information about the Pallet View Functions exposed by pallets.
- Information about the Config associated types used by each pallet.
- Support for V5 transactions, including support for a chain providing
multiple transaction extension versions and multiple supported
transaciton versions.
- Support for deprecation information, so that mostly anything defined
in a runtime can be marked as deprecated in order for runtime/pallet
authors to communicate to consumers about deprecated items.

To sanity check this, I've built a polkadot node with this change and
checked that using Subxt (this PR
paritytech/subxt#1999), v16 metadata is indeed
available, downloads and decodes correctly.

<!--
✄
-----------------------------------------------------------------------------

Thank you for your Pull Request! 🙏 Please make sure it follows the
contribution guidelines outlined in [this

document](https://github.com/paritytech/polkadot-sdk/blob/master/docs/contributor/CONTRIBUTING.md)
and fill out the
sections below. Once you're ready to submit your PR for review, please
delete this section and leave only the text under
the "Description" heading.

# Description

*A concise description of what your PR is doing, and what potential
issue it is solving. Use [Github semantic

linking](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)
to link the PR to an issue that must be closed once this is merged.*

## Integration

*In depth notes about how this PR should be integrated by downstream
projects. This part is mandatory, and should be
reviewed by reviewers, if the PR does NOT have the `R0-Silent` label. In
case of a `R0-Silent`, it can be ignored.*

## Review Notes

*In depth notes about the **implementation** details of your PR. This
should be the main guide for reviewers to
understand your approach and effectively review it. If too long, use

[`<details>`](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/details)*.

*Imagine that someone who is depending on the old code wants to
integrate your new code and the only information that
they get is this section. It helps to include example usage and default
value here, with a `diff` code-block to show
possibly integration.*

*Include your leftover TODOs, if any, here.*

# Checklist

* [ ] My PR includes a detailed description as outlined in the
"Description" and its two subsections above.
* [ ] My PR follows the [labeling requirements](

https://github.com/paritytech/polkadot-sdk/blob/master/docs/contributor/CONTRIBUTING.md#Process
) of this project (at minimum one label for `T` required)
* External contributors: ask maintainers to put the right label on your
PR.
* [ ] I have made corresponding changes to the documentation (if
applicable)
* [ ] I have added tests that prove my fix is effective or that my
feature works (if applicable)

You can remove the "Checklist" section once all have been checked. Thank
you for your contribution!

✄
-----------------------------------------------------------------------------
--.

---------

Co-authored-by: cmd[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: 2 people

Cargo.lock

@@ -804,7 +804,7 @@ frame-benchmarking-pallet-pov = { default-features = false, path = "substrate/fr
 frame-election-provider-solution-type = { path = "substrate/frame/election-provider-support/solution-type", default-features = false }
 frame-election-provider-support = { path = "substrate/frame/election-provider-support", default-features = false }
 frame-executive = { path = "substrate/frame/executive", default-features = false }
-frame-metadata = { version = "22.0.0", default-features = false }
+frame-metadata = { version = "23.0.0", default-features = false }
 frame-metadata-hash-extension = { path = "substrate/frame/metadata-hash-extension", default-features = false }
 frame-support = { path = "substrate/frame/support", default-features = false }
 frame-support-procedural = { path = "substrate/frame/support/procedural", default-features = false }

Cargo.toml

@@ -0,0 +1,30 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: Stabilize V16 metadata
+
+doc:
+  - audience: Node Dev
+    description: |
+      Metadata V16 is stabilized. V16 metadata exposes information about Pallet View Functions and V5 transactions,
+      and can be obtained, where applicable, by using the Runtime APIs `Metadata_metadata_at_version(16)`.
+
+crates:
+  # Update to support frame-metadata 23. Changes pub interfaces:
+  - name: sp-metadata-ir
+    bump: major
+  # Limit to fetching at latest v15 metadata:
+  - name: sc-runtime-utilities
+    bump: patch
+  # Use newer frame-metadata; should be no observable change (CI wants minor):
+  - name: substrate-wasm-builder
+    bump: minor
+  # frame-metadata bumped in this and exposed but via hidden docs. No code changes needed (CI wants minor):
+  - name: frame-support
+    bump: minor
+  # Uses newer frame-support / metadata-ir but no code change needed (CI wants minor):
+  - name: pallet-example-view-functions
+    bump: minor
+  # Avoid fetching V16 metadata:
+  - name: frame-benchmarking-cli
+    bump: minor

prdoc/pr_8443.prdoc

@@ -48,7 +48,9 @@ pub fn fetch_latest_metadata_from_code_blob<HF: HostFunctions>(
 			let supported_versions = Vec::<u32>::decode(&mut supported_versions.as_slice())?;
 			let latest_stable = supported_versions
 				.into_iter()
-				.filter(|v| *v != u32::MAX)
+				// TODO: Subxt doesn't support V16 metadata until v0.42.0, so don't try
+				// to fetch it here until we update to that version.
+				.filter(|v| *v != u32::MAX && *v < 16)
 				.max()
 				.ok_or(Error::StableMetadataVersionNotFound)?;
 

substrate/client/runtime-utilities/src/lib.rs

@@ -49,12 +49,6 @@ parameter_types! {
 	storage UpdateStorageVersion: bool = false;
 }
 
-/// Latest stable metadata version used for testing.
-const LATEST_METADATA_VERSION: u32 = 15;
-
-/// Unstable metadata version.
-const UNSTABLE_METADATA_VERSION: u32 = u32::MAX;
-
 pub struct SomeType1;
 impl From<SomeType1> for u64 {
 	fn from(_t: SomeType1) -> Self {
@@ -1500,7 +1494,7 @@ fn pallet_item_docs_in_metadata() {
 
 #[test]
 #[allow(deprecated)]
-fn metadata() {
+fn metadata_v15() {
 	use codec::Decode;
 	use frame_metadata::{v15::*, *};
 
@@ -1975,8 +1969,7 @@ fn metadata() {
 		_ => panic!("metadata has been bumped, test needs to be updated"),
 	};
 
-	let bytes = &Runtime::metadata_at_version(LATEST_METADATA_VERSION)
-		.expect("Metadata must be present; qed");
+	let bytes = &Runtime::metadata_at_version(15).expect("Metadata must be present; qed");
 
 	let actual_metadata: RuntimeMetadataPrefixed =
 		Decode::decode(&mut &bytes[..]).expect("Metadata encoded properly; qed");
@@ -2010,10 +2003,7 @@ fn metadata_at_version() {
 
 #[test]
 fn metadata_versions() {
-	assert_eq!(
-		vec![14, LATEST_METADATA_VERSION, UNSTABLE_METADATA_VERSION],
-		Runtime::metadata_versions()
-	);
+	assert_eq!(vec![14, 15, 16], Runtime::metadata_versions());
 }
 
 #[test]

substrate/frame/support/test/tests/pallet.rs

@@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 codec = { workspace = true }
-frame-metadata = { features = ["current", "unstable"], workspace = true }
+frame-metadata = { features = ["current"], workspace = true }
 scale-info = { features = ["derive"], workspace = true }
 
 [features]

substrate/primitives/metadata-ir/Cargo.toml

@@ -30,9 +30,9 @@ mod types;
 use frame_metadata::RuntimeMetadataPrefixed;
 pub use types::*;
 
-mod unstable;
 mod v14;
 mod v15;
+mod v16;
 
 /// Metadata V14.
 const V14: u32 = 14;
@@ -41,7 +41,7 @@ const V14: u32 = 14;
 const V15: u32 = 15;
 
 /// Unstable metadata V16.
-const UNSTABLE_V16: u32 = u32::MAX;
+const V16: u32 = 16;
 
 /// Transform the IR to the specified version.
 ///
@@ -54,19 +54,19 @@ pub fn into_version(metadata: MetadataIR, version: u32) -> Option<RuntimeMetadat
 		// `Metadata_metadata_at_version.
 		V14 => Some(into_v14(metadata)),
 
-		// Version V15 - latest stable.
-		V15 => Some(into_latest(metadata)),
+		// Version V15
+		V15 => Some(into_v15(metadata)),
 
-		// Unstable metadata under `u32::MAX`.
-		UNSTABLE_V16 => Some(into_unstable(metadata)),
+		// Version V16 - latest-stable.
+		V16 => Some(into_v16(metadata)),
 
 		_ => None,
 	}
 }
 
 /// Returns the supported metadata versions.
 pub fn supported_versions() -> alloc::vec::Vec<u32> {
-	alloc::vec![V14, V15, UNSTABLE_V16]
+	alloc::vec![V14, V15, V16]
 }
 
 /// Transform the IR to the latest stable metadata version.
@@ -81,8 +81,14 @@ pub fn into_v14(metadata: MetadataIR) -> RuntimeMetadataPrefixed {
 	latest.into()
 }
 
-/// Transform the IR to unstable metadata version 16.
-pub fn into_unstable(metadata: MetadataIR) -> RuntimeMetadataPrefixed {
+/// Transform the IR to metadata version 15.
+pub fn into_v15(metadata: MetadataIR) -> RuntimeMetadataPrefixed {
+	let latest: frame_metadata::v15::RuntimeMetadataV15 = metadata.into();
+	latest.into()
+}
+
+/// Transform the IR to metadata version 16.
+pub fn into_v16(metadata: MetadataIR) -> RuntimeMetadataPrefixed {
 	let latest: frame_metadata::v16::RuntimeMetadataV16 = metadata.into();
 	latest.into()
 }
@@ -144,4 +150,14 @@ mod test {
 
 		assert!(matches!(metadata.1, RuntimeMetadata::V15(_)));
 	}
+
+	#[test]
+	fn into_version_16() {
+		let ir = ir_metadata();
+		let metadata = into_version(ir, V16).expect("Should return prefixed metadata");
+
+		assert_eq!(metadata.0, META_RESERVED);
+
+		assert!(matches!(metadata.1, RuntimeMetadata::V16(_)));
+	}
 }

substrate/primitives/metadata-ir/src/lib.rs

@@ -44,7 +44,7 @@ impl From<MetadataIR> for RuntimeMetadataV16 {
 	fn from(ir: MetadataIR) -> Self {
 		RuntimeMetadataV16::new(
 			ir.pallets.into_iter().map(Into::into).collect(),
-			ir.extrinsic.into(),
+			ir.extrinsic.into_v16_with_call_ty(ir.outer_enums.call_enum_ty),
 			ir.apis.into_iter().map(Into::into).collect(),
 			ir.outer_enums.into(),
 			// Substrate does not collect yet the custom metadata fields.
@@ -179,18 +179,19 @@ impl From<TransactionExtensionMetadataIR> for TransactionExtensionMetadata {
 	}
 }
 
-impl From<ExtrinsicMetadataIR> for ExtrinsicMetadata {
-	fn from(ir: ExtrinsicMetadataIR) -> Self {
+impl ExtrinsicMetadataIR {
+	fn into_v16_with_call_ty(self, call_ty: scale_info::MetaType) -> ExtrinsicMetadata {
 		// Assume version 0 for all extensions.
-		let indexes = (0..ir.extensions.len()).map(|index| Compact(index as u32)).collect();
+		let indexes = (0..self.extensions.len()).map(|index| Compact(index as u32)).collect();
 		let transaction_extensions_by_version = [(0, indexes)].iter().cloned().collect();
 
 		ExtrinsicMetadata {
-			versions: ir.versions,
-			address_ty: ir.address_ty,
-			signature_ty: ir.signature_ty,
+			versions: self.versions,
+			address_ty: self.address_ty,
+			call_ty,
+			signature_ty: self.signature_ty,
 			transaction_extensions_by_version,
-			transaction_extensions: ir.extensions.into_iter().map(Into::into).collect(),
+			transaction_extensions: self.extensions.into_iter().map(Into::into).collect(),
 		}
 	}
 }

…e/primitives/metadata-ir/src/unstable.rs …strate/primitives/metadata-ir/src/v16.rssubstrate/primitives/metadata-ir/src/unstable.rs renamed to substrate/primitives/metadata-ir/src/v16.rs substrate/primitives/metadata-ir/src/unstable.rs renamed to substrate/primitives/metadata-ir/src/v16.rs

@@ -60,7 +60,9 @@ impl<C: Config<Hash = subxt::utils::H256>> DynamicRemarkBuilder<C> {
 
 			let latest = supported_metadata_versions
 				.into_iter()
-				.filter(|v| *v != u32::MAX)
+				// TODO: Subxt doesn't support V16 metadata until v0.42.0, so don't try
+				// to fetch it here until we update to that version.
+				.filter(|v| *v != u32::MAX && *v < 16)
 				.max()
 				.ok_or("No stable metadata versions supported".to_string())?;