From 777c077b01d1e599e81c3e33dc2e39bbf8a818f1 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:36:22 +0800 Subject: [PATCH] Update the pallet guide in `sdk-docs` (#4735) After using this tutorial in PBA, there was a few areas to improve it. Moreover, I have: - Improve `your_first_pallet`, link it in README, improve the parent `guide` section. - Updated the templates page, in light of recent efforts related to in https://github.com/paritytech/polkadot-sdk/issues/3155 - Added small ref docs about metadata, completed the one about native runtime, added one about host functions. - Remove a lot of unfinished stuff from sdk-docs - update diagram for `Hooks` --- Cargo.lock | 5 + README.md | 11 +- docs/mermaid/IA.mmd | 4 +- docs/sdk/Cargo.toml | 16 ++- docs/sdk/src/guides/mod.rs | 25 ++-- docs/sdk/src/guides/your_first_pallet/mod.rs | 109 +++++++++++------- docs/sdk/src/guides/your_first_runtime.rs | 2 + docs/sdk/src/polkadot_sdk/templates.rs | 70 ++++++----- .../reference_docs/blockchain_scalibility.rs | 0 .../src/reference_docs/consensus_swapping.rs | 6 - .../reference_docs/custom_host_functions.rs | 27 +++++ .../src/reference_docs/fee_less_runtime.rs | 1 + .../reference_docs/frame_offchain_workers.rs | 1 - .../reference_docs/frame_system_accounts.rs | 2 + docs/sdk/src/reference_docs/light_nodes.rs | 7 -- docs/sdk/src/reference_docs/metadata.rs | 24 ++++ docs/sdk/src/reference_docs/mod.rs | 19 +-- docs/sdk/src/reference_docs/wasm_memory.rs | 7 -- .../src/reference_docs/wasm_meta_protocol.rs | 79 ++++++++++--- substrate/frame/support/src/traits/hooks.rs | 33 +++--- substrate/primitives/io/Cargo.toml | 11 +- substrate/primitives/io/src/lib.rs | 1 + templates/minimal/node/src/service.rs | 2 + 23 files changed, 302 insertions(+), 160 deletions(-) delete mode 100644 docs/sdk/src/reference_docs/blockchain_scalibility.rs delete mode 100644 docs/sdk/src/reference_docs/consensus_swapping.rs create mode 100644 docs/sdk/src/reference_docs/custom_host_functions.rs delete mode 100644 docs/sdk/src/reference_docs/light_nodes.rs delete mode 100644 docs/sdk/src/reference_docs/wasm_memory.rs diff --git a/Cargo.lock b/Cargo.lock index 13d658b513518..a8b08d280158f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14328,6 +14328,7 @@ dependencies = [ "frame-support", "frame-system", "kitchensink-runtime", + "minimal-template-runtime", "pallet-assets", "pallet-aura", "pallet-authorship", @@ -14350,6 +14351,7 @@ dependencies = [ "pallet-transaction-payment", "pallet-uniques", "pallet-utility", + "parachain-template-runtime", "parity-scale-codec", "polkadot-sdk", "polkadot-sdk-frame", @@ -14369,6 +14371,7 @@ dependencies = [ "sc-service", "scale-info", "simple-mermaid 0.1.1", + "solochain-template-runtime", "sp-api", "sp-arithmetic", "sp-core", @@ -14377,6 +14380,7 @@ dependencies = [ "sp-keyring", "sp-offchain", "sp-runtime", + "sp-runtime-interface 24.0.0", "sp-version", "staging-chain-spec-builder", "staging-node-cli", @@ -20007,6 +20011,7 @@ name = "sp-io" version = "30.0.0" dependencies = [ "bytes", + "docify", "ed25519-dalek 2.1.1", "libsecp256k1", "log", diff --git a/README.md b/README.md index 0b027b2958c15..92901d070db08 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,12 @@ forks](https://img.shields.io/github/forks/paritytech/polkadot-sdk) ## 📚 Documentation * [🦀 rust-docs](https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/index.html) - * [Introduction](https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/polkadot_sdk/index.html) - to each component of the Polkadot SDK: Substrate, FRAME, Cumulus, and XCM + * [Introduction](https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/polkadot_sdk/index.html) + to each component of the Polkadot SDK: Substrate, FRAME, Cumulus, and XCM + * [Guides](https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/guides/index.html), + namely how to build your first FRAME pallet. + * [Templates](https://paritytech.github.io/polkadot-sdk/master/polkadot_sdk_docs/polkadot_sdk/templates/index.html) + for starting a new project. * Other Resources: * [Polkadot Wiki -> Build](https://wiki.polkadot.network/docs/build-guide) @@ -39,6 +43,9 @@ The Polkadot-SDK has two release channels: `stable` and `nightly`. Production so only use `stable`. `nightly` is meant for tinkerers to try out the latest features. The detailed release process is described in [RELEASE.md](docs/RELEASE.md). +You can use [`psvm`](https://github.com/paritytech/psvm) to manage your Polkadot-SDK dependency +versions in downstream projects. + ### 😌 Stable `stable` releases have a support duration of **three months**. In this period, the release will not diff --git a/docs/mermaid/IA.mmd b/docs/mermaid/IA.mmd index fe9a96bcafc00..37417497e1f84 100644 --- a/docs/mermaid/IA.mmd +++ b/docs/mermaid/IA.mmd @@ -1,6 +1,6 @@ flowchart parity[paritytech.github.io] --> devhub[polkadot_sdk_docs] - polkadot[polkadot.network] --> devhub[polkadot_sdk_docs] + polkadot_network[polkadot.network] --> devhub[polkadot_sdk_docs] devhub --> polkadot_sdk devhub --> reference_docs @@ -9,5 +9,5 @@ flowchart polkadot_sdk --> substrate polkadot_sdk --> frame polkadot_sdk --> cumulus - polkadot_sdk --> polkadot + polkadot_sdk --> polkadot[polkadot node] polkadot_sdk --> xcm diff --git a/docs/sdk/Cargo.toml b/docs/sdk/Cargo.toml index b0671623f48d4..10c0912116711 100644 --- a/docs/sdk/Cargo.toml +++ b/docs/sdk/Cargo.toml @@ -83,27 +83,31 @@ pallet-democracy = { path = "../../substrate/frame/democracy" } pallet-uniques = { path = "../../substrate/frame/uniques" } pallet-nfts = { path = "../../substrate/frame/nfts" } pallet-scheduler = { path = "../../substrate/frame/scheduler" } +pallet-referenda = { path = "../../substrate/frame/referenda" } +pallet-broker = { path = "../../substrate/frame/broker" } +pallet-babe = { path = "../../substrate/frame/babe" } # Primitives sp-io = { path = "../../substrate/primitives/io" } +sp-runtime-interface = { path = "../../substrate/primitives/runtime-interface" } sp-api = { path = "../../substrate/primitives/api" } sp-core = { path = "../../substrate/primitives/core" } sp-keyring = { path = "../../substrate/primitives/keyring" } sp-runtime = { path = "../../substrate/primitives/runtime" } sp-arithmetic = { path = "../../substrate/primitives/arithmetic" } sp-genesis-builder = { path = "../../substrate/primitives/genesis-builder" } - -# Misc pallet dependencies -pallet-referenda = { path = "../../substrate/frame/referenda" } -pallet-broker = { path = "../../substrate/frame/broker" } -pallet-babe = { path = "../../substrate/frame/babe" } - sp-offchain = { path = "../../substrate/primitives/offchain" } sp-version = { path = "../../substrate/primitives/version" } + # XCM xcm = { package = "staging-xcm", path = "../../polkadot/xcm" } xcm-docs = { path = "../../polkadot/xcm/docs" } # runtime guides chain-spec-guide-runtime = { path = "./src/reference_docs/chain_spec_runtime" } + +# Templates +minimal-template-runtime = { path = "../../templates/minimal/runtime" } +solochain-template-runtime = { path = "../../templates/solochain/runtime" } +parachain-template-runtime = { path = "../../templates/parachain/runtime" } diff --git a/docs/sdk/src/guides/mod.rs b/docs/sdk/src/guides/mod.rs index f5f6d2b5e0c07..485cdc30636f2 100644 --- a/docs/sdk/src/guides/mod.rs +++ b/docs/sdk/src/guides/mod.rs @@ -1,7 +1,16 @@ //! # Polkadot SDK Docs Guides //! -//! This crate contains a collection of guides that are foundational to the developers of -//! Polkadot SDK. They are common user-journeys that are traversed in the Polkadot ecosystem. +//! This crate contains a collection of guides that are foundational to the developers of Polkadot +//! SDK. They are common user-journeys that are traversed in the Polkadot ecosystem. +//! +//! 1. [`crate::guides::your_first_pallet`] is your starting point with Polkadot SDK. It contains +//! the basics of +//! building a simple crypto currency with FRAME. +//! 2. [`crate::guides::your_first_runtime`] is the next step in your journey. It contains the +//! basics of building a runtime that contains this pallet, plus a few common pallets from FRAME. +//! +//! +//! Other guides are related to other miscellaneous topics and are listed as modules below. /// Write your first simple pallet, learning the most most basic features of FRAME along the way. pub mod your_first_pallet; @@ -11,18 +20,18 @@ pub mod your_first_pallet; pub mod your_first_runtime; /// Running the given runtime with a node. No specific consensus mechanism is used at this stage. -pub mod your_first_node; - -/// How to change the consensus engine of both the node and the runtime. -pub mod changing_consensus; +// TODO +// pub mod your_first_node; /// How to enhance a given runtime and node to be cumulus-enabled, run it as a parachain and connect /// it to a relay-chain. -pub mod cumulus_enabled_parachain; +// TODO +// pub mod cumulus_enabled_parachain; /// How to make a given runtime XCM-enabled, capable of sending messages (`Transact`) between itself /// and the relay chain to which it is connected. -pub mod xcm_enabled_parachain; +// TODO +// pub mod xcm_enabled_parachain; /// How to enable storage weight reclaiming in a parachain node and runtime. pub mod enable_pov_reclaim; diff --git a/docs/sdk/src/guides/your_first_pallet/mod.rs b/docs/sdk/src/guides/your_first_pallet/mod.rs index c6e0dd0edf892..0a22b13df814d 100644 --- a/docs/sdk/src/guides/your_first_pallet/mod.rs +++ b/docs/sdk/src/guides/your_first_pallet/mod.rs @@ -14,18 +14,14 @@ //! > FRAME-based runtimes use various techniques to re-use a currency pallet instead of writing //! > one. Further advanced FRAME related topics are discussed in [`crate::reference_docs`]. //! -//! ## Topics Covered +//! ## Writing Your First Pallet //! -//! The following FRAME topics are covered in this guide: +//! To get started, use one of the templates mentioned in [`crate::polkadot_sdk::templates`]. We +//! recommend using the `polkadot-sdk-minimal-template`. You might need to change small parts of +//! this guide, namely the crate/package names, based on which tutorial you use. //! -//! - [Storage](frame::pallet_macros::storage) -//! - [Call](frame::pallet_macros::call) -//! - [Event](frame::pallet_macros::event) -//! - [Error](frame::pallet_macros::error) -//! - Basics of testing a pallet -//! - [Constructing a runtime](frame::runtime::prelude::construct_runtime) -//! -//! ## Writing Your First Pallet +//! > Be aware that you can read the entire source code backing this tutorial by clicking on the +//! > [`source`](./mod.rs.html) button at the top right of the page. //! //! You should have studied the following modules as a prelude to this guide: //! @@ -33,16 +29,28 @@ //! - [`crate::reference_docs::trait_based_programming`] //! - [`crate::polkadot_sdk::frame_runtime`] //! +//! ## Topics Covered +//! +//! The following FRAME topics are covered in this guide: +//! +//! - [`pallet::storage`] +//! - [`pallet::call`] +//! - [`pallet::event`] +//! - [`pallet::error`] +//! - Basics of testing a pallet +//! - [Constructing a runtime](frame::runtime::prelude::construct_runtime) +//! //! ### Shell Pallet //! //! Consider the following as a "shell pallet". We continue building the rest of this pallet based //! on this template. //! -//! [`pallet::config`](frame::pallet_macros::config) and -//! [`pallet::pallet`](frame::pallet_macros::pallet) are both mandatory parts of any pallet. Refer -//! to the documentation of each to get an overview of what they do. +//! [`pallet::config`] and [`pallet::pallet`] are both mandatory parts of any pallet. Refer to the +//! documentation of each to get an overview of what they do. #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", shell_pallet)] //! +//! All of the code that follows in this guide should live inside of the `mod pallet`. +//! //! ### Storage //! //! First, we will need to create two onchain storage declarations. @@ -55,15 +63,14 @@ //! > generic bounded type in the `Config` trait, and then specify it in the implementation. #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", Balance)] //! -//! The definition of these two storage items, based on [`frame::pallet_macros::storage`] details, -//! is as follows: +//! The definition of these two storage items, based on [`pallet::storage`] details, is as follows: #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", TotalIssuance)] #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", Balances)] //! //! ### Dispatchables //! -//! Next, we will define the dispatchable functions. As per [`frame::pallet_macros::call`], these -//! will be defined as normal `fn`s attached to `struct Pallet`. +//! Next, we will define the dispatchable functions. As per [`pallet::call`], these will be defined +//! as normal `fn`s attached to `struct Pallet`. #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", impl_pallet)] //! //! The logic of the functions is self-explanatory. Instead, we will focus on the FRAME-related @@ -79,7 +86,6 @@ //! was signed by `who`. #![doc = docify::embed!("../../substrate/frame/system/src/lib.rs", ensure_signed)] //! -//! //! - Where does `mutate`, `get` and `insert` and other storage APIs come from? All of them are //! explained in the corresponding `type`, for example, for `Balances::::insert`, you can look //! into [`frame::prelude::StorageMap::insert`]. @@ -95,8 +101,7 @@ //! //! - Why are all `get` and `mutate` functions returning an `Option`? This is the default behavior //! of FRAME storage APIs. You can learn more about how to override this by looking into -//! [`frame::pallet_macros::storage`], and -//! [`frame::prelude::ValueQuery`]/[`frame::prelude::OptionQuery`] +//! [`pallet::storage`], and [`frame::prelude::ValueQuery`]/[`frame::prelude::OptionQuery`] //! //! ### Improving Errors //! @@ -116,6 +121,25 @@ //! //! ### Your First (Test) Runtime //! +//! The typical testing code of a pallet lives in a module that imports some preludes useful for +//! testing, similar to: +//! +//! ``` +//! pub mod pallet { +//! // snip -- actually pallet code. +//! } +//! +//! #[cfg(test)] +//! mod tests { +//! // bring in the testing prelude of frame +//! use frame::testing_prelude::*; +//! // bring in all pallet items +//! use super::pallet::*; +//! +//! // snip -- rest of the testing code. +//! } +//! ``` +//! //! Next, we create a "test runtime" in order to test our pallet. Recall from //! [`crate::polkadot_sdk::frame_runtime`] that a runtime is a collection of pallets, expressed //! through [`frame::runtime::prelude::construct_runtime`]. All runtimes also have to include @@ -166,7 +190,6 @@ //! As noted above, the `T::AccountId` is now `u64`. Moreover, `Runtime` is replacing ``. //! This is why for example you see `Balances::::get(..)`. Finally, notice that the //! dispatchables are simply functions that can be called on top of the `Pallet` struct. -// TODO: hard to explain exactly `RuntimeOrigin::signed(ALICE)` at this point. //! //! Congratulations! You have written your first pallet and tested it! Next, we learn a few optional //! steps to improve our pallet. @@ -236,8 +259,7 @@ //! by one character. FRAME errors are exactly a solution to maintain readability, whilst fixing //! the drawbacks mentioned. In short, we use an enum to represent different variants of our //! error. These variants are then mapped in an efficient way (using only `u8` indices) to -//! [`sp_runtime::DispatchError::Module`]. Read more about this in -//! [`frame::pallet_macros::error`]. +//! [`sp_runtime::DispatchError::Module`]. Read more about this in [`pallet::error`]. //! //! - **Event**: Events are akin to the return type of dispatchables. They are mostly data blobs //! emitted by the runtime to let outside world know what is happening inside the pallet. Since @@ -246,20 +268,16 @@ //! use passive tense for event names (eg. `SomethingHappened`). This allows other sub-systems or //! external parties (eg. a light-node, a DApp) to listen to particular events happening, without //! needing to re-execute the whole state transition function. -// TODO: both need to be improved a lot at the pallet-macro rust-doc level. Also my explanation -// of event is probably not the best. //! //! With the explanation out of the way, let's see how these components can be added. Both follow a -//! fairly familiar syntax: normal Rust enums, with extra -//! [`#[frame::event]`](frame::pallet_macros::event) and -//! [`#[frame::error]`](frame::pallet_macros::error) attributes attached. +//! fairly familiar syntax: normal Rust enums, with extra [`pallet::event`] and [`pallet::error`] +//! attributes attached. #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", Event)] #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", Error)] //! -//! One slightly custom part of this is the [`#[pallet::generate_deposit(pub(super) fn -//! deposit_event)]`](frame::pallet_macros::generate_deposit) part. Without going into too -//! much detail, in order for a pallet to emit events to the rest of the system, it needs to do two -//! things: +//! One slightly custom part of this is the [`pallet::generate_deposit`] part. Without going into +//! too much detail, in order for a pallet to emit events to the rest of the system, it needs to do +//! two things: //! //! 1. Declare a type in its `Config` that refers to the overarching event type of the runtime. In //! short, by doing this, the pallet is expressing an important bound: `type RuntimeEvent: @@ -268,8 +286,8 @@ //! store it where needed. //! //! 2. But, doing this conversion and storing is too much to expect each pallet to define. FRAME -//! provides a default way of storing events, and this is what -//! [`pallet::generate_deposit`](frame::pallet_macros::generate_deposit) is doing. +//! provides a default way of storing events, and this is what [`pallet::generate_deposit`] is +//! doing. #![doc = docify::embed!("./src/guides/your_first_pallet/mod.rs", config_v2)] //! //! > These `Runtime*` types are better explained in @@ -297,10 +315,17 @@ //! - [`crate::reference_docs::defensive_programming`]. //! - [`crate::reference_docs::frame_origin`]. //! - [`crate::reference_docs::frame_runtime_types`]. -//! - The pallet we wrote in this guide was using `dev_mode`, learn more in -//! [`frame::pallet_macros::config`]. +//! - The pallet we wrote in this guide was using `dev_mode`, learn more in [`pallet::config`]. //! - Learn more about the individual pallet items/macros, such as event and errors and call, in //! [`frame::pallet_macros`]. +//! +//! [`pallet::storage`]: ../../../frame_support/pallet_macros/attr.config.html +//! [`pallet::call`]: ../../../frame_support/pallet_macros/attr.call.html +//! [`pallet::event`]: ../../../frame_support/pallet_macros/attr.event.html +//! [`pallet::error`]: ../../../frame_support/pallet_macros/attr.error.html +//! [`pallet::pallet`]: ../../../frame_support/pallet_macros/attr.pallet.html +//! [`pallet::config`]: ../../../frame_support/pallet_macros/attr.config.html +//! [`pallet::generate_deposit`]: ../../../frame_support/pallet_macros/attr.generate_deposit.html #[docify::export] #[frame::pallet(dev_mode)] @@ -418,16 +443,22 @@ pub mod pallet { #[cfg(any(test, doc))] pub(crate) mod tests { use crate::guides::your_first_pallet::pallet::*; + + #[docify::export(testing_prelude)] use frame::testing_prelude::*; - const ALICE: u64 = 1; - const BOB: u64 = 2; - const CHARLIE: u64 = 3; + + pub(crate) const ALICE: u64 = 1; + pub(crate) const BOB: u64 = 2; + pub(crate) const CHARLIE: u64 = 3; #[docify::export] + // This runtime is only used for testing, so it should be somewhere like `#[cfg(test)] mod + // tests { .. }` mod runtime { use super::*; // we need to reference our `mod pallet` as an identifier to pass to // `construct_runtime`. + // YOU HAVE TO CHANGE THIS LINE BASED ON YOUR TEMPLATE use crate::guides::your_first_pallet::pallet as pallet_currency; construct_runtime!( diff --git a/docs/sdk/src/guides/your_first_runtime.rs b/docs/sdk/src/guides/your_first_runtime.rs index 3e02ef1b1b28e..c58abc1120c13 100644 --- a/docs/sdk/src/guides/your_first_runtime.rs +++ b/docs/sdk/src/guides/your_first_runtime.rs @@ -1 +1,3 @@ //! # Your first Runtime +//! +//! 🚧 diff --git a/docs/sdk/src/polkadot_sdk/templates.rs b/docs/sdk/src/polkadot_sdk/templates.rs index 4bf0e839c798f..e87eb9c2bc8ab 100644 --- a/docs/sdk/src/polkadot_sdk/templates.rs +++ b/docs/sdk/src/polkadot_sdk/templates.rs @@ -1,19 +1,33 @@ //! # Templates //! -//! ### Internal +//! This document enumerates a non-exhaustive list of templates that one can use to get started with +//! polkadot-sdk. //! -//! The following templates are maintained as a part of the `polkadot-sdk` repository: +//! > Know more tools/templates that are not listed here? please contribute them by opening a PR. //! -//! - classic [`substrate-node-template`]: is a white-labeled substrate-based blockchain with a -//! moderate amount of features. It can act as a great starting point for those who want to learn -//! Substrate/FRAME and want to have a template that is already doing something. -//! - [`substrate-minimal-template`]: Same as the above, but it contains the least amount of code in -//! both the node and runtime. It is a great starting point for those who want to deeply learn -//! Substrate and FRAME. -//! - classic [`cumulus-parachain-template`], which is the de-facto parachain template shipped with -//! Cumulus. It is the parachain-enabled version of [`substrate-node-template`]. +//! ## Internal //! -//! ### External Templates +//! The following [templates](https://github.com/paritytech/polkadot-sdk/blob/master/templates) are +//! maintained as a part of the `polkadot-sdk` repository: +//! +//! - `minimal_template_node`/[`minimal_template_runtime`]: A minimal template that contains the +//! least amount of features to be a functioning blockchain. Suitable for learning, development +//! and testing. This template is not meant to be used in production. +//! - `solochain_template_node`/[`solochain_template_runtime`]: Formerly known as +//! "substrate-node-template", is a white-labeled substrate-based blockchain (aka. solochain) that +//! contains moderate features, such as a basic consensus engine and some FRAME pallets. This +//! template can act as a good starting point for those who want to launch a solochain. +//! - `parachain_template_node`/[`parachain_template_runtime`]: A parachain template ready to be +//! connected to a test relay-chain. +//! +//! These templates are always kept up to date, and are mirrored to external repositories for easy +//! forking: +//! +//! - +//! - +//! - +//! +//! ## External Templates //! //! Noteworthy templates outside of this repository. //! @@ -22,23 +36,17 @@ //! - [`frontier-parachain-template`](https://github.com/paritytech/frontier-parachain-template): A //! parachain template for launching EVM-compatible parachains. //! -//! [`minimal-template`]: https://github.com/paritytech/polkadot-sdk/blob/master/templates/minimal/ -//! [`parachain-template`]: https://github.com/paritytech/polkadot-sdk/blob/master/templates/parachain/ - -// TODO: in general, we need to make a deliberate choice here of moving a few key templates to this -// repo (nothing stays in `substrate-developer-hub`) and the everything else should be community -// maintained. https://github.com/paritytech/polkadot-sdk-docs/issues/67 - -// TODO: we should rename `substrate-node-template` to `substrate-basic-template`, -// `substrate-blockchain-template`. `node` is confusing in the name. -// `substrate-blockchain-template` and `cumulus-parachain-template` go well together 🤝. https://github.com/paritytech/polkadot-sdk-docs/issues/67 - -// NOTE: a super important detail that I am looking forward to here is -// and -// . Meaning that I would not spend time on -// teaching someone too much detail about the ugly thing we call "node" nowadays. In the future, I -// am sure we will either have a better "node-builder" code that can actually be tested, or an -// "omni-node" that can run (almost) any wasm file. We should already build tutorials in this -// direction IMO. This also affects all the templates. If we have a good neat runtime file, which we -// are moving toward, and a good node-builder, we don't need all of these damn templates. These -// templates are only there because the boilerplate is super horrible atm. +//! ## OpenZeppelin +//! +//! In June 2023, OpenZeppelin was awarded a grant from the [Polkadot +//! treasury](https://polkadot.polkassembly.io/treasury/406) for building a number of Polkadot-sdk +//! based templates. These templates are expected to be a great starting point for developers. +//! +//! - +//! +//! ## POP-CLI +//! +//! Is a CLI tool capable of scaffolding a new polkadot-sdk-based project, possibly removing the +//! need for templates. +//! +//! - diff --git a/docs/sdk/src/reference_docs/blockchain_scalibility.rs b/docs/sdk/src/reference_docs/blockchain_scalibility.rs deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docs/sdk/src/reference_docs/consensus_swapping.rs b/docs/sdk/src/reference_docs/consensus_swapping.rs deleted file mode 100644 index e639761ee97b4..0000000000000 --- a/docs/sdk/src/reference_docs/consensus_swapping.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Consensus Swapping -//! -//! Notes: -//! -//! - The typical workshop done by Joshy in some places where he swaps out the consensus to be PoW. -//! - This could also be a tutorial rather than a ref doc, depending on the size. diff --git a/docs/sdk/src/reference_docs/custom_host_functions.rs b/docs/sdk/src/reference_docs/custom_host_functions.rs new file mode 100644 index 0000000000000..719b208a2bff7 --- /dev/null +++ b/docs/sdk/src/reference_docs/custom_host_functions.rs @@ -0,0 +1,27 @@ +//! # Custom Host Functions +//! +//! Host functions are functions that the wasm instance can use to communicate with the node. Learn +//! more about this in [`crate::reference_docs::wasm_meta_protocol`]. +//! +//! ## Finding Host Functions +//! +//! To declare a set of functions as host functions, you need to use the `#[runtime_interface]` +//! ([`sp_runtime_interface`]) attribute macro. The most notable set of host functions are those +//! that allow the runtime to access the chain state, namely [`sp_io::storage`]. Some other notable +//! host functions are also defined in [`sp_io`]. +//! +//! ## Adding New Host Functions +//! +//! > Adding a new host function is a big commitment and should be done with care. Namely, the nodes +//! > in the network need to support all host functions forever in order to be able to sync +//! > historical blocks. +//! +//! Adding host functions is only possible when you are using a node-template, so that you have +//! access to the boilerplate of building your node. +//! +//! A group of host functions can always be grouped to gether as a tuple: +#![doc = docify::embed!("../../substrate/primitives/io/src/lib.rs", SubstrateHostFunctions)] +//! +//! The host functions are attached to the node side's [`sc_executor::WasmExecutor`]. For example in +//! the minimal template, the setup looks as follows: +#![doc = docify::embed!("../../templates/minimal/node/src/service.rs", FullClient)] diff --git a/docs/sdk/src/reference_docs/fee_less_runtime.rs b/docs/sdk/src/reference_docs/fee_less_runtime.rs index 1213c26282537..9146b30ec5774 100644 --- a/docs/sdk/src/reference_docs/fee_less_runtime.rs +++ b/docs/sdk/src/reference_docs/fee_less_runtime.rs @@ -1,5 +1,6 @@ //! # Fee-Less Runtime //! +//! 🚧 Work In Progress 🚧 //! //! Notes: //! diff --git a/docs/sdk/src/reference_docs/frame_offchain_workers.rs b/docs/sdk/src/reference_docs/frame_offchain_workers.rs index 7999707e5ee01..1ec9212e23066 100644 --- a/docs/sdk/src/reference_docs/frame_offchain_workers.rs +++ b/docs/sdk/src/reference_docs/frame_offchain_workers.rs @@ -58,7 +58,6 @@ //! [`frame::pallet_macros::hooks`]. //! //! ``` -//! //! #[frame::pallet] //! pub mod pallet { //! use frame::prelude::*; diff --git a/docs/sdk/src/reference_docs/frame_system_accounts.rs b/docs/sdk/src/reference_docs/frame_system_accounts.rs index ae9d2c9e0cb3c..523fe70430849 100644 --- a/docs/sdk/src/reference_docs/frame_system_accounts.rs +++ b/docs/sdk/src/reference_docs/frame_system_accounts.rs @@ -1,5 +1,7 @@ //! # FRAME Accounts //! +//! //! 🚧 Work In Progress 🚧 +//! //! How `frame_system` handles accountIds. Nonce. Consumers and Providers, reference counting. // - poorly understood topics, needs one great article to rul them all. diff --git a/docs/sdk/src/reference_docs/light_nodes.rs b/docs/sdk/src/reference_docs/light_nodes.rs deleted file mode 100644 index d6670bf03ab1a..0000000000000 --- a/docs/sdk/src/reference_docs/light_nodes.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! # Light Clients -//! -//! -//! Notes: should contain only high level information about light clients, then link to how to set -//! it up in PAPI and SubXT -//! -//! diff --git a/docs/sdk/src/reference_docs/metadata.rs b/docs/sdk/src/reference_docs/metadata.rs index 702c1c30fd9cf..96f92ac0c412b 100644 --- a/docs/sdk/src/reference_docs/metadata.rs +++ b/docs/sdk/src/reference_docs/metadata.rs @@ -1 +1,25 @@ //! # Metadata +//! +//! The existence of metadata in polkadot-sdk goes back to the (forkless) upgrade-ability of all +//! Substrate-based blockchains, which is achieved through +//! [`crate::reference_docs::wasm_meta_protocol`]. You can learn more about the details of how to +//! deal with these upgrades in [`crate::reference_docs::frame_runtime_upgrades_and_migrations`]. +//! +//! Another consequence of upgrade-ability is that as a UI, wallet, or generally an offchain entity, +//! it is hard to know the types internal to the runtime, specifically in light of the fact that +//! they can change at any point in time. +//! +//! This is why all Substrate-based runtimes must expose a [`sp_api::Metadata`] api, which mandates +//! the runtime to return a description of itself. The return type of this api is `Vec`, meaning +//! that it is up to the runtime developer to decide on the format of this. +//! +//! All [`crate::polkadot_sdk::frame_runtime`] based runtimes expose a specific metadata language, +//! maintained in which is adopted in the Polkadot +//! ecosystem. +//! +//! ## Metadata Explorers: +//! +//! A few noteworthy tools that inspect the (FRAME-based) metadata of a chain: +//! +//! +//! diff --git a/docs/sdk/src/reference_docs/mod.rs b/docs/sdk/src/reference_docs/mod.rs index 8e0431c48b6f6..51150a5583758 100644 --- a/docs/sdk/src/reference_docs/mod.rs +++ b/docs/sdk/src/reference_docs/mod.rs @@ -40,7 +40,6 @@ pub mod runtime_vs_smart_contract; pub mod extrinsic_encoding; /// Learn about the signed extensions that form a part of extrinsics. -// TODO: @jsdw https://github.com/paritytech/polkadot-sdk-docs/issues/42 pub mod signed_extensions; /// Learn about *Origins*, a topic in FRAME that enables complex account abstractions to be built. @@ -59,9 +58,11 @@ pub mod fee_less_runtime; /// Learn about metadata, the main means through which an upgradeable runtime communicates its /// properties to the outside world. -// TODO: @jsdw https://github.com/paritytech/polkadot-sdk-docs/issues/47 pub mod metadata; +/// Learn about how to add custom host functions to the node. +pub mod custom_host_functions; + /// Learn about how frame-system handles `account-ids`, nonces, consumers and providers. pub mod frame_system_accounts; @@ -78,26 +79,12 @@ pub mod frame_tokens; /// Learn about chain specification file and the genesis state of the blockchain. pub mod chain_spec_genesis; -/// Learn about all the memory limitations of the WASM runtime when it comes to memory usage. -// TODO: @kianenigma https://github.com/paritytech/polkadot-sdk-docs/issues/52 -pub mod wasm_memory; - /// Learn about Substrate's CLI, and how it can be extended. -// TODO: @kianenigma https://github.com/paritytech/polkadot-sdk-docs/issues/53 pub mod cli; -/// Learn about Substrate's consensus algorithms, and how you can switch between two. -// TODO: @JoshOrndorff @kianenigma https://github.com/paritytech/polkadot-sdk-docs/issues/54 -pub mod consensus_swapping; - /// Learn about Runtime Upgrades and best practices for writing Migrations. pub mod frame_runtime_upgrades_and_migrations; -/// Learn about light nodes, how they function, and how Substrate-based chains come -/// light-node-first out of the box. -// TODO: @jsdw @josepot https://github.com/paritytech/polkadot-sdk-docs/issues/68 -pub mod light_nodes; - /// Learn about the offchain workers, how they function, and how to use them, as provided by the /// [`frame`] APIs. pub mod frame_offchain_workers; diff --git a/docs/sdk/src/reference_docs/wasm_memory.rs b/docs/sdk/src/reference_docs/wasm_memory.rs deleted file mode 100644 index 4f4cda31094e4..0000000000000 --- a/docs/sdk/src/reference_docs/wasm_memory.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! # WASM Memory Limitations. -//! -//! Notes: -//! -//! - Stack: Need to use `Box<_>` -//! - Heap: Substrate imposes a limit. PvF execution has its own limits -//! - Heap: There is also a maximum amount that a single allocation can have. diff --git a/docs/sdk/src/reference_docs/wasm_meta_protocol.rs b/docs/sdk/src/reference_docs/wasm_meta_protocol.rs index 37d1460f0e1a3..0e91e65c55e36 100644 --- a/docs/sdk/src/reference_docs/wasm_meta_protocol.rs +++ b/docs/sdk/src/reference_docs/wasm_meta_protocol.rs @@ -1,11 +1,13 @@ //! # WASM Meta Protocol //! //! All Substrate based chains adhere to a unique architectural design novel to the Polkadot -//! ecosystem. We refer to this design as the "WASM Meta Protocol". +//! ecosystem. We refer to this design as the "**WASM Meta Protocol**". //! //! Consider the fact that a traditional blockchain software is usually a monolithic artifact. -//! Upgrading any part of the system implies upgrading the entire system. This has historically led -//! to cumbersome forkful upgrades to be the status quo in the blockchain ecosystem. +//! **Upgrading any part of the system implies upgrading the entire system**. This has historically +//! led to cumbersome forkful upgrades to be the status quo in blockchain ecosystems. In other +//! words, the entire node software is the specification of the blockchain's [`state transition +//! function`](crate::reference_docs::blockchain_state_machines). //! //! Moreover, the idea of "storing code in the state" is explored in the context of smart contracts //! platforms, but has not been expanded further. @@ -15,17 +17,16 @@ //! that a smart contract platform stores the code of individual contracts in its state. As noted in //! [`crate::reference_docs::blockchain_state_machines`], this state transition function is called //! the **Runtime**, and WASM is chosen as the bytecode. The Runtime is stored under a special key -//! in the state (see -//! [`sp_core::storage::well_known_keys`](../../../sp_core/index.html)) and can be -//! updated as a part of the state transition function's execution, just like a user's account -//! balance can be updated. +//! in the state (see [`sp_core::storage::well_known_keys`]) and can be updated as a part of the +//! state transition function's execution, just like a user's account balance can be updated. //! //! > Note that while we drew an analogy between smart contracts and runtimes in the above, there //! > are fundamental differences between the two, explained in //! > [`crate::reference_docs::runtime_vs_smart_contract`]. //! -//! The rest of the system that is NOT the state transition function is called the **node**, and -//! is a normal binary that is compiled from Rust to different hardware targets. +//! The rest of the system that is NOT the state transition function is called the +//! [**Node**](crate::reference_docs::glossary#node), and is a normal binary that is compiled from +//! Rust to different hardware targets. //! //! This design enables all Substrate-based chains to be fork-less-ly upgradeable, because the //! Runtime can be updates on the fly, within the execution of a block, and the node is (for the @@ -47,15 +48,18 @@ #![doc = simple_mermaid::mermaid!("../../../mermaid/substrate_client_runtime.mmd")] //! //! A runtime must have a set of runtime APIs in order to have any meaningful blockchain -//! functionality, but it can also expose more APIs. See TODO as an example of how to add custom -//! runtime APIs to your FRAME-based runtime. +//! functionality, but it can also expose more APIs. See +//! [`crate::reference_docs::custom_runtime_api_rpc`] as an example of how to add custom runtime +//! APIs to your FRAME-based runtime. //! //! Similarly, for a runtime to be "compatible" with a node, the node must implement the full set of //! host functions that the runtime at any point in time requires. Given the fact that a runtime can //! evolve in time, and a blockchain node (typically) wishes to be capable of re-executing all the //! previous blocks, this means that a node must always maintain support for the old host functions. -//! This also implies that adding a new host function is a big commitment and should be done with -//! care. This is why, for example, adding a new host function to Polkadot always requires an RFC. +//! **This implies that adding a new host function is a big commitment and should be done with +//! care**. This is why, for example, adding a new host function to Polkadot always requires an RFC. +//! Learn how to add a new host function to your runtime in +//! [`crate::reference_docs::custom_host_functions`]. //! //! ## Node vs. Runtime //! @@ -90,11 +94,11 @@ //! //! In fact, [`sp_core::storage::well_known_keys`] are the only state keys that the node side is //! aware of. The rest of the state, including what logic the runtime has, what balance each user -//! has and such are all only comprehensible to the runtime. +//! has and such, are all only comprehensible to the runtime. #![doc = simple_mermaid::mermaid!("../../../mermaid/state.mmd")] //! //! In the above diagram, all of the state keys and values are opaque bytes to the node. The node -//! does not know what they mean, and it does not now what is the type of the corresponding value +//! does not know what they mean, and it does not know what is the type of the corresponding value //! (e.g. if it is a number of a vector). Contrary, the runtime knows both the meaning of their //! keys, and the type of the values. //! @@ -105,9 +109,50 @@ //! //! ## Native Runtime //! -//! TODO +//! Historically, the node software also kept a native copy of the runtime at the time of +//! compilation within it. This used to be called the "Native Runtime". The main purpose of the +//! native runtime used to be leveraging the faster execution time and better debugging +//! infrastructure of native code. However, neither of the two arguments strongly hold and the +//! native runtime is being fully removed from the node-sdk. //! +//! See: +//! +//! > Also, note that the flags [`sc_cli::ExecutionStrategy::Native`] is already a noop and all +//! > chains built with Substrate only use WASM execution. +//! +//! ### Runtime Versions +//! +//! An important detail of the native execution worth learning about is that the node software, +//! obviously, only uses the native runtime if it is the same code as with the wasm blob stored +//! onchain. Else, nodes who run the native runtime will come to a different state transition. How +//! do nodes determine if two runtimes are the same? Through the very important +//! [`sp_version::RuntimeVersion`]. All runtimes expose their version via a runtime api +//! ([`sp_api::Core::version`]) that returns this struct. The node software, or other applications, +//! inspect this struct to examine the identity of a runtime, and to determine if two runtimes are +//! the same. Namely, [`sp_version::RuntimeVersion::spec_version`] is the main key that implies two +//! runtimes are the same. +//! +//! Therefore, it is utmost important to make sure before any runtime upgrade, the spec version is +//! updated. //! //! ## Example: Block Execution. //! -//! TODO +//! As a final example to recap, let's look at how Substrate-based nodes execute blocks. Blocks are +//! received in the node side software as opaque blobs and in the networking layer. +//! +//! At some point, based on the consensus algorithm's rules, the node decides to import (aka. +//! *validate*) a block. +//! +//! * First, the node will then fetch the state of the parent hash of the block that wishes to be +//! imported. +//! * The runtime is fetched from this state, and placed into a WASM execution environment. +//! * The [`sp_api::Core::execute_block`] runtime API is called and the blocked is passed in as an +//! argument. +//! * The runtime will then execute the block, and update the state accordingly. Any state update is +//! issues via the [`sp_io::storage`] host functions. +//! * Both the runtime and node will check the state-root of the state after the block execution to +//! match the one claimed in the block header. +//! +//! > Example taken from [this +//! > lecture](https://polkadot-blockchain-academy.github.io/pba-book/substrate/wasm/page.html#example-2-block-import-9) +//! > of the Polkadot Blockchain Academy. diff --git a/substrate/frame/support/src/traits/hooks.rs b/substrate/frame/support/src/traits/hooks.rs index ccccc50632866..1a687cade79f5 100644 --- a/substrate/frame/support/src/traits/hooks.rs +++ b/substrate/frame/support/src/traits/hooks.rs @@ -351,6 +351,7 @@ pub trait IntegrityTest { /// - [`crate::traits::misc::OffchainWorker`] /// - [`OnIdle`] /// - [`IntegrityTest`] +/// - [`OnPoll`] /// /// ## Ordering /// @@ -363,34 +364,32 @@ pub trait IntegrityTest { /// /// ```mermaid /// graph LR -/// Optional --> BeforeExtrinsics -/// BeforeExtrinsics --> Extrinsics -/// Extrinsics --> AfterExtrinsics -/// subgraph Optional +/// Optional --> Mandatory +/// Mandatory --> ExtrinsicsMandatory +/// ExtrinsicsMandatory --> Poll +/// Poll --> Extrinsics +/// Extrinsics --> AfterMandatory +/// AfterMandatory --> onIdle +/// +/// subgraph Optional /// OnRuntimeUpgrade /// end /// -/// subgraph BeforeExtrinsics +/// subgraph Mandatory /// OnInitialize /// end /// +/// subgraph ExtrinsicsMandatory +/// Inherent1 --> Inherent2 +/// end +/// /// subgraph Extrinsics /// direction TB -/// Inherent1 -/// Inherent2 -/// Extrinsic1 -/// Extrinsic2 -/// -/// Inherent1 --> Inherent2 -/// Inherent2 --> Extrinsic1 /// Extrinsic1 --> Extrinsic2 /// end /// -/// subgraph AfterExtrinsics -/// OnIdle +/// subgraph AfterMandatory /// OnFinalize -/// -/// OnIdle --> OnFinalize /// end /// ``` /// @@ -466,6 +465,8 @@ pub trait Hooks { /// /// Is not guaranteed to execute in a block and should therefore only be used in no-deadline /// scenarios. + /// + /// This is the non-mandatory version of [`Hooks::on_initialize`]. fn on_poll(_n: BlockNumber, _weight: &mut WeightMeter) {} /// Hook executed when a code change (aka. a "runtime upgrade") is detected by the FRAME diff --git a/substrate/primitives/io/Cargo.toml b/substrate/primitives/io/Cargo.toml index abb16d163da06..f6e157680f9c3 100644 --- a/substrate/primitives/io/Cargo.toml +++ b/substrate/primitives/io/Cargo.toml @@ -19,7 +19,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] bytes = { version = "1.1.0", default-features = false } -codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false, features = ["bytes"] } +codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false, features = [ + "bytes", +] } sp-core = { path = "../core", default-features = false } sp-crypto-hashing = { path = "../crypto/hashing", default-features = false } sp-keystore = { path = "../keystore", default-features = false, optional = true } @@ -31,13 +33,18 @@ sp-trie = { path = "../trie", default-features = false, optional = true } sp-externalities = { path = "../externalities", default-features = false } sp-tracing = { path = "../tracing", default-features = false } log = { optional = true, workspace = true, default-features = true } -secp256k1 = { version = "0.28.0", features = ["global-context", "recovery"], optional = true } +secp256k1 = { version = "0.28.0", features = [ + "global-context", + "recovery", +], optional = true } tracing = { version = "0.1.29", default-features = false } tracing-core = { version = "0.1.32", default-features = false } # Required for backwards compatibility reason, but only used for verifying when `UseDalekExt` is set. ed25519-dalek = { version = "2.1", default-features = false, optional = true } +docify = { version = "0.2.8" } + [target.'cfg(all(any(target_arch = "riscv32", target_arch = "riscv64"), substrate_runtime))'.dependencies] polkavm-derive = { workspace = true } diff --git a/substrate/primitives/io/src/lib.rs b/substrate/primitives/io/src/lib.rs index 8ef1f41ce0197..67e822ba7e248 100644 --- a/substrate/primitives/io/src/lib.rs +++ b/substrate/primitives/io/src/lib.rs @@ -1805,6 +1805,7 @@ pub type TestExternalities = sp_state_machine::TestExternalities>; + type FullBackend = sc_service::TFullBackend; type FullSelectChain = sc_consensus::LongestChain;